pax_global_header00006660000000000000000000000064152554333100014513gustar00rootroot0000000000000052 comment=c0af713163e78fe7d7717559fd4862adcb8162d7 tvondra-tdigest-c0af713/000077500000000000000000000000001525543331000152375ustar00rootroot00000000000000tvondra-tdigest-c0af713/.github/000077500000000000000000000000001525543331000165775ustar00rootroot00000000000000tvondra-tdigest-c0af713/.github/workflows/000077500000000000000000000000001525543331000206345ustar00rootroot00000000000000tvondra-tdigest-c0af713/.github/workflows/ci.yml000066400000000000000000000010031525543331000217440ustar00rootroot00000000000000name: make installcheck on: [push, pull_request] jobs: test: strategy: matrix: pg: [18, 17, 16, 15, 14, 13, 12, 11] name: PostgreSQL ${{ matrix.pg }} if: ${{ vars.CI_ENABLED == '1' }} runs-on: ubuntu-latest container: pgxn/pgxn-tools steps: - name: Start PostgreSQL ${{ matrix.pg }} run: pg-start ${{ matrix.pg }} - name: Check out the repo uses: actions/checkout@v6 - name: Test on PostgreSQL ${{ matrix.pg }} run: pg-build-test tvondra-tdigest-c0af713/.github/workflows/mirror.yml000066400000000000000000000016021525543331000226700ustar00rootroot00000000000000# Sync repo to the codeberg mirror name: Repo sync (GitHub -> Codeberg) on: push: branches: - '**' workflow_dispatch: # Manual dispatch schedule: - cron: "0 */6 * * *" jobs: sync-repo-to-codeberg: if: ${{ vars.CODEBERG_SYNC_ENABLED == '1' }} runs-on: ubuntu-latest steps: - name: "Checkout" uses: actions/checkout@v6 with: fetch-depth: 0 - name: "Gather facts" run: | echo "Repo variables:" echo " CODEBERG_URL: ${{ vars.CODEBERG_URL }}" echo " CODEBERG_USERNAME: ${{ vars.CODEBERG_USERNAME }}" - name: "Sync repo" uses: yesolutions/mirror-action@master with: REMOTE: "${{ vars.CODEBERG_URL }}" GIT_USERNAME: ${{ vars.CODEBERG_USERNAME }} GIT_SSH_PRIVATE_KEY: ${{ secrets.CODEBERG_SSH_KEY }} GIT_SSH_NO_VERIFY_HOST: "true" tvondra-tdigest-c0af713/.github/workflows/release.yml000066400000000000000000000026011525543331000227760ustar00rootroot00000000000000name: Release on: push: tags: [v*] permissions: contents: write jobs: release: name: Release on GitHub and PGXN if: ${{ vars.RELEASE_ENABLED == '1' }} runs-on: ubuntu-latest container: pgxn/pgxn-tools env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PGXN_USERNAME: ${{ secrets.PGXN_USERNAME }} PGXN_PASSWORD: ${{ secrets.PGXN_PASSWORD }} steps: - name: Check out the repo uses: actions/checkout@v6 - name: Install required packages run: | sudo apt-get update sudo apt-get install -y libpq-dev sudo apt-get install -y postgresql-server-dev-all sudo apt-get install -y postgresql-common - name: Bundle the Release id: bundle run: pgxn-bundle - name: Release on PGXN run: pgxn-release - name: Generate Release Changes run: make latest-changes.md - name: Create GitHub Release id: release uses: actions/create-release@v1 with: tag_name: ${{ github.ref }} release_name: Release ${{ github.ref }} body_path: latest-changes.md - name: Upload Release Asset uses: actions/upload-release-asset@v1 with: upload_url: ${{ steps.release.outputs.upload_url }} asset_path: ./${{ steps.bundle.outputs.bundle }} asset_name: ${{ steps.bundle.outputs.bundle }} asset_content_type: application/zip tvondra-tdigest-c0af713/.gitignore000066400000000000000000000002751525543331000172330ustar00rootroot00000000000000.deps/ results/ **/*.o **/*.so **/*.bc regression.diffs regression.out **/*.gcda **/*.gcno **/*.gcov **/*.gcov.out lcov_base.info lcov_test.info coverage/ coverage-html-stamp tdigest-*.zip tvondra-tdigest-c0af713/Changes000066400000000000000000000052131525543331000165330ustar00rootroot00000000000000Revision history for PostgreSQL extension tdigest. 1.4.7 - Keep percentile interpolation monotone for closely spaced means and ranks. - Use consistent inverse-percentile ranks at centroid boundaries with large counts. - Disable FP contraction for LLVM bitcode as well as for native code. - Reject text input that underflows to zero, while accepting nonzero subnormals. - Improve uninstall to cleanup this extension's control and SQL files. - Mark scalar functions operating on a tdigest value as parallel safe, to allow parallel queries in more cases. - Avoid division by zero when rebalancing and compacting digests (benign with IEEE-754). - Preserve reusable tdigest states during finalization (addresses `FINALFUNC_MODIFY = READ_ONLY`). - Improve error messages for arrays containing NULL values. - Fix duplicate keys in tdigest_to_json() output. - Reject NULL compression in the aggregate transition functions. - Reduce memory usage (grow/shrink aggregate state, eagerly free copies, read/write float8 arrays directly). - Corrections and clarifications of assorted comments and docs. - Various cleanups of stale boilerplate in the Makefile. 1.4.6 - Disable FP contraction to make tests pass on arm64 - Fix builds on 32-bit systems by defining strtoi64 - Fix validation of NULL / empty in several functions - Clamp results to valid range to handle rounding errors - Make possibly-long loops interruptible - Make centroid sort more resilient to invalid data 1.4.5 - Fix multiple memory-safety issues (OOB reads/writes, SIGSEGV crashes) - Fix multiple issues with float precision (NaN/infinity results) - Fix calculation of low/high thresholds in trimmed aggregates - Fix trimmed sum/avg aggregates to sort the centroids - Make sure processing does not generate digests with NaN/infinity float values - Make sure various int64 values do not overflow - Harden validation of function parameters to reject NULL/NaN/infinity values - Use overflow-resilient and/or numerically stable formulas in multiple functions - Improve test coverage by adding new regression tests - Speed up regression tests by using a parallel schedule - Add tdigest_is_valid() to check validity of existing t-digest values - Mitigate incorrect alignment of t-digest values. 1.4.4 - Add missing parts of automated release workflow. - Make regression tests pass on Postgres 19 branch - Fix various issues when parsing/validating input data - Fix combining t-digests with different parameters 1.4.3 - PGXN packaging (META etc.). - Automating releases to GitHub / PGXN tvondra-tdigest-c0af713/LICENSE000066400000000000000000000017011525543331000162430ustar00rootroot00000000000000Copyright (c) 2019, Tomas Vondra (tomas.vondra@postgresql.org). Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies. IN NO EVENT SHALL $ORGANISATION BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF TOMAS VONDRA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. TOMAS VONDRA SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND $ORGANISATION HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. tvondra-tdigest-c0af713/META.json000066400000000000000000000022071525543331000166610ustar00rootroot00000000000000{ "name": "t-digest", "abstract": "Aggregate for an on-line accumulation of rank-based statistics such as quantiles and trimmed means.", "description": "This PostgreSQL extension implements t-digest, a data structure for on-line accumulation of rank-based statistics such as quantiles and trimmed means. The algorithm is also very friendly to parallel programs.", "version": "1.4.7", "maintainer": [ "Tomas Vondra ", "Nils Dijk " ], "license": "bsd", "prereqs": { "runtime": { "requires": { "PostgreSQL": "13.0.0" } } }, "provides": { "tdigest": { "file": "tdigest--1.4.6--1.4.7.sql", "docfile" : "README.md", "version": "1.4.7" } }, "resources": { "repository": { "url": "https://github.com/tvondra/tdigest.git", "web": "http://github.com/tvondra/tdigest", "type": "git" } }, "tags" : ["quantile", "median", "mean", "aggregate"], "meta-spec": { "version": "1.0.0", "url": "http://pgxn.org/meta/spec.txt" }, "release_status" : "stable" } tvondra-tdigest-c0af713/Makefile000066400000000000000000000046251525543331000167060ustar00rootroot00000000000000# The version the distribution archive and the release notes are built for. # The extension name is not read from META.json - the name there is the PGXN # distribution name ("t-digest"), which is not the name of the extension. DISTVERSION = $(shell grep -m 1 '[[:space:]]\{3\}"version":' META.json | \ sed -e 's/[[:space:]]*"version":[[:space:]]*"\([^"]*\)",\{0,1\}/\1/') MODULE_big = tdigest OBJS = tdigest.o EXTENSION = tdigest DATA = tdigest--1.0.0.sql tdigest--1.0.0--1.0.1.sql tdigest--1.0.1--1.2.0.sql \ tdigest--1.2.0--1.3.0.sql tdigest--1.3.0--1.4.0.sql tdigest--1.4.0--1.4.1.sql \ tdigest--1.4.1--1.4.2.sql tdigest--1.4.2--1.4.3.sql tdigest--1.4.3--1.4.4.sql \ tdigest--1.4.4--1.4.5.sql tdigest--1.4.5--1.4.6.sql tdigest--1.4.6--1.4.7.sql REGRESS = --schedule=$(srcdir)/test/parallel_schedule REGRESS_OPTS = --inputdir=test PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) # Disable FP contraction, if supported, for native code and LLVM bitcode. # Append after PGXS so this overrides any earlier -ffp-contract option. FP_CONTRACT := $(shell $(CC) -ffp-contract=off -xc -E /dev/null > /dev/null 2>&1 \ && echo -ffp-contract=off) override CFLAGS += $(FP_CONTRACT) override BITCODE_CFLAGS += $(FP_CONTRACT) # The PGXS "uninstall" target only removes the files currently listed in DATA, # so scripts installed by an older build (e.g. upgrade scripts that were since # renamed or removed) would be left behind. Match the control file and versioned # SQL scripts, not just the name prefix, which other extensions may share. # # A recipe can't be appended to the PGXS "uninstall" rule, so this is hooked in # as a prerequisite (which means it runs before the PGXS part - that's fine, # the two steps are independent). uninstall: uninstall-stale .PHONY: uninstall-stale uninstall-stale: rm -f '$(DESTDIR)$(datadir)/extension/$(EXTENSION).control' \ '$(DESTDIR)$(datadir)/extension/$(EXTENSION)--'*.sql dist: git archive --format zip --prefix=$(EXTENSION)-$(DISTVERSION)/ -o $(EXTENSION)-$(DISTVERSION).zip HEAD # The output is tracked, so checkout timestamps do not establish whether its # contents match the release selected by META.json. .PHONY: latest-changes.md latest-changes.md: Changes META.json perl -e 'while (<>) {last if /^(v?\Q${DISTVERSION}\E)/; } print "Changes for v${DISTVERSION}:\n"; while (<>) { last if /^\s*$$/; s/^\s+//; print }' Changes > $@ tvondra-tdigest-c0af713/README.md000066400000000000000000001133441525543331000165240ustar00rootroot00000000000000# t-digest extension [![make installcheck](https://github.com/tvondra/tdigest/actions/workflows/ci.yml/badge.svg)](https://github.com/tvondra/tdigest/actions/workflows/ci.yml) This PostgreSQL extension implements t-digest, a data structure for on-line accumulation of rank-based statistics such as quantiles and trimmed means. The algorithm is also very friendly to parallel programs. The t-digest data structure was introduced by Ted Dunning in 2013, and a more detailed description and an example implementation are available in his GitHub repository [1]. In particular, see the paper [2] explaining the idea. Some of the code was inspired by tdigestc [3] and tdigest [4] by ajwerner. The accuracy of estimates produced by t-digests can be orders of magnitude more accurate than those produced by previous digest algorithms in spite of the fact that t-digests are much more compact when stored on disk. ## Basic usage For the basic use case the extension provides four aggregate functions. The `tdigest_percentile` ones can be seen as a replacement for the `percentile_cont` aggregate, while the `tdigest_percentile_of` ones perform the inverse operation, estimating the relative rank of a given value: * `tdigest_percentile(value double precision, compression int, quantile double precision)` * `tdigest_percentile(value double precision, compression int, quantiles double precision[])` * `tdigest_percentile_of(value double precision, compression int, hypothetical_value double precision)` * `tdigest_percentile_of(value double precision, compression int, hypothetical_values double precision[])` That is, instead of running ```sql SELECT percentile_cont(0.95) WITHIN GROUP (ORDER BY a) FROM t ``` you might now run ```sql SELECT tdigest_percentile(a, 100, 0.95) FROM t ``` and similarly for the variants with an array of percentiles. This should run much faster, as the t-digest does not require sorting all the data and can be parallelized. Also, the memory usage is very limited, depending on the compression parameter. ## Accuracy Functions building t-digests accept a `compression` parameter that controls the trade-off between accuracy, digest size, memory use and processing cost. Larger values generally retain more, smaller centroids. The accepted range is `[10, 10000]`; values outside this range are rejected with an error. Compression is neither the number of centroids nor an error bound. Accuracy depends on the data distribution, input order and history of merging digests. There is no general `1/N` error guarantee for a digest with `N` centroids, and compression 100 does not promise 1% error relative to the range of data values. Values such as 100, used in the examples, are starting points to evaluate against exact results on representative data. The algorithm allows smaller centroid weights near quantiles 0.0 and 1.0 than near the median. This concentrates resolution in the tails, but does not impose a fixed error bound in the units of the input values. ### Digest size and storage The centroid buffer holds at most `10 * compression` centroids, including uncompacted input. A compacted digest is usually much smaller. Each centroid stores an 8-byte `double precision` mean and an 8-byte integer count. With the 24-byte full header, a digest uses `24 + 16 * ncentroids` bytes before any TOAST processing. The largest permitted value therefore has 100000 centroids and occupies 1,600,024 bytes (about 1.53 MiB). The on-disk digests are typically much smaller than the centroid buffer, due to compaction which merges centroids depending on how close to the median of the dataset they lie. The type uses PostgreSQL's `EXTERNAL` storage policy by default. This allows large values to be stored out of line using TOAST, but does not compress them. A column can use `EXTENDED` storage to permit TOAST compression; changing that setting does not itself rewrite existing values. Tuple and TOAST overhead are additional to the size of the digest. Here is a table of sizes for digests with different compression values, built on random data: | compression | centroids | length (B) | external (B) | extended (B) | |--------------:|----------:|-----------:|-------------:|-------------:| | 10 | 18 | 305 | 309 | 308 | | 50 | 40 | 655 | 659 | 658 | | 100 | 61 | 993 | 997 | 997 | | 200 | 100 | 1616 | 1620 | 1624 | | 500 | 203 | 3275 | 3275 | 2238 | | 1000 | 357 | 5732 | 5732 | 3765 | | 2000 | 627 | 10058 | 10058 | 6432 | | 5000 | 1318 | 21113 | 21113 | 12646 | | 10000 | 2265 | 36260 | 36260 | 20177 | Where `centroids` is the number of centroids in a compacted digest, `length` is the "raw" size of the digest. `external` and `extended` are the on-disk sizes of centroid, depending on the storage policy set for the column. It's clear that `external` is almost the same as `length`, while `extended` is often much smaller thanks to compression. This is merely an example - the actual values depend on the data. For example digests on integer values tend to be much more compressible, cutting the `extended` size about in half. ## Advanced usage The extension also provides a `tdigest` data type, which makes it possible to precompute digests for subsets of data, and then quickly combine those "partial" digests into a digest representing the whole data set. The prebuilt digests should be much smaller compared to the original data set, allowing significantly faster response times. To compute a `t-digest`, use the `tdigest` aggregate function. The digests can then be stored on disk and later summarized using the `tdigest_percentile` functions (with `tdigest` as the first argument). * `tdigest(value double precision, compression int)` * `tdigest(digest tdigest)` * `tdigest_percentile(digest tdigest, quantile double precision)` * `tdigest_percentile(digest tdigest, quantiles double precision[])` * `tdigest_percentile_of(digest tdigest, hypothetical_value double precision)` * `tdigest_percentile_of(digest tdigest, hypothetical_values double precision[])` The `tdigest(digest tdigest)` variant is an aggregate merging multiple pre-computed digests into a single digest, which can be stored again. Digest-input aggregates accept digests with different compression settings. Each aggregate state takes its compression from its first non-`NULL` digest; with parallel aggregation, the final choice can depend on worker and combine order. Use a consistent compression across input digests when that choice matters. Merging cannot recover detail already lost by compaction. So for example you may do this: ```sql -- table with some random source data, with "a" usable as a count of -- occurrences (so it has to be positive) CREATE TABLE t (a int, b int, c double precision); INSERT INTO t SELECT 1 + 10 * random(), 10 * random(), random() FROM generate_series(1,10000000); -- table with pre-aggregated digests CREATE TABLE p AS SELECT a, b, tdigest(c, 100) AS d FROM t GROUP BY a, b; -- summarize the data from "p" (compute the 95th percentile) SELECT a, tdigest_percentile(d, 0.95) FROM p GROUP BY a ORDER BY a; ``` An example run produced a much smaller pre-aggregated table: ~~~ db=# \d+ List of relations Schema | Name | Type | Owner | Persistence | Size | Description --------+------+-------+-------+-------------+--------+------------- public | p | table | user | permanent | 120 kB | public | t | table | user | permanent | 422 MB | (2 rows) ~~~ On the same machine, the last query took about 1.5 ms. Compare that to the following example timings on the source data; sizes and timings will vary with the data, PostgreSQL version and hardware: ```sql \timing on -- exact results SELECT a, percentile_cont(0.95) WITHIN GROUP (ORDER BY c) FROM t GROUP BY a ORDER BY a; ... Time: 6956.566 ms (00:06.957) -- tdigest estimate (no parallelism) SET max_parallel_workers_per_gather = 0; SELECT a, tdigest_percentile(c, 100, 0.95) FROM t GROUP BY a ORDER BY a; ... Time: 2873.116 ms (00:02.873) -- tdigest estimate (4 workers) SET max_parallel_workers_per_gather = 4; SELECT a, tdigest_percentile(c, 100, 0.95) FROM t GROUP BY a ORDER BY a; ... Time: 893.538 ms ``` This illustrates how much faster the t-digest estimate can be than the exact query with `percentile_cont`. The difference can increase when sorting larger data sets requires spilling to disk. It also shows how effective the pre-aggregation can be. In this example, there are 121 rows in table `p`, so with 120kB disk space that's ~1kB per row, each representing about 80k values. With 8B per value, that's ~640kB, or a compression ratio of about 640:1. For a fixed number of groups and a fixed compression, digest storage is bounded while the raw data grows. ## Pre-aggregated data When dealing with data sets with a lot of redundancy (values repeating many times), it may be more efficient to partially pre-aggregate the data and use functions that allow specifying the number of occurrences for each value. This reduces the number of SQL-function calls. There are seven such aggregate functions: * `tdigest(value double precision, count bigint, compression int)` * `tdigest_percentile(value double precision, count bigint, compression int, quantile double precision)` * `tdigest_percentile(value double precision, count bigint, compression int, quantiles double precision[])` * `tdigest_percentile_of(value double precision, count bigint, compression int, hypothetical_value double precision)` * `tdigest_percentile_of(value double precision, count bigint, compression int, hypothetical_values double precision[])` * `tdigest_avg(value double precision, count bigint, compression int, low double precision, high double precision)` * `tdigest_sum(value double precision, count bigint, compression int, low double precision, high double precision)` A non-`NULL` `count` must be positive and determines how many times the value is added to the digest. A `NULL` count means one occurrence. The total count in a digest must fit in a `bigint`; exceeding 9223372036854775807 raises an error. See the "trimmed aggregates" section for the `low` and `high` parameters. ## Incremental updates An existing t-digest may be updated incrementally, either by adding a single value, or by merging-in a whole t-digest. The following examples use the table `p` with the pre-aggregated digests (in column `d`), built in [Advanced usage](#advanced-usage). Each example adds the same new values to every row of `p`; use a `WHERE` clause when updating only selected groups. For example, it's possible to add 1000 random values to the t-digests like this: ```sql DO LANGUAGE plpgsql $$ DECLARE r record; BEGIN FOR r IN (SELECT random() AS v FROM generate_series(1,1000)) LOOP UPDATE p SET d = tdigest_add(d, r.v); END LOOP; END $$; ``` The overhead of doing this is fairly high, though - the t-digest has to be deserialized and serialized over and over, for each value we're adding. That overhead may be reduced by pre-aggregating data, either into an array or a t-digest. ```sql DO LANGUAGE plpgsql $$ DECLARE vals double precision[]; BEGIN SELECT array_agg(random()) INTO vals FROM generate_series(1,1000); UPDATE p SET d = tdigest_add(d, vals); END $$; ``` Alternatively, it's possible to use pre-aggregated t-digest values instead of the arrays: ```sql WITH batch AS ( SELECT tdigest(random(), 100) AS d FROM generate_series(1,1000) ) UPDATE p SET d = tdigest_union(p.d, batch.d) FROM batch; ``` It may be undesirable to perform compaction after every incremental update, especially when adding values one by one. Setting `compact` to `false` skips compaction at the end of the call; compaction still occurs when adding to a full centroid buffer. The result may be unsorted and larger than a compacted digest, but remains subject to the `10 * compression` centroid limit. Use the multi-value functions with compaction after each batch when possible, or compact a stored digest by re-aggregating it: ```sql UPDATE p SET d = (SELECT tdigest(x) FROM (SELECT p.d) s(x)); ``` Adding a `NULL` value or a `NULL` array with `tdigest_add` returns the original digest unchanged. A non-`NULL` value or array with a `NULL` digest instead creates a new digest and requires a compression value. When either input to `tdigest_union` is `NULL`, it returns the other digest unchanged, without compaction; two `NULL` digests produce `NULL`. In all incremental functions, the `compact` flag itself must not be `NULL`, even for calls that otherwise do nothing. ## Trimmed aggregates The extension provides aggregate functions allowing to calculate trimmed (truncated) sum and average, either directly from the values or from a pre-computed digest: * `tdigest_sum(value double precision, compression int, low double precision, high double precision)` * `tdigest_sum(value double precision, count bigint, compression int, low double precision, high double precision)` * `tdigest_sum(digest tdigest, low double precision, high double precision)` * `tdigest_avg(value double precision, compression int, low double precision, high double precision)` * `tdigest_avg(value double precision, count bigint, compression int, low double precision, high double precision)` * `tdigest_avg(digest tdigest, low double precision, high double precision)` The `low` and `high` parameters specify where to truncate the data. They are percentiles (not values), so both have to be in `[0.0, 1.0]` with `low <= high`, otherwise an error is raised. For example `low = 0.1` and `high = 0.9` means the lowest and highest 10% of the values are discarded. There are also two non-aggregate functions, calculating the trimmed sum and average for a single `tdigest` value: * `tdigest_digest_sum(digest tdigest, low double precision DEFAULT 0.0, high double precision DEFAULT 1.0)` * `tdigest_digest_avg(digest tdigest, low double precision DEFAULT 0.0, high double precision DEFAULT 1.0)` The difference between `tdigest_sum(digest, low, high)` and `tdigest_digest_sum(digest, low, high)` is that the former is an aggregate (combining all the digests in a group first), while the latter is a plain function processing a single digest value (and thus may be combined with other columns without a `GROUP BY` clause). ## Functions The following list covers the aggregates and utility functions provided by this extension. Type I/O functions and internal aggregate support functions are not listed. The `accuracy` parameter in these descriptions is the compression used when building the t-digest, as described in the [Accuracy](#accuracy) section. The `tdigest`, `tdigest_percentile`, `tdigest_percentile_of`, `tdigest_avg` and `tdigest_sum` functions are aggregates (all of them parallel safe), while `tdigest_count`, `tdigest_add`, `tdigest_union`, `tdigest_json`, `tdigest_double_array`, `tdigest_digest_sum`, `tdigest_digest_avg` and `tdigest_is_valid` are plain functions operating on `tdigest` values. The examples use a table `t` with the values in column `c`, and - for the variants with a `count` parameter - the number of occurrences of each value in column `a`. Non-`NULL` counts must be positive; a `NULL` count means one occurrence. ### Common argument rules Aggregates ignore `NULL` input values or digests and return SQL `NULL` when there are no non-`NULL` inputs. This also applies to the array-returning aggregates: empty input produces `NULL`, not an empty array. Values added to a digest must be finite; `NaN` and positive or negative infinity are rejected. Compression, requested percentiles or hypothetical values, and trim thresholds must be non-`NULL` when the aggregate state is initialized. Keep these arguments constant within each group. The implementation captures them on the first non-`NULL` input of each state, rather than checking them on every row; later changes are ignored and can give order-dependent results, especially in parallel queries. The input value and its count may vary between rows. Requested percentiles must be in `[0, 1]`. Arrays of percentiles or hypothetical values must be nonempty, one-dimensional, and contain no `NULL` elements. Array results follow the order of the requested elements and have the usual lower bound of 1, regardless of the input array's lower bound. All `tdigest_percentile_of` variants estimate a smoothed relative rank, counting half of an equal-mean centroid group's weight at that mean. This is not an exact count of smaller values. Hypothetical values may be non-finite: `-Infinity`, `Infinity` and `NaN` return 0, 1 and `NaN`, respectively, when the aggregate has non-`NULL` input. The non-incremental scalar functions return `NULL` if any argument is `NULL`. The incremental functions have the initialization and no-op rules described in [Incremental updates](#incremental-updates). ### `tdigest_percentile(value, accuracy, percentile)` Computes a requested percentile from the data, using a t-digest with the specified accuracy. #### Synopsis ```sql SELECT tdigest_percentile(t.c, 100, 0.95) FROM t ``` #### Parameters - `value` - values to aggregate - `accuracy` - accuracy of the t-digest - `percentile` - value in [0, 1] specifying the percentile ### `tdigest_percentile(value, count, accuracy, percentile)` Computes a requested percentile from the data, using a t-digest with the specified accuracy. #### Synopsis ```sql SELECT tdigest_percentile(t.c, t.a, 100, 0.95) FROM t ``` #### Parameters - `value` - values to aggregate - `count` - number of occurrences of the value - `accuracy` - accuracy of the t-digest - `percentile` - value in [0, 1] specifying the percentile ### `tdigest_percentile(value, accuracy, percentile[])` Computes requested percentiles from the data, using a t-digest with the specified accuracy. #### Synopsis ```sql SELECT tdigest_percentile(t.c, 100, ARRAY[0.95, 0.99]) FROM t ``` #### Parameters - `value` - values to aggregate - `accuracy` - accuracy of the t-digest - `percentile[]` - array of values in [0, 1] specifying the percentiles ### `tdigest_percentile(value, count, accuracy, percentile[])` Computes requested percentiles from the data, using a t-digest with the specified accuracy. #### Synopsis ```sql SELECT tdigest_percentile(t.c, t.a, 100, ARRAY[0.95, 0.99]) FROM t ``` #### Parameters - `value` - values to aggregate - `count` - number of occurrences of the value - `accuracy` - accuracy of the t-digest - `percentile[]` - array of values in [0, 1] specifying the percentiles ### `tdigest_percentile_of(value, accuracy, hypothetical_value)` Computes relative rank of a hypothetical value, using a t-digest with the specified accuracy. #### Synopsis ```sql SELECT tdigest_percentile_of(t.c, 100, 139832.3) FROM t ``` #### Parameters - `value` - values to aggregate - `accuracy` - accuracy of the t-digest - `hypothetical_value` - hypothetical value ### `tdigest_percentile_of(value, count, accuracy, hypothetical_value)` Computes relative rank of a hypothetical value, using a t-digest with the specified accuracy. #### Synopsis ```sql SELECT tdigest_percentile_of(t.c, t.a, 100, 139832.3) FROM t ``` #### Parameters - `value` - values to aggregate - `count` - number of occurrences of the value - `accuracy` - accuracy of the t-digest - `hypothetical_value` - hypothetical value ### `tdigest_percentile_of(value, accuracy, hypothetical_value[])` Computes relative ranks of hypothetical values, using a t-digest with the specified accuracy. #### Synopsis ```sql SELECT tdigest_percentile_of(t.c, 100, ARRAY[6343.43, 139832.3]) FROM t ``` #### Parameters - `value` - values to aggregate - `accuracy` - accuracy of the t-digest - `hypothetical_value` - hypothetical values ### `tdigest_percentile_of(value, count, accuracy, hypothetical_value[])` Computes relative ranks of hypothetical values, using a t-digest with the specified accuracy. #### Synopsis ```sql SELECT tdigest_percentile_of(t.c, t.a, 100, ARRAY[6343.43, 139832.3]) FROM t ``` #### Parameters - `value` - values to aggregate - `count` - number of occurrences of the value - `accuracy` - accuracy of the t-digest - `hypothetical_value` - hypothetical values ### `tdigest(value, accuracy)` Computes t-digest with the specified accuracy. #### Synopsis ```sql SELECT tdigest(t.c, 100) FROM t ``` #### Parameters - `value` - values to aggregate - `accuracy` - accuracy of the t-digest ### `tdigest(value, count, accuracy)` Computes t-digest with the specified accuracy. The values are added with as many occurrences as determined by the count parameter. #### Synopsis ```sql SELECT tdigest(t.c, t.a, 100) FROM t ``` #### Parameters - `value` - values to aggregate - `count` - number of occurrences for each value - `accuracy` - accuracy of the t-digest ### `tdigest(digest)` Merges pre-computed t-digests into a single t-digest. This is also the way to force compaction of a digest built with `compact = false`. #### Synopsis ```sql SELECT tdigest(d) FROM ( SELECT tdigest(t.c, 100) AS d FROM t GROUP BY t.a ) foo ``` #### Parameters - `digest` - t-digests to merge ### `tdigest_count(tdigest)` Returns the number of items represented by the t-digest. This is a plain function, not an aggregate. #### Synopsis ```sql SELECT tdigest_count(d) FROM ( SELECT tdigest(t.c, 100) AS d FROM t ) foo ``` #### Parameters - `tdigest` - t-digest to inspect ### `tdigest_percentile(tdigest, percentile)` Computes requested percentile from the pre-computed t-digests. #### Synopsis ```sql SELECT tdigest_percentile(d, 0.99) FROM ( SELECT tdigest(t.c, 100) AS d FROM t ) foo ``` #### Parameters - `tdigest` - t-digest to aggregate and process - `percentile` - value in [0, 1] specifying the percentile ### `tdigest_percentile(tdigest, percentile[])` Computes requested percentiles from the pre-computed t-digests. #### Synopsis ```sql SELECT tdigest_percentile(d, ARRAY[0.95, 0.99]) FROM ( SELECT tdigest(t.c, 100) AS d FROM t ) foo ``` #### Parameters - `tdigest` - t-digest to aggregate and process - `percentile` - values in [0, 1] specifying the percentiles ### `tdigest_percentile_of(tdigest, hypothetical_value)` Estimates the relative rank of a hypothetical value using a pre-computed t-digest. At an exact centroid mean, half of the total weight of all centroids with that mean is counted. A digest containing only copies of one value therefore returns `0.5` at that value, not the fraction of rows strictly below it (`0.0`). This is a smoothed rank estimate, not an exact count of smaller values. #### Synopsis ```sql SELECT tdigest_percentile_of(d, 349834.1) FROM ( SELECT tdigest(t.c, 100) AS d FROM t ) foo ``` #### Parameters - `tdigest` - t-digest to aggregate and process - `hypothetical_value` - hypothetical value ### `tdigest_percentile_of(tdigest, hypothetical_value[])` Estimates relative ranks of hypothetical values using a pre-computed t-digest, with the same half-weight convention at centroid means as the scalar form. #### Synopsis ```sql SELECT tdigest_percentile_of(d, ARRAY[438.256, 349834.1]) FROM ( SELECT tdigest(t.c, 100) AS d FROM t ) foo ``` #### Parameters - `tdigest` - t-digest to aggregate and process - `hypothetical_value` - hypothetical values ### `tdigest_add(tdigest, double precision, compression = NULL, compact = true)` Performs incremental update of the t-digest by adding a single value. #### Synopsis ```sql UPDATE p SET d = tdigest_add(d, random()); ``` #### Parameters - `tdigest` - t-digest to update (may be `NULL`) - `element` - value to add; `NULL` leaves the digest unchanged - `compression` - required to initialize a digest from a non-`NULL` value; ignored for an existing digest (default: `NULL`) - `compact` - compact at the end of the call (default: true; must not be `NULL`) ### `tdigest_add(tdigest, double precision[], compression = NULL, compact = true)` Performs incremental update of the t-digest by adding values from an array. #### Synopsis ```sql UPDATE p SET d = tdigest_add(d, ARRAY[random(), random(), random()]); ``` #### Parameters - `tdigest` - t-digest to update (may be `NULL`) - `elements` - nonempty, one-dimensional array of non-`NULL` values; a `NULL` array leaves the digest unchanged - `compression` - required to initialize a digest from a non-`NULL` array; ignored for an existing digest (default: `NULL`) - `compact` - compact at the end of the call (default: true; must not be `NULL`) ### `tdigest_union(tdigest, tdigest, compact = true)` Performs incremental update of the t-digest by merging-in another digest. When either of the digests is `NULL`, the other one is returned unchanged (without compaction). When both are non-`NULL`, the result uses the compression of `digest1`, even if `digest2` has a different compression. #### Synopsis ```sql WITH x AS (SELECT tdigest(random(), 100) AS d FROM generate_series(1,1000)) UPDATE p SET d = tdigest_union(p.d, x.d) FROM x; ``` #### Parameters - `digest1` - t-digest to update - `digest2` - t-digest to merge into `digest1` - `compact` - compact at the end of the call (default: true; must not be `NULL`) ### `tdigest_json(tdigest)` Returns the t-digest as a JSON value. The function is also exposed as a cast from `tdigest` to `json`. The document has the flags, the total number of items (`count`), the compression and the number of centroids, followed by the per-centroid `means` and `counts` arrays. #### Synopsis ```sql SELECT tdigest_json(d) FROM ( SELECT tdigest(t.c, 100) AS d FROM t ) foo; SELECT CAST(d AS json) FROM ( SELECT tdigest(t.c, 100) AS d FROM t ) foo; ``` #### Parameters - `tdigest` - t-digest to cast to a `json` value ### `tdigest_double_array(tdigest)` Returns the t-digest as a `double precision[]` array. The function is also exposed as a cast from `tdigest` to `double precision[]`. The array contains the flags, the total number of items, the compression and the number of centroids, followed by a `(mean, count)` pair for each centroid. #### Synopsis ```sql SELECT tdigest_double_array(d) FROM ( SELECT tdigest(t.c, 100) AS d FROM t ) foo; SELECT CAST(d AS double precision[]) FROM ( SELECT tdigest(t.c, 100) AS d FROM t ) foo; ``` #### Parameters - `tdigest` - t-digest to cast to a `double precision[]` value ### `tdigest_avg(value, accuracy, low, high)` Computes trimmed mean of values, discarding values at the low and high end. The `low` and `high` values are percentiles in [0, 1] (with `low <= high`) specifying which part of the sample should be included in the mean, so e.g. `low = 0.1` and `high = 0.9` means 10% low and high values will be discarded. #### Synopsis ```sql SELECT tdigest_avg(t.c, 100, 0.1, 0.9) FROM t ``` #### Parameters - `value` - values to aggregate - `accuracy` - accuracy of the t-digest - `low` - low threshold percentile (values below are discarded) - `high` - high threshold percentile (values above are discarded) ### `tdigest_avg(value, count, accuracy, low, high)` Computes trimmed mean of values, discarding values at the low and high end. The `low` and `high` values are percentiles in [0, 1] (with `low <= high`) specifying which part of the sample should be included in the mean, so e.g. `low = 0.1` and `high = 0.9` means 10% low and high values will be discarded. #### Synopsis ```sql SELECT tdigest_avg(t.c, t.a, 100, 0.1, 0.9) FROM t ``` #### Parameters - `value` - values to aggregate - `count` - number of occurrences of the value - `accuracy` - accuracy of the t-digest - `low` - low threshold percentile (values below are discarded) - `high` - high threshold percentile (values above are discarded) ### `tdigest_avg(tdigest, low, high)` Computes trimmed mean of values, discarding values at the low and high end. The `low` and `high` values are percentiles in [0, 1] (with `low <= high`) specifying which part of the sample should be included in the mean, so e.g. `low = 0.1` and `high = 0.9` means 10% low and high values will be discarded. #### Synopsis ```sql SELECT tdigest_avg(d, 0.05, 0.95) FROM ( SELECT tdigest(t.c, 100) AS d FROM t ) foo; ``` #### Parameters - `tdigest` - tdigest to calculate mean from - `low` - low threshold percentile (values below are discarded) - `high` - high threshold percentile (values above are discarded) ### `tdigest_sum(value, accuracy, low, high)` Computes trimmed sum of values, discarding values at the low and high end. The `low` and `high` values are percentiles in [0, 1] (with `low <= high`) specifying which part of the sample should be included in the sum, so e.g. `low = 0.1` and `high = 0.9` means 10% low and high values will be discarded. #### Synopsis ```sql SELECT tdigest_sum(t.c, 100, 0.1, 0.9) FROM t ``` #### Parameters - `value` - values to aggregate - `accuracy` - accuracy of the t-digest - `low` - low threshold percentile (values below are discarded) - `high` - high threshold percentile (values above are discarded) ### `tdigest_sum(value, count, accuracy, low, high)` Computes trimmed sum of values, discarding values at the low and high end. The `low` and `high` values are percentiles in [0, 1] (with `low <= high`) specifying which part of the sample should be included in the sum, so e.g. `low = 0.1` and `high = 0.9` means 10% low and high values will be discarded. #### Synopsis ```sql SELECT tdigest_sum(t.c, t.a, 100, 0.1, 0.9) FROM t ``` #### Parameters - `value` - values to aggregate - `count` - number of occurrences of the value - `accuracy` - accuracy of the t-digest - `low` - low threshold percentile (values below are discarded) - `high` - high threshold percentile (values above are discarded) ### `tdigest_sum(tdigest, low, high)` Computes trimmed sum of values, discarding values at the low and high end. The `low` and `high` values are percentiles in [0, 1] (with `low <= high`) specifying which part of the sample should be included in the sum, so e.g. `low = 0.1` and `high = 0.9` means 10% low and high values will be discarded. #### Synopsis ```sql SELECT tdigest_sum(d, 0.05, 0.95) FROM ( SELECT tdigest(t.c, 100) AS d FROM t ) foo; ``` #### Parameters - `tdigest` - tdigest to calculate sum from - `low` - low threshold percentile (values below are discarded) - `high` - high threshold percentile (values above are discarded) ### `tdigest_digest_avg(tdigest, low, high)` Calculates trimmed mean for a single t-digest value. Unlike `tdigest_avg`, this is a plain function, not an aggregate. #### Synopsis ```sql SELECT tdigest_digest_avg(d, 0.25, 0.75) FROM ( SELECT tdigest(t.c, 100) AS d FROM t ) foo; ``` #### Parameters - `tdigest` - t-digest to calculate the mean for - `low` - low threshold percentile (values below are discarded, default: 0.0) - `high` - high threshold percentile (values above are discarded, default: 1.0) ### `tdigest_digest_sum(tdigest, low, high)` Calculates trimmed sum for a single t-digest value. Unlike `tdigest_sum`, this is a plain function, not an aggregate. #### Synopsis ```sql SELECT tdigest_digest_sum(d, 0.25, 0.75) FROM ( SELECT tdigest(t.c, 100) AS d FROM t ) foo; ``` #### Parameters - `tdigest` - t-digest to calculate the sum for - `low` - low threshold percentile (values below are discarded, default: 0.0) - `high` - high threshold percentile (values above are discarded, default: 1.0) ### `tdigest_is_valid(tdigest)` Checks whether the t-digest is valid, i.e. that it passes the same sanity checks as the input functions (parsing the text or binary representation). Returns `true` for valid digests, `false` otherwise. Digests produced by the extension are always valid, and it's not possible to construct an invalid one through the input functions. But digests stored by older versions of the extension (which did not have all the checks) may be broken in various ways, and the values are not re-validated when read back. This function makes it possible to find such digests. #### Synopsis ```sql SELECT a, b FROM p WHERE NOT tdigest_is_valid(p.d); ``` #### Parameters - `tdigest` - t-digest to check Notes ----- Input values and centroid means use `double precision`. PostgreSQL can convert other numeric types to it, as in the integer-valued examples, but those conversions can lose precision. The digest does not retain the native precision of `bigint` or `numeric` inputs. The estimates do depend on the order of incoming data, and so may differ between runs. This applies especially to parallel queries, for which the workers generally see different subsets of data for each run (and build different digests, which are then combined together). Security -------- If you believe you have found a security vulnerability in this repository, please report [this form](https://github.com/tvondra/tdigest/security/advisories/new) of this GitHub project. This creates a private communication channel between the reporter and the maintainers. If you are absolutely unable to or have strong reasons not to use GitHub's vulnerability reporting workflow, please reach out to the maintainer at [tomas@vondra.me](mailto:tomas@vondra.me). Notes: * The code assumes digests stored on-disk are valid and not corrupted. If the suspected vulnerability requires a corrupted digest, without a way to create such digests (using the current version), it's not a security issue. This is in line with general assumptions in the Postgres code. * A valid vulnerability must not require superuser privileges. A superuser can do almost anything (ultimately can read/write memory) and does not need to bother with vulnerabilities. Known issues ------------ ### incorrect alignment The SQL data type is defined without specifying the `ALIGNMENT` parameter, so it uses the default 4-byte alignment. Its C representation contains `double` and `int64` fields that can require 8-byte alignment. Accessing misaligned fields may incur a performance penalty on amd64/arm64 and can cause `SIGBUS` crashes on platforms with strict alignment requirements. The implementation handles this in `tdigest_detoast()` by making an aligned copy when necessary. Detoasting out-of-line values, compressed values or values with a short varlena header already produces an aligned allocation. Inline values with a 4-byte header need no copy during ordinary detoasting, so they may need the additional alignment copy. Whether a digest stays inline depends on its actual centroid count, the other columns in the tuple, and the column's storage settings. There is no compression-parameter threshold that guarantees out-of-line storage, and the default `EXTERNAL` policy does not permit TOAST compression. The extra copy requires an allocation and a single `memcpy()` of the digest. For small inline values, this overhead is usually modest. The SQL data type retains its original 4-byte alignment for compatibility with existing on-disk values. ### FINALFUNC_MODIFY = READ_ONLY The final functions mutate the aggregate state (they sort it, and most of them also compact it), which means `FINALFUNC_MODIFY` should not be `READ_ONLY`. It is, though, because that's what the aggregates were created with, and changing it would break upgrades of existing installations. Instead, each final function that would damage the state checks `AggStateIsShared()`, and works on a copy when the state may be needed again - that is, when the aggregate is used as a window function, or when several aggregates share a single transition state. So the results are correct in those cases, at the cost of copying the state. The two exceptions are the final functions of the trimmed `tdigest_sum()` and `tdigest_avg()` aggregates, which only sort the state. Sorting is just a permutation of the centroids, and the state is sorted anyway before it's used, so there's nothing to protect and no copy is made. ### fused multiply-add (FMA) Various places in the code use expressions of the form `a * b + c` (e.g. when calculating the mean of two merged centroids, or when interpolating between two centroid means). Compilers are allowed to contract such expressions into a single fused multiply-add (FMA) instruction, which rounds only once, and so produces slightly different results than a separate multiplication and addition. Whether that happens depends on the platform and on the compiler flags. FMA is part of the baseline instruction set on aarch64, so `gcc` contracts by default there (at `-O2` and higher - the contraction happens in a pass enabled only by `-O2`), while on x86-64 it does not, because FMA requires `-mfma` or a sufficiently recent `-march`. The results then differ in the last couple of digits, and the regression tests - which compare the exact float8 output - fail. For now, the `Makefile` builds with `-ffp-contract=off`, if the compiler understands the option, so that the results do not depend on which instructions happen to be available. The option is added to both `CFLAGS` and `BITCODE_CFLAGS`, because the LLVM bitcode used for JIT inlining is compiled separately and does not inherit `CFLAGS`. Compilers spelling the option differently (or not having it at all) may still produce digests that differ in the last digit or two. This is merely a workaround to make the tests pass. Ideally, we want to allow FMA, because it's expected to be faster and give more precise results (thanks to a single rounding). License ------- This software is distributed under the terms of the PostgreSQL license. See LICENSE or https://www.postgresql.org/about/licence/ for more details. [1] https://github.com/tdunning/t-digest [2] https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf [3] https://github.com/ajwerner/tdigestc [4] https://github.com/ajwerner/tdigest [5] https://github.com/tvondra/tdigest/security/advisories/new tvondra-tdigest-c0af713/TODO.md000066400000000000000000000021551525543331000163310ustar00rootroot00000000000000TODO ==== This is a simple list of possible improvements and enhancements, in mostly random order. So if you're thinking about contributing to the extension, this might provide inspiration. Of course, if you can think of yet another improvement, add it to this list. * Support other data types, not just "double precision". Supporting "numeric" seems natural, maybe we could support integer types too (possibly with rounding of the interpolated value). * Explore adding a "discrete" variant, similar to percentile_disc. I'm not sure this is actually possible, considering we're not keeping all the source data, forcing us to interpolate. * Maybe tdigest_compute_quantiles/tdigest_compute_quantiles_of should sum the counts of centroids matching the prev/next mean, and use that for the interpolation? See the comment at tdigest_compute_quantiles_of for more details. * When calculating trimmed aggregates, the code should interpolate between centroids just like quantiles/quantiles-of, instead of assuming all the items are located in the mean. See the comment at tdigest_trimmed_agg for additional details. tvondra-tdigest-c0af713/latest-changes.md000066400000000000000000000021241525543331000204620ustar00rootroot00000000000000Changes for v1.4.7: - Keep percentile interpolation monotone for closely spaced means and ranks. - Use consistent inverse-percentile ranks at centroid boundaries with large counts. - Disable FP contraction for LLVM bitcode as well as for native code. - Reject text input that underflows to zero, while accepting nonzero subnormals. - Improve uninstall to cleanup this extension's control and SQL files. - Mark scalar functions operating on a tdigest value as parallel safe, to allow parallel queries in more cases. - Avoid division by zero when rebalancing and compacting digests (benign with IEEE-754). - Preserve reusable tdigest states during finalization (addresses `FINALFUNC_MODIFY = READ_ONLY`). - Improve error messages for arrays containing NULL values. - Fix duplicate keys in tdigest_to_json() output. - Reject NULL compression in the aggregate transition functions. - Reduce memory usage (grow/shrink aggregate state, eagerly free copies, read/write float8 arrays directly). - Corrections and clarifications of assorted comments and docs. - Various cleanups of stale boilerplate in the Makefile. tvondra-tdigest-c0af713/scripts/000077500000000000000000000000001525543331000167265ustar00rootroot00000000000000tvondra-tdigest-c0af713/scripts/accuracy.sql000066400000000000000000000201661525543331000212460ustar00rootroot00000000000000drop table if exists t; create table t (v double precision); drop table if exists datasets; create table datasets (ds_name text, ds_sql text); insert into datasets values ('uniform', 'with d as (select pow(random(), 1) as v from generate_series(1,%s)) insert into t select v from (select v, generate_series(1, %s + (%s * random())::int) from d) foo'); insert into datasets values ('pow(2)', 'with d as (select pow(random(), 2) as v from generate_series(1,%s)) insert into t select v from (select v, generate_series(1, %s + (%s * random())::int) from d) foo'); insert into datasets values ('pow(4)', 'with d as (select pow(random(), 4) as v from generate_series(1,%s)) insert into t select v from (select v, generate_series(1, %s + (%s * random())::int) from d) foo'); insert into datasets values ('pow(0.5)', 'with d as (select pow(random(), 0.5) as v from generate_series(1,%s)) insert into t select v from (select v, generate_series(1, %s + (%s * random())::int) from d) foo'); insert into datasets values ('pow(0.25)', 'with d as (select pow(random(), 0.25) as v from generate_series(1,%s)) insert into t select v from (select v, generate_series(1, %s + (%s * random())::int) from d) foo'); insert into datasets values ('1 - pow(2)', 'with d as (select 1.0 - pow(random(), 2) as v from generate_series(1,%s)) insert into t select v from (select v, generate_series(1, %s + (%s * random())::int) from d) foo'); insert into datasets values ('1 - pow(4)', 'with d as (select 1.0 - pow(random(), 4) as v from generate_series(1,%s)) insert into t select v from (select v, generate_series(1, %s + (%s * random())::int) from d) foo'); insert into datasets values ('1 - pow(0.5)', 'with d as (select 1.0 - pow(random(), 0.5) as v from generate_series(1,%s)) insert into t select v from (select v, generate_series(1, %s + (%s * random())::int) from d) foo'); insert into datasets values ('1 - pow(0.25)', 'with d as (select 1.0 - pow(random(), 0.25) as v from generate_series(1,%s)) insert into t select v from (select v, generate_series(1, %s + (%s * random())::int) from d) foo'); create or replace function test_queries(npercentiles int, p double precision, nvalues int, minvalues int, maxvalues int, out dataset text, out simple_random double precision, out simple_asc double precision, out simple_desc double precision, out preagg_random double precision, out preagg_asc double precision, out preagg_desc double precision, out simple_asc_cmp double precision, out simple_desc_cmp double precision, out preagg_random_cmp double precision, out preagg_asc_cmp double precision, out preagg_desc_cmp double precision) returns setof record language plpgsql as $$ declare d record; perc_cont_percs double precision[]; simple_random_percs double precision[]; simple_asc_percs double precision[]; simple_desc_percs double precision[]; preagg_random_percs double precision[]; preagg_asc_percs double precision[]; preagg_desc_percs double precision[]; percs double precision[]; run int; tmp_simple_random double precision; tmp_simple_asc double precision; tmp_simple_desc double precision; tmp_preagg_random double precision; tmp_preagg_asc double precision; tmp_preagg_desc double precision; begin raise notice 'percentiles % range % values % min % max %', npercentiles, p, nvalues, minvalues, maxvalues; -- generate percentiles select array_agg(x) into percs from ( select i::double precision / npercentiles as x from generate_series(1,npercentiles) s(i) ) foo where x <= p or x > 1.0 - p; for d in (select * from datasets order by ds_name) loop simple_random := 0; simple_asc := 0; simple_desc := 0; preagg_random := 0; preagg_asc := 0; preagg_desc := 0; for run in 1..10 loop -- rebuild the table execute 'truncate t'; execute format(d.ds_sql, nvalues, minvalues, (maxvalues - minvalues)); execute 'analyze t'; dataset := d.ds_name; select percentile_cont(percs) within group (order by v) into perc_cont_percs from (select * from t) d; select tdigest_percentile(v, 100, percs) into simple_random_percs from (select * from t order by random()) d; select tdigest_percentile(v, 100, percs) into simple_asc_percs from (select * from t order by v) d; select tdigest_percentile(v, 100, percs) into simple_desc_percs from (select * from t order by v desc) d; select tdigest_percentile(v, c, 100, percs) into preagg_random_percs from (select v, count(*) as c from t group by v order by random()) d; select tdigest_percentile(v, c, 100, percs) into preagg_asc_percs from (select v, count(*) as c from t group by v order by v) d; select tdigest_percentile(v, c, 100, percs) into preagg_desc_percs from (select v, count(*) as c from t group by v order by v desc) d; select sqrt(sum(pow(a-b,2))) into tmp_simple_random from (select unnest(perc_cont_percs) as a, unnest(simple_random_percs) as b) d; select sqrt(sum(pow(a-b,2))) into tmp_simple_asc from (select unnest(perc_cont_percs) as a, unnest(simple_asc_percs) as b) d; select sqrt(sum(pow(a-b,2))) into tmp_simple_desc from (select unnest(perc_cont_percs) as a, unnest(simple_desc_percs) as b) d; select sqrt(sum(pow(a-b,2))) into tmp_preagg_random from (select unnest(perc_cont_percs) as a, unnest(preagg_random_percs) as b) d; select sqrt(sum(pow(a-b,2))) into tmp_preagg_asc from (select unnest(perc_cont_percs) as a, unnest(preagg_asc_percs) as b) d; select sqrt(sum(pow(a-b,2))) into tmp_preagg_desc from (select unnest(perc_cont_percs) as a, unnest(preagg_desc_percs) as b) d; simple_random := simple_random + tmp_simple_random / 10; simple_asc := simple_asc + tmp_simple_asc / 10; simple_desc := simple_desc + tmp_simple_desc / 10; preagg_random := preagg_random + tmp_preagg_random / 10; preagg_asc := preagg_asc + tmp_preagg_asc / 10; preagg_desc := preagg_desc + tmp_preagg_desc / 10; end loop; simple_asc_cmp := round((simple_asc / simple_random)::numeric, 2); simple_desc_cmp := round((simple_desc / simple_random)::numeric, 2); preagg_random_cmp := round((preagg_random / simple_random)::numeric, 2); preagg_asc_cmp := round((preagg_asc / simple_random)::numeric, 2); preagg_desc_cmp := round((preagg_desc / simple_random)::numeric, 2); simple_random := round(simple_random::numeric, 6); simple_asc := round(simple_asc::numeric, 6); simple_desc := round(simple_desc::numeric, 6); preagg_random := round(preagg_random::numeric, 6); preagg_asc := round(preagg_asc::numeric, 6); preagg_desc := round(preagg_desc::numeric, 6); return next; end loop; return; end; $$; select * from test_queries(1000, 0.01, 10000, 1, 1); select * from test_queries(1000, 0.05, 10000, 1, 1); select * from test_queries(1000, 0.1, 10000, 1, 1); select * from test_queries(1000, 0.2, 10000, 1, 1); select * from test_queries(1000, 0.3, 10000, 1, 1); select * from test_queries(1000, 0.4, 10000, 1, 1); select * from test_queries(1000, 0.5, 10000, 1, 1); select * from test_queries(1000, 0.01, 1000, 10, 20); select * from test_queries(1000, 0.05, 1000, 10, 20); select * from test_queries(1000, 0.1, 1000, 10, 20); select * from test_queries(1000, 0.2, 1000, 10, 20); select * from test_queries(1000, 0.3, 1000, 10, 20); select * from test_queries(1000, 0.4, 1000, 10, 20); select * from test_queries(1000, 0.5, 1000, 10, 20); select * from test_queries(1000, 0.01, 10000, 10, 20); select * from test_queries(1000, 0.05, 10000, 10, 20); select * from test_queries(1000, 0.1, 10000, 10, 20); select * from test_queries(1000, 0.2, 10000, 10, 20); select * from test_queries(1000, 0.3, 10000, 10, 20); select * from test_queries(1000, 0.4, 10000, 10, 20); select * from test_queries(1000, 0.5, 10000, 10, 20); tvondra-tdigest-c0af713/scripts/benchmark.sql000066400000000000000000000121051525543331000214000ustar00rootroot00000000000000drop table if exists t; create table t (v double precision); drop table if exists datasets; create table datasets (ds_name text, ds_sql text); insert into datasets values ('uniform', 'with d as (select pow(random(), 1) as v from generate_series(1,%s)) insert into t select v from (select v, generate_series(1, %s + (%s * random())::int) from d) foo'); insert into datasets values ('pow(2)', 'with d as (select pow(random(), 2) as v from generate_series(1,%s)) insert into t select v from (select v, generate_series(1, %s + (%s * random())::int) from d) foo'); insert into datasets values ('pow(4)', 'with d as (select pow(random(), 4) as v from generate_series(1,%s)) insert into t select v from (select v, generate_series(1, %s + (%s * random())::int) from d) foo'); insert into datasets values ('pow(0.5)', 'with d as (select pow(random(), 0.5) as v from generate_series(1,%s)) insert into t select v from (select v, generate_series(1, %s + (%s * random())::int) from d) foo'); insert into datasets values ('pow(0.25)', 'with d as (select pow(random(), 0.25) as v from generate_series(1,%s)) insert into t select v from (select v, generate_series(1, %s + (%s * random())::int) from d) foo'); insert into datasets values ('1 - pow(2)', 'with d as (select 1.0 - pow(random(), 2) as v from generate_series(1,%s)) insert into t select v from (select v, generate_series(1, %s + (%s * random())::int) from d) foo'); insert into datasets values ('1 - pow(4)', 'with d as (select 1.0 - pow(random(), 4) as v from generate_series(1,%s)) insert into t select v from (select v, generate_series(1, %s + (%s * random())::int) from d) foo'); insert into datasets values ('1 - pow(0.5)', 'with d as (select 1.0 - pow(random(), 0.5) as v from generate_series(1,%s)) insert into t select v from (select v, generate_series(1, %s + (%s * random())::int) from d) foo'); insert into datasets values ('1 - pow(0.25)', 'with d as (select 1.0 - pow(random(), 0.25) as v from generate_series(1,%s)) insert into t select v from (select v, generate_series(1, %s + (%s * random())::int) from d) foo'); create or replace function query_timing(query text, loops int = 10, out avg_time double precision, out stdev_time double precision) returns record language plpgsql as $$ declare timings double precision[] := NULL; i int; start_ts timestamptz; end_ts timestamptz; delta_ts double precision; total_ts double precision; r record; begin total_ts := 0; for i in 1..loops loop start_ts := clock_timestamp(); execute $1; end_ts := clock_timestamp(); delta_ts := 1000 * (extract(epoch from end_ts) - extract(epoch from start_ts)); timings := array_append(timings, delta_ts); total_ts := total_ts + delta_ts; end loop; avg_time := (total_ts / loops); stdev_time := 0.0; for r in select unnest(timings) as t loop stdev_time := stdev_time + pow(r.t - avg_time,2); end loop; stdev_time := sqrt(stdev_time / loops); avg_time := round(avg_time::numeric, 3); return; end; $$; create or replace function test_queries(nvalues int, minvalues int, maxvalues int, out dataset text, out simple_random double precision, out simple_asc double precision, out simple_desc double precision, out preagg_random double precision, out preagg_asc double precision, out preagg_desc double precision) returns setof record language plpgsql as $$ declare d record; begin raise notice 'values % min % max %', nvalues, minvalues, maxvalues; for d in (select * from datasets order by ds_name) loop -- rebuild the table execute 'truncate t'; execute format(d.ds_sql, nvalues, minvalues, (maxvalues - minvalues)); execute 'analyze t'; dataset := d.ds_name; select q.avg_time into simple_random from query_timing('select tdigest(v, 100) from (select * from t order by random()) d') q; select q.avg_time into simple_asc from query_timing('select tdigest(v, 100) from (select * from t order by v) d') q; select q.avg_time into simple_desc from query_timing('select tdigest(v, 100) from (select * from t order by v desc) d') q; select q.avg_time into preagg_random from query_timing('select tdigest(v, c, 100) from (select v, count(*) as c from t group by v order by random()) d') q; select q.avg_time into preagg_asc from query_timing('select tdigest(v, c, 100) from (select v, count(*) as c from t group by v order by v) d') q; select q.avg_time into preagg_desc from query_timing('select tdigest(v, c, 100) from (select v, count(*) as c from t group by v order by v desc) d') q; return next; end loop; return; end; $$; select * from test_queries(1000, 1, 1); select * from test_queries(10000, 1, 1); select * from test_queries(100000, 1, 1); select * from test_queries(1000, 5, 10); select * from test_queries(10000, 5, 10); select * from test_queries(100000, 5, 10); select * from test_queries(1000, 20, 40); select * from test_queries(10000, 20, 40); select * from test_queries(100000, 20, 40); tvondra-tdigest-c0af713/scripts/sizes.sql000066400000000000000000000077441525543331000206200ustar00rootroot00000000000000-- script to generate a table of digest sizes DROP EXTENSION IF EXISTS tdigest CASCADE; CREATE EXTENSION tdigest; CREATE CAST (tdigest AS bytea) WITHOUT FUNCTION; DROP TABLE IF EXISTS t_random; DROP TABLE IF EXISTS t_int; CREATE TABLE t_random (c int, d_external tdigest, d_extended tdigest); CREATE TABLE t_int (c int, d_external tdigest, d_extended tdigest); ALTER TABLE t_random ALTER COLUMN d_extended SET STORAGE extended; ALTER TABLE t_int ALTER COLUMN d_extended SET STORAGE extended; INSERT INTO t_int SELECT 10, d FROM (SELECT mod(i,10), tdigest(i, 10) d, NULL FROM generate_series(1, 1000000) s(i) GROUP BY mod(i, 10)); INSERT INTO t_int SELECT 50, d FROM (SELECT mod(i,10), tdigest(i, 50) d, NULL FROM generate_series(1, 1000000) s(i) GROUP BY mod(i, 10)); INSERT INTO t_int SELECT 100, d FROM (SELECT mod(i,10), tdigest(i, 100) d, NULL FROM generate_series(1, 1000000) s(i) GROUP BY mod(i, 10)); INSERT INTO t_int SELECT 200, d FROM (SELECT mod(i,10), tdigest(i, 200) d, NULL FROM generate_series(1, 1000000) s(i) GROUP BY mod(i, 10)); INSERT INTO t_int SELECT 500, d FROM (SELECT mod(i,10), tdigest(i, 500) d, NULL FROM generate_series(1, 1000000) s(i) GROUP BY mod(i, 10)); INSERT INTO t_int SELECT 1000, d FROM (SELECT mod(i,10), tdigest(i, 1000) d, NULL FROM generate_series(1, 1000000) s(i) GROUP BY mod(i, 10)); INSERT INTO t_int SELECT 2000, d FROM (SELECT mod(i,10), tdigest(i, 2000) d, NULL FROM generate_series(1, 1000000) s(i) GROUP BY mod(i, 10)); INSERT INTO t_int SELECT 5000, d FROM (SELECT mod(i,10), tdigest(i, 5000) d, NULL FROM generate_series(1, 1000000) s(i) GROUP BY mod(i, 10)); INSERT INTO t_int SELECT 10000, d FROM (SELECT mod(i,10), tdigest(i, 10000) d, NULL FROM generate_series(1, 1000000) s(i) GROUP BY mod(i, 10)); INSERT INTO t_random SELECT 10, d FROM (SELECT mod(i,10), tdigest(random(), 10) d, NULL FROM generate_series(1, 1000000) s(i) GROUP BY mod(i, 10)); INSERT INTO t_random SELECT 50, d FROM (SELECT mod(i,10), tdigest(random(), 50) d, NULL FROM generate_series(1, 1000000) s(i) GROUP BY mod(i, 10)); INSERT INTO t_random SELECT 100, d FROM (SELECT mod(i,10), tdigest(random(), 100) d, NULL FROM generate_series(1, 1000000) s(i) GROUP BY mod(i, 10)); INSERT INTO t_random SELECT 200, d FROM (SELECT mod(i,10), tdigest(random(), 200) d, NULL FROM generate_series(1, 1000000) s(i) GROUP BY mod(i, 10)); INSERT INTO t_random SELECT 500, d FROM (SELECT mod(i,10), tdigest(random(), 500) d, NULL FROM generate_series(1, 1000000) s(i) GROUP BY mod(i, 10)); INSERT INTO t_random SELECT 1000, d FROM (SELECT mod(i,10), tdigest(random(), 1000) d, NULL FROM generate_series(1, 1000000) s(i) GROUP BY mod(i, 10)); INSERT INTO t_random SELECT 2000, d FROM (SELECT mod(i,10), tdigest(random(), 2000) d, NULL FROM generate_series(1, 1000000) s(i) GROUP BY mod(i, 10)); INSERT INTO t_random SELECT 5000, d FROM (SELECT mod(i,10), tdigest(random(), 5000) d, NULL FROM generate_series(1, 1000000) s(i) GROUP BY mod(i, 10)); INSERT INTO t_random SELECT 10000, d FROM (SELECT mod(i,10), tdigest(random(), 10000) d, NULL FROM generate_series(1, 1000000) s(i) GROUP BY mod(i, 10)); UPDATE t_random SET d_extended = d_external; UPDATE t_int SET d_extended = d_external; SELECT c AS compression, AVG(tdigest_count(d_external))::int AS items, AVG(regexp_replace(d_external::text, '.*centroids ([0-9]*) .*', '\1')::int) ::int AS centroids, AVG(LENGTH(CAST(d_external AS bytea)))::int AS raw_length, AVG(pg_column_size(d_external))::int AS external_length, AVG(pg_column_size(d_extended))::int AS extended_length FROM t_int GROUP BY c ORDER BY c; SELECT c AS compression, AVG(tdigest_count(d_external))::int AS items, AVG(regexp_replace(d_external::text, '.*centroids ([0-9]*) .*', '\1')::int) ::int AS centroids, AVG(LENGTH(CAST(d_external AS bytea)))::int AS raw_length, AVG(pg_column_size(d_external))::int AS external_length, AVG(pg_column_size(d_extended))::int AS extended_length FROM t_random GROUP BY c ORDER BY c; tvondra-tdigest-c0af713/tdigest--1.0.0--1.0.1.sql000066400000000000000000000000001525543331000206300ustar00rootroot00000000000000tvondra-tdigest-c0af713/tdigest--1.0.0.sql000066400000000000000000000162721525543331000201420ustar00rootroot00000000000000/* tdigest for the double precision */ CREATE OR REPLACE FUNCTION tdigest_add_double(p_pointer internal, p_element double precision, p_compression int) RETURNS internal AS 'tdigest', 'tdigest_add_double' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_add_double(p_pointer internal, p_element double precision, p_compression int, p_quantile double precision) RETURNS internal AS 'tdigest', 'tdigest_add_double' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_add_double_array(p_pointer internal, p_element double precision, p_compression int, p_quantile double precision[]) RETURNS internal AS 'tdigest', 'tdigest_add_double_array' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_add_double_values(p_pointer internal, p_element double precision, p_compression int, p_value double precision) RETURNS internal AS 'tdigest', 'tdigest_add_double_values' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_add_double_array_values(p_pointer internal, p_element double precision, p_compression int, p_value double precision[]) RETURNS internal AS 'tdigest', 'tdigest_add_double_array_values' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_percentiles(p_pointer internal) RETURNS double precision AS 'tdigest', 'tdigest_percentiles' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_array_percentiles(p_pointer internal) RETURNS double precision[] AS 'tdigest', 'tdigest_array_percentiles' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_percentiles_of(p_pointer internal) RETURNS double precision AS 'tdigest', 'tdigest_percentiles_of' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_array_percentiles_of(p_pointer internal) RETURNS double precision[] AS 'tdigest', 'tdigest_array_percentiles_of' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_combine(a internal, b internal) RETURNS internal AS 'tdigest', 'tdigest_combine' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_serial(a internal) RETURNS bytea AS 'tdigest', 'tdigest_serial' LANGUAGE C IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION tdigest_deserial(a bytea, b internal) RETURNS internal AS 'tdigest', 'tdigest_deserial' LANGUAGE C IMMUTABLE STRICT; CREATE AGGREGATE tdigest_percentile(double precision, int, double precision) ( SFUNC = tdigest_add_double, STYPE = internal, FINALFUNC = tdigest_percentiles, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE AGGREGATE tdigest_percentile(double precision, int, double precision[]) ( SFUNC = tdigest_add_double_array, STYPE = internal, FINALFUNC = tdigest_array_percentiles, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE AGGREGATE tdigest_percentile_of(double precision, int, double precision) ( SFUNC = tdigest_add_double_values, STYPE = internal, FINALFUNC = tdigest_percentiles_of, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE AGGREGATE tdigest_percentile_of(double precision, int, double precision[]) ( SFUNC = tdigest_add_double_array_values, STYPE = internal, FINALFUNC = tdigest_array_percentiles_of, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE TYPE tdigest; CREATE OR REPLACE FUNCTION tdigest_in(cstring) RETURNS tdigest AS 'tdigest', 'tdigest_in' LANGUAGE C IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION tdigest_out(tdigest) RETURNS cstring AS 'tdigest', 'tdigest_out' LANGUAGE C IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION tdigest_send(tdigest) RETURNS bytea AS 'tdigest', 'tdigest_send' LANGUAGE C IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION tdigest_recv(internal) RETURNS tdigest AS 'tdigest', 'tdigest_recv' LANGUAGE C IMMUTABLE STRICT; CREATE TYPE tdigest ( INPUT = tdigest_in, OUTPUT = tdigest_out, RECEIVE = tdigest_recv, SEND = tdigest_send, INTERNALLENGTH = variable, STORAGE = external ); CREATE OR REPLACE FUNCTION tdigest_digest(p_pointer internal) RETURNS tdigest AS 'tdigest', 'tdigest_digest' LANGUAGE C IMMUTABLE; CREATE AGGREGATE tdigest(double precision, int) ( SFUNC = tdigest_add_double, STYPE = internal, FINALFUNC = tdigest_digest, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE OR REPLACE FUNCTION tdigest_add_digest(p_pointer internal, p_element tdigest) RETURNS internal AS 'tdigest', 'tdigest_add_digest' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_add_digest(p_pointer internal, p_element tdigest, p_quantile double precision) RETURNS internal AS 'tdigest', 'tdigest_add_digest' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_add_digest_array(p_pointer internal, p_element tdigest, p_quantile double precision[]) RETURNS internal AS 'tdigest', 'tdigest_add_digest_array' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_add_digest_values(p_pointer internal, p_element tdigest, p_value double precision) RETURNS internal AS 'tdigest', 'tdigest_add_digest_values' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_add_digest_array_values(p_pointer internal, p_element tdigest, p_value double precision[]) RETURNS internal AS 'tdigest', 'tdigest_add_digest_array_values' LANGUAGE C IMMUTABLE; CREATE AGGREGATE tdigest_percentile(tdigest, double precision) ( SFUNC = tdigest_add_digest, STYPE = internal, FINALFUNC = tdigest_percentiles, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE AGGREGATE tdigest_percentile(tdigest, double precision[]) ( SFUNC = tdigest_add_digest_array, STYPE = internal, FINALFUNC = tdigest_array_percentiles, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE AGGREGATE tdigest_percentile_of(tdigest, double precision) ( SFUNC = tdigest_add_digest_values, STYPE = internal, FINALFUNC = tdigest_percentiles_of, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE AGGREGATE tdigest_percentile_of(tdigest, double precision[]) ( SFUNC = tdigest_add_digest_array_values, STYPE = internal, FINALFUNC = tdigest_array_percentiles_of, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE AGGREGATE tdigest(tdigest) ( SFUNC = tdigest_add_digest, STYPE = internal, FINALFUNC = tdigest_digest, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE OR REPLACE FUNCTION tdigest_count(tdigest) RETURNS bigint AS 'tdigest', 'tdigest_count' LANGUAGE C IMMUTABLE STRICT; tvondra-tdigest-c0af713/tdigest--1.0.1--1.2.0.sql000066400000000000000000000067471525543331000206620ustar00rootroot00000000000000CREATE OR REPLACE FUNCTION tdigest_add_double_count(p_pointer internal, p_element double precision, p_count bigint, p_compression int) RETURNS internal AS 'tdigest', 'tdigest_add_double_count' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_add_double_count(p_pointer internal, p_element double precision, p_count bigint, p_compression int, p_quantile double precision) RETURNS internal AS 'tdigest', 'tdigest_add_double_count' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_add_double_array_count(p_pointer internal, p_element double precision, p_count bigint, p_compression int, p_quantile double precision[]) RETURNS internal AS 'tdigest', 'tdigest_add_double_array_count' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_add_double_values_count(p_pointer internal, p_element double precision, p_count bigint, p_compression int, p_value double precision) RETURNS internal AS 'tdigest', 'tdigest_add_double_values_count' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_add_double_array_values_count(p_pointer internal, p_element double precision, p_count bigint, p_compression int, p_value double precision[]) RETURNS internal AS 'tdigest', 'tdigest_add_double_array_values_count' LANGUAGE C IMMUTABLE; CREATE AGGREGATE tdigest(double precision, bigint, int) ( SFUNC = tdigest_add_double_count, STYPE = internal, FINALFUNC = tdigest_digest, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE AGGREGATE tdigest_percentile(double precision, bigint, int, double precision) ( SFUNC = tdigest_add_double_count, STYPE = internal, FINALFUNC = tdigest_percentiles, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE AGGREGATE tdigest_percentile(double precision, bigint, int, double precision[]) ( SFUNC = tdigest_add_double_array_count, STYPE = internal, FINALFUNC = tdigest_array_percentiles, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE AGGREGATE tdigest_percentile_of(double precision, bigint, int, double precision) ( SFUNC = tdigest_add_double_values_count, STYPE = internal, FINALFUNC = tdigest_percentiles_of, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE AGGREGATE tdigest_percentile_of(double precision, bigint, int, double precision[]) ( SFUNC = tdigest_add_double_array_values_count, STYPE = internal, FINALFUNC = tdigest_array_percentiles_of, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE OR REPLACE FUNCTION tdigest_add(p_digest tdigest, p_element double precision, p_compression int = NULL, p_compact bool = true) RETURNS tdigest AS 'tdigest', 'tdigest_add_double_increment' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_add(p_digest tdigest, p_elements double precision[], p_compression int = NULL, p_compact bool = true) RETURNS tdigest AS 'tdigest', 'tdigest_add_double_array_increment' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_union(p_digest1 tdigest, p_digest2 tdigest, p_compact bool = true) RETURNS tdigest AS 'tdigest', 'tdigest_union_double_increment' LANGUAGE C IMMUTABLE; tvondra-tdigest-c0af713/tdigest--1.2.0--1.3.0.sql000066400000000000000000000077231525543331000206570ustar00rootroot00000000000000CREATE FUNCTION tdigest_json(tdigest) RETURNS json AS 'tdigest', 'tdigest_to_json' LANGUAGE C IMMUTABLE STRICT; CREATE CAST (tdigest AS json) WITH FUNCTION tdigest_json(tdigest) AS ASSIGNMENT; CREATE FUNCTION tdigest_double_array(tdigest) RETURNS double precision[] AS 'tdigest', 'tdigest_to_array' LANGUAGE C IMMUTABLE STRICT; CREATE CAST (tdigest AS double precision[]) WITH FUNCTION tdigest_double_array(tdigest) AS ASSIGNMENT; -- trimmed aggregates CREATE OR REPLACE FUNCTION tdigest_add_double_trimmed(p_pointer internal, p_element double precision, p_compression int, p_low double precision, p_high double precision) RETURNS internal AS 'tdigest', 'tdigest_add_double_trimmed' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_add_double_count_trimmed(p_pointer internal, p_element double precision, p_count bigint, p_compression int, p_low double precision, p_high double precision) RETURNS internal AS 'tdigest', 'tdigest_add_double_count_trimmed' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_add_digest_trimmed(p_pointer internal, p_element tdigest, p_low double precision, p_high double precision) RETURNS internal AS 'tdigest', 'tdigest_add_digest_trimmed' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_trimmed_avg(p_pointer internal) RETURNS double precision AS 'tdigest', 'tdigest_trimmed_avg' LANGUAGE C IMMUTABLE; CREATE OR REPLACE FUNCTION tdigest_trimmed_sum(p_pointer internal) RETURNS double precision AS 'tdigest', 'tdigest_trimmed_sum' LANGUAGE C IMMUTABLE; CREATE AGGREGATE tdigest_avg(double precision, int, double precision, double precision) ( SFUNC = tdigest_add_double_trimmed, STYPE = internal, FINALFUNC = tdigest_trimmed_avg, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE AGGREGATE tdigest_avg(double precision, bigint, int, double precision, double precision) ( SFUNC = tdigest_add_double_count_trimmed, STYPE = internal, FINALFUNC = tdigest_trimmed_avg, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE AGGREGATE tdigest_avg(tdigest, double precision, double precision) ( SFUNC = tdigest_add_digest_trimmed, STYPE = internal, FINALFUNC = tdigest_trimmed_avg, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE AGGREGATE tdigest_sum(double precision, int, double precision, double precision) ( SFUNC = tdigest_add_double_trimmed, STYPE = internal, FINALFUNC = tdigest_trimmed_sum, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE AGGREGATE tdigest_sum(double precision, bigint, int, double precision, double precision) ( SFUNC = tdigest_add_double_count_trimmed, STYPE = internal, FINALFUNC = tdigest_trimmed_sum, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); CREATE AGGREGATE tdigest_sum(tdigest, double precision, double precision) ( SFUNC = tdigest_add_digest_trimmed, STYPE = internal, FINALFUNC = tdigest_trimmed_sum, SERIALFUNC = tdigest_serial, DESERIALFUNC = tdigest_deserial, COMBINEFUNC = tdigest_combine, PARALLEL = SAFE ); -- non-aggregate functions to extract trimmed sum/avg from a tdigest CREATE OR REPLACE FUNCTION tdigest_digest_sum(p_digest tdigest, p_low double precision = 0.0, p_high double precision = 1.0) RETURNS double precision AS 'tdigest', 'tdigest_digest_sum' LANGUAGE C IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION tdigest_digest_avg(p_digest tdigest, p_low double precision = 0.0, p_high double precision = 1.0) RETURNS double precision AS 'tdigest', 'tdigest_digest_avg' LANGUAGE C IMMUTABLE STRICT; tvondra-tdigest-c0af713/tdigest--1.3.0--1.4.0.sql000066400000000000000000000000001525543331000206360ustar00rootroot00000000000000tvondra-tdigest-c0af713/tdigest--1.4.0--1.4.1.sql000066400000000000000000000000001525543331000206400ustar00rootroot00000000000000tvondra-tdigest-c0af713/tdigest--1.4.1--1.4.2.sql000066400000000000000000000000001525543331000206420ustar00rootroot00000000000000tvondra-tdigest-c0af713/tdigest--1.4.2--1.4.3.sql000066400000000000000000000000001525543331000206440ustar00rootroot00000000000000tvondra-tdigest-c0af713/tdigest--1.4.3--1.4.4.sql000066400000000000000000000000001525543331000206460ustar00rootroot00000000000000tvondra-tdigest-c0af713/tdigest--1.4.4--1.4.5.sql000066400000000000000000000002141525543331000206570ustar00rootroot00000000000000CREATE OR REPLACE FUNCTION tdigest_is_valid(tdigest) RETURNS bool AS 'tdigest', 'tdigest_is_valid' LANGUAGE C IMMUTABLE STRICT; tvondra-tdigest-c0af713/tdigest--1.4.5--1.4.6.sql000066400000000000000000000000001525543331000206520ustar00rootroot00000000000000tvondra-tdigest-c0af713/tdigest--1.4.6--1.4.7.sql000066400000000000000000000030431525543331000206660ustar00rootroot00000000000000-- The plain functions operating on a tdigest value are all pure computations -- on their arguments, with no access to any shared state, so they are safe to -- run in a parallel worker. None of them were ever marked as such, which -- silently disabled parallelism for any query using them - a single unsafe -- function makes the whole plan serial, including the tdigest() aggregate -- building the digest they consume. ALTER FUNCTION tdigest_add(tdigest, double precision, int, bool) PARALLEL SAFE; ALTER FUNCTION tdigest_add(tdigest, double precision[], int, bool) PARALLEL SAFE; ALTER FUNCTION tdigest_union(tdigest, tdigest, bool) PARALLEL SAFE; ALTER FUNCTION tdigest_count(tdigest) PARALLEL SAFE; ALTER FUNCTION tdigest_json(tdigest) PARALLEL SAFE; ALTER FUNCTION tdigest_double_array(tdigest) PARALLEL SAFE; ALTER FUNCTION tdigest_is_valid(tdigest) PARALLEL SAFE; -- The type input/output functions are pure serialization routines, and every -- one of the built-in type I/O functions in PostgreSQL is marked parallel safe. -- These were not, which made any query merely casting a tdigest to text run -- serially. ALTER FUNCTION tdigest_in(cstring) PARALLEL SAFE; ALTER FUNCTION tdigest_out(tdigest) PARALLEL SAFE; ALTER FUNCTION tdigest_recv(internal) PARALLEL SAFE; ALTER FUNCTION tdigest_send(tdigest) PARALLEL SAFE; -- Mark the last two scalar functions as parallel safe. ALTER FUNCTION tdigest_digest_avg(tdigest, double precision, double precision) PARALLEL SAFE; ALTER FUNCTION tdigest_digest_sum(tdigest, double precision, double precision) PARALLEL SAFE; tvondra-tdigest-c0af713/tdigest.c000066400000000000000000004231531525543331000170560ustar00rootroot00000000000000/* * tdigest.c - implementation of t-digest for PostgreSQL, useful for estimation * of quantiles, percentiles, trimmed means, and various similar metrics. * * Copyright (C) Tomas Vondra, 2019 */ #include #include #include #include #include #include #include #include #include "postgres.h" #include "common/int.h" #include "libpq/pqformat.h" #include "miscadmin.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/memutils.h" #if PG_VERSION_NUM >= 120000 #include "utils/float.h" /* float8out_internal */ #endif #include "catalog/pg_type.h" PG_MODULE_MAGIC; /* * A centroid, used both for in-memory and on-disk storage. */ typedef struct centroid_t { double mean; int64 count; } centroid_t; /* * On-disk representation of the t-digest. */ typedef struct tdigest_t { int32 vl_len_; /* varlena header (do not touch directly!) */ int32 flags; /* on-disk format flags */ int64 count; /* number of items added to the t-digest */ int compression; /* compression used to build the digest */ int ncentroids; /* number of centroids in the array */ centroid_t centroids[FLEXIBLE_ARRAY_MEMBER]; } tdigest_t; /* * Centroids used to store (sum,count), but we want to store (mean,count) * because that allows us to prevent rounding errors e.g. when merging * centroids with the same mean, or adding the same value to the centroid. * * To handle existing tdigest data in backwards-compatible way, we have * a flag marking the new ones with mean, and we convert the old values. */ #define TDIGEST_STORES_MEAN 0x0001 /* All valid flags, OR-ed. */ #define TDIGEST_VALID_FLAGS (TDIGEST_STORES_MEAN) /* * An aggregate state, representing the t-digest and some additional info * (requested percentiles, ...). * * When adding new values to the t-digest, we add them as centroids into a * separate "uncompacted" part of the array. While centroids need more space * than plain points (16B vs. 8B), making the aggregate state quite a bit * larger, it does simplify the code quite a bit as it only needs to deal * with a single struct type instead of two (centroids + points). But maybe * we should separate those two things in the future. * * XXX We only ever use one of values/percentiles, never both at the same * time. In the future the values may use a different data type than double * (e.g. numeric), so we keep both fields. */ typedef struct tdigest_aggstate_t { /* basic t-digest fields (centroids at the end) */ int64 count; /* number of samples in the digest */ int ncompactions; /* number of merges/compactions */ int compression; /* compression parameter */ int maxcentroids; /* capacity of the centroids buffer */ int ncentroids; /* number of centroids */ int ncompacted; /* compacted part */ /* array of requested percentiles and values */ int npercentiles; /* number of percentiles */ int nvalues; /* number of values */ double trim_low; /* low threshold (for trimmed aggs) */ double trim_high; /* high threshold (for trimmed aggs) */ double *percentiles; /* array of percentiles (if any) */ double *values; /* array of values (if any) */ centroid_t *centroids; /* centroids for the digest */ } tdigest_aggstate_t; static int centroid_cmp(const void *a, const void *b); /* * Detoast a tdigest, making sure the result is properly aligned. * * The tdigest type is int4-aligned - that's the default for CREATE TYPE, and * changing it now would break on-disk compatibility, as typalign determines * where the value is placed in a tuple. But tdigest_t contains int64 and * double fields, which can require 8B alignment. Reading digest->count or * digest->centroids[i].mean from a value that is merely 4B aligned is * undefined behavior - it happens to work on x86-64, but it's a crash on * platforms not allowing unaligned access, and UBSan complains about it. * * Detoasting out-of-line, compressed and short-header values already makes * a freshly palloc-ed (and thus properly aligned) copy. But an inline value * with a 4B varlena header may still point into the tuple, so we have to make * an aligned copy ourselves if necessary. */ static tdigest_t * tdigest_detoast(Datum datum) { tdigest_t *digest = (tdigest_t *) PG_DETOAST_DATUM(datum); /* * Detoasting always produces a 4B header, so VARSIZE is fine here, and * reading it only needs the 4B alignment the type already guarantees. */ if (!PointerIsAligned(digest, int64)) { tdigest_t *aligned = (tdigest_t *) palloc(VARSIZE(digest)); memcpy(aligned, digest, VARSIZE(digest)); digest = aligned; } return digest; } #define PG_GETARG_TDIGEST(x) tdigest_detoast(PG_GETARG_DATUM(x)) /* * Size of buffer for incoming data, as a multiple of the compression value. * Quoting from the t-digest paper: * * The constant of proportionality should be determined by experiment, but * micro-benchmarks indicate that C2/C1 is in the range from 5 to 20 for * a single core of an Intel i7 processor. In these micro-benchmarks, * increasing the buffer size to (10 * delta) dramatically improves the * average speed but further buffer size increases have much less effect. * * XXX Maybe make the coefficient user-defined, with some reasonable limits * (say 2 - 20), so that users can pick the right trade-off between speed * and memory usage. */ #define BUFFER_SIZE(compression) (10 * (compression)) #define BUFFER_INITIAL_SIZE 16 #define AssertBounds(index, length) Assert((index) >= 0 && (index) < (length)) #define MIN_COMPRESSION 10 #define MAX_COMPRESSION 10000 /* prototypes */ PG_FUNCTION_INFO_V1(tdigest_add_double_array); PG_FUNCTION_INFO_V1(tdigest_add_double_array_count); PG_FUNCTION_INFO_V1(tdigest_add_double_array_values); PG_FUNCTION_INFO_V1(tdigest_add_double_array_values_count); PG_FUNCTION_INFO_V1(tdigest_add_double); PG_FUNCTION_INFO_V1(tdigest_add_double_count); PG_FUNCTION_INFO_V1(tdigest_add_double_values); PG_FUNCTION_INFO_V1(tdigest_add_double_values_count); PG_FUNCTION_INFO_V1(tdigest_add_digest_array); PG_FUNCTION_INFO_V1(tdigest_add_digest_array_values); PG_FUNCTION_INFO_V1(tdigest_add_digest); PG_FUNCTION_INFO_V1(tdigest_add_digest_values); PG_FUNCTION_INFO_V1(tdigest_array_percentiles); PG_FUNCTION_INFO_V1(tdigest_array_percentiles_of); PG_FUNCTION_INFO_V1(tdigest_percentiles); PG_FUNCTION_INFO_V1(tdigest_percentiles_of); PG_FUNCTION_INFO_V1(tdigest_digest); PG_FUNCTION_INFO_V1(tdigest_serial); PG_FUNCTION_INFO_V1(tdigest_deserial); PG_FUNCTION_INFO_V1(tdigest_combine); PG_FUNCTION_INFO_V1(tdigest_in); PG_FUNCTION_INFO_V1(tdigest_out); PG_FUNCTION_INFO_V1(tdigest_send); PG_FUNCTION_INFO_V1(tdigest_recv); PG_FUNCTION_INFO_V1(tdigest_is_valid); PG_FUNCTION_INFO_V1(tdigest_count); PG_FUNCTION_INFO_V1(tdigest_to_json); PG_FUNCTION_INFO_V1(tdigest_to_array); PG_FUNCTION_INFO_V1(tdigest_add_double_increment); PG_FUNCTION_INFO_V1(tdigest_add_double_array_increment); PG_FUNCTION_INFO_V1(tdigest_union_double_increment); PG_FUNCTION_INFO_V1(tdigest_add_double_trimmed); PG_FUNCTION_INFO_V1(tdigest_add_double_count_trimmed); PG_FUNCTION_INFO_V1(tdigest_add_digest_trimmed); PG_FUNCTION_INFO_V1(tdigest_trimmed_avg); PG_FUNCTION_INFO_V1(tdigest_trimmed_sum); PG_FUNCTION_INFO_V1(tdigest_digest_sum); PG_FUNCTION_INFO_V1(tdigest_digest_avg); Datum tdigest_add_double_array(PG_FUNCTION_ARGS); Datum tdigest_add_double_array_count(PG_FUNCTION_ARGS); Datum tdigest_add_double_array_values(PG_FUNCTION_ARGS); Datum tdigest_add_double_array_values_count(PG_FUNCTION_ARGS); Datum tdigest_add_double(PG_FUNCTION_ARGS); Datum tdigest_add_double_count(PG_FUNCTION_ARGS); Datum tdigest_add_double_values(PG_FUNCTION_ARGS); Datum tdigest_add_double_values_count(PG_FUNCTION_ARGS); Datum tdigest_add_digest_array(PG_FUNCTION_ARGS); Datum tdigest_add_digest_array_values(PG_FUNCTION_ARGS); Datum tdigest_add_digest(PG_FUNCTION_ARGS); Datum tdigest_add_digest_values(PG_FUNCTION_ARGS); Datum tdigest_array_percentiles(PG_FUNCTION_ARGS); Datum tdigest_array_percentiles_of(PG_FUNCTION_ARGS); Datum tdigest_percentiles(PG_FUNCTION_ARGS); Datum tdigest_percentiles_of(PG_FUNCTION_ARGS); Datum tdigest_digest(PG_FUNCTION_ARGS); Datum tdigest_serial(PG_FUNCTION_ARGS); Datum tdigest_deserial(PG_FUNCTION_ARGS); Datum tdigest_combine(PG_FUNCTION_ARGS); Datum tdigest_in(PG_FUNCTION_ARGS); Datum tdigest_out(PG_FUNCTION_ARGS); Datum tdigest_send(PG_FUNCTION_ARGS); Datum tdigest_recv(PG_FUNCTION_ARGS); Datum tdigest_is_valid(PG_FUNCTION_ARGS); Datum tdigest_count(PG_FUNCTION_ARGS); Datum tdigest_add_double_increment(PG_FUNCTION_ARGS); Datum tdigest_add_double_array_increment(PG_FUNCTION_ARGS); Datum tdigest_union_double_increment(PG_FUNCTION_ARGS); Datum tdigest_to_json(PG_FUNCTION_ARGS); Datum tdigest_to_array(PG_FUNCTION_ARGS); Datum tdigest_add_double_trimmed(PG_FUNCTION_ARGS); Datum tdigest_add_double_count_trimmed(PG_FUNCTION_ARGS); Datum tdigest_add_digest_trimmed(PG_FUNCTION_ARGS); Datum tdigest_trimmed_avg(PG_FUNCTION_ARGS); Datum tdigest_trimmed_sum(PG_FUNCTION_ARGS); Datum tdigest_digest_sum(PG_FUNCTION_ARGS); Datum tdigest_digest_avg(PG_FUNCTION_ARGS); static ArrayType *double_array_allocate(int nitems); static const double *array_to_double(ArrayType *v, const char *what, int *len); static int64 double_to_int64(double value, int64 maxvalue); static tdigest_aggstate_t *tdigest_copy(tdigest_aggstate_t *state); static void tdigest_aggstate_enlarge(tdigest_aggstate_t *state); static void tdigest_aggstate_shrink(tdigest_aggstate_t *state); #if PG_VERSION_NUM < 150000 /* * Thin wrappers that convert strings to exactly 64-bit integers, matching our * definition of int64. (For the naming, compare that POSIX has * strtoimax()/strtoumax() which return intmax_t/uintmax_t.) * * XXX Backward compatibility */ #ifdef HAVE_LONG_INT_64 /* int64 is "long int", so strtol() returns exactly the right width */ #define strtoi64(str, endptr, base) ((int64) strtol(str, endptr, base)) #else /* int64 is "long long int" (C99 guarantees it is at least 64 bits) */ #define strtoi64(str, endptr, base) ((int64) strtoll(str, endptr, base)) #endif #endif /* PG_VERSION_NUM < 150000 */ /* basic checks on the t-digest (proper sum of counts, ...) */ static void AssertCheckTDigest(tdigest_t *digest) { #ifdef USE_ASSERT_CHECKING int i; int64 cnt; Assert(digest->flags == 0 || digest->flags == TDIGEST_STORES_MEAN); Assert((digest->compression >= MIN_COMPRESSION) && (digest->compression <= MAX_COMPRESSION)); Assert(digest->count >= 0); Assert(digest->ncentroids >= 0); Assert(digest->ncentroids <= BUFFER_SIZE(digest->compression)); cnt = 0; for (i = 0; i < digest->ncentroids; i++) { Assert(digest->centroids[i].count > 0); Assert(isfinite(digest->centroids[i].mean)); cnt += digest->centroids[i].count; /* FIXME also check this does work with the scale function */ } Assert(VARSIZE_ANY(digest) == offsetof(tdigest_t, centroids) + digest->ncentroids * sizeof(centroid_t)); Assert(digest->count == cnt); #endif } static void AssertCheckTDigestAggState(tdigest_aggstate_t *state) { #ifdef USE_ASSERT_CHECKING int i; int64 cnt; Assert(state->npercentiles >= 0); Assert(((state->npercentiles == 0) && (state->percentiles == NULL)) || ((state->npercentiles > 0) && (state->percentiles != NULL))); for (i = 0; i < state->npercentiles; i++) Assert((state->percentiles[i] >= 0.0) && (state->percentiles[i] <= 1.0)); Assert((state->compression >= MIN_COMPRESSION) && (state->compression <= MAX_COMPRESSION)); Assert(state->count >= 0); Assert(state->ncentroids >= 0); Assert(state->ncentroids <= state->maxcentroids); Assert(state->maxcentroids <= BUFFER_SIZE(state->compression)); Assert(state->ncentroids <= BUFFER_SIZE(state->compression)); cnt = 0; for (i = 0; i < state->ncentroids; i++) { Assert(state->centroids[i].count > 0); Assert(isfinite(state->centroids[i].mean)); cnt += state->centroids[i].count; /* XXX maybe check this does work with the scale function */ } Assert(state->count == cnt); #endif } static void reverse_centroids(centroid_t *centroids, int ncentroids) { int start = 0, end = (ncentroids - 1); while (start < end) { centroid_t tmp = centroids[start]; centroids[start] = centroids[end]; centroids[end] = tmp; start++; end--; } } static void rebalance_centroids(centroid_t *centroids, int ncentroids, int64 weight_before, int64 weight_after) { double ratio = weight_before / (double) weight_after; int64 count_before = 0; int64 count_after = 0; int start = 0; int end = (ncentroids - 1); int i; centroid_t *scratch = palloc(sizeof(centroid_t) * ncentroids); Assert(weight_after > 0); i = 0; while (i < ncentroids) { while (i < ncentroids) { scratch[start] = centroids[i]; count_before += centroids[i].count; i++; start++; if (count_before > count_after * ratio) break; } while (i < ncentroids) { scratch[end] = centroids[i]; count_after += centroids[i].count; i++; end--; if (count_before < count_after * ratio) break; } } memcpy(centroids, scratch, sizeof(centroid_t) * ncentroids); pfree(scratch); } /* * Sort an array of centroids, with the given total count. * * We have to sort the whole array, because we don't just simply sort the * centroids - we do the rebalancing of items with the same mean too. */ static void tdigest_sort_centroids(centroid_t *centroids, int ncentroids, int64 count) { int i; int64 count_so_far; int64 next_group; int64 median_count; /* sort the whole array before rebalancing equal-mean groups */ pg_qsort(centroids, ncentroids, sizeof(centroid_t), centroid_cmp); /* * The centroids are sorted by (mean,count). That's fine for centroids up * to median, but above median this ordering is incorrect for centroids * with the same mean (or for groups crossing the median boundary). To fix * this we 'rebalance' those groups. Those entirely above median can be * simply sorted in the opposite order, while those crossing the median * need to be rebalanced depending on what part is below/above median. */ count_so_far = 0; next_group = 0; /* includes count_so_far */ median_count = (count / 2); /* * Split the centroids into groups with the same mean, process each group * depending on whether it falls before/after median. */ i = 0; while (i < ncentroids) { int j = i; int group_size = 0; CHECK_FOR_INTERRUPTS(); /* * Consume the first centroid of the group unconditionally. It is what * defines the group, so comparing it against itself decides nothing, * and for a NaN mean the comparison would be false, leaving "j" at "i" * and the outer loop without any way to advance. This way guarantees * forward progress. */ next_group += centroids[j].count; group_size++; j++; /* determine the end of the group */ while ((j < ncentroids) && (centroids[i].mean == centroids[j].mean)) { next_group += centroids[j].count; group_size++; j++; } /* * We can ignore groups of size 1 (number of centroids, not counts), as * those are trivially sorted. */ if (group_size > 1) { if (count_so_far >= median_count) { /* group fully above median - reverse the order */ reverse_centroids(¢roids[i], group_size); } else if (next_group > median_count) /* group split by median */ { rebalance_centroids(¢roids[i], group_size, median_count - count_so_far, next_group - median_count); } } /* * We should be making forward progress. If not, we're in an infinite * loop. With properly formed digests that should not happen. */ Assert(i < j); i = j; count_so_far = next_group; } } /* * Sort centroids in the aggregate state. */ static void tdigest_sort(tdigest_aggstate_t *state) { tdigest_sort_centroids(state->centroids, state->ncentroids, state->count); } /* * Perform compaction of the t-digest, i.e. merge the centroids as required * by the compression parameter. * * We always keep the data sorted in ascending order. This way we can reuse * the sort between compactions, and also when computing the quantiles. * * The regular compaction is not guaranteed to make any progress. The size * limits are calculated in double, and may end up too low to allow merging any * centroids. The limits are computed from exact integer remainders, which * makes this very unlikely, but if a compaction still leaves the buffer full, * we raise an error rather than continue with a digest that has no room left. * * XXX Switch the direction regularly, to eliminate possible bias and improve * accuracy, as mentioned in the paper. * * XXX This initially used the k1 scale function, but the implementation was * not limiting the number of centroids for some reason (it might have been * a bug in the implementation, of course). The current code is a modified * copy from ajwerner [1], and AFAIK it's the k2 function, it's much simpler * and generally works quite nicely. * * [1] https://github.com/ajwerner/tdigestc/blob/master/go/tdigest.c */ static void tdigest_compact(tdigest_aggstate_t *state) { int i; int cur; /* current centroid */ int64 count_so_far; int64 total_count; double denom; double normalizer; int start; int step; int n; AssertCheckTDigestAggState(state); /* if the digest is fully compacted, it's been already compacted */ if (state->ncompacted == state->ncentroids) return; /* * If there's just a single centroid, there's nothing to compact (or * sort). And we'd also end up with a division by zero below, because * log(1) = 0. It'd work out in the end, because 1/0 = infinity, and * so we'd merge nothing. But it's sloppy. * * XXX We're checking ncentroids, while the log() is on total_count. * But that's fine. Compaction/sort is pointless no matter how large * the single centroid is. And with 2+ centroids, the total_count has * to be 2+ too. */ if (state->ncentroids == 1) { state->ncompacted = state->ncentroids; return; } tdigest_sort(state); state->ncompactions++; if (state->ncompactions % 2 == 0) { start = 0; step = 1; } else { start = state->ncentroids - 1; step = -1; } total_count = state->count; denom = 2 * M_PI * total_count * log(total_count); normalizer = state->compression / denom; cur = start; count_so_far = 0; n = 1; for (i = start + step; (i >= 0) && (i < state->ncentroids); i += step) { int64 proposed_count; double q0; double q2; double z; bool should_add; CHECK_FOR_INTERRUPTS(); proposed_count = state->centroids[cur].count + state->centroids[i].count; z = proposed_count * normalizer; q0 = count_so_far / (double) total_count; q2 = (count_so_far + proposed_count) / (double) total_count; /* * Calculate the (1 - q) factors from the exact integer remainders, * instead of subtracting the quotients from 1. * * The two are equivalent with exact arithmetic, but not in double. If * a single centroid holds almost the whole weight of the digest, the * quotient rounds to exactly 1.0, and (1 - q) cancels to exactly 0. * The size limit then says no two centroids may be merged, even * though the exact limit is small but positive - and the compaction * ends up not making any progress at all. */ should_add = (z <= (q0 * ((double) (total_count - count_so_far) / (double) total_count))) && (z <= (q2 * ((double) (total_count - count_so_far - proposed_count) / (double) total_count))); if (should_add) { /* * If both centroids have the same mean, don't calculate it again. * The recalculation may cause rounding errors, so that the means * would drift apart over time. We want to keep them equal for as * long as possible. */ if (state->centroids[cur].mean != state->centroids[i].mean) { double mean; int64 count; /* count can't overflow int64 (total is within INT64_MAX) */ count = state->centroids[i].count; count += state->centroids[cur].count; /* calculate the mean in a way that should not overflow */ mean = state->centroids[i].mean * (state->centroids[i].count / (double) count); mean += state->centroids[cur].mean * (state->centroids[cur].count / (double) count); /* should not happen for finite inputs */ Assert(!isnan(mean)); /* * paranoia: clamp to not underflow/overflow the inputs * * We may be walking the centroids forward or backwards, which * determines whether (cur < i) or (cur > i). */ if (step > 0) { Assert(cur < i); mean = Min(Max(state->centroids[cur].mean, mean), state->centroids[i].mean); } else { Assert(cur > i); mean = Min(Max(state->centroids[i].mean, mean), state->centroids[cur].mean); } state->centroids[cur].mean = mean; } /* XXX Do this after possibly recalculating the mean. */ state->centroids[cur].count += state->centroids[i].count; } else { count_so_far += state->centroids[cur].count; cur += step; n++; state->centroids[cur] = state->centroids[i]; } if (cur != i) { state->centroids[i].count = 0; state->centroids[i].mean = 0; } } state->ncentroids = n; state->ncompacted = state->ncentroids; if (step < 0) memmove(state->centroids, &state->centroids[cur], n * sizeof(centroid_t)); /* * The compaction should have reduced the number of digest centroids. If * that did not succeed, and the buffer is still full, something must have * gone wrong. It's not safe to proceed. * * XXX Could be useful to include the digest through errdetail(), but with * large digests it might generate a significant log volume. */ if (state->ncentroids == BUFFER_SIZE(state->compression)) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("digest compaction failed to reduce number of centroids"), errhint("This should be impossible. Please report this to maintainers " "of the extension (ideally with a reproducer)."))); /* Maybe reclaim some of the centroid buffer. */ tdigest_aggstate_shrink(state); AssertCheckTDigestAggState(state); /* Must have freed some space in the buffer. */ Assert(state->ncentroids < BUFFER_SIZE(state->compression)); } /* * Interpolate between ordered finite endpoints without losing monotonicity. * * For endpoints of the same sign, separately rounded weighted products can * make (1 - q) * a + q * b decrease as q increases. Interpolating the * difference avoids that. * * Anchor negative endpoints at b to avoid subtracting nearly equal large * magnitudes when q is close to 1. That's necessary because of widely * separated negative means, like centroids at -2^54 and -1. Subtracting * almost equal large magnitudes exposes the rounding error in (b - a). * The right-anchored formula (b - (1 - q) * (b - a)) solves that. It's * monotone and does not have the cancellation issue. * * Positive intervals keep the existing left-anchored formula. * * For opposite signs, use the overflow-safe weighted sum. The difference may * overflow, but both weighted terms are nondecreasing, so use that. * * Return the endpoints explicitly when q is zero or one. A rounded difference * may not reconstruct either endpoint, and clamping cannot repair a result * that remains inside the bracket. */ static double tdigest_interpolate(double a, double b, double q) { double result; Assert(isfinite(a) && isfinite(b) && a <= b); Assert(q >= 0.0 && q <= 1.0); /* Rounded differences need not reproduce the endpoints exactly. */ if (q == 0.0) return a; if (q == 1.0) return b; /* * There are two approaches to calculating linear interpolation: * * (1 - q) * a + q * b and a + q * (b - a) * * The first formula is monotonic only when (a * b < 0). We know that * (a <= b), so this means (a < 0) and (b > 0). We use it also when * either value is 0. * * The second formula is monotonic, but does not guarantee the result * to be "b" when "q = 1", due to floating-point arithmetic errors. It * also has a risk of cancellation when subtracting almost equal large * magnitudes. To deal with that, we use a variation anchored to the * right endpoint. * * We switch between the options, to pick the better one. */ if (a <= 0.0 && b >= 0.0) result = (1 - q) * a + q * b; else if (b < 0.0) /* anchored to right endpoint */ result = b - (1 - q) * (b - a); else /* anchored to left endpoint */ result = a + q * (b - a); /* Final rounding must not move the result outside the endpoints. */ return Max(a, Min(b, result)); } /* * Estimate requested quantiles from the t-digest agg state. */ static void tdigest_compute_quantiles(tdigest_aggstate_t *state, double *result) { int i, j; AssertCheckTDigestAggState(state); /* * Trigger a compaction, which also sorts the data. * * XXX It might seem we can just do a sort here, and maybe get a bit more * accurate results. However, sorting single-item centroids in between much * larger centroids seems to interfere with the interpolation, making the * errors much larger. */ tdigest_compact(state); /* * Determine the two centroids the quantile lies between, and calculate the * estimate using linear interpolation. * * XXX All of this works fine for t-digests with non-extreme counts, up to * about 2^52. At that point the double precision ULP gets > 1.0, and some * of the calculations here start misbehaving a little. For example the * (count * 0.9999...) can get higher than count, etc. We try to prevent * obviously bogus results, but it's futile to try to fix this perfectly. * The cases are extremely rare, and we're calculating estimates anyway. * If we wanted to fix this properly, we'd need to use some sort of large * float data type (there seems to be "long double" and binary128). * * XXX The rounding/precision issues affect only accuracy of results, not * correctness of the code. For example, it must not result in OOB access * to bogus centroids etc. */ for (i = 0; i < state->npercentiles; i++) { int64 count; double goal = (state->count * state->percentiles[i]); bool is_before = false; centroid_t *c = NULL, *prev, *next; /* integer and fractional parts of half-centroids before/after */ double distance, total_distance, q; /* first centroid for percentile 0.0 */ if (state->percentiles[i] == 0.0) { c = &state->centroids[0]; result[i] = c->mean; continue; } /* last centroid for percentile 1.0 */ if (state->percentiles[i] == 1.0) { c = &state->centroids[state->ncentroids - 1]; result[i] = c->mean; continue; } /* * Walk the centroids and calculate running sum of counts. Stop before * adding a centroid that would exceed the goal - we don't know if the * goal falls before/after the mean yet. * * FIXME There can be multiple centroids with the same mean, in which * case we should use the total count for all of them. Not sure how * likely it's to have centroids with exactly the same mean. But it * might affect the interpolation later. */ count = 0; for (j = 0; j < state->ncentroids; j++) { CHECK_FOR_INTERRUPTS(); c = &state->centroids[j]; /* Adding the centroid would exceed the goal, so stop. */ if (count + c->count >= goal) break; count += c->count; } /* * Adding the whole centroid would exceed the goal, but we don't know * on which side of the mean the value lies yet. We might have also * hit the mean exactly. Let's figure that out. * * This will determine which centroids we'll look at for linear * interpolation (previous/following one) later. * * We know centroid "c" exceeds the goal, but did we hit the mean, * or are we to the left/right? We assume half the items are before * the mean, half after. */ is_before = goal < (count + c->count / 2.0); /* * Pick centroids for linear interpolation, depending on which side * of the "current" centroid we fell on. Either use the previous or * the following centroid. * * For extreme percentile values (or somehow weird digests) we can * end up before/after the last centroid, in which case we need to * be careful to not access OOB. */ if (is_before) { /* no previous centroid, use the current (first) one */ if (j == 0) { result[i] = c->mean; continue; } prev = &state->centroids[j - 1]; next = &state->centroids[j]; Assert(next == c); /* * Undo the centroid already added above (count is integer, * so we can't undo half of it without possibly losing half * of the count). We'll deal with that later. */ count -= prev->count; } else { /* no following centroid, use the current (last) one */ if (j == (state->ncentroids - 1)) { result[i] = c->mean; continue; } prev = &state->centroids[j]; next = &state->centroids[j + 1]; Assert(prev == c); } /* paranoia: make sure the prev/next centroids are valid */ Assert(prev >= &state->centroids[0]); Assert(next <= &state->centroids[state->ncentroids - 1]); Assert((prev + 1) == next); /* * Now we know the quantile lies somewhere between the centroids, * we need to calculate the correct value. (We know it's not at * either mean, that's what the above branches are for.) * * We will calculate the distance from the first mean, the total * distance between the means. And we'll do linear interpolation. */ /* distance to the first mean (of the previous centroid) */ distance = (double) (goal - count) - prev->count / 2.0; /* distance between the means of the two centroids */ total_distance = (prev->count / 2.0) + (next->count / 2.0); /* * We should be "to the right" of the first centroid, and should not * be so far ahead to exceed the next one. So in principle, this * should be true: * * Assert((distance >= 0) && (distance <= total_distance)); * * But, it's tricky due to precision and rounding. We're switching * from int64 to double, and double has much lower precision close * to INT64_MAX (ULP >> 1.0). With high goal and/or count values we * can end up with distance outside the [0, total_distance] range, * or just hit the centroids exactly. * * XXX Try uncommenting the assert, there's a test triggering it. */ /* * Clamp distance to [0, total_distance], to mitigate unexpected * rounding / precision errors. */ distance = Max(0.0, Min(total_distance, distance)); /* Keep the interpolation fraction within the centroid bracket. */ q = Max(0.0, Min(1.0, distance / total_distance)); result[i] = tdigest_interpolate(prev->mean, next->mean, q); } } /* * Calculate the rank at the midpoint of a centroid or equal-mean group. * * Form twice the midpoint exactly before converting to double. Rounding the * prefix and half-weight separately can reverse the order of nearby ranks. * Since count_before + count_at <= total_count <= INT64_MAX, the doubled * midpoint fits in uint64, even when it would overflow int64. */ static double tdigest_compute_rank(int64 count_before, int64 count_at, int64 total_count) { uint64 count_twice; double rank; Assert(count_at > 0 && count_at <= total_count); Assert(count_before >= 0 && count_before <= total_count - count_at); count_twice = 2 * (uint64) count_before + (uint64) count_at; rank = ((double) count_twice / 2.0) / (double) total_count; return Max(0.0, Min(1.0, rank)); } /* * Estimate inverse quantiles for values using a t-digest agg state. * * Essentially an inverse to tdigest_compute_quantiles. * * XXX I suspect this may not be doing the interpolation quite right when * there are multiple centroids with the same mean (compute_quantiles has * essentially the same issue in the opposite direction, but it's easier * to describe it for rank). The current code walks centroids until we find * a centroid with a higher mean - let's assume the value is between two * centroids (not equal to any centroid mean). It takes the previous and * following centroids, and interpolates those. * * But there can be multiple centroids with the same mean and different * counts. If the value falls before the median, we could see e.g. this: * * (1.0, 1000), (2.0, 1), (2.0, 999) * * Both means have the same weight, but the interprolation will use only * the (1.0, 1000) and (2.0, 1) centroids, i.e. 2.0 will have much less * weight. And then at 2.0 the CDF jumps up suddenly. * * This may not be very common, due to merging/compaction. We usually merge * the small centroids into the large ones quickly, so the differences * tend to be less extreme, which limits the effect on interpolation. Also, * the merging does floating point math with rounding errors, so means get * to diverge at some point. But it can happen e.g. when merging digests. * * There's also a bit of inconsistency (both here and in the reference * implementation), because we sum all the counts for the case when the * value matches a mean exactly, but not in the other cases. */ static void tdigest_compute_quantiles_of(tdigest_aggstate_t *state, double *result) { int i; AssertCheckTDigestAggState(state); /* * Trigger a compaction, which also sorts the data. * * XXX It might seem we can just do a sort here, and maybe get a bit more * accurate results. However, sorting single-item centroids in between much * larger centroids seems to interfere with the interpolation, making the * errors much larger. */ tdigest_compact(state); for (i = 0; i < state->nvalues; i++) { int j; int64 count; double value = state->values[i]; double d, q, q1, q2; /* next and previous centroids */ centroid_t *curr = NULL; centroid_t *prev = NULL; /* handle infinity/NaN values by mapping them to 0.0, 1.0 and NaN */ if (!isfinite(value)) { if (isnan(value)) result[i] = NAN; else if (value < 0) /* -infinity */ result[i] = 0.0; else /* infinity */ result[i] = 1.0; continue; } /* * Find the first centroid with (mean >= value), and remember the * last centroid before that - if the value is in between, we will * be calculating the percentile by linear approximation. */ count = 0; for (j = 0; j < state->ncentroids; j++) { CHECK_FOR_INTERRUPTS(); /* remember the previous centroid, grab the next one */ prev = curr; curr = &state->centroids[j]; if (curr->mean >= value) break; count += curr->count; } /* * If the value exactly matches the mean of the current centroid, * we're almost there. There may be multiple centroids with the same * mean, so we just need to advance past those. */ if (value == curr->mean) { int64 count_at_value = 0; /* * There may be multiple centroids with this mean (i.e. containing * this value), so find all of them and sum their weights. */ while ((j < state->ncentroids) && (state->centroids[j].mean == value)) { count_at_value += state->centroids[j].count; j++; } result[i] = tdigest_compute_rank(count, count_at_value, state->count); /* the next centroid has a higher mean, so we're done */ continue; } /* * If (value > curr->mean), it means we went through all centroids * without finding one with a larger mean. So the value is above * all centroids, and so it's 1.0 percentile. */ if (value > curr->mean) /* past the largest centroid */ { result[i] = 1; continue; } /* * It's also possible even the first centroid has a higher mean, in * which case the value is 0.0 percentile. */ if (prev == NULL) /* before the smallest centroid */ { result[i] = 0; continue; } /* we have two distinct centroids */ Assert((prev != NULL) && (curr != NULL) && (prev != curr)); Assert(prev->mean < curr->mean); /* * The value lies somewhere between two centroids. We want to figure out * where along the line from the prev node to this node the value is. * * FIXME What if there are multiple centroids with the same mean as the * prev/curr centroid? This probably needs to look up all of them and sum * their counts, just like we did in case of the exact mean equality, no? * Both for the current and previous centroids, so that the approximation * works well. */ d = (curr->mean - prev->mean); /* Use the same exact midpoint arithmetic as the exact-mean branch. */ q1 = tdigest_compute_rank(count - prev->count, prev->count, state->count); q2 = tdigest_compute_rank(count, curr->count, state->count); Assert(q1 <= q2); /* * Calculate the linear interpolation of q1/q2 percentiles. * * We need to be careful about infinity/NaN during calculation. The * means may be so close to +/- DBL_MAX that "d" becomes infinite. * That's equivalent to 0 slope, but we can do a bit better - if this * happens, we halve the values, which makes the difference finite * again (in exchange for loss of precision, but that's acceptable). */ if (isfinite(d)) q = (value - prev->mean) / d; else { /* trick - halve the means, so the difference can't overflow */ q = (value / 2.0 - prev->mean / 2.0) / (curr->mean / 2.0 - prev->mean / 2.0); } result[i] = tdigest_interpolate(q1, q2, q); } } /* * Make sure the aggregate state has space for more centroids. Double the * capacity up to the allowed maximum determined by the compression. */ static void tdigest_aggstate_enlarge(tdigest_aggstate_t *state) { Assert(state->ncentroids <= state->maxcentroids); Assert(state->maxcentroids <= BUFFER_SIZE(state->compression)); /* only enlarge if actually full */ if (state->ncentroids < state->maxcentroids) return; /* double the capacity, but cap it to BUFFER_SIZE */ state->maxcentroids = Min(2 * state->maxcentroids, BUFFER_SIZE(state->compression)); /* repalloc keeps the buffer in its original memory context */ state->centroids = repalloc(state->centroids, state->maxcentroids * sizeof(centroid_t)); /* make sure we have space for the value */ Assert(state->ncentroids < state->maxcentroids); Assert(state->maxcentroids <= BUFFER_SIZE(state->compression)); } /* * Try to reclaim some of the centroid buffer after compaction. * * After compaction, the centroid buffer is guaranteed to be mostly empty, * and there's no guarantee it'll ever be used again. Reclaim large, mostly * unused allocations, keeping headroom for new centroids. * * We only do this for large buffers, so that the chunks are allocted as * separate oversized chunks, which means we actually do free() on them, * instead of stashing them to a freelist. * * And we only do that for buffers that are at least 75% empty, and thus * outside the usual "doubling" strategy. * * This is not free, but it only happens after compaction, which is rather * rare and already fairly expensive, so the additional cost should be * rather acceptable. * * We have to enforce the power-of-2 sizing even during shrinking. */ static void tdigest_aggstate_shrink(tdigest_aggstate_t *state) { /* the buffer has to be at least 75% empty */ if (state->maxcentroids <= 4 * state->ncentroids) return; /* the buffer has to be at least 8KB */ if (state->maxcentroids * sizeof(centroid_t) <= 8192) return; /* * Find the first power-of-2 capacity above ncentroids. It has to be * strictly above, so that the buffer we just resized is not immediately * full again - the next centroid would have to grow it right back. */ state->maxcentroids = BUFFER_INITIAL_SIZE; while (state->maxcentroids <= state->ncentroids) state->maxcentroids *= 2; Assert(state->ncentroids < state->maxcentroids); Assert(state->maxcentroids <= BUFFER_SIZE(state->compression)); state->centroids = repalloc(state->centroids, state->maxcentroids * sizeof(centroid_t)); } /* add a value to the t-digest, trigger a compaction if full */ static void tdigest_add(tdigest_aggstate_t *state, double v) { int compression = state->compression; /* make sure we're not adding bogus NaN/infinity values as centroids */ if (!isfinite(v)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("all values added to t-digest must be finite"))); /* * If the buffer is full, trigger compaction here so that we have * free space for the new value. */ if (state->ncentroids == BUFFER_SIZE(compression)) tdigest_compact(state); /* ensure there's free space in the aggregate state */ tdigest_aggstate_enlarge(state); /* for a single point, the value is both sum and mean */ state->centroids[state->ncentroids].count = 1; state->centroids[state->ncentroids].mean = v; state->ncentroids++; /* make sure the total does not overflow */ if (pg_add_s64_overflow(state->count, 1, &state->count)) ereport(ERROR, (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), errmsg("tdigest count overflow"))); } /* * Add a centroid (possibly with count not equal to 1) to the t-digest, * triggering a compaction when the buffer is full. */ static void tdigest_add_centroid(tdigest_aggstate_t *state, double mean, int64 count) { int compression = state->compression; /* we should not have an infinite/NaN mean in a digest */ Assert(isfinite(mean)); /* * If the buffer is full, trigger compaction here so that we have * free space for the new value. */ if (state->ncentroids == BUFFER_SIZE(compression)) tdigest_compact(state); /* ensure there's free space in the aggregate state */ tdigest_aggstate_enlarge(state); state->centroids[state->ncentroids].count = count; state->centroids[state->ncentroids].mean = mean; state->ncentroids++; /* make sure the total does not overflow */ if (pg_add_s64_overflow(state->count, count, &state->count)) ereport(ERROR, (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), errmsg("tdigest count overflow"))); } /* allocate t-digest with enough space for a requested number of centroids */ static tdigest_t * tdigest_allocate(int ncentroids) { Size len; tdigest_t *digest; char *ptr; len = offsetof(tdigest_t, centroids) + ncentroids * sizeof(centroid_t); ptr = palloc(len); SET_VARSIZE(ptr, len); digest = (tdigest_t *) ptr; digest->flags = 0; digest->ncentroids = 0; digest->count = 0; digest->compression = 0; /* new t-digests automatically store means */ digest->flags |= TDIGEST_STORES_MEAN; return digest; } /* * tdigest_update_format * Update t-digest format to represent centroids as (mean,count). * * Switches the centroids from (sum,count) to (mean,count), so that all * the places processing centroids can use just the new format. * * The caller must own the value before converting a legacy digest, and * the digest is modified in place (without creating a copy). Current * format values are left untouched. */ static void tdigest_update_format(tdigest_t *digest) { int i; /* if already new format, we're done */ if (digest->flags & TDIGEST_STORES_MEAN) return; for (i = 0; i < digest->ncentroids; i++) { CHECK_FOR_INTERRUPTS(); digest->centroids[i].mean = digest->centroids[i].mean / digest->centroids[i].count; } digest->flags |= TDIGEST_STORES_MEAN; } /* * tdigest_prepare * Prepare the digest for additional processing (adding values, ...). * * Detoast, align and normalize a digest, optionally sorting its centroids. * * Digests with the centroids in an arbitrary order are perfectly valid - the * incremental API can leave digests uncompacted (and possibly unsorted) when * compact is false, and the input functions accept such digests too. So the * places walking the centroids in mean order have to do the sort themselves. * * Reuse an owned detoasted/aligned copy for conversion and sorting. Otherwise * copy only before the first modification. The returned value is either the * original datum or one owned allocation, releasable with PG_FREE_IF_COPY. */ static tdigest_t * tdigest_prepare(Datum datum, bool sort) { tdigest_t *digest = tdigest_detoast(datum); int i; /* * If the digest uses the old format, switch to the new one (and make * sure we have a copy, as required by tdigest_update_format). */ if (!(digest->flags & TDIGEST_STORES_MEAN)) { /* if not a copy already, make one */ if ((Pointer) digest == DatumGetPointer(datum)) digest = (tdigest_t *) PG_DETOAST_DATUM_COPY(datum); tdigest_update_format(digest); } /* if not requested to sort centroids, we're done */ if (!sort) return digest; /* if the centroids are already sorted, we're done */ for (i = 1; i < digest->ncentroids; i++) { CHECK_FOR_INTERRUPTS(); /* * XXX Not quite right, it needs to consider the count too, if * the centroids have the same mean (and whether we're below or * above the mean of the whole digest. */ if (digest->centroids[i - 1].mean > digest->centroids[i].mean) break; } /* if the digest is already sorted, bail out */ if (i >= digest->ncentroids) return digest; /* if not a copy already, make one */ if ((Pointer) digest == DatumGetPointer(datum)) digest = (tdigest_t *) PG_DETOAST_DATUM_COPY(datum); tdigest_sort_centroids(digest->centroids, digest->ncentroids, digest->count); return digest; } /* * Allocate a tdigest aggregate state and its query parameters. The centroid * buffer grows separately, leaving the state pointer stable for the executor. * ncentroids is an initial capacity hint, not the number of populated slots. */ static tdigest_aggstate_t * tdigest_aggstate_allocate(int npercentiles, int nvalues, int compression, int ncentroids) { Size len; tdigest_aggstate_t *state; char *ptr; /* at least one of those values is 0 */ Assert(nvalues == 0 || npercentiles == 0); Assert(ncentroids >= 0 && ncentroids <= BUFFER_SIZE(compression)); /* * Allocate a single chunk for the struct, the optional percentile or * hypothetical-value array. */ len = MAXALIGN(sizeof(tdigest_aggstate_t)) + MAXALIGN(sizeof(double) * npercentiles) + MAXALIGN(sizeof(double) * nvalues); ptr = palloc0(len); state = (tdigest_aggstate_t *) ptr; ptr += MAXALIGN(sizeof(tdigest_aggstate_t)); state->nvalues = nvalues; state->npercentiles = npercentiles; state->compression = compression; if (npercentiles > 0) { state->percentiles = (double *) ptr; ptr += MAXALIGN(sizeof(double) * npercentiles); } if (nvalues > 0) { state->values = (double *) ptr; ptr += MAXALIGN(sizeof(double) * nvalues); } Assert(ptr == (char *) state + len); /* * ncentroids is an arbitrary value, but we want to stick to power-of-2 * sizes, to match the size classes used by AllocSet */ state->maxcentroids = BUFFER_INITIAL_SIZE; while (state->maxcentroids < ncentroids) state->maxcentroids *= 2; /* don't use buffers larger than BUFFER_SIZE */ state->maxcentroids = Min(BUFFER_SIZE(compression), state->maxcentroids); state->centroids = palloc(state->maxcentroids * sizeof(centroid_t)); return state; } static void tdigest_aggstate_free(tdigest_aggstate_t *state) { pfree(state->centroids); pfree(state); } static tdigest_t * tdigest_aggstate_to_digest(tdigest_aggstate_t *state, bool compact) { int i; tdigest_t *digest; if (compact) tdigest_compact(state); digest = tdigest_allocate(state->ncentroids); digest->count = state->count; digest->ncentroids = state->ncentroids; digest->compression = state->compression; for (i = 0; i < state->ncentroids; i++) { CHECK_FOR_INTERRUPTS(); digest->centroids[i].mean = state->centroids[i].mean; digest->centroids[i].count = state->centroids[i].count; } return digest; } /* check that the requested percentiles are valid */ static void check_percentiles(const double *percentiles, int npercentiles) { int i; for (i = 0; i < npercentiles; i++) { if (!((percentiles[i] >= 0.0) && (percentiles[i] <= 1.0))) elog(ERROR, "invalid percentile value %f, should be in [0.0, 1.0]", percentiles[i]); } } static void check_compression(int compression) { if (compression < MIN_COMPRESSION || compression > MAX_COMPRESSION) elog(ERROR, "invalid compression value %d", compression); } static void check_trim_values(double low, double high) { if (!((low >= 0.0) && (low <= 1.0))) elog(ERROR, "invalid low percentile value %f, should be in [0.0, 1.0]", low); if (!((high >= 0.0) && (high <= 1.0))) elog(ERROR, "invalid high percentile value %f, should be in [0.0, 1.0]", high); if (low > high) elog(ERROR, "invalid low/high percentile values %f/%f, should be low <= high", low, high); } /* * Add an input value to the aggregate state, creating it if needed. * Shared by tdigest and tdigest_percentile with a single percentile. * * Transition functions capture compression and query parameters when the * first non-NULL input creates the state. Callers should keep those * parameters constant within a group; later rows are not checked for changes. */ Datum tdigest_add_double(PG_FUNCTION_ARGS) { tdigest_aggstate_t *state; MemoryContext aggcontext; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_add_double called in non-aggregate context"); /* * We want to skip NULL values altogether - we return either the existing * t-digest (if it already exists) or NULL. */ if (PG_ARGISNULL(1)) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); /* if there already is a state accumulated, don't forget it */ PG_RETURN_DATUM(PG_GETARG_DATUM(0)); } /* if there's no digest allocated, create it now */ if (PG_ARGISNULL(0)) { int compression; double *percentiles = NULL; int npercentiles = 0; MemoryContext oldcontext; if (PG_ARGISNULL(2)) elog(ERROR, "compression must not be NULL"); compression = PG_GETARG_INT32(2); check_compression(compression); oldcontext = MemoryContextSwitchTo(aggcontext); if (PG_NARGS() >= 4) { /* the percentile is required to create the aggregate state */ if (PG_ARGISNULL(3)) elog(ERROR, "percentile must not be NULL"); percentiles = (double *) palloc(sizeof(double)); percentiles[0] = PG_GETARG_FLOAT8(3); npercentiles = 1; check_percentiles(percentiles, npercentiles); } state = tdigest_aggstate_allocate(npercentiles, 0, compression, 0); if (percentiles) { memcpy(state->percentiles, percentiles, sizeof(double) * npercentiles); pfree(percentiles); } MemoryContextSwitchTo(oldcontext); } else state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); tdigest_add(state, PG_GETARG_FLOAT8(1)); AssertCheckTDigestAggState(state); PG_RETURN_POINTER(state); } /* * Add a value with a given count to the t-digest, as a sequence of properly * sized centroids. * * This is an alternative to adding a single centroid, representing all the * points with the same value. It follows all the rules on centroid sizes, * etc. * * The centroids are handed over to the aggregate state as they are computed, * instead of building a t-digest first. The number of centroids the loop * produces is not bounded by the compression (on the tails the calculated * size drops below 1, and gets clamped), so there is no size of a centroid * array that would be guaranteed to be sufficient. */ static void tdigest_add_generated(tdigest_aggstate_t *state, double value, int64 count) { int64 count_so_far; int64 count_remaining; double denom; double normalizer; int compression = state->compression; /* make sure we're not adding bogus NaN/infinity values as centroids */ if (!isfinite(value)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("all values added to t-digest must be finite"))); denom = 2 * M_PI * count * log(count); normalizer = compression / denom; count_so_far = 0; /* does not include current centroid */ count_remaining = count; /* * Create largest possible centroids, until we run out of items. In each * step we need to find the largest possible well-formed centroid, i.e. one * that matches the two conditions: * * z <= q0 * (1 - q0) where q0 = (count_so_far / count) * * z <= q2 * (1 - q2) where q2 = (count_so_far + X) / count; * * with z = (X * normalizer). X being the value we need to determine. Solving * q0 is trivial, while q2 leads to a quadratic equation with two roots. */ while (count_remaining > 0) { int64 proposed_count; double q0; double b, c, d; double r1, r2; CHECK_FOR_INTERRUPTS(); /* * Solving z <= q0 * (1 - q0) is trivial. * * Just like in tdigest_compact, we must not calculate (1 - q0) by * subtracting the two doubles - for q0 close to 1 that cancels out all * the significant digits. We already have count_remaining, which is * exactly (count - count_so_far), so use that remainder instead. */ q0 = count_so_far / (double) count; r1 = (q0 * (count_remaining / (double) count) / normalizer); /* * Solve z <= q2 * (1 - q2) as a quadratic equation. The inequality we * need to solve is * * 0 <= a * x^2 + b * x + c * * with (a = -1) and the following coefficients. * * XXX The counts may be very high values (int64), so we need to be * careful to prevent overflows by doing everything with double. * * XXX c is mathematically (count_so_far * (count - count_so_far)), so * calculate it as a plain product of the two exact integers. The * expanded form (count_so_far * count - count_so_far * count_so_far) * is a difference of two huge and nearly equal values, which loses * almost all the precision. */ b = ((double) count - 2 * (double) count_so_far - (double) count * (double) count * normalizer); c = ((double) count_so_far * (double) count_remaining); /* * As this is an "upside down" parabola, the values between the roots * are positive - we're looking for the larger of the two roots, which * for a = -1 is (b + sqrt(b*b + 4*c)) / 2. * * XXX Evaluating that expression directly is only safe for b >= 0. * For b < 0 the sqrt is very close to -b, so the addition cancels out * all the significant digits (and often yields exactly zero, forcing * us to emit a single-item centroid). Use the equivalent "conjugate" * form 2*c / (sqrt(b*b + 4*c) - b) in that case, which only ever adds * values of the same sign. Both branches are hit in practice - b is * positive whenever compression < 2*pi*ln(count). * * XXX c is never negative, so the discriminant is a sum of two * non-negative values and the sqrt is always well defined. */ d = sqrt(b * b + 4 * c); if (b >= 0) r2 = (b + d) / 2; else r2 = (2 * c) / (d - b); /* * paranoia: We should not be dealing with NaN values here. Crash in * debug build, double_to_int64 will mitigate it in regular builds. */ Assert(isfinite(r1) && isfinite(r2)); /* * We need to meet both conditions, so use the smaller solution. The * value may be large (or NaN), so clamp it - we must not add more * than what remains anyway. */ proposed_count = double_to_int64(floor(Min(r1, r2)), count_remaining); /* * It's possible to get very low values on the tails, but we must add * at least something, otherwise we'd get infinite loops. */ proposed_count = Max(proposed_count, 1); tdigest_add_centroid(state, value, proposed_count); count_so_far += proposed_count; count_remaining -= proposed_count; } } /* * Add an input value with a count to the aggregate state, creating it if * needed. Shared by tdigest and tdigest_percentile with a single percentile. */ Datum tdigest_add_double_count(PG_FUNCTION_ARGS) { int64 i; int64 count; tdigest_aggstate_t *state; MemoryContext aggcontext; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_add_double_count called in non-aggregate context"); /* * We want to skip NULL values altogether - we return either the existing * t-digest (if it already exists) or NULL. */ if (PG_ARGISNULL(1)) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); /* if there already is a state accumulated, don't forget it */ PG_RETURN_DATUM(PG_GETARG_DATUM(0)); } /* if there's no digest allocated, create it now */ if (PG_ARGISNULL(0)) { int compression; double *percentiles = NULL; int npercentiles = 0; MemoryContext oldcontext; if (PG_ARGISNULL(3)) elog(ERROR, "compression must not be NULL"); compression = PG_GETARG_INT32(3); check_compression(compression); oldcontext = MemoryContextSwitchTo(aggcontext); if (PG_NARGS() >= 5) { /* the percentile is required to create the aggregate state */ if (PG_ARGISNULL(4)) elog(ERROR, "percentile must not be NULL"); percentiles = (double *) palloc(sizeof(double)); percentiles[0] = PG_GETARG_FLOAT8(4); npercentiles = 1; check_percentiles(percentiles, npercentiles); } state = tdigest_aggstate_allocate(npercentiles, 0, compression, 0); if (percentiles) { memcpy(state->percentiles, percentiles, sizeof(double) * npercentiles); pfree(percentiles); } MemoryContextSwitchTo(oldcontext); } else state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); if (PG_ARGISNULL(2)) { count = 1; } else count = PG_GETARG_INT64(2); /* can't add values with non-positive counts */ if (count <= 0) elog(ERROR, "invalid count value %lld, must be a positive value", (long long) count); /* * When adding more values than would fit into an empty buffer (and * thus likely causing too many compactions), we instead add them as * properly sized centroids. * * This is much faster, because the centroids can be generated in one go, * so there are only very few compactions. */ if (count > BUFFER_SIZE(state->compression)) { tdigest_add_generated(state, PG_GETARG_FLOAT8(1), count); count = 0; } /* * If there are only a few values, just add them one by one, so that * we do proper compaction and sizing of centroids. Otherwise we might end * up with oversized centroids on the tails etc. */ for (i = 0; i < count; i++) tdigest_add(state, PG_GETARG_FLOAT8(1)); AssertCheckTDigestAggState(state); PG_RETURN_POINTER(state); } /* * Add an input value to the aggregate state for tdigest_percentile_of * with a single hypothetical value. */ Datum tdigest_add_double_values(PG_FUNCTION_ARGS) { tdigest_aggstate_t *state; MemoryContext aggcontext; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_add_double_values called in non-aggregate context"); /* * We want to skip NULL values altogether - we return either the existing * t-digest (if it already exists) or NULL. */ if (PG_ARGISNULL(1)) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); /* if there already is a state accumulated, don't forget it */ PG_RETURN_DATUM(PG_GETARG_DATUM(0)); } /* if there's no digest allocated, create it now */ if (PG_ARGISNULL(0)) { int compression; double *values = NULL; int nvalues = 0; MemoryContext oldcontext; if (PG_ARGISNULL(2)) elog(ERROR, "compression must not be NULL"); compression = PG_GETARG_INT32(2); check_compression(compression); oldcontext = MemoryContextSwitchTo(aggcontext); if (PG_NARGS() >= 4) { /* the value is required to create the aggregate state */ if (PG_ARGISNULL(3)) elog(ERROR, "value must not be NULL"); values = (double *) palloc(sizeof(double)); values[0] = PG_GETARG_FLOAT8(3); nvalues = 1; } state = tdigest_aggstate_allocate(0, nvalues, compression, 0); if (values) { memcpy(state->values, values, sizeof(double) * nvalues); pfree(values); } MemoryContextSwitchTo(oldcontext); } else state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); tdigest_add(state, PG_GETARG_FLOAT8(1)); AssertCheckTDigestAggState(state); PG_RETURN_POINTER(state); } /* * Add an input value with a count to the aggregate state for * tdigest_percentile_of with a single hypothetical value. */ Datum tdigest_add_double_values_count(PG_FUNCTION_ARGS) { int64 i; int64 count; tdigest_aggstate_t *state; MemoryContext aggcontext; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_add_double_values_count called in non-aggregate context"); /* * We want to skip NULL values altogether - we return either the existing * t-digest (if it already exists) or NULL. */ if (PG_ARGISNULL(1)) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); /* if there already is a state accumulated, don't forget it */ PG_RETURN_DATUM(PG_GETARG_DATUM(0)); } /* if there's no digest allocated, create it now */ if (PG_ARGISNULL(0)) { int compression; double *values = NULL; int nvalues = 0; MemoryContext oldcontext; if (PG_ARGISNULL(3)) elog(ERROR, "compression must not be NULL"); compression = PG_GETARG_INT32(3); check_compression(compression); oldcontext = MemoryContextSwitchTo(aggcontext); if (PG_NARGS() >= 5) { /* the value is required to create the aggregate state */ if (PG_ARGISNULL(4)) elog(ERROR, "value must not be NULL"); values = (double *) palloc(sizeof(double)); values[0] = PG_GETARG_FLOAT8(4); nvalues = 1; } state = tdigest_aggstate_allocate(0, nvalues, compression, 0); if (values) { memcpy(state->values, values, sizeof(double) * nvalues); pfree(values); } MemoryContextSwitchTo(oldcontext); } else state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); if (PG_ARGISNULL(2)) { count = 1; } else count = PG_GETARG_INT64(2); /* can't add values with non-positive counts */ if (count <= 0) elog(ERROR, "invalid count value %lld, must be a positive value", (long long) count); /* * When adding more values than would fit into an empty buffer (and * thus likely causing too many compactions), we instead add them as * properly sized centroids. * * This is much faster, because the centroids can be generated in one go, * so there are only very few compactions. */ if (count > BUFFER_SIZE(state->compression)) { tdigest_add_generated(state, PG_GETARG_FLOAT8(1), count); count = 0; } /* * If there are only a few values, just add them one by one, so that * we do proper compaction and sizing of centroids. Otherwise we might end * up with oversized centroids on the tails etc. */ for (i = 0; i < count; i++) tdigest_add(state, PG_GETARG_FLOAT8(1)); AssertCheckTDigestAggState(state); PG_RETURN_POINTER(state); } /* * Merge an input digest into the aggregate state, creating it if needed. * Shared by tdigest and tdigest_percentile with a single percentile. */ Datum tdigest_add_digest(PG_FUNCTION_ARGS) { int i; tdigest_aggstate_t *state; tdigest_t *digest; MemoryContext aggcontext; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_add_digest called in non-aggregate context"); /* * We want to skip NULL values altogether - we return either the existing * t-digest (if it already exists) or NULL. */ if (PG_ARGISNULL(1)) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); /* if there already is a state accumulated, don't forget it */ PG_RETURN_DATUM(PG_GETARG_DATUM(0)); } /* make sure we have a detoasted copy */ digest = tdigest_prepare(PG_GETARG_DATUM(1), false); /* if there's no aggregate state allocated, create it now */ if (PG_ARGISNULL(0)) { double *percentiles = NULL; int npercentiles = 0; MemoryContext oldcontext; oldcontext = MemoryContextSwitchTo(aggcontext); if (PG_NARGS() >= 3) { /* the percentile is required to create the aggregate state */ if (PG_ARGISNULL(2)) elog(ERROR, "percentile must not be NULL"); percentiles = (double *) palloc(sizeof(double)); percentiles[0] = PG_GETARG_FLOAT8(2); npercentiles = 1; check_percentiles(percentiles, npercentiles); } state = tdigest_aggstate_allocate(npercentiles, 0, digest->compression, digest->ncentroids); if (percentiles) { memcpy(state->percentiles, percentiles, sizeof(double) * npercentiles); pfree(percentiles); } MemoryContextSwitchTo(oldcontext); } else state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); /* * Keep the compression chosen when the state was created, even if the * input uses a different setting. Compaction can merge incoming * centroids, but cannot split them to recover lost detail. */ /* copy data from the tdigest into the aggstate */ for (i = 0; i < digest->ncentroids; i++) { CHECK_FOR_INTERRUPTS(); tdigest_add_centroid(state, digest->centroids[i].mean, digest->centroids[i].count); } AssertCheckTDigestAggState(state); PG_FREE_IF_COPY(digest, 1); PG_RETURN_POINTER(state); } /* * Merge an input digest into the aggregate state for tdigest_percentile_of * with a single hypothetical value. */ Datum tdigest_add_digest_values(PG_FUNCTION_ARGS) { int i; tdigest_aggstate_t *state; tdigest_t *digest; MemoryContext aggcontext; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_add_digest_values called in non-aggregate context"); /* * We want to skip NULL values altogether - we return either the existing * t-digest (if it already exists) or NULL. */ if (PG_ARGISNULL(1)) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); /* if there already is a state accumulated, don't forget it */ PG_RETURN_DATUM(PG_GETARG_DATUM(0)); } /* make sure we have a detoasted copy */ digest = tdigest_prepare(PG_GETARG_DATUM(1), false); /* if there's no aggregate state allocated, create it now */ if (PG_ARGISNULL(0)) { double *values = NULL; int nvalues = 0; MemoryContext oldcontext; oldcontext = MemoryContextSwitchTo(aggcontext); if (PG_NARGS() >= 3) { /* the value is required to create the aggregate state */ if (PG_ARGISNULL(2)) elog(ERROR, "value must not be NULL"); values = (double *) palloc(sizeof(double)); values[0] = PG_GETARG_FLOAT8(2); nvalues = 1; } state = tdigest_aggstate_allocate(0, nvalues, digest->compression, digest->ncentroids); if (values) { memcpy(state->values, values, sizeof(double) * nvalues); pfree(values); } MemoryContextSwitchTo(oldcontext); } else state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); /* * Keep the compression chosen when the state was created, even if the * input uses a different setting. Compaction can merge incoming * centroids, but cannot split them to recover lost detail. */ for (i = 0; i < digest->ncentroids; i++) { CHECK_FOR_INTERRUPTS(); tdigest_add_centroid(state, digest->centroids[i].mean, digest->centroids[i].count); } AssertCheckTDigestAggState(state); PG_FREE_IF_COPY(digest, 1); PG_RETURN_POINTER(state); } /* * Add an input value to the aggregate state for tdigest_percentile with * an array of requested percentiles. */ Datum tdigest_add_double_array(PG_FUNCTION_ARGS) { tdigest_aggstate_t *state; MemoryContext aggcontext; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_add_double_array called in non-aggregate context"); /* * We want to skip NULL values altogether - we return either the existing * t-digest or NULL. */ if (PG_ARGISNULL(1)) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); /* if there already is a state accumulated, don't forget it */ PG_RETURN_DATUM(PG_GETARG_DATUM(0)); } /* if there's no digest allocated, create it now */ if (PG_ARGISNULL(0)) { int compression; const double *percentiles; int npercentiles; MemoryContext oldcontext; ArrayType *array; if (PG_ARGISNULL(2)) elog(ERROR, "compression must not be NULL"); compression = PG_GETARG_INT32(2); check_compression(compression); /* Percentiles are required in order to create the aggregate state. */ if (PG_ARGISNULL(3)) elog(ERROR, "percentiles must not be NULL"); /* process the input array in caller's memory context */ array = PG_GETARG_ARRAYTYPE_P(3); percentiles = array_to_double(array, "a percentile value", &npercentiles); check_percentiles(percentiles, npercentiles); oldcontext = MemoryContextSwitchTo(aggcontext); state = tdigest_aggstate_allocate(npercentiles, 0, compression, 0); memcpy(state->percentiles, percentiles, sizeof(double) * npercentiles); MemoryContextSwitchTo(oldcontext); /* free the parsed data, possibly detoasted copy of the array */ PG_FREE_IF_COPY(array, 3); } else state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); tdigest_add(state, PG_GETARG_FLOAT8(1)); AssertCheckTDigestAggState(state); PG_RETURN_POINTER(state); } /* * Add an input value with a count to the aggregate state for * tdigest_percentile with an array of requested percentiles. */ Datum tdigest_add_double_array_count(PG_FUNCTION_ARGS) { int64 i; int64 count; tdigest_aggstate_t *state; MemoryContext aggcontext; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_add_double_array_count called in non-aggregate context"); /* * We want to skip NULL values altogether - we return either the existing * t-digest or NULL. */ if (PG_ARGISNULL(1)) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); /* if there already is a state accumulated, don't forget it */ PG_RETURN_DATUM(PG_GETARG_DATUM(0)); } /* if there's no digest allocated, create it now */ if (PG_ARGISNULL(0)) { int compression; const double *percentiles; int npercentiles; MemoryContext oldcontext; ArrayType *array; if (PG_ARGISNULL(3)) elog(ERROR, "compression must not be NULL"); compression = PG_GETARG_INT32(3); check_compression(compression); /* Percentiles are required in order to create the aggregate state. */ if (PG_ARGISNULL(4)) elog(ERROR, "percentiles must not be NULL"); /* process the input array in caller's memory context */ array = PG_GETARG_ARRAYTYPE_P(4); percentiles = array_to_double(array, "a percentile value", &npercentiles); check_percentiles(percentiles, npercentiles); oldcontext = MemoryContextSwitchTo(aggcontext); state = tdigest_aggstate_allocate(npercentiles, 0, compression, 0); memcpy(state->percentiles, percentiles, sizeof(double) * npercentiles); MemoryContextSwitchTo(oldcontext); /* free the parsed data, possibly detoasted copy of the array */ PG_FREE_IF_COPY(array, 4); } else state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); if (PG_ARGISNULL(2)) { count = 1; } else count = PG_GETARG_INT64(2); /* can't add values with non-positive counts */ if (count <= 0) elog(ERROR, "invalid count value %lld, must be a positive value", (long long) count); /* * When adding more values than would fit into an empty buffer (and * thus likely causing too many compactions), we instead add them as * properly sized centroids. * * This is much faster, because the centroids can be generated in one go, * so there are only very few compactions. */ if (count > BUFFER_SIZE(state->compression)) { tdigest_add_generated(state, PG_GETARG_FLOAT8(1), count); count = 0; } /* * Add the values one by one, not as one large centroid with the count. * We do it like this to allow proper compaction and sizing of centroids, * otherwise we might end up with oversized centroids on the tails etc. * * XXX If this turns out a bit too expensive, we may try determining the * size by looking for the smallest centroid covering this value. */ for (i = 0; i < count; i++) tdigest_add(state, PG_GETARG_FLOAT8(1)); AssertCheckTDigestAggState(state); PG_RETURN_POINTER(state); } /* * Add an input value to the aggregate state for tdigest_percentile_of * with an array of hypothetical values. */ Datum tdigest_add_double_array_values(PG_FUNCTION_ARGS) { tdigest_aggstate_t *state; MemoryContext aggcontext; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_add_double_array called in non-aggregate context"); /* * We want to skip NULL values altogether - we return either the existing * t-digest or NULL. */ if (PG_ARGISNULL(1)) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); /* if there already is a state accumulated, don't forget it */ PG_RETURN_DATUM(PG_GETARG_DATUM(0)); } /* if there's no digest allocated, create it now */ if (PG_ARGISNULL(0)) { int compression; const double *values; int nvalues; MemoryContext oldcontext; ArrayType *array; if (PG_ARGISNULL(2)) elog(ERROR, "compression must not be NULL"); compression = PG_GETARG_INT32(2); check_compression(compression); /* Values are required in order to create the aggregate state. */ if (PG_ARGISNULL(3)) elog(ERROR, "values must not be NULL"); /* process the input array in caller's memory context */ array = PG_GETARG_ARRAYTYPE_P(3); values = array_to_double(array, "a value", &nvalues); oldcontext = MemoryContextSwitchTo(aggcontext); state = tdigest_aggstate_allocate(0, nvalues, compression, 0); memcpy(state->values, values, sizeof(double) * nvalues); MemoryContextSwitchTo(oldcontext); /* free the parsed data, possibly detoasted copy of the array */ PG_FREE_IF_COPY(array, 3); } else state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); tdigest_add(state, PG_GETARG_FLOAT8(1)); AssertCheckTDigestAggState(state); PG_RETURN_POINTER(state); } /* * Add an input value with a count to the aggregate state for * tdigest_percentile_of with an array of hypothetical values. */ Datum tdigest_add_double_array_values_count(PG_FUNCTION_ARGS) { int64 i; int64 count; tdigest_aggstate_t *state; MemoryContext aggcontext; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_add_double_array_values_count called in non-aggregate context"); /* * We want to skip NULL values altogether - we return either the existing * t-digest or NULL. */ if (PG_ARGISNULL(1)) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); /* if there already is a state accumulated, don't forget it */ PG_RETURN_DATUM(PG_GETARG_DATUM(0)); } /* if there's no digest allocated, create it now */ if (PG_ARGISNULL(0)) { int compression; const double *values; int nvalues; MemoryContext oldcontext; ArrayType *array; if (PG_ARGISNULL(3)) elog(ERROR, "compression must not be NULL"); compression = PG_GETARG_INT32(3); check_compression(compression); /* Values are required in order to create the aggregate state. */ if (PG_ARGISNULL(4)) elog(ERROR, "values must not be NULL"); /* process the input array in caller's memory context */ array = PG_GETARG_ARRAYTYPE_P(4); values = array_to_double(array, "a value", &nvalues); oldcontext = MemoryContextSwitchTo(aggcontext); state = tdigest_aggstate_allocate(0, nvalues, compression, 0); memcpy(state->values, values, sizeof(double) * nvalues); MemoryContextSwitchTo(oldcontext); /* free the parsed data, possibly detoasted copy of the array */ PG_FREE_IF_COPY(array, 4); } else state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); if (PG_ARGISNULL(2)) { count = 1; } else count = PG_GETARG_INT64(2); /* can't add values with non-positive counts */ if (count <= 0) elog(ERROR, "invalid count value %lld, must be a positive value", (long long) count); /* * When adding more values than would fit into an empty buffer (and * thus likely causing too many compactions), we instead add them as * properly sized centroids. * * This is much faster, because the centroids can be generated in one go, * so there are only very few compactions. */ if (count > BUFFER_SIZE(state->compression)) { tdigest_add_generated(state, PG_GETARG_FLOAT8(1), count); count = 0; } /* * Add the values one by one, not as one large centroid with the count. * We do it like this to allow proper compaction and sizing of centroids, * otherwise we might end up with oversized centroids on the tails etc. * * XXX If this turns out a bit too expensive, we may try determining the * size by looking for the smallest centroid covering this value. */ for (i = 0; i < count; i++) tdigest_add(state, PG_GETARG_FLOAT8(1)); AssertCheckTDigestAggState(state); PG_RETURN_POINTER(state); } /* * Merge an input digest into the aggregate state for tdigest_percentile * with an array of requested percentiles. */ Datum tdigest_add_digest_array(PG_FUNCTION_ARGS) { int i; tdigest_aggstate_t *state; tdigest_t *digest; MemoryContext aggcontext; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_add_digest_array called in non-aggregate context"); /* * We want to skip NULL values altogether - we return either the existing * t-digest (if it already exists) or NULL. */ if (PG_ARGISNULL(1)) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); /* if there already is a state accumulated, don't forget it */ PG_RETURN_DATUM(PG_GETARG_DATUM(0)); } /* make sure we have a detoasted copy */ digest = tdigest_prepare(PG_GETARG_DATUM(1), false); /* if there's no aggregate state allocated, create it now */ if (PG_ARGISNULL(0)) { const double *percentiles; int npercentiles; MemoryContext oldcontext; ArrayType *array; /* Percentiles are required in order to create the aggregate state. */ if (PG_ARGISNULL(2)) elog(ERROR, "percentiles must not be NULL"); /* process the input array in caller's memory context */ array = PG_GETARG_ARRAYTYPE_P(2); percentiles = array_to_double(array, "a percentile value", &npercentiles); check_percentiles(percentiles, npercentiles); oldcontext = MemoryContextSwitchTo(aggcontext); state = tdigest_aggstate_allocate(npercentiles, 0, digest->compression, digest->ncentroids); memcpy(state->percentiles, percentiles, sizeof(double) * npercentiles); MemoryContextSwitchTo(oldcontext); /* free the parsed data, possibly detoasted copy of the array */ PG_FREE_IF_COPY(array, 2); } else state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); /* * Keep the compression chosen when the state was created, even if the * input uses a different setting. Compaction can merge incoming * centroids, but cannot split them to recover lost detail. */ for (i = 0; i < digest->ncentroids; i++) { CHECK_FOR_INTERRUPTS(); tdigest_add_centroid(state, digest->centroids[i].mean, digest->centroids[i].count); } AssertCheckTDigestAggState(state); PG_FREE_IF_COPY(digest, 1); PG_RETURN_POINTER(state); } /* * Merge an input digest into the aggregate state for tdigest_percentile_of * with an array of hypothetical values. */ Datum tdigest_add_digest_array_values(PG_FUNCTION_ARGS) { int i; tdigest_aggstate_t *state; tdigest_t *digest; MemoryContext aggcontext; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_add_digest_array_values called in non-aggregate context"); /* * We want to skip NULL values altogether - we return either the existing * t-digest (if it already exists) or NULL. */ if (PG_ARGISNULL(1)) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); /* if there already is a state accumulated, don't forget it */ PG_RETURN_DATUM(PG_GETARG_DATUM(0)); } /* make sure we have a detoasted copy */ digest = tdigest_prepare(PG_GETARG_DATUM(1), false); /* if there's no aggregate state allocated, create it now */ if (PG_ARGISNULL(0)) { const double *values; int nvalues; MemoryContext oldcontext; ArrayType *array; /* Values are required in order to create the aggregate state. */ if (PG_ARGISNULL(2)) elog(ERROR, "values must not be NULL"); /* process the input array in caller's memory context */ array = PG_GETARG_ARRAYTYPE_P(2); values = array_to_double(array, "a value", &nvalues); oldcontext = MemoryContextSwitchTo(aggcontext); state = tdigest_aggstate_allocate(0, nvalues, digest->compression, digest->ncentroids); memcpy(state->values, values, sizeof(double) * nvalues); MemoryContextSwitchTo(oldcontext); /* free the parsed data, possibly detoasted copy of the array */ PG_FREE_IF_COPY(array, 2); } else state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); /* * Keep the compression chosen when the state was created, even if the * input uses a different setting. Compaction can merge incoming * centroids, but cannot split them to recover lost detail. */ for (i = 0; i < digest->ncentroids; i++) { CHECK_FOR_INTERRUPTS(); tdigest_add_centroid(state, digest->centroids[i].mean, digest->centroids[i].count); } AssertCheckTDigestAggState(state); PG_FREE_IF_COPY(digest, 1); PG_RETURN_POINTER(state); } /* * Compute a percentile from the aggregate state. Final function for * tdigest_percentile with a single requested percentile. */ Datum tdigest_percentiles(PG_FUNCTION_ARGS) { tdigest_aggstate_t *state; MemoryContext aggcontext; double ret; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_percentiles called in non-aggregate context"); /* if there's no digest, return NULL */ if (PG_ARGISNULL(0)) PG_RETURN_NULL(); state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); /* windows and shared aggregates may need the original state again. */ if (AggStateIsShared(fcinfo)) { tdigest_aggstate_t *copy = tdigest_copy(state); tdigest_compute_quantiles(copy, &ret); tdigest_aggstate_free(copy); } else tdigest_compute_quantiles(state, &ret); PG_RETURN_FLOAT8(ret); } /* * Compute an inverse percentile from the aggregate state. Final function * for tdigest_percentile_of with a single hypothetical value. */ Datum tdigest_percentiles_of(PG_FUNCTION_ARGS) { tdigest_aggstate_t *state; MemoryContext aggcontext; double ret; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_percentiles_of called in non-aggregate context"); /* if there's no digest, return NULL */ if (PG_ARGISNULL(0)) PG_RETURN_NULL(); state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); /* windows and shared aggregates may need the original state again. */ if (AggStateIsShared(fcinfo)) { tdigest_aggstate_t *copy = tdigest_copy(state); tdigest_compute_quantiles_of(copy, &ret); tdigest_aggstate_free(copy); } else tdigest_compute_quantiles_of(state, &ret); PG_RETURN_FLOAT8(ret); } /* * Build a t-digest varlena value from the aggregate state. */ Datum tdigest_digest(PG_FUNCTION_ARGS) { tdigest_t *digest; tdigest_aggstate_t *state; MemoryContext aggcontext; bool shared; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_digest called in non-aggregate context"); /* if there's no digest, return NULL */ if (PG_ARGISNULL(0)) PG_RETURN_NULL(); state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); /* windows and shared aggregates may need the original state again. */ shared = AggStateIsShared(fcinfo); if (shared) state = tdigest_copy(state); digest = tdigest_aggstate_to_digest(state, true); if (shared) tdigest_aggstate_free(state); PG_RETURN_POINTER(digest); } /* * Compute percentiles from the aggregate state. Final function for * tdigest_percentile with an array of requested percentiles. */ Datum tdigest_array_percentiles(PG_FUNCTION_ARGS) { double *result; ArrayType *array; MemoryContext aggcontext; tdigest_aggstate_t *state; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_array_percentiles called in non-aggregate context"); if (PG_ARGISNULL(0)) PG_RETURN_NULL(); state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); /* write results directly into a correctly sized ArrayType */ array = double_array_allocate(state->npercentiles); result = (double *) ARR_DATA_PTR(array); /* windows and shared aggregates may need the original state again. */ if (AggStateIsShared(fcinfo)) { tdigest_aggstate_t *copy = tdigest_copy(state); tdigest_compute_quantiles(copy, result); tdigest_aggstate_free(copy); } else tdigest_compute_quantiles(state, result); PG_RETURN_ARRAYTYPE_P(array); } /* * Compute inverse percentiles from the aggregate state. Final function * for tdigest_percentile_of with an array of hypothetical values. */ Datum tdigest_array_percentiles_of(PG_FUNCTION_ARGS) { double *result; ArrayType *array; MemoryContext aggcontext; tdigest_aggstate_t *state; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_array_percentiles_of called in non-aggregate context"); if (PG_ARGISNULL(0)) PG_RETURN_NULL(); state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); /* write results directly into a correctly sized ArrayType */ array = double_array_allocate(state->nvalues); result = (double *) ARR_DATA_PTR(array); /* windows and shared aggregates may need the original state again. */ if (AggStateIsShared(fcinfo)) { tdigest_aggstate_t *copy = tdigest_copy(state); tdigest_compute_quantiles_of(copy, result); tdigest_aggstate_free(copy); } else tdigest_compute_quantiles_of(state, result); PG_RETURN_ARRAYTYPE_P(array); } Datum tdigest_serial(PG_FUNCTION_ARGS) { bytea *v; tdigest_aggstate_t *state; Size len; char *ptr; state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); len = offsetof(tdigest_aggstate_t, percentiles) + state->npercentiles * sizeof(double) + state->nvalues * sizeof(double) + state->ncentroids * sizeof(centroid_t); v = palloc(len + VARHDRSZ); SET_VARSIZE(v, len + VARHDRSZ); ptr = VARDATA(v); memcpy(ptr, state, offsetof(tdigest_aggstate_t, percentiles)); ptr += offsetof(tdigest_aggstate_t, percentiles); if (state->npercentiles > 0) { memcpy(ptr, state->percentiles, sizeof(double) * state->npercentiles); ptr += sizeof(double) * state->npercentiles; } if (state->nvalues > 0) { memcpy(ptr, state->values, sizeof(double) * state->nvalues); ptr += sizeof(double) * state->nvalues; } memcpy(ptr, state->centroids, sizeof(centroid_t) * state->ncentroids); ptr += sizeof(centroid_t) * state->ncentroids; Assert(VARDATA(v) + len == ptr); PG_RETURN_POINTER(v); } /* * XXX Unlike the other "input" functions (tdigest_in/tdigest_recv), this * does not validate the digest at all. We assume this function is used only * on data we created in the same query (possibly in a parallel worker), * and not on untrusted values controlled by the user (which is why the other * input functions need the validation). */ Datum tdigest_deserial(PG_FUNCTION_ARGS) { bytea *v = (bytea *) PG_GETARG_POINTER(0); char *ptr = VARDATA_ANY(v); tdigest_aggstate_t tmp; tdigest_aggstate_t *state; /* copy aggstate header into a local variable */ memcpy(&tmp, ptr, offsetof(tdigest_aggstate_t, percentiles)); ptr += offsetof(tdigest_aggstate_t, percentiles); state = tdigest_aggstate_allocate(tmp.npercentiles, tmp.nvalues, tmp.compression, tmp.ncentroids); if (tmp.npercentiles > 0) { memcpy(state->percentiles, ptr, tmp.npercentiles * sizeof(double)); ptr += tmp.npercentiles * sizeof(double); } if (tmp.nvalues > 0) { memcpy(state->values, ptr, tmp.nvalues * sizeof(double)); ptr += tmp.nvalues * sizeof(double); } /* * copy the aggstate header (except for the maxcentroids, which would * corrupt the aggstate we just allocated) */ state->count = tmp.count; state->ncompactions = tmp.ncompactions; state->compression = tmp.compression; /* skip maxcentroids */ state->ncentroids = tmp.ncentroids; state->ncompacted = tmp.ncompacted; state->npercentiles = tmp.npercentiles; state->nvalues = tmp.nvalues; state->trim_low = tmp.trim_low; state->trim_high = tmp.trim_high; /* we don't need to move the pointer */ /* copy the centroids back */ memcpy(state->centroids, ptr, sizeof(centroid_t) * state->ncentroids); ptr += sizeof(centroid_t) * state->ncentroids; Assert(ptr == VARDATA_ANY(v) + VARSIZE_ANY_EXHDR(v)); PG_RETURN_POINTER(state); } static tdigest_aggstate_t * tdigest_copy(tdigest_aggstate_t *state) { tdigest_aggstate_t *copy; copy = tdigest_aggstate_allocate(state->npercentiles, state->nvalues, state->compression, state->ncentroids); /* * copy the aggstate header (except for the maxcentroids, which would * corrupt the aggstate we just allocated) */ copy->count = state->count; copy->ncompactions = state->ncompactions; copy->compression = state->compression; /* skip maxcentroids */ copy->ncentroids = state->ncentroids; copy->ncompacted = state->ncompacted; copy->npercentiles = state->npercentiles; copy->nvalues = state->nvalues; copy->trim_low = state->trim_low; copy->trim_high = state->trim_high; if (state->nvalues > 0) memcpy(copy->values, state->values, sizeof(double) * state->nvalues); if (state->npercentiles > 0) memcpy(copy->percentiles, state->percentiles, sizeof(double) * state->npercentiles); memcpy(copy->centroids, state->centroids, state->ncentroids * sizeof(centroid_t)); return copy; } Datum tdigest_combine(PG_FUNCTION_ARGS) { tdigest_aggstate_t *src; tdigest_aggstate_t *dst; MemoryContext aggcontext; MemoryContext oldcontext; int i; if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_combine called in non-aggregate context"); /* if no "merged" state yet, try creating it */ if (PG_ARGISNULL(0)) { /* nope, the second argument is NULL too, so return NULL */ if (PG_ARGISNULL(1)) PG_RETURN_NULL(); /* the second argument is not NULL, so copy it */ src = (tdigest_aggstate_t *) PG_GETARG_POINTER(1); /* copy the digest into the right long-lived memory context */ oldcontext = MemoryContextSwitchTo(aggcontext); src = tdigest_copy(src); MemoryContextSwitchTo(oldcontext); PG_RETURN_POINTER(src); } /* * If the second argument is NULL, just return the first one (we know * it's not NULL at this point). */ if (PG_ARGISNULL(1)) PG_RETURN_DATUM(PG_GETARG_DATUM(0)); /* Now we know neither argument is NULL, so merge them. */ src = (tdigest_aggstate_t *) PG_GETARG_POINTER(1); dst = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); AssertCheckTDigestAggState(dst); AssertCheckTDigestAggState(src); /* * Keep the compression chosen when the state was created, even if the * input uses a different setting. Compaction can merge incoming * centroids, but cannot split them to recover lost detail. */ /* copy data from the tdigest into the aggstate */ for (i = 0; i < src->ncentroids; i++) { CHECK_FOR_INTERRUPTS(); tdigest_add_centroid(dst, src->centroids[i].mean, src->centroids[i].count); } AssertCheckTDigestAggState(dst); PG_RETURN_POINTER(dst); } /* API for incremental updates */ /* * expand the t-digest into an in-memory aggregate state */ static tdigest_aggstate_t * tdigest_digest_to_aggstate(tdigest_t *digest) { int i; tdigest_aggstate_t *state; /* * The digest should have gone through tdigest_prepare(), which means * we should have a local copy in the new format. */ Assert(digest->flags & TDIGEST_STORES_MEAN); state = tdigest_aggstate_allocate(0, 0, digest->compression, digest->ncentroids); /* copy data from the tdigest into the aggstate */ for (i = 0; i < digest->ncentroids; i++) { CHECK_FOR_INTERRUPTS(); tdigest_add_centroid(state, digest->centroids[i].mean, digest->centroids[i].count); } AssertCheckTDigestAggState(state); return state; } /* * Add a single value to the t-digest. This is not very efficient, as it has * to deserialize the t-digest into the in-memory aggstate representation * and serialize it back for each call, but it's convenient and acceptable * for some use cases. * * When efficiency is important, it may be possible to use the batch variant * by first aggregating the updates into a t-digest, and then merging that * into an existing t-digest in one step using tdigest_union_double_increment. * * The compact flag controls final compaction only. Adding to a full buffer * still triggers compaction through tdigest_add or tdigest_add_centroid. * * This is similar to hll_add, while the "union" is more like hll_union. */ Datum tdigest_add_double_increment(PG_FUNCTION_ARGS) { tdigest_aggstate_t *state; tdigest_t *result; bool compact; /* the flag determines whether the result gets compacted */ if (PG_ARGISNULL(3)) elog(ERROR, "compact flag must not be NULL"); compact = PG_GETARG_BOOL(3); /* * We want to skip NULL values altogether - we return either the existing * t-digest (if it already exists) or NULL. */ if (PG_ARGISNULL(1)) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); /* if there already is a state accumulated, don't forget it */ PG_RETURN_DATUM(PG_GETARG_DATUM(0)); } /* if there's no digest allocated, create it now */ if (PG_ARGISNULL(0)) { int compression; /* * We don't require compression, but only when there is an existing * t-digest value. Make sure the value was supplied. */ if (PG_ARGISNULL(2)) elog(ERROR, "compression value not supplied, but t-digest is NULL"); compression = PG_GETARG_INT32(2); check_compression(compression); state = tdigest_aggstate_allocate(0, 0, compression, 0); } else { tdigest_t *digest; digest = tdigest_prepare(PG_GETARG_DATUM(0), false); state = tdigest_digest_to_aggstate(digest); PG_FREE_IF_COPY(digest, 0); } tdigest_add(state, PG_GETARG_FLOAT8(1)); AssertCheckTDigestAggState(state); result = tdigest_aggstate_to_digest(state, compact); tdigest_aggstate_free(state); PG_RETURN_POINTER(result); } /* * Add an array of values to the t-digest. This amortizes the overhead of * deserializing and serializing the t-digest, compared to the per-value * version. * * When efficiency is important, it may be possible to use the batch variant * by first aggregating the updates into a t-digest, and then merging that * into an existing t-digest in one step using tdigest_union_double_increment. * * The compact flag controls final compaction only. Adding to a full buffer * still triggers compaction through tdigest_add or tdigest_add_centroid. * * This is similar to hll_add, while the "union" is more like hll_union. */ Datum tdigest_add_double_array_increment(PG_FUNCTION_ARGS) { tdigest_aggstate_t *state; tdigest_t *result; bool compact; ArrayType *array; const double *values; int nvalues; int i; /* the flag determines whether the result gets compacted */ if (PG_ARGISNULL(3)) elog(ERROR, "compact flag must not be NULL"); compact = PG_GETARG_BOOL(3); /* * We want to skip NULL values altogether - we return either the existing * t-digest (if it already exists) or NULL. */ if (PG_ARGISNULL(1)) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); /* if there already is a state accumulated, don't forget it */ PG_RETURN_DATUM(PG_GETARG_DATUM(0)); } /* if there's no digest allocated, create it now */ if (PG_ARGISNULL(0)) { int compression; /* * We don't require compression, but only when there is an existing * t-digest value. Make sure the value was supplied. */ if (PG_ARGISNULL(2)) elog(ERROR, "compression value not supplied, but t-digest is NULL"); compression = PG_GETARG_INT32(2); check_compression(compression); state = tdigest_aggstate_allocate(0, 0, compression, 0); } else { tdigest_t *digest; digest = tdigest_prepare(PG_GETARG_DATUM(0), false); state = tdigest_digest_to_aggstate(digest); PG_FREE_IF_COPY(digest, 0); } array = PG_GETARG_ARRAYTYPE_P(1); values = array_to_double(array, "an element", &nvalues); for (i = 0; i < nvalues; i++) tdigest_add(state, values[i]); PG_FREE_IF_COPY(array, 1); AssertCheckTDigestAggState(state); result = tdigest_aggstate_to_digest(state, compact); tdigest_aggstate_free(state); PG_RETURN_POINTER(result); } /* * Merge a t-digest into another t-digest. This is somewhat inefficient, as * it has to deserialize the t-digests into the in-memory aggstate values, * and serialize it back for each call, but it's better than doing it for * each individual value (like tdigest_add_double_increment). * * The compact flag controls final compaction only. Adding to a full buffer * still triggers compaction through tdigest_add or tdigest_add_centroid. * * This is similar to hll_union. */ Datum tdigest_union_double_increment(PG_FUNCTION_ARGS) { int i; tdigest_aggstate_t *state; tdigest_t *digest; tdigest_t *result; bool compact; /* the flag determines whether the result gets compacted */ if (PG_ARGISNULL(2)) elog(ERROR, "compact flag must not be NULL"); compact = PG_GETARG_BOOL(2); if (PG_ARGISNULL(0) && PG_ARGISNULL(1)) PG_RETURN_NULL(); else if (PG_ARGISNULL(0)) PG_RETURN_POINTER(PG_GETARG_POINTER(1)); else if (PG_ARGISNULL(1)) PG_RETURN_POINTER(PG_GETARG_POINTER(0)); /* now we know both arguments are non-null */ /* parse the first digest (we'll merge the other one into this) */ digest = tdigest_prepare(PG_GETARG_DATUM(0), false); state = tdigest_digest_to_aggstate(digest); PG_FREE_IF_COPY(digest, 0); AssertCheckTDigestAggState(state); /* parse the second digest */ digest = tdigest_prepare(PG_GETARG_DATUM(1), false); AssertCheckTDigest(digest); /* copy data from the tdigest into the aggstate */ for (i = 0; i < digest->ncentroids; i++) { CHECK_FOR_INTERRUPTS(); tdigest_add_centroid(state, digest->centroids[i].mean, digest->centroids[i].count); } PG_FREE_IF_COPY(digest, 1); AssertCheckTDigestAggState(state); result = tdigest_aggstate_to_digest(state, compact); tdigest_aggstate_free(state); PG_RETURN_POINTER(result); } /* * Comparator, ordering the centroids by mean value. * * When the mean is the same, we try ordering the centroids by count. * * In principle, centroids with the same mean represent the same value, * but we still need to care about the count to allow rebalancing the * centroids later. */ static int centroid_cmp(const void *a, const void *b) { double ma, mb; centroid_t *ca = (centroid_t *) a; centroid_t *cb = (centroid_t *) b; ma = ca->mean; mb = cb->mean; if (ma < mb) return -1; else if (ma > mb) return 1; if (ca->count < cb->count) return -1; else if (ca->count > cb->count) return 1; return 0; } /* * Parsing of the textual t-digest representation. * * We can't use sscanf, because it does not report overflows in any way - the * value simply saturates to the maximum for the data type. That's a problem * for the count, where the saturated value is a perfectly valid count, so we * can't detect it after the fact. Use strtoll/strtod, which do set errno. * * All of these advance the pointer past the parsed part on success, and never * return on failure. */ /* * Match a literal string, after skipping (optional) leading space. */ static void parse_str(char **ptr, const char *value, bool space) { char *str = *ptr; size_t len = strlen(value); /* if requested, skip the one initial space character */ if (space) { if (isspace((unsigned char) *str)) str++; else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("failed to parse t-digest value, missing space"))); } /* at this point there must be no whitespace */ if (isspace((unsigned char) *str)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("failed to parse t-digest value, unexpected space"))); /* the prefix should match our string */ if (strncmp(str, value, len) != 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("failed to parse t-digest value, expected \"%s\"", value))); *ptr = str + len; } /* * Parse an int64 value, and make sure it's in range. */ static int64 parse_int64(char **ptr, const char *field) { char *endptr; int64 value; errno = 0; value = strtoi64(*ptr, &endptr, 10); if (endptr == *ptr) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("failed to parse %s of a t-digest", field))); if (errno == ERANGE) ereport(ERROR, (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), errmsg("%s of a t-digest is out of range for bigint", field))); *ptr = endptr; return value; } /* * Parse an int32 value, and make sure it's in range. * * Parse it as int64 first, so that we can range check it before narrowing it * down, instead of relying on an implementation-defined narrowing conversion. */ static int32 parse_int32(char **ptr, const char *field) { int64 value = parse_int64(ptr, field); if (value < PG_INT32_MIN || value > PG_INT32_MAX) ereport(ERROR, (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), errmsg("%s of a t-digest is out of range for integer", field))); return (int32) value; } /* * Parse a double value, and make sure it's in range. */ static double parse_double(char **ptr, const char *field) { char *endptr; double value; errno = 0; value = strtod(*ptr, &endptr); if (endptr == *ptr) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("failed to parse %s of a t-digest", field))); /* ERANGE may also signal a nonzero subnormal, which we can store. */ if ((errno == ERANGE) && ((value == 0.0) || !isfinite(value))) ereport(ERROR, (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), errmsg("%s of a t-digest is out of range for double precision", field))); *ptr = endptr; return value; } Datum tdigest_in(PG_FUNCTION_ARGS) { int i; char *str = PG_GETARG_CSTRING(0); tdigest_t *digest = NULL; size_t slen; /* t-digest header fields */ int32 flags; int64 count, total_count; int compression; int ncentroids; char *ptr; slen = strlen(str); ptr = str; parse_str(&ptr, "flags", false); flags = parse_int32(&ptr, "flags"); parse_str(&ptr, "count", true); count = parse_int64(&ptr, "count"); parse_str(&ptr, "compression", true); compression = parse_int32(&ptr, "compression"); parse_str(&ptr, "centroids", true); ncentroids = parse_int32(&ptr, "number of centroids"); if ((flags & ~TDIGEST_VALID_FLAGS) != 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid flags for t-digest"))); if ((compression < MIN_COMPRESSION) || (compression > MAX_COMPRESSION)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("compression for t-digest must be in [%d, %d]", MIN_COMPRESSION, MAX_COMPRESSION))); if (count <= 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("count value for the t-digest must be positive"))); if (ncentroids <= 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("number of centroids for the t-digest must be positive"))); if (ncentroids > BUFFER_SIZE(compression)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("number of centroids for the t-digest exceeds buffer size"))); digest = tdigest_allocate(ncentroids); digest->flags = flags; digest->count = count; digest->ncentroids = ncentroids; digest->compression = compression; total_count = 0; ncentroids = 0; for (i = 0; i < digest->ncentroids; i++) { double mean; CHECK_FOR_INTERRUPTS(); parse_str(&ptr, "(", true); mean = parse_double(&ptr, "mean of a centroid"); parse_str(&ptr, ",", false); count = parse_int64(&ptr, "count of a centroid"); parse_str(&ptr, ")", false); if (!isfinite(mean)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("mean value for all centroids in a t-digest must be valid"))); if (count <= 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("count value for all centroids in a t-digest must be positive"))); else if (count > digest->count) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("count value of a centroid exceeds total count"))); digest->centroids[i].count = count; digest->centroids[i].mean = mean; /* * track the total count so that we can check later * * Make sure the count does not overflow at any point. It could * overflow and then wrap around to the expected total, but it would * still cause an issue. */ if (pg_add_s64_overflow(total_count, count, &total_count)) ereport(ERROR, (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), errmsg("tdigest count overflow"))); /* * This can't overflow - each centroid has a positive count, and if * the total_count does not overflow, this can't either. */ ncentroids++; /* * The parsing already moved the pointer past the closing parenthesis. * If this is the end of the string, stop parsing, even if we failed to * parse the right number of centroids. */ if (*ptr == '\0') break; /* must not scan past the end of the input string */ Assert(ptr <= str + slen); } /* * Malformed inputs may have the wrong number of centroids, in which case * we either don't consume the whole input (ncentroids too low), or we * don't get all the expected centroids (ncentroids too high). */ if (ptr < str + slen) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("input t-digest value too long"))); if (ncentroids != digest->ncentroids) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("input t-digest value too short"))); /* * If we consumed just the right number of centroids, we must have read * the whole input value exactly. */ Assert(ptr == str + slen); /* check that the total matches */ if (total_count != digest->count) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("total count does not match the data (%lld != %lld)", (long long) total_count, (long long) digest->count))); /* * Make sure we return digest with the new format (it might be the * old format, in which case "mean" fields actually store "sum"). */ tdigest_update_format(digest); AssertCheckTDigest(digest); PG_RETURN_POINTER(digest); } Datum tdigest_out(PG_FUNCTION_ARGS) { int i; tdigest_t *digest = PG_GETARG_TDIGEST(0); StringInfoData str; AssertCheckTDigest(digest); initStringInfo(&str); appendStringInfo(&str, "flags %d count " INT64_FORMAT " compression %d centroids %d", digest->flags, digest->count, digest->compression, digest->ncentroids); /* * If this is an old tdigest with sum values, we'll send those, and * it's up to the reader to fix it. It'll be indicated by not having * the TDIGEST_STORES_MEAN flag. */ for (i = 0; i < digest->ncentroids; i++) { char *tmp = float8out_internal(digest->centroids[i].mean); CHECK_FOR_INTERRUPTS(); appendStringInfo(&str, " (%s, " INT64_FORMAT ")", tmp, digest->centroids[i].count); pfree(tmp); } PG_FREE_IF_COPY(digest, 0); PG_RETURN_CSTRING(str.data); } Datum tdigest_recv(PG_FUNCTION_ARGS) { StringInfo buf = (StringInfo) PG_GETARG_POINTER(0); tdigest_t *digest; int i; int64 count; int64 total_count; int32 flags; int32 compression; int32 ncentroids; flags = pq_getmsgint(buf, sizeof(int32)); /* make sure the t-digest format is supported */ if ((flags != 0) && (flags != TDIGEST_STORES_MEAN)) elog(ERROR, "unsupported t-digest on-disk format"); count = pq_getmsgint64(buf); compression = pq_getmsgint(buf, sizeof(int32)); ncentroids = pq_getmsgint(buf, sizeof(int32)); if ((compression < MIN_COMPRESSION) || (compression > MAX_COMPRESSION)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("compression for t-digest must be in [%d, %d]", MIN_COMPRESSION, MAX_COMPRESSION))); if (count <= 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("count value for the t-digest must be positive"))); if (ncentroids <= 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("number of centroids for the t-digest must be positive"))); if (ncentroids > BUFFER_SIZE(compression)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("number of centroids for the t-digest exceeds buffer size"))); digest = tdigest_allocate(ncentroids); digest->flags = flags; digest->count = count; digest->compression = compression; digest->ncentroids = ncentroids; total_count = 0; for (i = 0; i < digest->ncentroids; i++) { CHECK_FOR_INTERRUPTS(); digest->centroids[i].mean = pq_getmsgfloat8(buf); digest->centroids[i].count = pq_getmsgint64(buf); if (!isfinite(digest->centroids[i].mean)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("mean value for all centroids in a t-digest must be valid"))); if (digest->centroids[i].count <= 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("count value for all centroids in a t-digest must be positive"))); else if (digest->centroids[i].count > digest->count) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("count value of a centroid exceeds total count"))); /* * track the total count so that we can check later * * Make sure the count does not overflow at any point. It could * overflow and then wrap around to the expected total, but it would * still cause an issue. */ if (pg_add_s64_overflow(total_count, digest->centroids[i].count, &total_count)) ereport(ERROR, (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), errmsg("tdigest count overflow"))); } /* check that the total matches */ if (total_count != digest->count) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("total count does not match the data (%lld != %lld)", (long long) total_count, (long long) digest->count))); /* * Make sure we return digest with the new format (it might be the * old format, in which case "mean" fields actually store "sum"). */ tdigest_update_format(digest); AssertCheckTDigest(digest); PG_RETURN_POINTER(digest); } Datum tdigest_send(PG_FUNCTION_ARGS) { tdigest_t *digest = PG_GETARG_TDIGEST(0); StringInfoData buf; int i; pq_begintypsend(&buf); pq_sendint(&buf, digest->flags, 4); pq_sendint64(&buf, digest->count); pq_sendint(&buf, digest->compression, 4); pq_sendint(&buf, digest->ncentroids, 4); for (i = 0; i < digest->ncentroids; i++) { CHECK_FOR_INTERRUPTS(); pq_sendfloat8(&buf, digest->centroids[i].mean); pq_sendint64(&buf, digest->centroids[i].count); } PG_FREE_IF_COPY(digest, 0); PG_RETURN_BYTEA_P(pq_endtypsend(&buf)); } /* * tdigest_is_valid * Check the t-digest passes the same sanity checks as the input funcs. * * The input functions (tdigest_in and tdigest_recv) validate the values, so * it's not possible to construct an invalid digest through them. But digests * stored by older versions of the extension (which did not have all those * checks) are not re-validated when read back, so they may be broken in * various ways - bogus flags, compression out of range, centroid counts not * adding up to the total count, and so on. This function performs the same * checks on an existing value, allowing users to find such digests. * * Returns true if the digest passes all the checks, false otherwise. Never * raises an error for an invalid digest (that's the whole point). * * The one check the input functions don't need to do explicitly is on the * length of the value - they build the digest from the parsed header, so it * always matches. For an existing value we have to check the value really is * long enough for the centroids the header promises, otherwise we'd read past * the end of it. * * XXX Keep this in sync with the checks in tdigest_in and tdigest_recv. */ static bool tdigest_is_valid_internal(tdigest_t *digest) { int i; int64 total_count; Size vlen; Size expected; vlen = VARSIZE_ANY(digest); /* * We need at least the header, otherwise we can't even look at the fields * describing the rest of the value. */ if (vlen < offsetof(tdigest_t, centroids)) return false; /* make sure the t-digest format is supported */ if ((digest->flags & ~TDIGEST_VALID_FLAGS) != 0) return false; if ((digest->compression < MIN_COMPRESSION) || (digest->compression > MAX_COMPRESSION)) return false; if (digest->count <= 0) return false; if (digest->ncentroids <= 0) return false; if (digest->ncentroids > BUFFER_SIZE(digest->compression)) return false; /* * The header determines how long the value has to be, so make sure it * really is that long before reading any of the centroids. * * The number of centroids is limited by the buffer size (checked above), * so this can't overflow. */ expected = offsetof(tdigest_t, centroids) + digest->ncentroids * sizeof(centroid_t); if (vlen != expected) return false; total_count = 0; for (i = 0; i < digest->ncentroids; i++) { CHECK_FOR_INTERRUPTS(); if (!isfinite(digest->centroids[i].mean)) return false; if (digest->centroids[i].count <= 0) return false; else if (digest->centroids[i].count > digest->count) return false; /* * track the total count so that we can check later * * Make sure the count does not overflow at any point. It could * overflow and then wrap around to the expected total, but it would * still cause an issue. */ if (pg_add_s64_overflow(total_count, digest->centroids[i].count, &total_count)) return false; } /* check that the total matches */ if (total_count != digest->count) return false; return true; } /* * Wrapper for tdigest_is_valid_internal(), so that we can free the digest. */ Datum tdigest_is_valid(PG_FUNCTION_ARGS) { tdigest_t *digest = PG_GETARG_TDIGEST(0); bool valid = tdigest_is_valid_internal(digest); PG_FREE_IF_COPY(digest, 0); PG_RETURN_BOOL(valid); } Datum tdigest_count(PG_FUNCTION_ARGS) { int64 result; tdigest_t *digest = PG_GETARG_TDIGEST(0); result = digest->count; PG_FREE_IF_COPY(digest, 0); PG_RETURN_INT64(result); } /* * tdigest_to_json * Transform the tdigest into a JSON value. * * We make sure to always print mean, even for tdigests in the older format * storing sum for centroids. Otherwise the "means" key would be confusing. * But we don't call tdigest_update_format, and instead we simply update the * flags and convert the sum/mean values. * * The centroids are stored in two separate arrays - one for means, one for * counts. That makes it easier to process, because it's clear the i-th * in each array is for i-th centroid. We might store it in a single array, * but then we'd have to walk it in pairs. And it'd mix float and int * values in the same array. * * The arrays are named in plural ("means" and "counts"), so that the array * of per-centroid counts does not collide with the total count. Duplicate * keys are not strictly forbidden by JSON, but most parsers keep just one * of the values (jsonb keeps the last one), losing the other. */ Datum tdigest_to_json(PG_FUNCTION_ARGS) { int i; StringInfoData str; text *result; tdigest_t *digest = PG_GETARG_TDIGEST(0); int32 flags = digest->flags; initStringInfo(&str); appendStringInfoChar(&str, '{'); flags |= TDIGEST_STORES_MEAN; appendStringInfo(&str, "\"flags\": %d, ", flags); appendStringInfo(&str, "\"count\": " INT64_FORMAT ", ", digest->count); appendStringInfo(&str, "\"compression\": %d, ", digest->compression); appendStringInfo(&str, "\"centroids\": %d, ", digest->ncentroids); appendStringInfoString(&str, "\"means\": ["); for (i = 0; i < digest->ncentroids; i++) { double mean = digest->centroids[i].mean; char *tmp; CHECK_FOR_INTERRUPTS(); if (i > 0) appendStringInfoString(&str, ", "); /* * When the TDIGEST_STORES_MEAN flag is not set, the value is * actually a sum, so convert it to mean now. We have to check the * digest->flags, not the local variable. */ if (! (digest->flags & TDIGEST_STORES_MEAN)) mean = mean / digest->centroids[i].count; tmp = float8out_internal(mean); /* don't print insignificant zeroes to the right of decimal point */ appendStringInfo(&str, "%s", tmp); pfree(tmp); } appendStringInfoString(&str, "], "); appendStringInfoString(&str, "\"counts\": ["); for (i = 0; i < digest->ncentroids; i++) { CHECK_FOR_INTERRUPTS(); if (i > 0) appendStringInfoString(&str, ", "); appendStringInfo(&str, INT64_FORMAT, digest->centroids[i].count); } appendStringInfoString(&str, "]"); appendStringInfoChar(&str, '}'); result = cstring_to_text(str.data); /* free the local buffer, and possibly the detoasted digest copy */ pfree(str.data); PG_FREE_IF_COPY(digest, 0); PG_RETURN_TEXT_P(result); } /* * tdigest_to_array * Transform the tdigest into an array of double values. * * The whole digest is stored in a single "double precision" array, which * may be a bit confusing and perhaps fragile if more fields need to be * added in the future. The initial elements are flags, count (number of * items added to the digest), compression (determines the limit on number * of centroids) and current number of centroids, followed by a stream of values * encoding the centroids in pairs of (mean, count). * * We make sure to always print mean, even for tdigests in the older format * storing sum for centroids. Otherwise the "mean" key would be confusing. * But we don't call tdigest_update_format, and instead we simply update the * flags and convert the sum/mean values. */ Datum tdigest_to_array(PG_FUNCTION_ARGS) { int i, idx; tdigest_t *digest = PG_GETARG_TDIGEST(0); int32 flags = digest->flags; ArrayType *array; double *values; int nvalues; flags |= TDIGEST_STORES_MEAN; /* number of values to store in the array */ nvalues = 4 + (digest->ncentroids * 2); /* write results directly into a correctly sized ArrayType */ array = double_array_allocate(nvalues); values = (double *) ARR_DATA_PTR(array); idx = 0; values[idx++] = flags; values[idx++] = digest->count; values[idx++] = digest->compression; values[idx++] = digest->ncentroids; for (i = 0; i < digest->ncentroids; i++) { double mean = digest->centroids[i].mean; CHECK_FOR_INTERRUPTS(); /* * When the TDIGEST_STORES_MEAN flag is not set, the value is * actually a sum, so convert it to mean now. We have to check the * digest->flags, not the local variable. */ if (! (digest->flags & TDIGEST_STORES_MEAN)) mean = mean / digest->centroids[i].count; /* don't print insignificant zeroes to the right of decimal point */ values[idx++] = mean; values[idx++] = digest->centroids[i].count; } Assert(idx == nvalues); PG_FREE_IF_COPY(digest, 0); PG_RETURN_ARRAYTYPE_P(array); } Datum tdigest_add_double_trimmed(PG_FUNCTION_ARGS) { tdigest_aggstate_t *state; MemoryContext aggcontext; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_add_double_trimmed called in non-aggregate context"); /* * We want to skip NULL values altogether - we return either the existing * t-digest (if it already exists) or NULL. */ if (PG_ARGISNULL(1)) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); /* if there already is a state accumulated, don't forget it */ PG_RETURN_DATUM(PG_GETARG_DATUM(0)); } /* if there's no digest allocated, create it now */ if (PG_ARGISNULL(0)) { MemoryContext oldcontext; int compression; double low, high; /* the trim thresholds are required to create the aggregate state */ if (PG_ARGISNULL(3) || PG_ARGISNULL(4)) elog(ERROR, "trim thresholds must not be NULL"); low = PG_GETARG_FLOAT8(3); high = PG_GETARG_FLOAT8(4); if (PG_ARGISNULL(2)) elog(ERROR, "compression must not be NULL"); compression = PG_GETARG_INT32(2); check_compression(compression); check_trim_values(low, high); oldcontext = MemoryContextSwitchTo(aggcontext); state = tdigest_aggstate_allocate(0, 0, compression, 0); state->trim_low = low; state->trim_high = high; MemoryContextSwitchTo(oldcontext); } else state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); tdigest_add(state, PG_GETARG_FLOAT8(1)); AssertCheckTDigestAggState(state); PG_RETURN_POINTER(state); } Datum tdigest_add_double_count_trimmed(PG_FUNCTION_ARGS) { int64 i; int64 count; tdigest_aggstate_t *state; MemoryContext aggcontext; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_add_double_count_trimmed called in non-aggregate context"); /* * We want to skip NULL values altogether - we return either the existing * t-digest (if it already exists) or NULL. */ if (PG_ARGISNULL(1)) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); /* if there already is a state accumulated, don't forget it */ PG_RETURN_DATUM(PG_GETARG_DATUM(0)); } /* if there's no digest allocated, create it now */ if (PG_ARGISNULL(0)) { MemoryContext oldcontext; int compression; double low, high; /* the trim thresholds are required to create the aggregate state */ if (PG_ARGISNULL(4) || PG_ARGISNULL(5)) elog(ERROR, "trim thresholds must not be NULL"); low = PG_GETARG_FLOAT8(4); high = PG_GETARG_FLOAT8(5); if (PG_ARGISNULL(3)) elog(ERROR, "compression must not be NULL"); compression = PG_GETARG_INT32(3); check_compression(compression); check_trim_values(low, high); oldcontext = MemoryContextSwitchTo(aggcontext); state = tdigest_aggstate_allocate(0, 0, compression, 0); state->trim_low = low; state->trim_high = high; MemoryContextSwitchTo(oldcontext); } else state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); if (PG_ARGISNULL(2)) count = 1; else count = PG_GETARG_INT64(2); /* can't add values with non-positive counts */ if (count <= 0) elog(ERROR, "invalid count value %lld, must be a positive value", (long long) count); /* * When adding more values than would fit into an empty buffer (and * thus likely causing too many compactions), we instead add them as * properly sized centroids. * * This is much faster, because the centroids can be generated in one go, * so there are only very few compactions. */ if (count > BUFFER_SIZE(state->compression)) { tdigest_add_generated(state, PG_GETARG_FLOAT8(1), count); count = 0; } /* * If there are only a few values, just add them one by one, so that * we do proper compaction and sizing of centroids. Otherwise we might end * up with oversized centroids on the tails etc. */ for (i = 0; i < count; i++) tdigest_add(state, PG_GETARG_FLOAT8(1)); AssertCheckTDigestAggState(state); PG_RETURN_POINTER(state); } /* * Add a value to the tdigest (create one if needed). Transition function * for tdigest aggregate with a single value. */ Datum tdigest_add_digest_trimmed(PG_FUNCTION_ARGS) { int i; tdigest_aggstate_t *state; tdigest_t *digest; MemoryContext aggcontext; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_add_digest_trimmed called in non-aggregate context"); /* * We want to skip NULL values altogether - we return either the existing * t-digest (if it already exists) or NULL. */ if (PG_ARGISNULL(1)) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); /* if there already is a state accumulated, don't forget it */ PG_RETURN_DATUM(PG_GETARG_DATUM(0)); } /* make sure we have a detoasted copy */ digest = tdigest_prepare(PG_GETARG_DATUM(1), false); /* if there's no aggregate state allocated, create it now */ if (PG_ARGISNULL(0)) { MemoryContext oldcontext; double low, high; /* the trim thresholds are required to create the aggregate state */ if (PG_ARGISNULL(2) || PG_ARGISNULL(3)) elog(ERROR, "trim thresholds must not be NULL"); low = PG_GETARG_FLOAT8(2); high = PG_GETARG_FLOAT8(3); check_trim_values(low, high); oldcontext = MemoryContextSwitchTo(aggcontext); state = tdigest_aggstate_allocate(0, 0, digest->compression, digest->ncentroids); state->trim_low = low; state->trim_high = high; MemoryContextSwitchTo(oldcontext); } else state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); /* * Keep the compression chosen when the state was created, even if the * input uses a different setting. Compaction can merge incoming * centroids, but cannot split them to recover lost detail. */ for (i = 0; i < digest->ncentroids; i++) { CHECK_FOR_INTERRUPTS(); tdigest_add_centroid(state, digest->centroids[i].mean, digest->centroids[i].count); } AssertCheckTDigestAggState(state); PG_FREE_IF_COPY(digest, 1); PG_RETURN_POINTER(state); } /* * Convert a (count * fraction) product back to a count. * * The product is calculated in double, so for very high counts it may end up * outside the int64 range - even for frac = 1.0. For example * * 1.000000 * 9223372036854775296 = 9223372036854775808.000000 * * which is above both the count and INT64_MAX (9223372036854775807). That * makes the conversion undefined. It can't lead to underflow/overflow, as * we're not using this to access memory, but it might lead to bogus results * (e.g. NULL instead of the correct result). * * Clamp it to the [0, maxvalue] range, which is the only range that makes * sense anyway. * * Note: We know the frac value is in [0.0, 1.0], but we don't rely on that * here. We'll clamp it to [0, maxvalue]. */ static int64 double_to_int64(double value, int64 maxvalue) { /* paranoia: we should not get NaN values here */ if (isnan(value)) return 0; /* clamp it to the [0, count] range */ if (value < 0) return 0; /* * The comparison is done in double on purpose. If we did it as int64, * it might already overflow and wrap. Converting count to double may * round it up to 2^63, but that should be fine - the comparison is * still correct, and we return count for anything that large. */ if (value >= (double) maxvalue) return maxvalue; /* ok, should be safe to cast */ return (int64) value; } /* * How many items of a centroid to use for the aggregate? * * Calculates the number of items of a centroid that fall into the * [count_low, count_high) range of items, with count_done items preceding * the centroid. * * The centroids in the middle of the range are included as a whole, but the * first and last one may be cut in half by the boundary, in which case only * part of the centroid is included. */ static int64 tdigest_trimmed_count(centroid_t *centroid, int64 count_done, int64 count_low, int64 count_high) { int64 count_add; /* Assume the whole centroid falls into the range. */ count_add = centroid->count; /* * If we haven't reached the low threshold yet, skip appropriate * part of the centroid. * * (count_low - count_done) is how far we're from the low threshold, so a * positive value is how many items we still need to "skip" (capped to * size of the centroid). A negative value means we're past the low * threshold, and the centroid may be in the range (unless it's past the * high threshold too). */ count_add -= Min(Max(0, count_low - count_done), count_add); /* * If we have reached the upper threshold, ignore the overflowing * part of the centroid. * * The items we still have start at count_low (or at the beginning of * the centroid, whichever comes later), not at count_done - the part * below count_low was already removed by the preceding step. Don't * count that part a second time, i.e. don't start at the beginning * of the centroid. */ count_add = Min(Max(0, count_high - Max(count_done, count_low)), count_add); return count_add; } /* * Calculate trimmed aggregates from centroids. * * Returns the trimmed mean, the trimmed sum, and the number of items in the * trimmed range. * * The obvious way to calculate the mean is to add (mean * count) for all the * centroids in the range, and then divide the sum by the number of items. * That however may overflow to infinity, even when the mean is perfectly * representable - the average of 1e307 values is 1e307, but the sum of many * such values is not. And once one partial sum saturates to +Infinity and * another one to -Infinity, the accumulator turns into NaN. * * The mean of a digest is always within the range of the centroid means, so * it should never overflow. We just have to calculate it in a way that does * not overflow either, i.e. as a weighted average of the centroid means, * similarly to how tdigest_compact() does when merging centroids * * mean += centroids[i].mean * (count_add / total_count) * * The weights are non-negative and add up to 1.0 (albeit maybe not perfectly, * due to limited precision of float8), so the running mean stays within the * range of the centroid means. * * This needs the total number of items in the range up front, so we walk the * centroids twice. The first pass determines the first and last centroid of * the range, and adds up the (int64) counts - that can't overflow, as the * counts add up to the total count of the digest. * * The sum, on the other hand, may legitimately exceed the float8 range, so we * keep accumulating it the simple way (which is also exact), and leave it to * the caller to complain about the overflow. * * FIXME I believe this has two issues: * * First, the count_low/count_high round in opposite direction, but may round * to the same value when the product hits an integer exactly (when low==high). * If this happens, we end up returning NULL a bit later (count_total==0). * Maybe we should require (low < high)? Would that solve the issue, or are * there other ways to still hit this? Or maybe we should do something like * count_high = Max(count_high, count_low + 1), to force non-empty ranges? * * Second, the code assumes the whole centroid is located at the mean, and does * not use the interpolation (assuming half the items are below/above the mean). * Maybe it should do something like the quantile/quantile-of code? */ static void tdigest_trimmed_agg(centroid_t *centroids, int ncentroids, int64 count, double low, double high, double *meanp, double *sump, int64 *countp) { int i; int first = 0, last = -1; double mean = 0, sum = 0; int64 count_done = 0, count_first = 0, count_total = 0, count_low, count_high; /* translate the percentiles to counts */ count_low = double_to_int64(floor(count * low), count); count_high = double_to_int64(ceil(count * high), count); /* verify sane range */ Assert((count_low <= count_high) && (0 <= count_low) && (count_high <= count)); *meanp = 0; *sump = 0; *countp = 0; /* * Find the first and last centroid of the range, and the number of items * the range contains. */ for (i = 0; i < ncentroids; i++) { int64 count_add = tdigest_trimmed_count(¢roids[i], count_done, count_low, count_high); CHECK_FOR_INTERRUPTS(); if (count_add > 0) { /* remember where the range starts, including the item offset */ if (last == -1) { first = i; count_first = count_done; } last = i; count_total += count_add; } /* consider the whole centroid processed */ count_done += centroids[i].count; /* break once we cross the high threshold */ if (count_done >= count_high) break; } /* no items in the range, the callers return NULL in that case */ if (count_total == 0) return; /* now walk just the centroids in the range, and calculate mean / sum */ count_done = count_first; for (i = first; i <= last; i++) { int64 count_add = tdigest_trimmed_count(¢roids[i], count_done, count_low, count_high); CHECK_FOR_INTERRUPTS(); /* consider the whole centroid processed */ count_done += centroids[i].count; /* increment the mean / sum */ mean += centroids[i].mean * (count_add / (double) count_total); sum += centroids[i].mean * count_add; Assert(!isnan(mean)); } // Assert((centroids[first].mean <= mean) && (mean <= centroids[last].mean)); /* * paranoia: handle possible overflow of the mean by clamping it to the * valid range using the mean of the first/last centroid * * Maybe overflow is not the right term, but what can happen easily is * the value "drifting" outside the valid range with very high counts, * even if the centroids have the exact same mean. For example with * centroids like (1, 22522773787004704) and (1, 22627252715671912) the * mean will drift to 1.0000000000000002. * * To confirm, uncomment the assert above, and rerun tests. There's a * query that triggers it. */ mean = Max(Min(centroids[last].mean, mean), centroids[first].mean); *meanp = mean; *sump = sum; *countp = count_total; } /* * Return the trimmed sum, and complain if it overflowed - just like the * regular float8 arithmetic does, instead of silently returning infinity. * * Infinity is a perfectly valid result if some of the input values were * infinite, in which case the mean is infinite too. * * XXX This is what float8_mul() does, but that's only available on PG12+. */ static double tdigest_trimmed_sum_value(double sum, double mean, int64 count) { /* * The accumulator may overflow even when the sum itself is perfectly * representable (e.g. with values of the opposite sign). The mean can't * overflow, so recalculate the sum from that. */ if (!isfinite(sum) && isfinite(mean)) sum = mean * (double) count; if (isinf(sum) && !isinf(mean)) ereport(ERROR, (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), errmsg("value out of range: overflow"))); return sum; } /* * Compute percentile from a tdigest. Final function for tdigest aggregate * with a single percentile. */ Datum tdigest_trimmed_avg(PG_FUNCTION_ARGS) { tdigest_aggstate_t *state; MemoryContext aggcontext; double mean; double sum; int64 count; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_trimmed_avg called in non-aggregate context"); /* if there's no digest, return NULL */ if (PG_ARGISNULL(0)) PG_RETURN_NULL(); state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); /* make sure the centroids are sorted */ tdigest_sort(state); tdigest_trimmed_agg(state->centroids, state->ncentroids, state->count, state->trim_low, state->trim_high, &mean, &sum, &count); if (count > 0) PG_RETURN_FLOAT8(mean); PG_RETURN_NULL(); } /* * Compute percentile from a tdigest. Final function for tdigest aggregate * with a single percentile. */ Datum tdigest_trimmed_sum(PG_FUNCTION_ARGS) { tdigest_aggstate_t *state; MemoryContext aggcontext; double mean; double sum; int64 count; /* cannot be called directly because of internal-type argument */ if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "tdigest_trimmed_sum called in non-aggregate context"); /* if there's no digest, return NULL */ if (PG_ARGISNULL(0)) PG_RETURN_NULL(); state = (tdigest_aggstate_t *) PG_GETARG_POINTER(0); /* make sure the centroids are sorted */ tdigest_sort(state); tdigest_trimmed_agg(state->centroids, state->ncentroids, state->count, state->trim_low, state->trim_high, &mean, &sum, &count); if (count > 0) PG_RETURN_FLOAT8(tdigest_trimmed_sum_value(sum, mean, count)); PG_RETURN_NULL(); } /* * Trimmed sum of a single digest (non-aggregate function). */ Datum tdigest_digest_sum(PG_FUNCTION_ARGS) { tdigest_t *digest; double low = PG_GETARG_FLOAT8(1); double high = PG_GETARG_FLOAT8(2); double mean; double sum; int64 count; check_trim_values(low, high); /* tdigest_trimmed_agg expects the centroids sorted by mean */ digest = tdigest_prepare(PG_GETARG_DATUM(0), true); AssertCheckTDigest(digest); tdigest_trimmed_agg(digest->centroids, digest->ncentroids, digest->count, low, high, &mean, &sum, &count); PG_FREE_IF_COPY(digest, 0); if (count > 0) PG_RETURN_FLOAT8(tdigest_trimmed_sum_value(sum, mean, count)); PG_RETURN_NULL(); } /* * Trimmed average of a single digest (non-aggregate function) */ Datum tdigest_digest_avg(PG_FUNCTION_ARGS) { tdigest_t *digest; double low = PG_GETARG_FLOAT8(1); double high = PG_GETARG_FLOAT8(2); double mean; double sum; int64 count; check_trim_values(low, high); /* tdigest_trimmed_agg expects the centroids sorted by mean */ digest = tdigest_prepare(PG_GETARG_DATUM(0), true); AssertCheckTDigest(digest); tdigest_trimmed_agg(digest->centroids, digest->ncentroids, digest->count, low, high, &mean, &sum, &count); PG_FREE_IF_COPY(digest, 0); if (count > 0) PG_RETURN_FLOAT8(mean); PG_RETURN_NULL(); } /* * Return a read-only view of an input FLOAT8 SQL array as C doubles. * * This expects a single-dimensional float8 array, fails otherwise. * * "what" names a single element of the array, with the article, and is used * when reporting a NULL element. The callers pass arrays of different things * (percentiles, hypothetical values, values to add to a digest), and a message * naming the wrong one points at the wrong argument. * * The caller must keep the detoasted array alive while using the view. */ static const double * array_to_double(ArrayType *v, const char *what, int *len) { int nitems, *dims, ndims; Oid element_type; ndims = ARR_NDIM(v); dims = ARR_DIMS(v); nitems = ArrayGetNItems(ndims, dims); /* * Reject empty arrays explicitly. An empty array has ndims = 0, so * without this it would be caught by the single-dimension check below * and reported as a dimensionality problem, which is misleading - the * array is well-formed, it just has nothing in it. */ if (nitems == 0) elog(ERROR, "the array must not be empty"); /* this is a special-purpose function for single-dimensional arrays */ if (ndims != 1) elog(ERROR, "expected a single-dimensional array (dims = %d)", ndims); element_type = ARR_ELEMTYPE(v); /* XXX not sure if really needed (can it actually happen?) */ if (element_type != FLOAT8OID) elog(ERROR, "array_to_double expects FLOAT8 array"); if (array_contains_nulls(v)) elog(ERROR, "NULL not allowed as %s", what); (*len) = nitems; /* Non-NULL float8 elements have the same layout as a C double array. */ return (const double *) ARR_DATA_PTR(v); } /* * Allocate a one-dimensional, non-NULL float8 array for direct result writes. * Array storage uses native doubles even on platforms with pass-by-reference * float8 Datums, so no per-element Datum allocations are needed. */ static ArrayType * double_array_allocate(int nitems) { ArrayType *array; Size size; /* should not happen */ if (nitems <= 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid array size (%d)", nitems))); if (nitems > MaxArraySize) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("array size exceeds the maximum allowed (%d)", (int) MaxArraySize))); size = ARR_OVERHEAD_NONULLS(1) + nitems * sizeof(double); array = (ArrayType *) palloc(size); SET_VARSIZE(array, size); ARR_NDIM(array) = 1; array->dataoffset = 0; ARR_ELEMTYPE(array) = FLOAT8OID; ARR_DIMS(array)[0] = nitems; ARR_LBOUND(array)[0] = 1; return array; } tvondra-tdigest-c0af713/tdigest.control000066400000000000000000000001361525543331000203040ustar00rootroot00000000000000comment = 'Provides tdigest aggregate function.' default_version = '1.4.7' relocatable = true tvondra-tdigest-c0af713/test/000077500000000000000000000000001525543331000162165ustar00rootroot00000000000000tvondra-tdigest-c0af713/test/expected/000077500000000000000000000000001525543331000200175ustar00rootroot00000000000000tvondra-tdigest-c0af713/test/expected/add_overflow.out000066400000000000000000000025071525543331000232270ustar00rootroot00000000000000-- Regression test for the count overflow in tdigest_add(), which adds a -- single value (a centroid with count 1) to the aggregate state. -- -- The total count is int64, so adding one more value to a t-digest with -- count INT64_MAX has to fail instead of wrapping around to a negative -- total. The incremental API is the simplest way to get a state with such -- a total count. \set VERBOSITY terse -- adding a single value overflows the total count SELECT tdigest_add('flags 1 count 9223372036854775807 compression 100 centroids 1 (1, 9223372036854775807)'::tdigest, 2.0); ERROR: tdigest count overflow -- the same thing when adding an array of values SELECT tdigest_add('flags 1 count 9223372036854775807 compression 100 centroids 1 (1, 9223372036854775807)'::tdigest, ARRAY[2.0]); ERROR: tdigest count overflow -- the last value that still fits must not fail, though SELECT tdigest_add('flags 1 count 9223372036854775806 compression 100 centroids 1 (1, 9223372036854775806)'::tdigest, 2.0); tdigest_add ----------------------------------------------------------------------------------------------- flags 1 count 9223372036854775807 compression 100 centroids 2 (1, 9223372036854775806) (2, 1) (1 row) tvondra-tdigest-c0af713/test/expected/array_count_checks.out000066400000000000000000000032341525543331000244200ustar00rootroot00000000000000-- Regression test for the count check in tdigest_add_double_array_count() -- and tdigest_add_double_array_values_count(), the transition functions of -- the aggregates accepting an explicit number of occurrences of a value and -- an array of percentiles (or values). -- -- A non-NULL count must be positive. A NULL count means one occurrence -- of the input value. \set VERBOSITY terse -- zero count SELECT tdigest_percentile(v, 0::bigint, 100, ARRAY[0.5]) FROM (VALUES (1.0)) AS t(v); ERROR: invalid count value 0, must be a positive value SELECT tdigest_percentile_of(v, 0::bigint, 100, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v); ERROR: invalid count value 0, must be a positive value -- negative count SELECT tdigest_percentile(v, -1::bigint, 100, ARRAY[0.5]) FROM (VALUES (1.0)) AS t(v); ERROR: invalid count value -1, must be a positive value SELECT tdigest_percentile_of(v, -1::bigint, 100, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v); ERROR: invalid count value -1, must be a positive value -- the smallest valid count has to work SELECT tdigest_percentile(v, 1::bigint, 100, ARRAY[0.5]) FROM (VALUES (1.0)) AS t(v); tdigest_percentile -------------------- {1} (1 row) SELECT tdigest_percentile_of(v, 1::bigint, 100, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v); tdigest_percentile_of ----------------------- {0.5} (1 row) -- NULL count means a single occurrence, so it's valid too SELECT tdigest_percentile(v, NULL::bigint, 100, ARRAY[0.5]) FROM (VALUES (1.0)) AS t(v); tdigest_percentile -------------------- {1} (1 row) SELECT tdigest_percentile_of(v, NULL::bigint, 100, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v); tdigest_percentile_of ----------------------- {0.5} (1 row) tvondra-tdigest-c0af713/test/expected/basic.out000066400000000000000000000254441525543331000216420ustar00rootroot00000000000000-- validation of percentile values -- correct values SELECT tdigest_percentile(1.0, 10, 0.0); tdigest_percentile -------------------- 1 (1 row) SELECT tdigest_percentile(1.0, 10, 0.1); tdigest_percentile -------------------- 1 (1 row) SELECT tdigest_percentile(1.0, 10, 0.5); tdigest_percentile -------------------- 1 (1 row) SELECT tdigest_percentile(1.0, 10, 0.9); tdigest_percentile -------------------- 1 (1 row) SELECT tdigest_percentile(1.0, 10, 1.0); tdigest_percentile -------------------- 1 (1 row) SELECT tdigest_percentile(1.0, 10, ARRAY[0.0, 0.1, 0.5, 0.9, 1.0]); tdigest_percentile -------------------- {1,1,1,1,1} (1 row) -- out of range values SELECT tdigest_percentile(1.0, 10, -1.0); ERROR: invalid percentile value -1.000000, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0, 10, 1.5); ERROR: invalid percentile value 1.500000, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0, 10, ARRAY[0.0, -1.0, 0.5, 0.9, 1.0]); ERROR: invalid percentile value -1.000000, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0, 10, ARRAY[0.0, 0.1, 0.5, 0.9, 1.5]); ERROR: invalid percentile value 1.500000, should be in [0.0, 1.0] -- NaN SELECT tdigest_percentile(1.0, 10, 'NaN'::double precision); ERROR: invalid percentile value NaN, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0, 10, ARRAY[0.0, 'NaN'::double precision, 0.5, 0.9, 1.0]); ERROR: invalid percentile value NaN, should be in [0.0, 1.0] -- infinite values SELECT tdigest_percentile(1.0, 10, 'infinity'::double precision); ERROR: invalid percentile value Infinity, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0, 10, '-infinity'::double precision); ERROR: invalid percentile value -Infinity, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0, 10, ARRAY[0.0, 'infinity'::double precision, 0.5, 0.9, 1.0]); ERROR: invalid percentile value Infinity, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0, 10, ARRAY[0.0, '-infinity'::double precision, 0.5, 0.9, 1.0]); ERROR: invalid percentile value -Infinity, should be in [0.0, 1.0] -- more tests SELECT tdigest_percentile(1.0::double precision, 10, 'NaN'::double precision); ERROR: invalid percentile value NaN, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0::double precision, 10, '-infinity'::double precision); ERROR: invalid percentile value -Infinity, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0::double precision, 10, 'infinity'::double precision); ERROR: invalid percentile value Infinity, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0::double precision, 10, ARRAY[0.5, 'NaN']::double precision[]); ERROR: invalid percentile value NaN, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0::double precision, 10, ARRAY[0.5, '-infinity']::double precision[]); ERROR: invalid percentile value -Infinity, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0::double precision, 10, ARRAY[0.5, 'infinity']::double precision[]); ERROR: invalid percentile value Infinity, should be in [0.0, 1.0] SELECT tdigest_percentile('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, 'NaN'::double precision); ERROR: invalid percentile value NaN, should be in [0.0, 1.0] SELECT tdigest_percentile('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, '-infinity'::double precision); ERROR: invalid percentile value -Infinity, should be in [0.0, 1.0] SELECT tdigest_percentile('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, 'infinity'::double precision); ERROR: invalid percentile value Infinity, should be in [0.0, 1.0] SELECT tdigest_percentile('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, ARRAY[0.5, 'NaN']::double precision[]); ERROR: invalid percentile value NaN, should be in [0.0, 1.0] SELECT tdigest_percentile('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, ARRAY[0.5, '-infinity']::double precision[]); ERROR: invalid percentile value -Infinity, should be in [0.0, 1.0] SELECT tdigest_percentile('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, ARRAY[0.5, 'infinity']::double precision[]); ERROR: invalid percentile value Infinity, should be in [0.0, 1.0] -- trim thresholds are checked too SELECT tdigest_avg(1.0::double precision, 10, 'NaN'::double precision, 1.0); ERROR: invalid low percentile value NaN, should be in [0.0, 1.0] SELECT tdigest_avg(1.0::double precision, 10, '-infinity'::double precision, 1.0); ERROR: invalid low percentile value -Infinity, should be in [0.0, 1.0] SELECT tdigest_avg(1.0::double precision, 10, 'infinity'::double precision, 1.0); ERROR: invalid low percentile value Infinity, should be in [0.0, 1.0] SELECT tdigest_digest_sum('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, 0.0, 'NaN'::double precision); ERROR: invalid high percentile value NaN, should be in [0.0, 1.0] SELECT tdigest_digest_sum('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, 0.0, '-infinity'::double precision); ERROR: invalid high percentile value -Infinity, should be in [0.0, 1.0] SELECT tdigest_digest_sum('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, 0.0, 'infinity'::double precision); ERROR: invalid high percentile value Infinity, should be in [0.0, 1.0] -- validation of input values -- input functions must reject NaN / infinity means in various places SELECT 'flags 1 count 1 compression 10 centroids 1 (NaN, 1)'::tdigest; ERROR: mean value for all centroids in a t-digest must be valid LINE 1: SELECT 'flags 1 count 1 compression 10 centroids 1 (NaN, 1)'... ^ SELECT 'flags 1 count 1 compression 10 centroids 1 (infinity, 1)'::tdigest; ERROR: mean value for all centroids in a t-digest must be valid LINE 1: SELECT 'flags 1 count 1 compression 10 centroids 1 (infinity... ^ SELECT 'flags 1 count 1 compression 10 centroids 1 (-infinity, 1)'::tdigest; ERROR: mean value for all centroids in a t-digest must be valid LINE 1: SELECT 'flags 1 count 1 compression 10 centroids 1 (-infinit... ^ -- the aggregates building the digest have to reject NaN / infinity too SELECT tdigest('NaN'::float8, 10); ERROR: all values added to t-digest must be finite SELECT tdigest('infinity'::float8, 10); ERROR: all values added to t-digest must be finite SELECT tdigest('-infinity'::float8, 10); ERROR: all values added to t-digest must be finite -- same for the value/count API SELECT tdigest('NaN'::float8, 50::bigint, 10); ERROR: all values added to t-digest must be finite SELECT tdigest('NaN'::float8, 200::bigint, 10); ERROR: all values added to t-digest must be finite SELECT tdigest('Infinity'::float8, 200::bigint, 10); ERROR: all values added to t-digest must be finite SELECT tdigest('-Infinity'::float8, 200::bigint, 10); ERROR: all values added to t-digest must be finite -- t-digest extreme enough for the percentile to fall to the right of the last centroid SELECT tdigest_percentile('flags 1 count 4611686018427388928 compression 10 centroids 3 (1, 4611686018427387904) (2, 512) (3, 512)'::tdigest, 0.9999999999999998::double precision); tdigest_percentile -------------------- 2 (1 row) -- same digest, but the percentile falls in between the second and third centroid SELECT tdigest_percentile('flags 1 count 4611686018427388928 compression 10 centroids 3 (1, 4611686018427387904) (2, 512) (3, 512)'::tdigest, ARRAY[0.0, 0.5, 0.9999999999999996, 1.0]::double precision[]); tdigest_percentile --------------------------------------------- {1,1.0000000000000002,1.9999999999999996,3} (1 row) -- the same digest with other compressions, to show the compression does not affect this -- the compaction leaves the digest alone for all compression values SELECT c AS compression, tdigest_percentile(('flags 1 count 4611686018427388928 compression ' || c || ' centroids 3 (1, 4611686018427387904) (2, 512) (3, 512)')::tdigest, 0.9999999999999998::double precision) FROM (VALUES (10), (100), (1000), (10000)) v(c) GROUP BY c ORDER BY c; compression | tdigest_percentile -------------+-------------------- 10 | 2 100 | 2 1000 | 2 10000 | 2 (4 rows) -- make sure finite inputs do not produce infinite centroids SELECT tdigest_count(tdigest(v, 10)) FROM (SELECT (CASE WHEN i % 2 = 0 THEN 1e308 ELSE 1e307 END)::float8 AS v FROM generate_series(1,200) i) x; tdigest_count --------------- 200 (1 row) -- make sure finite inputs do not result in NaN means WITH x AS (SELECT (CASE i % 4 WHEN 0 THEN -1e308 WHEN 1 THEN -1e307 WHEN 2 THEN 1e307 ELSE 1e308 END)::float8 AS v FROM generate_series(1,400) i) SELECT tdigest_percentile(v,10,0.5) FROM x; tdigest_percentile ------------------------- 1.8260099550716747e+306 (1 row) -- extreme digest - make sure we can read the output we produced SELECT tdigest_count((SELECT tdigest(v, c, 10) FROM (VALUES (1e307::float8, 1000000::bigint), (1e308::float8, 1000000::bigint)) x(v, c))::text::tdigest); tdigest_count --------------- 2000000 (1 row) -- extreme digest - overflow/rounding issue in tdigest_compute_quantiles SELECT tdigest_percentile('flags 1 count 204745659738676291 compression 100 centroids 3 (5360.6421513780951, 96854112897858161) (5670.4376627340216, 36597233557612361) (5670.4376627340216, 71294313283205769)'::tdigest, 0.56241841621276667); tdigest_percentile -------------------- 5670.437662734022 (1 row) -- extreme digest - overflow/rounding issue in tdigest_compute_quantiles_of SELECT tdigest_percentile_of('flags 1 count 467525031770889061 compression 100 centroids 5 (-714.32532319527991, 372151463885287745) (-649.22210009079686, 12983637447676689) (-613.86327584091737, 82389930437924625) (-554.7460856469786, 1) (-497.74083276256061, 1)'::tdigest, -517.61603495911652); tdigest_percentile_of ----------------------- 1 (1 row) -- most user-facing scalar functions should be strict, except for functions -- that need to accept NULL as initial digest value (e.g. for the incremental -- API, or when combining digests) SELECT p.proname, p.proisstrict FROM pg_proc p JOIN pg_depend d ON (d.objid = p.oid AND d.deptype = 'e') JOIN pg_extension e ON (e.oid = d.refobjid) WHERE e.extname = 'tdigest' AND p.proargtypes::oid[] && ARRAY['internal'::regtype]::oid[] IS NOT TRUE AND p.prorettype <> 'internal'::regtype AND p.proisstrict <> true AND NOT EXISTS (SELECT 1 FROM pg_aggregate WHERE aggfnoid = p.oid) ORDER BY 1; proname | proisstrict ---------------+------------- tdigest_add | f tdigest_add | f tdigest_union | f (3 rows) tvondra-tdigest-c0af713/test/expected/basic_1.out000066400000000000000000000251721525543331000220600ustar00rootroot00000000000000-- validation of percentile values -- correct values SELECT tdigest_percentile(1.0, 10, 0.0); tdigest_percentile -------------------- 1 (1 row) SELECT tdigest_percentile(1.0, 10, 0.1); tdigest_percentile -------------------- 1 (1 row) SELECT tdigest_percentile(1.0, 10, 0.5); tdigest_percentile -------------------- 1 (1 row) SELECT tdigest_percentile(1.0, 10, 0.9); tdigest_percentile -------------------- 1 (1 row) SELECT tdigest_percentile(1.0, 10, 1.0); tdigest_percentile -------------------- 1 (1 row) SELECT tdigest_percentile(1.0, 10, ARRAY[0.0, 0.1, 0.5, 0.9, 1.0]); tdigest_percentile -------------------- {1,1,1,1,1} (1 row) -- out of range values SELECT tdigest_percentile(1.0, 10, -1.0); ERROR: invalid percentile value -1.000000, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0, 10, 1.5); ERROR: invalid percentile value 1.500000, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0, 10, ARRAY[0.0, -1.0, 0.5, 0.9, 1.0]); ERROR: invalid percentile value -1.000000, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0, 10, ARRAY[0.0, 0.1, 0.5, 0.9, 1.5]); ERROR: invalid percentile value 1.500000, should be in [0.0, 1.0] -- NaN SELECT tdigest_percentile(1.0, 10, 'NaN'::double precision); ERROR: invalid percentile value nan, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0, 10, ARRAY[0.0, 'NaN'::double precision, 0.5, 0.9, 1.0]); ERROR: invalid percentile value nan, should be in [0.0, 1.0] -- infinite values SELECT tdigest_percentile(1.0, 10, 'infinity'::double precision); ERROR: invalid percentile value inf, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0, 10, '-infinity'::double precision); ERROR: invalid percentile value -inf, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0, 10, ARRAY[0.0, 'infinity'::double precision, 0.5, 0.9, 1.0]); ERROR: invalid percentile value inf, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0, 10, ARRAY[0.0, '-infinity'::double precision, 0.5, 0.9, 1.0]); ERROR: invalid percentile value -inf, should be in [0.0, 1.0] -- more tests SELECT tdigest_percentile(1.0::double precision, 10, 'NaN'::double precision); ERROR: invalid percentile value nan, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0::double precision, 10, '-infinity'::double precision); ERROR: invalid percentile value -inf, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0::double precision, 10, 'infinity'::double precision); ERROR: invalid percentile value inf, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0::double precision, 10, ARRAY[0.5, 'NaN']::double precision[]); ERROR: invalid percentile value nan, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0::double precision, 10, ARRAY[0.5, '-infinity']::double precision[]); ERROR: invalid percentile value -inf, should be in [0.0, 1.0] SELECT tdigest_percentile(1.0::double precision, 10, ARRAY[0.5, 'infinity']::double precision[]); ERROR: invalid percentile value inf, should be in [0.0, 1.0] SELECT tdigest_percentile('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, 'NaN'::double precision); ERROR: invalid percentile value nan, should be in [0.0, 1.0] SELECT tdigest_percentile('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, '-infinity'::double precision); ERROR: invalid percentile value -inf, should be in [0.0, 1.0] SELECT tdigest_percentile('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, 'infinity'::double precision); ERROR: invalid percentile value inf, should be in [0.0, 1.0] SELECT tdigest_percentile('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, ARRAY[0.5, 'NaN']::double precision[]); ERROR: invalid percentile value nan, should be in [0.0, 1.0] SELECT tdigest_percentile('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, ARRAY[0.5, '-infinity']::double precision[]); ERROR: invalid percentile value -inf, should be in [0.0, 1.0] SELECT tdigest_percentile('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, ARRAY[0.5, 'infinity']::double precision[]); ERROR: invalid percentile value inf, should be in [0.0, 1.0] -- trim thresholds are checked too SELECT tdigest_avg(1.0::double precision, 10, 'NaN'::double precision, 1.0); ERROR: invalid low percentile value nan, should be in [0.0, 1.0] SELECT tdigest_avg(1.0::double precision, 10, '-infinity'::double precision, 1.0); ERROR: invalid low percentile value -inf, should be in [0.0, 1.0] SELECT tdigest_avg(1.0::double precision, 10, 'infinity'::double precision, 1.0); ERROR: invalid low percentile value inf, should be in [0.0, 1.0] SELECT tdigest_digest_sum('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, 0.0, 'NaN'::double precision); ERROR: invalid high percentile value nan, should be in [0.0, 1.0] SELECT tdigest_digest_sum('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, 0.0, '-infinity'::double precision); ERROR: invalid high percentile value -inf, should be in [0.0, 1.0] SELECT tdigest_digest_sum('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, 0.0, 'infinity'::double precision); ERROR: invalid high percentile value inf, should be in [0.0, 1.0] -- validation of input values -- input functions must reject NaN / infinity means in various places SELECT 'flags 1 count 1 compression 10 centroids 1 (NaN, 1)'::tdigest; ERROR: mean value for all centroids in a t-digest must be valid LINE 1: SELECT 'flags 1 count 1 compression 10 centroids 1 (NaN, 1)'... ^ SELECT 'flags 1 count 1 compression 10 centroids 1 (infinity, 1)'::tdigest; ERROR: mean value for all centroids in a t-digest must be valid LINE 1: SELECT 'flags 1 count 1 compression 10 centroids 1 (infinity... ^ SELECT 'flags 1 count 1 compression 10 centroids 1 (-infinity, 1)'::tdigest; ERROR: mean value for all centroids in a t-digest must be valid LINE 1: SELECT 'flags 1 count 1 compression 10 centroids 1 (-infinit... ^ -- the aggregates building the digest have to reject NaN / infinity too SELECT tdigest('NaN'::float8, 10); ERROR: all values added to t-digest must be finite SELECT tdigest('infinity'::float8, 10); ERROR: all values added to t-digest must be finite SELECT tdigest('-infinity'::float8, 10); ERROR: all values added to t-digest must be finite -- same for the value/count API SELECT tdigest('NaN'::float8, 50::bigint, 10); ERROR: all values added to t-digest must be finite SELECT tdigest('NaN'::float8, 200::bigint, 10); ERROR: all values added to t-digest must be finite SELECT tdigest('Infinity'::float8, 200::bigint, 10); ERROR: all values added to t-digest must be finite SELECT tdigest('-Infinity'::float8, 200::bigint, 10); ERROR: all values added to t-digest must be finite -- t-digest extreme enough for the percentile to fall to the right of the last centroid SELECT tdigest_percentile('flags 1 count 4611686018427388928 compression 10 centroids 3 (1, 4611686018427387904) (2, 512) (3, 512)'::tdigest, 0.9999999999999998::double precision); tdigest_percentile -------------------- 2 (1 row) -- same digest, but the percentile falls in between the second and third centroid SELECT tdigest_percentile('flags 1 count 4611686018427388928 compression 10 centroids 3 (1, 4611686018427387904) (2, 512) (3, 512)'::tdigest, ARRAY[0.0, 0.5, 0.9999999999999996, 1.0]::double precision[]); tdigest_percentile -------------------- {1,1,2,3} (1 row) -- the same digest with other compressions, to show the compression does not affect this -- the compaction leaves the digest alone for all compression values SELECT c AS compression, tdigest_percentile(('flags 1 count 4611686018427388928 compression ' || c || ' centroids 3 (1, 4611686018427387904) (2, 512) (3, 512)')::tdigest, 0.9999999999999998::double precision) FROM (VALUES (10), (100), (1000), (10000)) v(c) GROUP BY c ORDER BY c; compression | tdigest_percentile -------------+-------------------- 10 | 2 100 | 2 1000 | 2 10000 | 2 (4 rows) -- make sure finite inputs do not produce infinite centroids SELECT tdigest_count(tdigest(v, 10)) FROM (SELECT (CASE WHEN i % 2 = 0 THEN 1e308 ELSE 1e307 END)::float8 AS v FROM generate_series(1,200) i) x; tdigest_count --------------- 200 (1 row) -- make sure finite inputs do not result in NaN means WITH x AS (SELECT (CASE i % 4 WHEN 0 THEN -1e308 WHEN 1 THEN -1e307 WHEN 2 THEN 1e307 ELSE 1e308 END)::float8 AS v FROM generate_series(1,400) i) SELECT tdigest_percentile(v,10,0.5) FROM x; tdigest_percentile ----------------------- 1.82600995507167e+306 (1 row) -- extreme digest - make sure we can read the output we produced SELECT tdigest_count((SELECT tdigest(v, c, 10) FROM (VALUES (1e307::float8, 1000000::bigint), (1e308::float8, 1000000::bigint)) x(v, c))::text::tdigest); tdigest_count --------------- 2000000 (1 row) -- extreme digest - overflow/rounding issue in tdigest_compute_quantiles SELECT tdigest_percentile('flags 1 count 204745659738676291 compression 100 centroids 3 (5360.6421513780951, 96854112897858161) (5670.4376627340216, 36597233557612361) (5670.4376627340216, 71294313283205769)'::tdigest, 0.56241841621276667); tdigest_percentile -------------------- 5670.43766273402 (1 row) -- extreme digest - overflow/rounding issue in tdigest_compute_quantiles_of SELECT tdigest_percentile_of('flags 1 count 467525031770889061 compression 100 centroids 5 (-714.32532319527991, 372151463885287745) (-649.22210009079686, 12983637447676689) (-613.86327584091737, 82389930437924625) (-554.7460856469786, 1) (-497.74083276256061, 1)'::tdigest, -517.61603495911652); tdigest_percentile_of ----------------------- 1 (1 row) -- most user-facing scalar functions should be strict, except for functions -- that need to accept NULL as initial digest value (e.g. for the incremental -- API, or when combining digests) SELECT p.proname, p.proisstrict FROM pg_proc p JOIN pg_depend d ON (d.objid = p.oid AND d.deptype = 'e') JOIN pg_extension e ON (e.oid = d.refobjid) WHERE e.extname = 'tdigest' AND p.proargtypes::oid[] && ARRAY['internal'::regtype]::oid[] IS NOT TRUE AND p.prorettype <> 'internal'::regtype AND p.proisstrict <> true AND NOT EXISTS (SELECT 1 FROM pg_aggregate WHERE aggfnoid = p.oid) ORDER BY 1; proname | proisstrict ---------------+------------- tdigest_add | f tdigest_add | f tdigest_union | f (3 rows) tvondra-tdigest-c0af713/test/expected/cast.out000066400000000000000000000202751525543331000215100ustar00rootroot00000000000000-- test casting to json SELECT cast(tdigest(i / 1000.0, 10) as json) from generate_series(1,1000) s(i); tdigest ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ {"flags": 1, "count": 1000, "compression": 10, "centroids": 13, "means": [0.001, 0.002, 0.0045, 0.012999999999999998, 0.0405, 0.135, 0.4640000000000002, 0.7929999999999998, 0.9159999999999996, 0.9794999999999999, 0.9959999999999999, 0.999, 1], "counts": [1, 1, 4, 13, 42, 147, 511, 147, 99, 28, 5, 1, 1]} (1 row) SELECT cast(tdigest(i / 1000.0, 25) as json) from generate_series(1,1000) s(i); tdigest ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ {"flags": 1, "count": 1000, "compression": 25, "centroids": 18, "means": [0.001, 0.002, 0.003, 0.0055000000000000005, 0.011999999999999997, 0.0265, 0.05749999999999998, 0.11499999999999992, 0.232, 0.4720000000000001, 0.727, 0.8775, 0.9489999999999998, 0.9764999999999998, 0.9914999999999998, 0.997, 0.999, 1], "counts": [1, 1, 1, 4, 9, 20, 42, 73, 161, 319, 191, 110, 33, 22, 8, 3, 1, 1]} (1 row) SELECT cast(tdigest(i / 1000.0, 100) as json) from generate_series(1,1000) s(i); tdigest ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- {"flags": 1, "count": 1000, "compression": 100, "centroids": 40, "means": [0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.0075, 0.009999999999999998, 0.013499999999999998, 0.018000000000000002, 0.0245, 0.03400000000000001, 0.04699999999999999, 0.065, 0.08999999999999998, 0.12450000000000006, 0.17099999999999987, 0.23149999999999998, 0.30749999999999994, 0.39850000000000024, 0.5010000000000003, 0.6034999999999998, 0.6944999999999996, 0.7704999999999996, 0.8309999999999995, 0.8774999999999997, 0.9119999999999999, 0.9369999999999997, 0.9550000000000003, 0.968, 0.9775, 0.9840000000000001, 0.9884999999999999, 0.992, 0.9944999999999999, 0.996, 0.997, 0.998, 0.999, 1], "counts": [1, 1, 1, 1, 1, 1, 2, 3, 4, 5, 8, 11, 15, 21, 29, 40, 53, 68, 84, 98, 107, 98, 84, 68, 53, 40, 29, 21, 15, 11, 8, 5, 4, 3, 2, 1, 1, 1, 1, 1]} (1 row) -- test casting to double precision array SELECT array_agg(round(v::numeric,3)) FROM ( SELECT unnest(cast(tdigest(i / 1000.0, 10) as double precision[])) AS v from generate_series(1,1000) s(i) ) foo; array_agg ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ {1.000,1000.000,10.000,13.000,0.001,1.000,0.002,1.000,0.005,4.000,0.013,13.000,0.041,42.000,0.135,147.000,0.464,511.000,0.793,147.000,0.916,99.000,0.980,28.000,0.996,5.000,0.999,1.000,1.000,1.000} (1 row) SELECT array_agg(round(v::numeric,3)) FROM ( SELECT unnest(cast(tdigest(i / 1000.0, 25) as double precision[])) AS v from generate_series(1,1000) s(i) ) foo; array_agg --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- {1.000,1000.000,25.000,18.000,0.001,1.000,0.002,1.000,0.003,1.000,0.006,4.000,0.012,9.000,0.027,20.000,0.058,42.000,0.115,73.000,0.232,161.000,0.472,319.000,0.727,191.000,0.878,110.000,0.949,33.000,0.977,22.000,0.992,8.000,0.997,3.000,0.999,1.000,1.000,1.000} (1 row) SELECT array_agg(round(v::numeric,3)) FROM ( SELECT unnest(cast(tdigest(i / 1000.0, 100) as double precision[])) AS v from generate_series(1,1000) s(i) ) foo; array_agg ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- {1.000,1000.000,100.000,40.000,0.001,1.000,0.002,1.000,0.003,1.000,0.004,1.000,0.005,1.000,0.006,1.000,0.008,2.000,0.010,3.000,0.014,4.000,0.018,5.000,0.025,8.000,0.034,11.000,0.047,15.000,0.065,21.000,0.090,29.000,0.125,40.000,0.171,53.000,0.232,68.000,0.308,84.000,0.399,98.000,0.501,107.000,0.604,98.000,0.695,84.000,0.771,68.000,0.831,53.000,0.878,40.000,0.912,29.000,0.937,21.000,0.955,15.000,0.968,11.000,0.978,8.000,0.984,5.000,0.989,4.000,0.992,3.000,0.995,2.000,0.996,1.000,0.997,1.000,0.998,1.000,0.999,1.000,1.000,1.000} (1 row) tvondra-tdigest-c0af713/test/expected/cast_1.out000066400000000000000000000153771525543331000217370ustar00rootroot00000000000000-- test casting to json SELECT cast(tdigest(i / 1000.0, 10) as json) from generate_series(1,1000) s(i); tdigest ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- {"flags": 1, "count": 1000, "compression": 10, "centroids": 13, "means": [0.001, 0.002, 0.0045, 0.013, 0.0405, 0.135, 0.464, 0.793, 0.916, 0.9795, 0.996, 0.999, 1], "counts": [1, 1, 4, 13, 42, 147, 511, 147, 99, 28, 5, 1, 1]} (1 row) SELECT cast(tdigest(i / 1000.0, 25) as json) from generate_series(1,1000) s(i); tdigest ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- {"flags": 1, "count": 1000, "compression": 25, "centroids": 18, "means": [0.001, 0.002, 0.003, 0.0055, 0.012, 0.0265, 0.0575, 0.115, 0.232, 0.472, 0.727, 0.8775, 0.949, 0.9765, 0.9915, 0.997, 0.999, 1], "counts": [1, 1, 1, 4, 9, 20, 42, 73, 161, 319, 191, 110, 33, 22, 8, 3, 1, 1]} (1 row) SELECT cast(tdigest(i / 1000.0, 100) as json) from generate_series(1,1000) s(i); tdigest ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- {"flags": 1, "count": 1000, "compression": 100, "centroids": 40, "means": [0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.0075, 0.01, 0.0135, 0.018, 0.0245, 0.034, 0.047, 0.065, 0.09, 0.1245, 0.171, 0.2315, 0.3075, 0.3985, 0.501, 0.6035, 0.6945, 0.7705, 0.831, 0.8775, 0.912, 0.937, 0.955, 0.968, 0.9775, 0.984, 0.9885, 0.992, 0.9945, 0.996, 0.997, 0.998, 0.999, 1], "counts": [1, 1, 1, 1, 1, 1, 2, 3, 4, 5, 8, 11, 15, 21, 29, 40, 53, 68, 84, 98, 107, 98, 84, 68, 53, 40, 29, 21, 15, 11, 8, 5, 4, 3, 2, 1, 1, 1, 1, 1]} (1 row) -- test casting to double precision array SELECT array_agg(round(v::numeric,3)) FROM ( SELECT unnest(cast(tdigest(i / 1000.0, 10) as double precision[])) AS v from generate_series(1,1000) s(i) ) foo; array_agg ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ {1.000,1000.000,10.000,13.000,0.001,1.000,0.002,1.000,0.005,4.000,0.013,13.000,0.041,42.000,0.135,147.000,0.464,511.000,0.793,147.000,0.916,99.000,0.980,28.000,0.996,5.000,0.999,1.000,1.000,1.000} (1 row) SELECT array_agg(round(v::numeric,3)) FROM ( SELECT unnest(cast(tdigest(i / 1000.0, 25) as double precision[])) AS v from generate_series(1,1000) s(i) ) foo; array_agg --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- {1.000,1000.000,25.000,18.000,0.001,1.000,0.002,1.000,0.003,1.000,0.006,4.000,0.012,9.000,0.027,20.000,0.058,42.000,0.115,73.000,0.232,161.000,0.472,319.000,0.727,191.000,0.878,110.000,0.949,33.000,0.977,22.000,0.992,8.000,0.997,3.000,0.999,1.000,1.000,1.000} (1 row) SELECT array_agg(round(v::numeric,3)) FROM ( SELECT unnest(cast(tdigest(i / 1000.0, 100) as double precision[])) AS v from generate_series(1,1000) s(i) ) foo; array_agg ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- {1.000,1000.000,100.000,40.000,0.001,1.000,0.002,1.000,0.003,1.000,0.004,1.000,0.005,1.000,0.006,1.000,0.008,2.000,0.010,3.000,0.014,4.000,0.018,5.000,0.025,8.000,0.034,11.000,0.047,15.000,0.065,21.000,0.090,29.000,0.125,40.000,0.171,53.000,0.232,68.000,0.308,84.000,0.399,98.000,0.501,107.000,0.604,98.000,0.695,84.000,0.771,68.000,0.831,53.000,0.878,40.000,0.912,29.000,0.937,21.000,0.955,15.000,0.968,11.000,0.978,8.000,0.984,5.000,0.989,4.000,0.992,3.000,0.995,2.000,0.996,1.000,0.997,1.000,0.998,1.000,0.999,1.000,1.000,1.000} (1 row) tvondra-tdigest-c0af713/test/expected/combine.out000066400000000000000000000066261525543331000221760ustar00rootroot00000000000000-- partitioned table: two partitions, each containing a t-digest with vastly -- different compression values CREATE TABLE digest_combine_test(k int, d tdigest) PARTITION BY LIST (k); CREATE TABLE digest_combine_test_1 PARTITION OF digest_combine_test FOR VALUES IN (1); CREATE TABLE digest_combine_test_2 PARTITION OF digest_combine_test FOR VALUES IN (2); DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '0', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '0', false); END IF; -- parallel_leader_participation and enable_partitionwise_aggregate -- were introduced in 11 IF v_version >= 11 THEN PERFORM set_config('parallel_leader_participation', 'off', false); PERFORM set_config('enable_partitionwise_aggregate', 'on', false); END IF; END; $$ LANGUAGE plpgsql; DO $$ DECLARE v_scale_1 INT; v_scale_2 INT; v_compress INT; v_rows INT; v_rec RECORD; BEGIN FOR v_scale_1 IN 1..3 LOOP FOR v_scale_2 IN 1..3 LOOP FOR r IN 1..5 LOOP TRUNCATE digest_combine_test; -- first random t-digest v_compress := pow(10, v_scale_1) + (random() * pow(10, (v_scale_1 + 1)))::int; v_compress := GREATEST(10, LEAST(v_compress, 10000)); v_rows := (random() * v_compress * 10); v_rows := GREATEST(100, LEAST(v_rows, 25000)); -- RAISE NOTICE '% %', v_compress, v_rows; INSERT INTO digest_combine_test SELECT 1, tdigest(v * random(), v_compress) FROM generate_series(1, v_rows) v; -- second random t-digest v_compress := pow(10, v_scale_2) + (random() * pow(10, (v_scale_2 + 1)))::int; v_compress := GREATEST(10, LEAST(v_compress, 10000)); v_rows := (random() * v_compress * 10); v_rows := GREATEST(100, LEAST(v_rows, 25000)); -- RAISE NOTICE '% %', v_compress, v_rows; INSERT INTO digest_combine_test SELECT 2, tdigest(v * random(), v_compress) FROM generate_series(1, v_rows) v; -- duplicate the data enough times to reliably trigger partitionwise aggregate INSERT INTO digest_combine_test SELECT * FROM digest_combine_test; INSERT INTO digest_combine_test SELECT * FROM digest_combine_test; INSERT INTO digest_combine_test SELECT * FROM digest_combine_test; INSERT INTO digest_combine_test SELECT * FROM digest_combine_test; ANALYZE digest_combine_test; -- FOR v_rec IN EXPLAIN SELECT tdigest(d) FROM digest_combine_test LOOP -- RAISE NOTICE '%', v_rec."QUERY PLAN"; -- END LOOP; -- tdigest_combine(d1, d2) PERFORM tdigest(d) FROM digest_combine_test; END LOOP; END LOOP; END LOOP; END; $$ LANGUAGE plpgsql; DROP TABLE digest_combine_test; tvondra-tdigest-c0af713/test/expected/combine_1.out000066400000000000000000000101531525543331000224040ustar00rootroot00000000000000-- partitioned table: two partitions, each containing a t-digest with vastly -- different compression values CREATE TABLE digest_combine_test(k int, d tdigest) PARTITION BY LIST (k); ERROR: syntax error at or near "PARTITION" LINE 1: ...REATE TABLE digest_combine_test(k int, d tdigest) PARTITION ... ^ CREATE TABLE digest_combine_test_1 PARTITION OF digest_combine_test FOR VALUES IN (1); ERROR: syntax error at or near "PARTITION" LINE 1: CREATE TABLE digest_combine_test_1 PARTITION OF digest_combi... ^ CREATE TABLE digest_combine_test_2 PARTITION OF digest_combine_test FOR VALUES IN (2); ERROR: syntax error at or near "PARTITION" LINE 1: CREATE TABLE digest_combine_test_2 PARTITION OF digest_combi... ^ DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '0', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '0', false); END IF; -- parallel_leader_participation and enable_partitionwise_aggregate -- were introduced in 11 IF v_version >= 11 THEN PERFORM set_config('parallel_leader_participation', 'off', false); PERFORM set_config('enable_partitionwise_aggregate', 'on', false); END IF; END; $$ LANGUAGE plpgsql; DO $$ DECLARE v_scale_1 INT; v_scale_2 INT; v_compress INT; v_rows INT; v_rec RECORD; BEGIN FOR v_scale_1 IN 1..3 LOOP FOR v_scale_2 IN 1..3 LOOP FOR r IN 1..5 LOOP TRUNCATE digest_combine_test; -- first random t-digest v_compress := pow(10, v_scale_1) + (random() * pow(10, (v_scale_1 + 1)))::int; v_compress := GREATEST(10, LEAST(v_compress, 10000)); v_rows := (random() * v_compress * 10); v_rows := GREATEST(100, LEAST(v_rows, 25000)); -- RAISE NOTICE '% %', v_compress, v_rows; INSERT INTO digest_combine_test SELECT 1, tdigest(v * random(), v_compress) FROM generate_series(1, v_rows) v; -- second random t-digest v_compress := pow(10, v_scale_2) + (random() * pow(10, (v_scale_2 + 1)))::int; v_compress := GREATEST(10, LEAST(v_compress, 10000)); v_rows := (random() * v_compress * 10); v_rows := GREATEST(100, LEAST(v_rows, 25000)); -- RAISE NOTICE '% %', v_compress, v_rows; INSERT INTO digest_combine_test SELECT 2, tdigest(v * random(), v_compress) FROM generate_series(1, v_rows) v; -- duplicate the data enough times to reliably trigger partitionwise aggregate INSERT INTO digest_combine_test SELECT * FROM digest_combine_test; INSERT INTO digest_combine_test SELECT * FROM digest_combine_test; INSERT INTO digest_combine_test SELECT * FROM digest_combine_test; INSERT INTO digest_combine_test SELECT * FROM digest_combine_test; ANALYZE digest_combine_test; -- FOR v_rec IN EXPLAIN SELECT tdigest(d) FROM digest_combine_test LOOP -- RAISE NOTICE '%', v_rec."QUERY PLAN"; -- END LOOP; -- tdigest_combine(d1, d2) PERFORM tdigest(d) FROM digest_combine_test; END LOOP; END LOOP; END LOOP; END; $$ LANGUAGE plpgsql; ERROR: relation "digest_combine_test" does not exist CONTEXT: SQL statement "TRUNCATE digest_combine_test" PL/pgSQL function inline_code_block line 16 at SQL statement DROP TABLE digest_combine_test; ERROR: table "digest_combine_test" does not exist tvondra-tdigest-c0af713/test/expected/combine_crash.out000066400000000000000000000056671525543331000233620ustar00rootroot00000000000000-- partitioned table: two partitions, each containing a t-digest with vastly -- different compression values CREATE TABLE combine_crash_test(k int, d tdigest) PARTITION BY LIST (k); CREATE TABLE combine_crash_test_1 PARTITION OF combine_crash_test FOR VALUES IN (1); CREATE TABLE combine_crash_test_2 PARTITION OF combine_crash_test FOR VALUES IN (2); -- comp=10: BUFFER_SIZE=100 slots INSERT INTO combine_crash_test SELECT 1, tdigest(v::float8, 10) FROM generate_series(1,100) v; -- comp=10000: BUFFER_SIZE=100000 slots, with 10000 input values INSERT INTO combine_crash_test SELECT 2, tdigest(v::float8, 10000) FROM generate_series(1,10000) v; -- enough data to reliably trigger partitionwise aggregate INSERT INTO combine_crash_test SELECT * FROM combine_crash_test; INSERT INTO combine_crash_test SELECT * FROM combine_crash_test; INSERT INTO combine_crash_test SELECT * FROM combine_crash_test; ANALYZE combine_crash_test; -- force partitionwise parallel aggregate SET enable_partitionwise_aggregate = on; SET max_parallel_workers_per_gather = 2; SET parallel_leader_participation = off; SET parallel_setup_cost = 0; SET parallel_tuple_cost = 0; SET min_parallel_table_scan_size = 0; -- tdigest_combine(small, huge) EXPLAIN (COSTS OFF) SELECT tdigest(d) FROM combine_crash_test; QUERY PLAN ------------------------------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Parallel Append -> Partial Aggregate -> Seq Scan on combine_crash_test_1 combine_crash_test -> Partial Aggregate -> Seq Scan on combine_crash_test_2 combine_crash_test_1 (8 rows) -- stabilize the output (some randomness due to parallelism) SET extra_float_digits = 0; SELECT tdigest(d) FROM combine_crash_test; tdigest -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- flags 1 count 80800 compression 10 centroids 15 (1, 1) (1, 1) (1, 8) (7.51428571428572, 70) (73.1499999999999, 560) (553.977563383441, 4457) (2361.85970535021, 19345) (5389.71486469791, 40465) (7942.99078997343, 10532) (9201.27863855728, 3937) (9702.28774928776, 1053) (9928.82692307693, 312) (9990.64705882353, 51) (9999.28571428571, 7) (10000, 1) (1 row) DROP TABLE combine_crash_test; tvondra-tdigest-c0af713/test/expected/combine_crash_1.out000066400000000000000000000053661525543331000235760ustar00rootroot00000000000000-- partitioned table: two partitions, each containing a t-digest with vastly -- different compression values CREATE TABLE combine_crash_test(k int, d tdigest) PARTITION BY LIST (k); CREATE TABLE combine_crash_test_1 PARTITION OF combine_crash_test FOR VALUES IN (1); CREATE TABLE combine_crash_test_2 PARTITION OF combine_crash_test FOR VALUES IN (2); -- comp=10: BUFFER_SIZE=100 slots INSERT INTO combine_crash_test SELECT 1, tdigest(v::float8, 10) FROM generate_series(1,100) v; -- comp=10000: BUFFER_SIZE=100000 slots, with 10000 input values INSERT INTO combine_crash_test SELECT 2, tdigest(v::float8, 10000) FROM generate_series(1,10000) v; -- enough data to reliably trigger partitionwise aggregate INSERT INTO combine_crash_test SELECT * FROM combine_crash_test; INSERT INTO combine_crash_test SELECT * FROM combine_crash_test; INSERT INTO combine_crash_test SELECT * FROM combine_crash_test; ANALYZE combine_crash_test; -- force partitionwise parallel aggregate SET enable_partitionwise_aggregate = on; ERROR: unrecognized configuration parameter "enable_partitionwise_aggregate" SET max_parallel_workers_per_gather = 2; SET parallel_leader_participation = off; ERROR: unrecognized configuration parameter "parallel_leader_participation" SET parallel_setup_cost = 0; SET parallel_tuple_cost = 0; SET min_parallel_table_scan_size = 0; -- tdigest_combine(small, huge) EXPLAIN (COSTS OFF) SELECT tdigest(d) FROM combine_crash_test; QUERY PLAN -------------------------------------------------------------- Aggregate -> Gather Workers Planned: 1 -> Append -> Parallel Seq Scan on combine_crash_test_1 -> Parallel Seq Scan on combine_crash_test_2 (6 rows) SELECT tdigest(d) FROM combine_crash_test; tdigest ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- flags 1 count 80800 compression 10 centroids 15 (1.000000, 1) (1.000000, 1) (1.000000, 8) (7.514286, 70) (72.955357, 560) (553.198115, 4457) (2362.044921, 19345) (5389.714865, 40465) (7942.990790, 10532) (9201.278639, 3937) (9702.287749, 1053) (9928.826923, 312) (9990.647059, 51) (9999.285714, 7) (10000.000000, 1) (1 row) DROP TABLE combine_crash_test; tvondra-tdigest-c0af713/test/expected/combine_crash_2.out000066400000000000000000000055451525543331000235760ustar00rootroot00000000000000-- partitioned table: two partitions, each containing a t-digest with vastly -- different compression values CREATE TABLE combine_crash_test(k int, d tdigest) PARTITION BY LIST (k); CREATE TABLE combine_crash_test_1 PARTITION OF combine_crash_test FOR VALUES IN (1); CREATE TABLE combine_crash_test_2 PARTITION OF combine_crash_test FOR VALUES IN (2); -- comp=10: BUFFER_SIZE=100 slots INSERT INTO combine_crash_test SELECT 1, tdigest(v::float8, 10) FROM generate_series(1,100) v; -- comp=10000: BUFFER_SIZE=100000 slots, with 10000 input values INSERT INTO combine_crash_test SELECT 2, tdigest(v::float8, 10000) FROM generate_series(1,10000) v; -- enough data to reliably trigger partitionwise aggregate INSERT INTO combine_crash_test SELECT * FROM combine_crash_test; INSERT INTO combine_crash_test SELECT * FROM combine_crash_test; INSERT INTO combine_crash_test SELECT * FROM combine_crash_test; ANALYZE combine_crash_test; -- force partitionwise parallel aggregate SET enable_partitionwise_aggregate = on; SET max_parallel_workers_per_gather = 2; SET parallel_leader_participation = off; SET parallel_setup_cost = 0; SET parallel_tuple_cost = 0; SET min_parallel_table_scan_size = 0; -- tdigest_combine(small, huge) EXPLAIN (COSTS OFF) SELECT tdigest(d) FROM combine_crash_test; QUERY PLAN ---------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Parallel Append -> Partial Aggregate -> Seq Scan on combine_crash_test_1 -> Partial Aggregate -> Seq Scan on combine_crash_test_2 (8 rows) -- stabilize the output (some randomness due to parallelism) SET extra_float_digits = 0; SELECT tdigest(d) FROM combine_crash_test; tdigest -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- flags 1 count 80800 compression 10 centroids 15 (1, 1) (1, 1) (1, 8) (7.51428571428572, 70) (73.1499999999999, 560) (553.977563383441, 4457) (2361.85970535021, 19345) (5389.71486469791, 40465) (7942.99078997343, 10532) (9201.27863855728, 3937) (9702.28774928776, 1053) (9928.82692307693, 312) (9990.64705882353, 51) (9999.28571428571, 7) (10000, 1) (1 row) DROP TABLE combine_crash_test; tvondra-tdigest-c0af713/test/expected/combine_crash_3.out000066400000000000000000000061751525543331000235770ustar00rootroot00000000000000-- partitioned table: two partitions, each containing a t-digest with vastly -- different compression values CREATE TABLE combine_crash_test(k int, d tdigest) PARTITION BY LIST (k); ERROR: syntax error at or near "PARTITION" LINE 1: ...REATE TABLE combine_crash_test(k int, d tdigest) PARTITION ... ^ CREATE TABLE combine_crash_test_1 PARTITION OF combine_crash_test FOR VALUES IN (1); ERROR: syntax error at or near "PARTITION" LINE 1: CREATE TABLE combine_crash_test_1 PARTITION OF digest_combi... ^ CREATE TABLE combine_crash_test_2 PARTITION OF combine_crash_test FOR VALUES IN (2); ERROR: syntax error at or near "PARTITION" LINE 1: CREATE TABLE combine_crash_test_2 PARTITION OF digest_combi... ^ -- comp=10: BUFFER_SIZE=100 slots INSERT INTO combine_crash_test SELECT 1, tdigest(v::float8, 10) FROM generate_series(1,100) v; ERROR: relation "combine_crash_test" does not exist LINE 1: INSERT INTO combine_crash_test SELECT 1, tdigest(v::float8,... ^ -- comp=10000: BUFFER_SIZE=100000 slots, with 10000 input values INSERT INTO combine_crash_test SELECT 2, tdigest(v::float8, 10000) FROM generate_series(1,10000) v; ERROR: relation "combine_crash_test" does not exist LINE 1: INSERT INTO combine_crash_test SELECT 2, tdigest(v::float8,... ^ -- enough data to reliably trigger partitionwise aggregate INSERT INTO combine_crash_test SELECT * FROM combine_crash_test; ERROR: relation "combine_crash_test" does not exist LINE 1: INSERT INTO combine_crash_test SELECT * FROM digest_combine... ^ INSERT INTO combine_crash_test SELECT * FROM combine_crash_test; ERROR: relation "combine_crash_test" does not exist LINE 1: INSERT INTO combine_crash_test SELECT * FROM digest_combine... ^ INSERT INTO combine_crash_test SELECT * FROM combine_crash_test; ERROR: relation "combine_crash_test" does not exist LINE 1: INSERT INTO combine_crash_test SELECT * FROM digest_combine... ^ ANALYZE combine_crash_test; ERROR: relation "combine_crash_test" does not exist -- force partitionwise parallel aggregate SET enable_partitionwise_aggregate = on; ERROR: unrecognized configuration parameter "enable_partitionwise_aggregate" SET max_parallel_workers_per_gather = 2; SET parallel_leader_participation = off; ERROR: unrecognized configuration parameter "parallel_leader_participation" SET parallel_setup_cost = 0; SET parallel_tuple_cost = 0; SET min_parallel_table_scan_size = 0; ERROR: unrecognized configuration parameter "min_parallel_table_scan_size" -- tdigest_combine(small, huge) EXPLAIN (COSTS OFF) SELECT tdigest(d) FROM combine_crash_test; ERROR: relation "combine_crash_test" does not exist LINE 2: SELECT tdigest(d) FROM combine_crash_test; ^ SELECT tdigest(d) FROM combine_crash_test; ERROR: relation "combine_crash_test" does not exist LINE 1: SELECT tdigest(d) FROM combine_crash_test; ^ DROP TABLE combine_crash_test; ERROR: table "combine_crash_test" does not exist tvondra-tdigest-c0af713/test/expected/compact_no_progress.out000066400000000000000000000035201525543331000246160ustar00rootroot00000000000000-- Use terse verbosity, so that the expected output does not depend on the -- error context, which differs between PostgreSQL versions. \set VERBOSITY terse -- check tdigest_compact() makes progress even for extreme digests -- -- tdigest_compact() walks the sorted centroids and merges each one into the -- current output centroid while the merged count stays below the size limit -- derived from the quantile position. With the former subtraction-based -- calculation of (1-q), a centroid holding nearly all the weight could make -- q0 round to 1 for everything following it. The size limit then became zero -- and compaction returned with ncentroids unchanged, still at BUFFER_SIZE. -- The current code uses exact integer remainders for these factors, and -- raises an error if a compaction still leaves the buffer full. -- -- Note that nothing here overflows - all the counts are perfectly valid and -- the digests are well-formed. The old code assumed compaction always freed -- some space. These regressions check progress and preservation of the count. -- -- 1) small number of values SELECT tdigest_count(tdigest(d)) FROM ( SELECT ('flags 1 count ' || c || ' compression 10 centroids 1 (' || i || ', ' || c || ')')::tdigest AS d FROM generate_series(1, 500) s(i), LATERAL (SELECT CASE WHEN i = 1 THEN 9000000000000000000::bigint ELSE 1 END AS c) x ) q; tdigest_count --------------------- 9000000000000000499 (1 row) -- 2) more values, requiring additional compactions SELECT tdigest_count(tdigest(d)) FROM ( SELECT ('flags 1 count ' || c || ' compression 10 centroids 1 (' || i || ', ' || c || ')')::tdigest AS d FROM generate_series(1, 2000) s(i), LATERAL (SELECT CASE WHEN i = 1 THEN 9000000000000000000::bigint ELSE 1 END AS c) x ) q; tdigest_count --------------------- 9000000000000001999 (1 row) tvondra-tdigest-c0af713/test/expected/compression_checks.out000066400000000000000000000020201525543331000244230ustar00rootroot00000000000000-- Regression test for check_compression(), which rejects compression -- values outside the [MIN_COMPRESSION, MAX_COMPRESSION] = [10, 10000] -- range accepted by the t-digest implementation. \set VERBOSITY terse -- compression below the minimum SELECT tdigest_percentile(v, 9, 0.5) FROM (VALUES (1.0)) AS t(v); ERROR: invalid compression value 9 -- compression above the maximum SELECT tdigest_percentile(v, 10001, 0.5) FROM (VALUES (1.0)) AS t(v); ERROR: invalid compression value 10001 -- the incremental API checks the compression too SELECT tdigest_add(NULL::tdigest, 1.0, 9); ERROR: invalid compression value 9 SELECT tdigest_add(NULL::tdigest, ARRAY[1.0], 10001); ERROR: invalid compression value 10001 -- both ends of the accepted range have to work SELECT tdigest_percentile(v, 10, 0.5) FROM (VALUES (1.0)) AS t(v); tdigest_percentile -------------------- 1 (1 row) SELECT tdigest_percentile(v, 10000, 0.5) FROM (VALUES (1.0)) AS t(v); tdigest_percentile -------------------- 1 (1 row) tvondra-tdigest-c0af713/test/expected/conversions.out000066400000000000000000000040021525543331000231140ustar00rootroot00000000000000-- test input function, and conversion from old to new format SELECT 'flags 0 count 20 compression 10 centroids 8 (1000.000000, 1) (2000.000000, 1) (7000.000000, 2) (26000.000000, 4) (84000.000000, 7) (51000.000000, 3) (19000.000000, 1) (20000.000000, 1)'::tdigest; tdigest --------------------------------------------------------------------------------------------------------------------------------- flags 1 count 20 compression 10 centroids 8 (1000, 1) (2000, 1) (3500, 2) (6500, 4) (12000, 7) (17000, 3) (19000, 1) (20000, 1) (1 row) -- test input of invalid data -- negative count SELECT 'flags 0 count -20 compression 10 centroids 8 (1000.000000, 1) (2000.000000, 1) (7000.000000, 2) (26000.000000, 4) (84000.000000, 7) (51000.000000, 3) (19000.000000, 1) (20000.000000, 1)'::tdigest; ERROR: count value for the t-digest must be positive LINE 1: SELECT 'flags 0 count -20 compression 10 centroids 8 (1000.0... ^ -- mismatching count SELECT 'flags 0 count 21 compression 10 centroids 8 (1000.000000, 1) (2000.000000, 1) (7000.000000, 2) (26000.000000, 4) (84000.000000, 7) (51000.000000, 3) (19000.000000, 1) (20000.000000, 1)'::tdigest; ERROR: total count does not match the data (20 != 21) LINE 1: SELECT 'flags 0 count 21 compression 10 centroids 8 (1000.00... ^ -- unsorted centroids SELECT 'flags 0 count 20 compression 10 centroids 8 (1000.000000, 1) (2000.000000, 1) (1000.000000, 2) (26000.000000, 4) (84000.000000, 7) (51000.000000, 3) (19000.000000, 1) (20000.000000, 1)'::tdigest; tdigest -------------------------------------------------------------------------------------------------------------------------------- flags 1 count 20 compression 10 centroids 8 (1000, 1) (2000, 1) (500, 2) (6500, 4) (12000, 7) (17000, 3) (19000, 1) (20000, 1) (1 row) tvondra-tdigest-c0af713/test/expected/copy.out000066400000000000000000000036071525543331000215300ustar00rootroot00000000000000CREATE TABLE tdigest_src (id int, s tdigest); CREATE TABLE tdigest_dst (id int, s tdigest); -- generate 100 random sketches, export/import them in text/binary mode DO $$ DECLARE compression int; BEGIN FOR i IN 1..100 LOOP -- random compression value compression := 100 + random() * 1000; INSERT INTO tdigest_src SELECT i, tdigest(random(), compression) FROM generate_series(1, compression * 10) s(x); END LOOP; END; $$ LANGUAGE plpgsql; -- export in text and binary formats COPY tdigest_src TO '/tmp/tdigest_copy_text.out'; COPY tdigest_src TO '/tmp/tdigest_copy_binary.out' WITH (FORMAT BINARY); -- import in text and binary formats COPY tdigest_dst FROM '/tmp/tdigest_copy_text.out'; COPY tdigest_dst FROM '/tmp/tdigest_copy_binary.out' WITH (FORMAT BINARY); -- all imported values have to match the source SELECT COUNT(*) AS count_all, COUNT(CASE WHEN (src.s::text != dst.s::text) THEN 1 ELSE NULL END) AS count_mismatching FROM tdigest_src src JOIN tdigest_dst dst ON (src.id = dst.id); count_all | count_mismatching -----------+------------------- 200 | 0 (1 row) DROP TABLE tdigest_src; DROP TABLE tdigest_dst; -- tdigest_recv() has to accept digests with unsorted centroids -- -- Test through a binary COPY, which goes through the receive function. -- We build the input with the send functions of the individual fields. CREATE TABLE tdigest_dst (s tdigest); COPY (SELECT int4send(1) -- flags || int8send(3::bigint) -- count || int4send(10) -- compression || int4send(2) -- ncentroids || float8send(5) || int8send(2::bigint) || float8send(1) || int8send(1::bigint)) TO '/tmp/tdigest_recv_unsorted.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_unsorted.bin' WITH (FORMAT binary); DROP TABLE tdigest_dst; tvondra-tdigest-c0af713/test/expected/count_checks.out000066400000000000000000000021471525543331000232240ustar00rootroot00000000000000-- Regression test for the count check in tdigest_add_double_count(), the -- transition function of the aggregates accepting an explicit number of -- occurrences of a value. -- -- A non-NULL count must be positive. A NULL count means one occurrence -- of the input value. \set VERBOSITY terse -- zero count SELECT tdigest(v, 0::bigint, 100) FROM (VALUES (1.0)) AS t(v); ERROR: invalid count value 0, must be a positive value -- negative count SELECT tdigest(v, -1::bigint, 100) FROM (VALUES (1.0)) AS t(v); ERROR: invalid count value -1, must be a positive value -- the smallest valid count has to work SELECT tdigest(v, 1::bigint, 100) FROM (VALUES (1.0)) AS t(v); tdigest ---------------------------------------------------- flags 1 count 1 compression 100 centroids 1 (1, 1) (1 row) -- NULL count means a single occurrence, so it's valid too SELECT tdigest(v, NULL::bigint, 100) FROM (VALUES (1.0)) AS t(v); tdigest ---------------------------------------------------- flags 1 count 1 compression 100 centroids 1 (1, 1) (1 row) tvondra-tdigest-c0af713/test/expected/count_generated.out000066400000000000000000000034311525543331000237170ustar00rootroot00000000000000-- Regression test for adding a value with a count exceeding BUFFER_SIZE in -- tdigest_add_double_array_values_count(), the transition function for -- inverse percentiles of an array of hypothetical values. -- -- Adding the values one by one would mean too many compactions, so counts -- larger than the buffer (BUFFER_SIZE = 10 * compression, i.e. 100 for the -- smallest compression) are added as pre-generated centroids instead. Both -- paths have to produce comparable results. -- a single distinct value, so the result is exact on both sides of the -- BUFFER_SIZE threshold SELECT tdigest_percentile_of(v, 100::bigint, 10, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v); tdigest_percentile_of ----------------------- {0.5} (1 row) SELECT tdigest_percentile_of(v, 101::bigint, 10, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v); tdigest_percentile_of ----------------------- {0.5} (1 row) SELECT tdigest_percentile_of(v, 1000000::bigint, 10, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v); tdigest_percentile_of ----------------------- {0.5} (1 row) -- two distinct values, with the counts at and just above the -- threshold SELECT tdigest_percentile_of(v, 100::bigint, 10, ARRAY[1.0, 2.0]) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_percentile_of ----------------------- {0.0725,0.775} (1 row) SELECT tdigest_percentile_of(v, 101::bigint, 10, ARRAY[1.0, 2.0]) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_percentile_of ----------------------- {0.25,0.75} (1 row) -- a larger compression means a larger buffer, so the very same count now -- goes through the one-by-one path SELECT tdigest_percentile_of(v, 101::bigint, 100, ARRAY[1.0, 2.0]) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_percentile_of ------------------------------------------ {0.24504950495049505,0.7846534653465347} (1 row) tvondra-tdigest-c0af713/test/expected/count_generated_0.out000066400000000000000000000034201525543331000241340ustar00rootroot00000000000000-- Regression test for adding a value with a count exceeding BUFFER_SIZE in -- tdigest_add_double_array_values_count(), the transition function for -- inverse percentiles of an array of hypothetical values. -- -- Adding the values one by one would mean too many compactions, so counts -- larger than the buffer (BUFFER_SIZE = 10 * compression, i.e. 100 for the -- smallest compression) are added as pre-generated centroids instead. Both -- paths have to produce comparable results. -- a single distinct value, so the result is exact on both sides of the -- BUFFER_SIZE threshold SELECT tdigest_percentile_of(v, 100::bigint, 10, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v); tdigest_percentile_of ----------------------- {0.5} (1 row) SELECT tdigest_percentile_of(v, 101::bigint, 10, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v); tdigest_percentile_of ----------------------- {0.5} (1 row) SELECT tdigest_percentile_of(v, 1000000::bigint, 10, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v); tdigest_percentile_of ----------------------- {0.5} (1 row) -- two distinct values, with the counts at and just above the -- threshold SELECT tdigest_percentile_of(v, 100::bigint, 10, ARRAY[1.0, 2.0]) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_percentile_of ----------------------- {0.0725,0.775} (1 row) SELECT tdigest_percentile_of(v, 101::bigint, 10, ARRAY[1.0, 2.0]) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_percentile_of ----------------------- {0.25,0.75} (1 row) -- a larger compression means a larger buffer, so the very same count now -- goes through the one-by-one path SELECT tdigest_percentile_of(v, 101::bigint, 100, ARRAY[1.0, 2.0]) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_percentile_of --------------------------------------- {0.245049504950495,0.784653465346535} (1 row) tvondra-tdigest-c0af713/test/expected/count_overflow.out000066400000000000000000000156571525543331000236410ustar00rootroot00000000000000-- Use terse verbosity, so that the expected output does not depend on the -- error context, which differs between PostgreSQL versions (particularly -- for the COPY ... FROM statements below). \set VERBOSITY terse CREATE TABLE tdigest_overflow_test (d tdigest); -- Regression test for the count overflow in tdigest_in() and tdigest_recv(). -- -- Both input functions sum the centroid counts and compare the result to the -- total count stored in the header. Both the centroid counts and the total -- are int64, so the sum may overflow - and if it happens to wrap around to -- exactly the value stored in the header, the "total count does not match" -- check passes and a corrupted t-digest gets accepted. -- -- Five centroids with count 2^62 are the smallest such case: the centroids -- weigh 5 * 2^62 in total, which is 2^62 modulo 2^64. So a digest claiming -- "count 2^62" passes the check, while the centroids actually weigh five -- times as much. -- tdigest_in() has to reject the wrapping value SELECT 'flags 1 count 4611686018427387904 compression 10 centroids 5 (1, 4611686018427387904) (2, 4611686018427387904) (3, 4611686018427387904) (4, 4611686018427387904) (5, 4611686018427387904)'::tdigest; ERROR: tdigest count overflow at character 8 -- tdigest_recv() has to reject it too. There's no way to call the receive -- function directly, so go through a binary COPY. A binary COPY file with a -- single bytea column is byte-for-byte the same as a file with a single -- tdigest column holding the same bytes, so we can build the input with the -- send functions of the individual header/centroid fields. COPY (SELECT int4send(1) -- flags || int8send(4611686018427387904::bigint) -- count || int4send(10) -- compression || int4send(5) -- ncentroids || float8send(1) || int8send(4611686018427387904::bigint) || float8send(2) || int8send(4611686018427387904::bigint) || float8send(3) || int8send(4611686018427387904::bigint) || float8send(4) || int8send(4611686018427387904::bigint) || float8send(5) || int8send(4611686018427387904::bigint)) TO '/tmp/tdigest_count_overflow.bin' WITH (FORMAT binary); COPY tdigest_overflow_test FROM '/tmp/tdigest_count_overflow.bin' WITH (FORMAT binary); ERROR: tdigest count overflow -- a t-digest with a total count of exactly INT64_MAX is still valid, though, -- and has to be accepted by both input functions SELECT 'flags 1 count 9223372036854775807 compression 10 centroids 2 (1, 4611686018427387903) (2, 4611686018427387904)'::tdigest; tdigest ---------------------------------------------------------------------------------------------------------------- flags 1 count 9223372036854775807 compression 10 centroids 2 (1, 4611686018427387903) (2, 4611686018427387904) (1 row) COPY (SELECT int4send(1) || int8send(9223372036854775807::bigint) || int4send(10) || int4send(2) || float8send(1) || int8send(4611686018427387903::bigint) || float8send(2) || int8send(4611686018427387904::bigint)) TO '/tmp/tdigest_count_overflow.bin' WITH (FORMAT binary); COPY tdigest_overflow_test FROM '/tmp/tdigest_count_overflow.bin' WITH (FORMAT binary); SELECT * FROM tdigest_overflow_test; d ---------------------------------------------------------------------------------------------------------------- flags 1 count 9223372036854775807 compression 10 centroids 2 (1, 4611686018427387903) (2, 4611686018427387904) (1 row) -- Regression test for the count overflow in tdigest_add() and tdigest_add_centroid(). -- A couple examples triggering overflows of total count when adding values -- or centroids to a digest (or aggregate state) WITH source(value) AS ( SELECT * FROM (VALUES ('flags 1 count 9223372036854775807 compression 10 centroids 1 (-1000, 9223372036854775807)'::tdigest), ('flags 1 count 1 compression 10 centroids 1 (-1000, 1)'::tdigest)) AS t(v) ) SELECT tdigest(value) FROM source; ERROR: tdigest count overflow WITH source(value) AS ( SELECT * FROM (VALUES ('flags 1 count 4611686018427387904 compression 10 centroids 1 (-1000, 4611686018427387904)'::tdigest), ('flags 1 count 4611686018427387904 compression 10 centroids 1 (-1000, 4611686018427387904)'::tdigest)) AS t(v) ) SELECT tdigest(value) FROM source; ERROR: tdigest count overflow WITH source(value) AS ( SELECT 'flags 1 count 5000000000000000000 compression 10 centroids 1 (-1000, 5000000000000000000)'::tdigest FROM generate_series(1, 2) ) SELECT tdigest(value) FROM source; ERROR: tdigest count overflow -- the negative cases (no overflows, hitting INT64_MAX exactly) should still pass WITH source(value) AS ( SELECT * FROM (VALUES ('flags 1 count 9223372036854775806 compression 10 centroids 1 (-1000, 9223372036854775806)'::tdigest), ('flags 1 count 1 compression 10 centroids 1 (-1000, 1)'::tdigest)) AS t(v) ) SELECT tdigest(value) FROM source; tdigest ------------------------------------------------------------------------------------------------------ flags 1 count 9223372036854775807 compression 10 centroids 2 (-1000, 1) (-1000, 9223372036854775806) (1 row) WITH source(value) AS ( SELECT * FROM (VALUES ('flags 1 count 4611686018427387903 compression 10 centroids 1 (-1000, 4611686018427387903)'::tdigest), ('flags 1 count 4611686018427387904 compression 10 centroids 1 (-1000, 4611686018427387904)'::tdigest)) AS t(v) ) SELECT tdigest(value) FROM source; tdigest ------------------------------------------------------------------------------------------------------------------------ flags 1 count 9223372036854775807 compression 10 centroids 2 (-1000, 4611686018427387903) (-1000, 4611686018427387904) (1 row) WITH source(value) AS ( SELECT 'flags 1 count 4611686018427387903 compression 10 centroids 1 (-1000, 4611686018427387903)'::tdigest FROM generate_series(1, 2) UNION ALL SELECT 'flags 1 count 1 compression 10 centroids 1 (-1000, 1)'::tdigest ) SELECT tdigest(value) FROM source; tdigest ----------------------------------------------------------------------------------------------------------------------------------- flags 1 count 9223372036854775807 compression 10 centroids 3 (-1000, 1) (-1000, 4611686018427387903) (-1000, 4611686018427387903) (1 row) DROP TABLE tdigest_overflow_test; tvondra-tdigest-c0af713/test/expected/digest_is_valid.out000066400000000000000000000304361525543331000237070ustar00rootroot00000000000000-- Regression test for tdigest_is_valid(), which performs the same checks as -- the input functions (tdigest_in and tdigest_recv), but on an existing value -- and without raising an error. -- -- Digests written by older versions of the extension (which did not have all -- of the checks) may be broken in various ways, and nothing re-validates the -- values when reading them back from disk. This function allows finding such -- digests. -- -- The input functions reject invalid values, so we can't use them to build -- the test data. Instead we build the on-disk representation by hand, and use -- a binary-coercible cast to turn it into a t-digest. Note the forged values -- must not be passed to anything except tdigest_is_valid() - the whole point -- is that they are broken, so e.g. printing them might read past the end of -- the value. \set VERBOSITY terse -- casts allowing us to inspect and forge the on-disk representation CREATE CAST (tdigest AS bytea) WITHOUT FUNCTION; CREATE CAST (bytea AS tdigest) WITHOUT FUNCTION; -- The fields of the on-disk representation use the host byte order, while -- the send functions produce network byte order (big endian), so we may need -- to reverse the bytes. Determine the host byte order by looking at the flags -- of a digest we know has flags set to 1. CREATE FUNCTION is_little_endian() RETURNS bool AS $$ SELECT substring('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest::bytea from 1 for 4) = '\x01000000'::bytea; $$ LANGUAGE sql IMMUTABLE; CREATE FUNCTION swap_bytes(b bytea) RETURNS bytea AS $$ SELECT CASE WHEN is_little_endian() THEN (SELECT string_agg(substring(b from i for 1), ''::bytea ORDER BY i DESC) FROM generate_series(1, length(b)) AS i) ELSE b END; $$ LANGUAGE sql IMMUTABLE; -- Transform values to big endian (if needed) CREATE FUNCTION be_int4(v int) RETURNS bytea AS $$ SELECT swap_bytes(int4send(v)); $$ LANGUAGE sql IMMUTABLE; CREATE FUNCTION be_int8(v bigint) RETURNS bytea AS $$ SELECT swap_bytes(int8send(v)); $$ LANGUAGE sql IMMUTABLE; CREATE FUNCTION be_float8(v double precision) RETURNS bytea AS $$ SELECT swap_bytes(float8send(v)); $$ LANGUAGE sql IMMUTABLE; -- the layout has to be (flags, count, compression, ncentroids) followed by -- the centroids, each stored as (mean, count) SELECT length('flags 1 count 3 compression 10 centroids 2 (1, 1) (2, 2)'::tdigest::bytea) AS bytes; bytes ------- 52 (1 row) -- a forged value has to be the same as the parsed one (sanity check of the -- forging, not of the validation) SELECT (be_int4(1) || be_int8(3::bigint) || be_int4(10) || be_int4(2) || be_float8(1) || be_int8(1::bigint) || be_float8(2) || be_int8(2::bigint))::tdigest; tdigest ---------------------------------------------------------- flags 1 count 3 compression 10 centroids 2 (1, 1) (2, 2) (1 row) -- -- valid values, to make sure the checks don't reject correct digests -- SELECT tdigest_is_valid('flags 1 count 3 compression 10 centroids 2 (1, 1) (2, 2)'::tdigest); tdigest_is_valid ------------------ t (1 row) -- the old on-disk format, storing sums instead of means SELECT tdigest_is_valid('flags 0 count 3 compression 10 centroids 2 (1, 1) (4, 2)'::tdigest); tdigest_is_valid ------------------ t (1 row) -- centroids not sorted by mean (produced by the incremental API) SELECT tdigest_is_valid('flags 1 count 3 compression 10 centroids 2 (2, 2) (1, 1)'::tdigest); tdigest_is_valid ------------------ t (1 row) -- the extreme compression values SELECT tdigest_is_valid('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest); tdigest_is_valid ------------------ t (1 row) SELECT tdigest_is_valid('flags 1 count 1 compression 10000 centroids 1 (1, 1)'::tdigest); tdigest_is_valid ------------------ t (1 row) -- digests built by the extension itself SELECT tdigest_is_valid(tdigest(i::double precision, 10)) FROM generate_series(1, 1000) AS s(i); tdigest_is_valid ------------------ t (1 row) SELECT tdigest_is_valid(tdigest(i::double precision, 100)) FROM generate_series(1, 1000) AS s(i); tdigest_is_valid ------------------ t (1 row) SELECT tdigest_is_valid(tdigest_add(NULL::tdigest, ARRAY[1.0, 2.0, 3.0], 10, false)); tdigest_is_valid ------------------ t (1 row) SELECT tdigest_is_valid(tdigest_union(tdigest_add(NULL::tdigest, 1.0, 10), tdigest_add(NULL::tdigest, 2.0, 10))); tdigest_is_valid ------------------ t (1 row) -- -- sanity checks on the header -- -- unknown flags SELECT tdigest_is_valid((be_int4(2) || be_int8(1::bigint) || be_int4(10) || be_int4(1) || be_float8(1) || be_int8(1::bigint))::tdigest); tdigest_is_valid ------------------ f (1 row) -- compression out of the supported range SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(9) || be_int4(1) || be_float8(1) || be_int8(1::bigint))::tdigest); tdigest_is_valid ------------------ f (1 row) SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10001) || be_int4(1) || be_float8(1) || be_int8(1::bigint))::tdigest); tdigest_is_valid ------------------ f (1 row) -- non-positive total count SELECT tdigest_is_valid((be_int4(1) || be_int8(0::bigint) || be_int4(10) || be_int4(1) || be_float8(1) || be_int8(1::bigint))::tdigest); tdigest_is_valid ------------------ f (1 row) SELECT tdigest_is_valid((be_int4(1) || be_int8((-1)::bigint) || be_int4(10) || be_int4(1) || be_float8(1) || be_int8(1::bigint))::tdigest); tdigest_is_valid ------------------ f (1 row) -- non-positive number of centroids SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(0))::tdigest); tdigest_is_valid ------------------ f (1 row) SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(-1))::tdigest); tdigest_is_valid ------------------ f (1 row) -- more centroids than fit into the buffer (10 * compression) SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(101) || be_float8(1) || be_int8(1::bigint))::tdigest); tdigest_is_valid ------------------ f (1 row) -- -- sanity checks on the length of the value -- -- The input functions build the digest from the parsed header, so the length -- always matches. For an existing value we have to check it really is long -- enough for the centroids the header promises. -- -- value too short to even contain the header SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10))::tdigest); tdigest_is_valid ------------------ f (1 row) -- fewer centroids than the header says SELECT tdigest_is_valid((be_int4(1) || be_int8(3::bigint) || be_int4(10) || be_int4(2) || be_float8(1) || be_int8(1::bigint))::tdigest); tdigest_is_valid ------------------ f (1 row) -- more centroids than the header says SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(1) || be_float8(1) || be_int8(1::bigint) || be_float8(2) || be_int8(2::bigint))::tdigest); tdigest_is_valid ------------------ f (1 row) -- incomplete centroid at the end of the value SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(1) || be_float8(1))::tdigest); tdigest_is_valid ------------------ f (1 row) -- -- sanity checks on the centroids -- -- centroid mean that is not a valid number SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(1) || be_float8('NaN') || be_int8(1::bigint))::tdigest); tdigest_is_valid ------------------ f (1 row) SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(1) || be_float8('Infinity') || be_int8(1::bigint))::tdigest); tdigest_is_valid ------------------ f (1 row) SELECT tdigest_is_valid((be_int4(1) || be_int8(3::bigint) || be_int4(10) || be_int4(2) || be_float8(1) || be_int8(1::bigint) || be_float8('-Infinity') || be_int8(2::bigint))::tdigest); tdigest_is_valid ------------------ f (1 row) -- non-positive centroid count SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(1) || be_float8(1) || be_int8(0::bigint))::tdigest); tdigest_is_valid ------------------ f (1 row) SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(2) || be_float8(1) || be_int8(1::bigint) || be_float8(2) || be_int8((-1)::bigint))::tdigest); tdigest_is_valid ------------------ f (1 row) -- centroid count exceeding the total count SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(1) || be_float8(1) || be_int8(2::bigint))::tdigest); tdigest_is_valid ------------------ f (1 row) -- centroid counts overflowing the total count SELECT tdigest_is_valid((be_int4(1) || be_int8(9223372036854775807::bigint) || be_int4(10) || be_int4(2) || be_float8(1) || be_int8(9223372036854775807::bigint) || be_float8(2) || be_int8(9223372036854775807::bigint))::tdigest); tdigest_is_valid ------------------ f (1 row) -- -- sanity checks on the value as a whole -- -- centroid counts not adding up to the total count SELECT tdigest_is_valid((be_int4(1) || be_int8(3::bigint) || be_int4(10) || be_int4(1) || be_float8(1) || be_int8(1::bigint))::tdigest); tdigest_is_valid ------------------ f (1 row) SELECT tdigest_is_valid((be_int4(1) || be_int8(3::bigint) || be_int4(10) || be_int4(2) || be_float8(1) || be_int8(1::bigint) || be_float8(2) || be_int8(3::bigint))::tdigest); tdigest_is_valid ------------------ f (1 row) -- -- the checks have to work on values read back from a table too (which may -- use a different varlena header, or be TOASTed) -- CREATE TABLE tdigest_values (id int, d tdigest); INSERT INTO tdigest_values SELECT 1, 'flags 1 count 3 compression 10 centroids 2 (1, 1) (2, 2)'::tdigest; INSERT INTO tdigest_values SELECT 2, (be_int4(1) || be_int8(3::bigint) || be_int4(10) || be_int4(1) || be_float8(1) || be_int8(1::bigint))::tdigest; INSERT INTO tdigest_values SELECT 3, tdigest(i::double precision, 1000) FROM generate_series(1, 100000) AS s(i); SELECT id, tdigest_is_valid(d) FROM tdigest_values ORDER BY id; id | tdigest_is_valid ----+------------------ 1 | t 2 | f 3 | t (3 rows) -- NULL input produces NULL SELECT tdigest_is_valid(NULL::tdigest) IS NULL; ?column? ---------- t (1 row) DROP TABLE tdigest_values; DROP FUNCTION be_float8(double precision); DROP FUNCTION be_int8(bigint); DROP FUNCTION be_int4(int); DROP FUNCTION swap_bytes(bytea); DROP FUNCTION is_little_endian(); DROP CAST (bytea AS tdigest); DROP CAST (tdigest AS bytea); tvondra-tdigest-c0af713/test/expected/empty_input.out000066400000000000000000000022121525543331000231220ustar00rootroot00000000000000-- Regression test for the NULL handling in the final functions. -- -- None of the final functions is strict, so an aggregate over an empty -- input calls them with a NULL state, and they have to return NULL. Each -- final function is reached through the matching aggregate, with a WHERE -- clause filtering out all the rows. -- tdigest_percentiles SELECT tdigest_percentile(v, 100, 0.5) FROM (VALUES (1.0)) AS t(v) WHERE v > 2.0; tdigest_percentile -------------------- (1 row) -- tdigest_percentiles_of SELECT tdigest_percentile_of(v, 100, 1.0) FROM (VALUES (1.0)) AS t(v) WHERE v > 2.0; tdigest_percentile_of ----------------------- (1 row) -- tdigest_digest SELECT tdigest(v, 100) FROM (VALUES (1.0)) AS t(v) WHERE v > 2.0; tdigest --------- (1 row) -- tdigest_array_percentiles SELECT tdigest_percentile(v, 100, ARRAY[0.5]) FROM (VALUES (1.0)) AS t(v) WHERE v > 2.0; tdigest_percentile -------------------- (1 row) -- tdigest_array_percentiles_of SELECT tdigest_percentile_of(v, 100, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v) WHERE v > 2.0; tdigest_percentile_of ----------------------- (1 row) tvondra-tdigest-c0af713/test/expected/generate_overflow.out000066400000000000000000000022351525543331000242670ustar00rootroot00000000000000-- Use terse verbosity, so that the expected output does not depend on the -- error context, which differs between PostgreSQL versions. \set VERBOSITY terse -- via the tdigest(value, count, compression) aggregate SELECT length(tdigest(1.0::float8, 9223372036854775807::bigint, 10000)::text); length -------- 45786 (1 row) -- the count does not have to be anywhere near the int64 limit, a large enough -- count with a moderate compression is enough SELECT length(tdigest(1.0::float8, 1000000000000000000::bigint, 1000)::text); length -------- 5370 (1 row) -- via the percentile aggregates on the value/count API SELECT tdigest_percentile(1.0::float8, 9223372036854775807::bigint, 10000, 0.5); tdigest_percentile -------------------- 1 (1 row) -- via the trimmed aggregates on the value/count API -- -- The exact value depends on where exactly the centroid boundaries end up, -- so don't compare it to an exact value - it'd be a 1 ULP difference away -- from failing. It just has to be (close to) 1.0, not 0.9 or NULL. SELECT abs(tdigest_avg(1.0::float8, 9223372036854775807::bigint, 10000, 0.1, 0.9) - 1.0) < 1e-12; ?column? ---------- t (1 row) tvondra-tdigest-c0af713/test/expected/generate_overflow_1.out000066400000000000000000000022351525543331000245070ustar00rootroot00000000000000-- Use terse verbosity, so that the expected output does not depend on the -- error context, which differs between PostgreSQL versions. \set VERBOSITY terse -- via the tdigest(value, count, compression) aggregate SELECT length(tdigest(1.0::float8, 9223372036854775807::bigint, 10000)::text); length -------- 45786 (1 row) -- the count does not have to be anywhere near the int64 limit, a large enough -- count with a moderate compression is enough SELECT length(tdigest(1.0::float8, 1000000000000000000::bigint, 1000)::text); length -------- 5370 (1 row) -- via the percentile aggregates on the value/count API SELECT tdigest_percentile(1.0::float8, 9223372036854775807::bigint, 10000, 0.5); tdigest_percentile -------------------- 1 (1 row) -- via the trimmed aggregates on the value/count API -- -- The exact value depends on where exactly the centroid boundaries end up, -- so don't compare it to an exact value - it'd be a 1 ULP difference away -- from failing. It just has to be (close to) 1.0, not 0.9 or NULL. SELECT abs(tdigest_avg(1.0::float8, 9223372036854775807::bigint, 10000, 0.1, 0.9) - 1.0) < 1e-12; ?column? ---------- t (1 row) tvondra-tdigest-c0af713/test/expected/increment_compression.out000066400000000000000000000042321525543331000251560ustar00rootroot00000000000000-- Regression test for the compression check in tdigest_add_double_increment() -- and tdigest_add_double_array_increment(), the functions of the incremental -- API. -- -- The compression is only optional when there already is a t-digest to add -- the value to, because that determines the compression. Without a digest -- there is nothing to derive the compression from, so it has to be supplied. \set VERBOSITY terse -- NULL digest and NULL compression (which is the default) SELECT tdigest_add(NULL::tdigest, 1.0::double precision); ERROR: compression value not supplied, but t-digest is NULL SELECT tdigest_add(NULL::tdigest, 1.0::double precision, NULL::int); ERROR: compression value not supplied, but t-digest is NULL SELECT tdigest_add(NULL::tdigest, ARRAY[1.0]::double precision[]); ERROR: compression value not supplied, but t-digest is NULL SELECT tdigest_add(NULL::tdigest, ARRAY[1.0]::double precision[], NULL::int); ERROR: compression value not supplied, but t-digest is NULL -- with the compression supplied it has to work SELECT tdigest_add(NULL::tdigest, 1.0::double precision, 100); tdigest_add ---------------------------------------------------- flags 1 count 1 compression 100 centroids 1 (1, 1) (1 row) SELECT tdigest_add(NULL::tdigest, ARRAY[1.0]::double precision[], 100); tdigest_add ---------------------------------------------------- flags 1 count 1 compression 100 centroids 1 (1, 1) (1 row) -- with an existing digest the compression is not needed SELECT tdigest_add('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest, 2.0::double precision); tdigest_add ----------------------------------------------------------- flags 1 count 2 compression 100 centroids 2 (1, 1) (2, 1) (1 row) SELECT tdigest_add('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest, ARRAY[2.0]::double precision[]); tdigest_add ----------------------------------------------------------- flags 1 count 2 compression 100 centroids 2 (1, 1) (2, 1) (1 row) tvondra-tdigest-c0af713/test/expected/incremental.out000066400000000000000000000065131525543331000230560ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('extra_float_digits', '0', false); PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; -- test incremental API (adding values one by one) CREATE TABLE test_incremental (d tdigest); INSERT INTO test_incremental VALUES (NULL); -- check this produces the same result as building the tdigest at once, but we -- need to be careful about feeding the data in the same order, and we must -- not compact the t-digest after each increment DO LANGUAGE plpgsql $$ DECLARE r RECORD; BEGIN FOR r IN (SELECT i FROM generate_series(1,1000) s(i) ORDER BY md5(i::text)) LOOP UPDATE test_incremental SET d = tdigest_add(d, r.i, 100, false); END LOOP; END$$; -- compare the results, but do force a compaction of the incremental result WITH x AS (SELECT i FROM generate_series(1,1000) s(i) ORDER BY md5(i::text)) SELECT (SELECT tdigest(d)::text FROM test_incremental) = (SELECT tdigest(x.i, 100)::text FROM x) AS match; match ------- t (1 row) -- now try the same thing with bulk incremental update (using arrays) TRUNCATE test_incremental; INSERT INTO test_incremental VALUES (NULL); DO LANGUAGE plpgsql $$ DECLARE r RECORD; BEGIN FOR r IN (SELECT a, array_agg(i::double precision) AS v FROM (SELECT mod(i,5) AS a, i FROM generate_series(1,1000) s(i) ORDER BY mod(i,5), md5(i::text)) foo GROUP BY a ORDER BY a) LOOP UPDATE test_incremental SET d = tdigest_add(d, r.v, 100, false); END LOOP; END$$; -- compare the results, but do force a compaction of the incremental result WITH x AS (SELECT mod(i,5) AS a, i::double precision AS d FROM generate_series(1,1000) s(i) ORDER BY mod(i,5), i) SELECT (SELECT tdigest(d)::text FROM test_incremental) = (SELECT tdigest(x.d, 100)::text FROM x); ?column? ---------- t (1 row) -- now try the same thing with bulk incremental update (using t-digests) TRUNCATE test_incremental; INSERT INTO test_incremental VALUES (NULL); DO LANGUAGE plpgsql $$ DECLARE r RECORD; BEGIN FOR r IN (SELECT a, tdigest(i,100) AS d FROM (SELECT mod(i,5) AS a, i FROM generate_series(1,1000) s(i) ORDER BY mod(i,5), md5(i::text)) foo GROUP BY a ORDER BY a) LOOP UPDATE test_incremental SET d = tdigest_union(d, r.d, false); END LOOP; END$$; -- compare the results, but do force a compaction of the incremental result WITH x AS (SELECT a, tdigest(i,100) AS d FROM (SELECT mod(i,5) AS a, i FROM generate_series(1,1000) s(i) ORDER BY mod(i,5), md5(i::text)) foo GROUP BY a ORDER BY a) SELECT (SELECT tdigest(d)::text FROM test_incremental) = (SELECT tdigest(x.d)::text FROM x); ?column? ---------- t (1 row) DROP TABLE test_incremental; tvondra-tdigest-c0af713/test/expected/init.out000066400000000000000000000031421525543331000215130ustar00rootroot00000000000000\set ECHO none -- Set-returning function implementing a simple deterministic PRNG CREATE OR REPLACE FUNCTION prng(nrows int, seed int = 23982, p1 bigint = 16807, p2 bigint = 0, n bigint = 2147483647) RETURNS SETOF double precision AS $$ DECLARE val INT := seed; BEGIN FOR i IN 1..nrows LOOP val := (val * p1 + p2) % n; RETURN NEXT (val::double precision / n); END LOOP; RETURN; END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION random_normal(nrows int, mean double precision = 0.5, stddev double precision = 0.1, minval double precision = 0.0, maxval double precision = 1.0, seed int = 23982, p1 bigint = 16807, p2 bigint = 0, n bigint = 2147483647) RETURNS SETOF double precision AS $$ DECLARE v BIGINT := seed; x DOUBLE PRECISION; y DOUBLE PRECISION; s DOUBLE PRECISION; r INT := nrows; BEGIN WHILE true LOOP -- random x v := (v * p1 + p2) % n; x := 2 * v / n::double precision - 1.0; -- random y v := (v * p1 + p2) % n; y := 2 * v / n::double precision - 1.0; s := x^2 + y^2; IF s != 0.0 AND s < 1.0 THEN s = sqrt(-2 * ln(s) / s); x := mean + stddev * s * x; IF x >= minval AND x <= maxval THEN RETURN NEXT x; r := r - 1; END IF; EXIT WHEN r = 0; y := mean + stddev * s * y; IF y >= minval AND y <= maxval THEN RETURN NEXT y; r := r - 1; END IF; EXIT WHEN r = 0; END IF; END LOOP; END; $$ LANGUAGE plpgsql; tvondra-tdigest-c0af713/test/expected/input_saturation.out000066400000000000000000000070761525543331000241720ustar00rootroot00000000000000-- Reproducer: tdigest_in() accepts out-of-range numbers, because sscanf() has -- no way to report a range error. -- -- This one does not crash either, it silently accepts input it should reject -- and stores something different from what was written. -- -- tdigest_in() parses the whole value with sscanf() (tdigest.c around lines -- 2646 and 2694) and only checks that the parsed values make sense relative to -- each other. But sscanf() has undefined behaviour when the number does not -- fit the target type (C99 7.21.6.2p10), and glibc's implementation does not -- report it: -- -- %lld saturates: "99999999999999999999" -> INT64_MAX -- "-99999999999999999999" -> INT64_MIN -- %d wraps: "2147483648" -> -2147483648 -- "4294967296" -> 0 -- "4294967306" -> 10 -- "99999999999" -> 1215752191 -- -- The count overflow checks added in d030f1f run on the parsed (already -- mangled) values, so they do not catch any of this. \set VERBOSITY terse -- 1) the counts saturate to INT64_MAX. Both the total and the centroid count -- saturate to the same value, so they still agree with each other and the -- digest is accepted. Note the output does not match the input. SELECT 'flags 1 count 99999999999999999999 compression 10 centroids 1 (1, 99999999999999999999)'::tdigest; ERROR: count of a t-digest is out of range for bigint at character 8 -- 2) worse, the compression is parsed with %d and wraps modulo 2^32. 4294967306 -- is 2^32 + 10, so this is accepted and stored as compression 10 - well inside -- the allowed range, so the range check that follows sees nothing wrong. SELECT 'flags 1 count 3 compression 4294967306 centroids 1 (1, 3)'::tdigest; ERROR: compression of a t-digest is out of range for integer at character 8 -- and 2^33 + 10 gives the same thing SELECT 'flags 1 count 3 compression 8589934602 centroids 1 (1, 3)'::tdigest; ERROR: compression of a t-digest is out of range for integer at character 8 -- this means the text output of tdigest_in() is not the text that was fed to -- it, i.e. the type is not round-trip stable and a dump/restore of such a -- value produces a different digest SELECT 'flags 1 count 3 compression 4294967306 centroids 1 (1, 3)'::tdigest::text = 'flags 1 count 3 compression 4294967306 centroids 1 (1.000000, 3)' AS round_trips; ERROR: compression of a t-digest is out of range for integer at character 8 -- 3) values that happen to wrap to something outside the allowed range are -- still rejected, but by the range check and with a misleading message - the -- parse itself never fails SELECT 'flags 1 count 3 compression 4294967296 centroids 1 (1, 3)'::tdigest; ERROR: compression of a t-digest is out of range for integer at character 8 SELECT 'flags 1 count 3 compression 2147483648 centroids 1 (1, 3)'::tdigest; ERROR: compression of a t-digest is out of range for integer at character 8 -- 4) negative counts saturate to INT64_MIN, and are then caught by the -- "count value for the t-digest must be positive" check SELECT 'flags 1 count -99999999999999999999 compression 10 centroids 1 (1, 3)'::tdigest; ERROR: count of a t-digest is out of range for bigint at character 8 -- subnormal means should be accepted (and treated as 0.0) SELECT 'flags 1 count 2 compression 10 centroids 2 (1e-320, 1) (2e-320, 1)'::tdigest; tdigest -------------------------------------------------------------------- flags 1 count 2 compression 10 centroids 2 (1e-320, 1) (2e-320, 1) (1 row) tvondra-tdigest-c0af713/test/expected/input_saturation_1.out000066400000000000000000000072301525543331000244020ustar00rootroot00000000000000-- Reproducer: tdigest_in() accepts out-of-range numbers, because sscanf() has -- no way to report a range error. -- -- This one does not crash either, it silently accepts input it should reject -- and stores something different from what was written. -- -- tdigest_in() parses the whole value with sscanf() (tdigest.c around lines -- 2646 and 2694) and only checks that the parsed values make sense relative to -- each other. But sscanf() has undefined behaviour when the number does not -- fit the target type (C99 7.21.6.2p10), and glibc's implementation does not -- report it: -- -- %lld saturates: "99999999999999999999" -> INT64_MAX -- "-99999999999999999999" -> INT64_MIN -- %d wraps: "2147483648" -> -2147483648 -- "4294967296" -> 0 -- "4294967306" -> 10 -- "99999999999" -> 1215752191 -- -- The count overflow checks added in d030f1f run on the parsed (already -- mangled) values, so they do not catch any of this. \set VERBOSITY terse -- 1) the counts saturate to INT64_MAX. Both the total and the centroid count -- saturate to the same value, so they still agree with each other and the -- digest is accepted. Note the output does not match the input. SELECT 'flags 1 count 99999999999999999999 compression 10 centroids 1 (1, 99999999999999999999)'::tdigest; ERROR: count of a t-digest is out of range for bigint at character 8 -- 2) worse, the compression is parsed with %d and wraps modulo 2^32. 4294967306 -- is 2^32 + 10, so this is accepted and stored as compression 10 - well inside -- the allowed range, so the range check that follows sees nothing wrong. SELECT 'flags 1 count 3 compression 4294967306 centroids 1 (1, 3)'::tdigest; ERROR: compression of a t-digest is out of range for integer at character 8 -- and 2^33 + 10 gives the same thing SELECT 'flags 1 count 3 compression 8589934602 centroids 1 (1, 3)'::tdigest; ERROR: compression of a t-digest is out of range for integer at character 8 -- this means the text output of tdigest_in() is not the text that was fed to -- it, i.e. the type is not round-trip stable and a dump/restore of such a -- value produces a different digest SELECT 'flags 1 count 3 compression 4294967306 centroids 1 (1, 3)'::tdigest::text = 'flags 1 count 3 compression 4294967306 centroids 1 (1.000000, 3)' AS round_trips; ERROR: compression of a t-digest is out of range for integer at character 8 -- 3) values that happen to wrap to something outside the allowed range are -- still rejected, but by the range check and with a misleading message - the -- parse itself never fails SELECT 'flags 1 count 3 compression 4294967296 centroids 1 (1, 3)'::tdigest; ERROR: compression of a t-digest is out of range for integer at character 8 SELECT 'flags 1 count 3 compression 2147483648 centroids 1 (1, 3)'::tdigest; ERROR: compression of a t-digest is out of range for integer at character 8 -- 4) negative counts saturate to INT64_MIN, and are then caught by the -- "count value for the t-digest must be positive" check SELECT 'flags 1 count -99999999999999999999 compression 10 centroids 1 (1, 3)'::tdigest; ERROR: count of a t-digest is out of range for bigint at character 8 -- subnormal means should be accepted (and treated as 0.0) SELECT 'flags 1 count 2 compression 10 centroids 2 (1e-320, 1) (2e-320, 1)'::tdigest; tdigest -------------------------------------------------------------------------------------------------- flags 1 count 2 compression 10 centroids 2 (9.99988867182683e-321, 1) (1.99997773436537e-320, 1) (1 row) tvondra-tdigest-c0af713/test/expected/interpolation_monotonicity.out000066400000000000000000000047061525543331000262610ustar00rootroot00000000000000\set ECHO none NOTICE: Percentiles are monotone and in range NOTICE: Scalar and array percentiles agree NOTICE: Percentiles are monotone and in range NOTICE: Scalar and array percentiles agree NOTICE: Percentiles are monotone and in range NOTICE: Scalar and array percentiles agree NOTICE: Percentiles are monotone and in range NOTICE: Scalar and array percentiles agree NOTICE: Percentiles are monotone and in range NOTICE: Scalar and array percentiles agree NOTICE: Percentiles are monotone and in range NOTICE: Scalar and array percentiles agree NOTICE: Raw and weighted percentiles are monotone NOTICE: Raw and weighted percentiles are monotone NOTICE: Inverse interpolation is monotone NOTICE: Scalar and array inverse percentiles agree -- At p = 0.75 - 2^-53, q = 1 - 2^-52 and the result rounds to -5. -- Subtracting nearly equal magnitudes used to return -4 instead. SELECT tdigest_percentile(v, 10000, 0.7499999999999999::double precision) AS raw_scalar, tdigest_percentile(v, 10000, ARRAY[0.7499999999999999]::double precision[]) AS raw_array, tdigest_percentile(v, 1::bigint, 10000, 0.7499999999999999::double precision) AS weighted_scalar, tdigest_percentile(v, 1::bigint, 10000, ARRAY[0.7499999999999999]::double precision[]) AS weighted_array FROM (VALUES (-18014398509481984::double precision), (-1::double precision)) AS input(v); raw_scalar | raw_array | weighted_scalar | weighted_array ------------+-----------+-----------------+---------------- -5 | {-5} | -5 | {-5} (1 row) -- Also cover stored digests and count rounding that makes q exactly 1. -- The endpoint case used to return -4, inside the bracket but not equal to -3. WITH inputs(label, d, p) AS ( VALUES ( 'precision', 'flags 1 count 2 compression 10000 centroids 2 (-18014398509481984, 1) (-1, 1)'::tdigest, 0.7499999999999999::double precision ), ( 'endpoint', 'flags 1 count 9007199254743615 compression 10000 centroids 3 (-10000000000000000, 9007199254740839) (-3, 290) (-1, 2486)'::tdigest, 0.9999999999997077::double precision ) ) SELECT label, tdigest_percentile(d, p) AS scalar_result, tdigest_percentile(d, ARRAY[p]) AS array_result FROM inputs GROUP BY label ORDER BY label; label | scalar_result | array_result -----------+---------------+-------------- endpoint | -3 | {-3} precision | -5 | {-5} (2 rows) tvondra-tdigest-c0af713/test/expected/inverse_rank_boundaries.out000066400000000000000000000042521525543331000254540ustar00rootroot00000000000000\set ECHO none NOTICE: Inverse ranks increase at a centroid boundary NOTICE: Scalar and array inverse percentiles agree NOTICE: Inverse ranks increase at a centroid boundary NOTICE: Scalar and array inverse percentiles agree NOTICE: Inverse ranks increase at a centroid boundary NOTICE: Scalar and array inverse percentiles agree NOTICE: Inverse ranks increase at a centroid boundary NOTICE: Scalar and array inverse percentiles agree NOTICE: Equal-mean ranks follow the half-weight convention SET extra_float_digits = 2; -- The CDF used to decrease from 0.375 to 0.37499999999999994 just above 1. -- Both equal-mean centroids survive compaction at this compression. WITH ranks AS ( SELECT tdigest_percentile_of(d, 1::double precision) AS at_mean, tdigest_percentile_of(d, 1.0000000000000002::double precision) AS above_mean, tdigest_percentile_of(d, ARRAY[1, 1.0000000000000002]::double precision[]) AS array_ranks FROM (VALUES ( 'flags 1 count 36028797018963981 compression 10000 centroids 4 (0, 9007199254740995) (1, 2) (1, 9007199254740993) (10, 18014398509481991)'::tdigest )) AS input(d) ) SELECT at_mean, above_mean, array_ranks, at_mean <= above_mean AS monotone, array_ranks = ARRAY[at_mean, above_mean] AS scalar_array_agree FROM ranks; at_mean | above_mean | array_ranks | monotone | scalar_array_agree ---------------------+------------+-----------------------------+----------+-------------------- 0.37499999999999994 | 0.375 | {0.37499999999999994,0.375} | t | t (1 row) -- Preserve half-items without rounding the integer part and half separately. -- In particular, (double) (18014398509481987 / 2) + 0.5 double-rounds. WITH counts(n) AS ( VALUES (1::bigint), (3), (18014398509481987), (9223372036854775807) ) SELECT n, tdigest_percentile_of( format('flags 1 count %s compression 10000 centroids 1 (1, %s)', n, n)::tdigest, 1::double precision) AS rank FROM counts GROUP BY n ORDER BY n; n | rank ---------------------+------ 1 | 0.5 3 | 0.5 18014398509481987 | 0.5 9223372036854775807 | 0.5 (4 rows) tvondra-tdigest-c0af713/test/expected/memory_usage.out000066400000000000000000000033641525543331000232520ustar00rootroot00000000000000-- check the number of batches and amount of memory used by the HashAgg node CREATE FUNCTION check_memory_usage(text) RETURNS TABLE (nbatches INT, memorykb INT) LANGUAGE plpgsql AS $$ DECLARE ln TEXT; tmp TEXT[]; found_match BOOL := false; BEGIN FOR LN IN EXECUTE format('EXPLAIN ANALYZE %s', $1) LOOP IF NOT found_match THEN tmp := regexp_match(ln, '.*Batches: (\d*) .* Memory Usage: (\d*)')kB; IF tmp IS NOT NULL THEN found_match := true; RETURN query SELECT tmp[1]::int, tmp[2]::int; END IF; END IF; END LOOP; END; $$; -- make sure we use hash aggregate, with a single batch SET max_parallel_workers_per_gather = 0; SET work_mem = '256MB'; SET enable_hashagg = on; SET enable_sort = off; -- tiny table with just 256 rows CREATE TABLE tdigest_memory AS SELECT i % 128 AS g, i::float8 AS v FROM generate_series(1, 256) AS s(i); ANALYZE tdigest_memory; -- check we're actually using hash aggregate EXPLAIN (COSTS OFF) SELECT g, tdigest_count(tdigest(v, 10000)) AS count FROM tdigest_memory GROUP BY g; QUERY PLAN ---------------------------------- HashAggregate Group Key: g -> Seq Scan on tdigest_memory (3 rows) -- Hash aggregation holds all 128 groups at once: single batch should be enough, -- and we compare the memory to 200kB, in case older releases or some platforms -- happen to use a bit more memory etc. SELECT (nbatches = 1) AS batches_ok, (memorykb < 200) AS memory_ok FROM check_memory_usage('SELECT g, tdigest_count(tdigest(v, 10000)) AS count FROM tdigest_memory GROUP BY g'); batches_ok | memory_ok ------------+----------- t | t (1 row) DROP TABLE tdigest_memory; DROP FUNCTION check_memory_usage; tvondra-tdigest-c0af713/test/expected/memory_usage_1.out000066400000000000000000000033451525543331000234710ustar00rootroot00000000000000-- check the number of batches and amount of memory used by the HashAgg node CREATE FUNCTION check_memory_usage(text) RETURNS TABLE (nbatches INT, memorykb INT) LANGUAGE plpgsql AS $$ DECLARE ln TEXT; tmp TEXT[]; found_match BOOL := false; BEGIN FOR LN IN EXECUTE format('EXPLAIN ANALYZE %s', $1) LOOP IF NOT found_match THEN tmp := regexp_match(ln, '.*Batches: (\d*) .* Memory Usage: (\d*)')kB; IF tmp IS NOT NULL THEN found_match := true; RETURN query SELECT tmp[1]::int, tmp[2]::int; END IF; END IF; END LOOP; END; $$; -- make sure we use hash aggregate, with a single batch SET max_parallel_workers_per_gather = 0; SET work_mem = '256MB'; SET enable_hashagg = on; SET enable_sort = off; -- tiny table with just 256 rows CREATE TABLE tdigest_memory AS SELECT i % 128 AS g, i::float8 AS v FROM generate_series(1, 256) AS s(i); ANALYZE tdigest_memory; -- check we're actually using hash aggregate EXPLAIN (COSTS OFF) SELECT g, tdigest_count(tdigest(v, 10000)) AS count FROM tdigest_memory GROUP BY g; QUERY PLAN ---------------------------------- HashAggregate Group Key: g -> Seq Scan on tdigest_memory (3 rows) -- Hash aggregation holds all 128 groups at once: single batch should be enough, -- and we compare the memory to 200kB, in case older releases or some platforms -- happen to use a bit more memory etc. SELECT (nbatches = 1) AS batches_ok, (memorykb < 200) AS memory_ok FROM check_memory_usage('SELECT g, tdigest_count(tdigest(v, 10000)) AS count FROM tdigest_memory GROUP BY g'); batches_ok | memory_ok ------------+----------- (0 rows) DROP TABLE tdigest_memory; DROP FUNCTION check_memory_usage; tvondra-tdigest-c0af713/test/expected/null_arrays.out000066400000000000000000000135051525543331000231070ustar00rootroot00000000000000-- Non-STRICT transition functions must reject NULL percentile or hypothetical -- value arguments when a non-NULL input creates the aggregate state. NULL -- input values themselves are skipped instead. \set VERBOSITY terse -- The requested percentile/value is only read while the aggregate state is -- still NULL, so the first non-NULL input is enough to exercise each check. -- tdigest_add_double(internal, double precision, int, double precision) SELECT tdigest_percentile(v, 100, NULL::double precision) FROM (VALUES (1.0::double precision), (2.0)) s(v); ERROR: percentile must not be NULL -- tdigest_add_double_array(internal, double precision, int, double precision[]) SELECT tdigest_percentile(v, 100, NULL::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); ERROR: percentiles must not be NULL -- tdigest_add_double_count(internal, double precision, bigint, int, double precision) SELECT tdigest_percentile(v, 2, 100, NULL::double precision) FROM (VALUES (1.0::double precision), (2.0)) s(v); ERROR: percentile must not be NULL -- tdigest_add_double_array_count(internal, double precision, bigint, int, double precision[]) SELECT tdigest_percentile(v, 2, 100, NULL::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); ERROR: percentiles must not be NULL -- tdigest_add_digest(internal, tdigest, double precision) SELECT tdigest_percentile(d, NULL::double precision) FROM (SELECT tdigest(v, 100) AS d FROM (VALUES (1.0::double precision), (2.0)) s(v)) t; ERROR: percentile must not be NULL -- tdigest_add_digest_array(internal, tdigest, double precision[]) SELECT tdigest_percentile(d, NULL::double precision[]) FROM (SELECT tdigest(v, 100) AS d FROM (VALUES (1.0::double precision), (2.0)) s(v)) t; ERROR: percentiles must not be NULL -- tdigest_add_double_values(internal, double precision, int, double precision) SELECT tdigest_percentile_of(v, 100, NULL::double precision) FROM (VALUES (1.0::double precision), (2.0)) s(v); ERROR: value must not be NULL -- tdigest_add_double_array_values(internal, double precision, int, double precision[]) SELECT tdigest_percentile_of(v, 100, NULL::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); ERROR: values must not be NULL -- tdigest_add_double_values_count(internal, double precision, bigint, int, double precision) SELECT tdigest_percentile_of(v, 2, 100, NULL::double precision) FROM (VALUES (1.0::double precision), (2.0)) s(v); ERROR: value must not be NULL -- tdigest_add_double_array_values_count(internal, double precision, bigint, int, double precision[]) SELECT tdigest_percentile_of(v, 2, 100, NULL::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); ERROR: values must not be NULL -- tdigest_add_digest_values(internal, tdigest, double precision) SELECT tdigest_percentile_of(d, NULL::double precision) FROM (SELECT tdigest(v, 100) AS d FROM (VALUES (1.0::double precision), (2.0)) s(v)) t; ERROR: value must not be NULL -- tdigest_add_digest_array_values(internal, tdigest, double precision[]) SELECT tdigest_percentile_of(d, NULL::double precision[]) FROM (SELECT tdigest(v, 100) AS d FROM (VALUES (1.0::double precision), (2.0)) s(v)) t; ERROR: values must not be NULL -- A NULL array on a later row is harmless (the state already exists), and -- tdigest_add_double_array_increment() guards the same call correctly. Both -- work before and after the fix. SELECT tdigest_percentile(v, 100, CASE WHEN v = 1.0 THEN ARRAY[0.5] END) FROM (VALUES (1.0::double precision), (2.0)) s(v); tdigest_percentile -------------------- {1.5} (1 row) SELECT tdigest_count(tdigest_add(NULL::tdigest, NULL::double precision[], 100)); tdigest_count --------------- (1 row) -- The arrays however should not be empty {}. -- tdigest_add_double_array(internal, double precision, int, double precision[]) SELECT tdigest_percentile(v, 100, '{}'::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); ERROR: the array must not be empty -- tdigest_add_double_array_count(internal, double precision, bigint, int, double precision[]) SELECT tdigest_percentile(v, 2, 100, '{}'::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); ERROR: the array must not be empty -- tdigest_add_digest_array(internal, tdigest, double precision[]) SELECT tdigest_percentile(d, '{}'::double precision[]) FROM (SELECT tdigest(v, 100) AS d FROM (VALUES (1.0::double precision), (2.0)) s(v)) t; ERROR: the array must not be empty -- tdigest_add_double_array_values(internal, double precision, int, double precision[]) SELECT tdigest_percentile_of(v, 100, '{}'::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); ERROR: the array must not be empty -- tdigest_add_double_array_values_count(internal, double precision, bigint, int, double precision[]) SELECT tdigest_percentile_of(v, 2, 100, '{}'::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); ERROR: the array must not be empty -- tdigest_add_digest_array_values(internal, tdigest, double precision[]) SELECT tdigest_percentile_of(d, '{}'::double precision[]) FROM (SELECT tdigest(v, 100) AS d FROM (VALUES (1.0::double precision), (2.0)) s(v)) t; ERROR: the array must not be empty -- a NULL element inside an array is rejected too, and the message has to name -- the argument the array was passed as, not always a percentile SELECT tdigest_percentile(v, 100, ARRAY[0.5, NULL]::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); ERROR: NULL not allowed as a percentile value SELECT tdigest_percentile_of(v, 100, ARRAY[1.0, NULL]::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); ERROR: NULL not allowed as a value SELECT tdigest_count(tdigest_add(NULL::tdigest, ARRAY[1.0, NULL]::double precision[], 100)); ERROR: NULL not allowed as an element tvondra-tdigest-c0af713/test/expected/null_count.out000066400000000000000000000044141525543331000227350ustar00rootroot00000000000000-- Regression test for a NULL "count" argument in the transition functions -- accepting an explicit number of occurrences of a value. -- -- The count argument is not required to be non-NULL, and a NULL count is -- interpreted as a single occurrence of the value - exactly like the -- variants without the count argument. -- tdigest_add_double_values_count SELECT tdigest_percentile_of(v, NULL::bigint, 100, 1.0) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_percentile_of ----------------------- 0.25 (1 row) SELECT tdigest_percentile_of(v, 1::bigint, 100, 1.0) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_percentile_of ----------------------- 0.25 (1 row) -- tdigest_add_double_array_count SELECT tdigest_percentile(v, NULL::bigint, 100, ARRAY[0.5]) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_percentile -------------------- {1.5} (1 row) SELECT tdigest_percentile(v, 1::bigint, 100, ARRAY[0.5]) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_percentile -------------------- {1.5} (1 row) -- tdigest_add_double_array_values_count SELECT tdigest_percentile_of(v, NULL::bigint, 100, ARRAY[1.0]) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_percentile_of ----------------------- {0.25} (1 row) SELECT tdigest_percentile_of(v, 1::bigint, 100, ARRAY[1.0]) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_percentile_of ----------------------- {0.25} (1 row) -- tdigest_add_double_count_trimmed SELECT tdigest_avg(v, NULL::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_avg ------------- 1.5 (1 row) SELECT tdigest_avg(v, 1::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_avg ------------- 1.5 (1 row) -- tdigest_add_digest_trimmed has no count argument - the counts come from -- the centroids of the digest - so the closest equivalent is a NULL digest, -- which contributes no values at all SELECT tdigest_avg(d, 0.0, 1.0) FROM (VALUES (NULL::tdigest), (NULL)) AS t(d); tdigest_avg ------------- (1 row) SELECT tdigest_avg(d, 0.0, 1.0) FROM (VALUES ('flags 1 count 2 compression 100 centroids 2 (1, 1) (2, 1)'::tdigest), (NULL), ('flags 1 count 2 compression 100 centroids 2 (3, 1) (4, 1)'::tdigest)) AS t(d); tdigest_avg ------------- 2.5 (1 row) tvondra-tdigest-c0af713/test/expected/null_increment.out000066400000000000000000000062651525543331000235770ustar00rootroot00000000000000-- Regression test for the NULL handling in the incremental API. -- -- None of the incremental functions is strict, so NULL arguments do reach -- them. Adding a NULL value (or merging a NULL digest) is a no-op, and -- returns the digest passed in - possibly NULL, if that was NULL too. -- tdigest_add_double_increment SELECT tdigest_add(NULL::tdigest, NULL::double precision); tdigest_add ------------- (1 row) SELECT tdigest_add('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest, NULL::double precision); tdigest_add ---------------------------------------------------- flags 1 count 1 compression 100 centroids 1 (1, 1) (1 row) SELECT tdigest_add(NULL::tdigest, 1.0::double precision, 100); tdigest_add ---------------------------------------------------- flags 1 count 1 compression 100 centroids 1 (1, 1) (1 row) -- tdigest_add_double_array_increment SELECT tdigest_add(NULL::tdigest, NULL::double precision[]); tdigest_add ------------- (1 row) SELECT tdigest_add('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest, NULL::double precision[]); tdigest_add ---------------------------------------------------- flags 1 count 1 compression 100 centroids 1 (1, 1) (1 row) SELECT tdigest_add(NULL::tdigest, ARRAY[1.0]::double precision[], 100); tdigest_add ---------------------------------------------------- flags 1 count 1 compression 100 centroids 1 (1, 1) (1 row) -- tdigest_union_double_increment SELECT tdigest_union(NULL::tdigest, NULL::tdigest); tdigest_union --------------- (1 row) SELECT tdigest_union('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest, NULL::tdigest); tdigest_union ---------------------------------------------------- flags 1 count 1 compression 100 centroids 1 (1, 1) (1 row) SELECT tdigest_union(NULL::tdigest, 'flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest); tdigest_union ---------------------------------------------------- flags 1 count 1 compression 100 centroids 1 (1, 1) (1 row) -- The compact flag is used on every code path that builds a result, so unlike -- the value/digest arguments it is not something that can be skipped. Passing -- NULL explicitly has to be rejected rather than silently taken as false. SELECT tdigest_add(NULL::tdigest, 1.0::double precision, 100, NULL); ERROR: compact flag must not be NULL SELECT tdigest_add('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest, 1.0::double precision, NULL, NULL); ERROR: compact flag must not be NULL SELECT tdigest_add(NULL::tdigest, ARRAY[1.0]::double precision[], 100, NULL); ERROR: compact flag must not be NULL SELECT tdigest_add('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest, ARRAY[1.0]::double precision[], NULL, NULL); ERROR: compact flag must not be NULL SELECT tdigest_union(NULL::tdigest, NULL::tdigest, NULL); ERROR: compact flag must not be NULL SELECT tdigest_union('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest, NULL::tdigest, NULL); ERROR: compact flag must not be NULL tvondra-tdigest-c0af713/test/expected/null_values.out000066400000000000000000000113511525543331000231020ustar00rootroot00000000000000-- Regression test for the NULL handling in the transition functions. -- -- None of the transition functions is strict, so NULL rows do reach them -- and have to be ignored. Each function is exercised twice: once with all -- input rows NULL (so the state stays NULL and the transition function -- returns NULL), and once with a NULL row following a non-NULL one (so the -- transition function returns the existing state unmodified). -- tdigest_add_double SELECT tdigest_percentile(v, 100, 0.5) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); tdigest_percentile -------------------- (1 row) SELECT tdigest_percentile(v, 100, 0.5) FROM (VALUES (1.0), (NULL)) AS t(v); tdigest_percentile -------------------- 1 (1 row) -- tdigest_add_double_count SELECT tdigest(v, 1::bigint, 100) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); tdigest --------- (1 row) SELECT tdigest(v, 1::bigint, 100) FROM (VALUES (1.0), (NULL)) AS t(v); tdigest ---------------------------------------------------- flags 1 count 1 compression 100 centroids 1 (1, 1) (1 row) -- tdigest_add_double_values SELECT tdigest_percentile_of(v, 100, 1.0) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); tdigest_percentile_of ----------------------- (1 row) SELECT tdigest_percentile_of(v, 100, 1.0) FROM (VALUES (1.0), (NULL)) AS t(v); tdigest_percentile_of ----------------------- 0.5 (1 row) -- tdigest_add_double_values_count SELECT tdigest_percentile_of(v, 1::bigint, 100, 1.0) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); tdigest_percentile_of ----------------------- (1 row) SELECT tdigest_percentile_of(v, 1::bigint, 100, 1.0) FROM (VALUES (1.0), (NULL)) AS t(v); tdigest_percentile_of ----------------------- 0.5 (1 row) -- tdigest_add_digest SELECT tdigest_percentile(d, 0.5) FROM (VALUES (NULL::tdigest), (NULL)) AS t(d); tdigest_percentile -------------------- (1 row) SELECT tdigest_percentile(d, 0.5) FROM (VALUES ('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest), (NULL)) AS t(d); tdigest_percentile -------------------- 1 (1 row) -- tdigest_add_digest_values SELECT tdigest_percentile_of(d, 1.0) FROM (VALUES (NULL::tdigest), (NULL)) AS t(d); tdigest_percentile_of ----------------------- (1 row) SELECT tdigest_percentile_of(d, 1.0) FROM (VALUES ('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest), (NULL)) AS t(d); tdigest_percentile_of ----------------------- 0.5 (1 row) -- tdigest_add_double_array SELECT tdigest_percentile(v, 100, ARRAY[0.5]) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); tdigest_percentile -------------------- (1 row) SELECT tdigest_percentile(v, 100, ARRAY[0.5]) FROM (VALUES (1.0), (NULL)) AS t(v); tdigest_percentile -------------------- {1} (1 row) -- tdigest_add_double_array_count SELECT tdigest_percentile(v, 1::bigint, 100, ARRAY[0.5]) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); tdigest_percentile -------------------- (1 row) SELECT tdigest_percentile(v, 1::bigint, 100, ARRAY[0.5]) FROM (VALUES (1.0), (NULL)) AS t(v); tdigest_percentile -------------------- {1} (1 row) -- tdigest_add_double_array_values SELECT tdigest_percentile_of(v, 100, ARRAY[1.0]) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); tdigest_percentile_of ----------------------- (1 row) SELECT tdigest_percentile_of(v, 100, ARRAY[1.0]) FROM (VALUES (1.0), (NULL)) AS t(v); tdigest_percentile_of ----------------------- {0.5} (1 row) -- tdigest_add_double_array_values_count SELECT tdigest_percentile_of(v, 1::bigint, 100, ARRAY[1.0]) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); tdigest_percentile_of ----------------------- (1 row) SELECT tdigest_percentile_of(v, 1::bigint, 100, ARRAY[1.0]) FROM (VALUES (1.0), (NULL)) AS t(v); tdigest_percentile_of ----------------------- {0.5} (1 row) -- tdigest_add_digest_array SELECT tdigest_percentile(d, ARRAY[0.5]) FROM (VALUES (NULL::tdigest), (NULL)) AS t(d); tdigest_percentile -------------------- (1 row) SELECT tdigest_percentile(d, ARRAY[0.5]) FROM (VALUES ('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest), (NULL)) AS t(d); tdigest_percentile -------------------- {1} (1 row) -- tdigest_add_digest_array_values SELECT tdigest_percentile_of(d, ARRAY[1.0]) FROM (VALUES (NULL::tdigest), (NULL)) AS t(d); tdigest_percentile_of ----------------------- (1 row) SELECT tdigest_percentile_of(d, ARRAY[1.0]) FROM (VALUES ('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest), (NULL)) AS t(d); tdigest_percentile_of ----------------------- {0.5} (1 row) tvondra-tdigest-c0af713/test/expected/parallel_query.out000066400000000000000000000402001525543331000235650ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('extra_float_digits', '0', false); PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; -- the user-facing functions must all be parallel safe, otherwise queries -- using them lose parallelism (the aggregate support functions take or -- return "internal" and cannot be called from SQL, so they are exempt) SELECT p.proname, p.proparallel FROM pg_proc p JOIN pg_depend d ON (d.objid = p.oid AND d.deptype = 'e') JOIN pg_extension e ON (e.oid = d.refobjid) WHERE e.extname = 'tdigest' AND p.proargtypes::oid[] && ARRAY['internal'::regtype]::oid[] IS NOT TRUE AND p.prorettype <> 'internal'::regtype AND p.proparallel <> 's' ORDER BY 1; proname | proparallel ---------+------------- (0 rows) -- test parallel query CREATE TABLE test_parallel (v double precision, c int, d int); INSERT INTO test_parallel SELECT 1000 * random(), 1 + mod(i,7), mod(i,113) FROM generate_series(1,100000) s(i); ANALYZE test_parallel; CREATE TABLE test_parallel_2 (d tdigest); INSERT INTO test_parallel_2 SELECT tdigest(v, 100) FROM test_parallel GROUP BY d; ANALYZE test_parallel_2; -- individual values EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, 100, 0.95) AS b FROM test_parallel) foo; QUERY PLAN ------------------------------------------------------------------------------ Subquery Scan on foo -> Finalize Aggregate InitPlan 1 (returns $1) -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel test_parallel_1 -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (11 rows) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, 100, 0.95) AS b FROM test_parallel) foo; ?column? | ?column? ----------+---------- 0.95 | t (1 row) EXPLAIN (COSTS OFF) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(v, 100, 950) AS b FROM test_parallel) foo; QUERY PLAN ------------------------------------------------------------ Subquery Scan on foo -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (6 rows) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(v, 100, 950) AS b FROM test_parallel) foo; ?column? | ?column? ----------+---------- 950 | t (1 row) EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(d, 0.95) AS b FROM test_parallel_2) foo; QUERY PLAN -------------------------------------------------------------- Subquery Scan on foo -> Finalize Aggregate InitPlan 1 (returns $1) -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (11 rows) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(d, 0.95) AS b FROM test_parallel_2) foo; ?column? | ?column? ----------+---------- 0.95 | t (1 row) EXPLAIN (COSTS OFF) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(d, 950) AS b FROM test_parallel_2) foo; QUERY PLAN -------------------------------------------------------------- Subquery Scan on foo -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (6 rows) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(d, 950) AS b FROM test_parallel_2) foo; ?column? | ?column? ----------+---------- 950 | t (1 row) -- array of percentiles / values EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel) foo; QUERY PLAN ------------------------------------------------------------------------------ Subquery Scan on foo -> ProjectSet InitPlan 1 (returns $1) -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel test_parallel_1 -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (12 rows) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel) foo; p | ?column? ------+---------- 0.0 | t 0.95 | t 0.99 | t 1.0 | t (4 rows) EXPLAIN (COSTS OFF) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(v, 100, ARRAY[950, 990])) AS b FROM test_parallel) foo; QUERY PLAN -------------------------------------------------------------------------------------- Subquery Scan on foo -> ProjectSet InitPlan 2 (returns $2) -> Aggregate -> Function Scan on unnest f SubPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel test_parallel_1 -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (15 rows) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(v, 100, ARRAY[950, 990])) AS b FROM test_parallel) foo; p | ?column? -----+---------- 950 | t 990 | t (2 rows) EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(d, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel_2) foo; QUERY PLAN -------------------------------------------------------------------- Subquery Scan on foo -> ProjectSet InitPlan 1 (returns $1) -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (12 rows) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(d, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel_2) foo; p | ?column? ------+---------- 0.0 | t 0.95 | t 0.99 | t 1.0 | t (4 rows) EXPLAIN (COSTS OFF) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(d, ARRAY[950, 990])) AS b FROM test_parallel_2) foo; QUERY PLAN ---------------------------------------------------------------------- Subquery Scan on foo -> ProjectSet InitPlan 2 (returns $2) -> Aggregate -> Function Scan on unnest f SubPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (15 rows) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(d, ARRAY[950, 990])) AS b FROM test_parallel_2) foo; p | ?column? -----+---------- 950 | t 990 | t (2 rows) -- trimmed aggregates EXPLAIN (COSTS OFF) SELECT tdigest_sum(v, 100, 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) -- trimming nothing has to reproduce the exact sum SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_sum(v, 100, 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_avg(v, 100, 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_avg(v, 100, 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_sum(d, 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_sum(d, 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_avg(d, 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_avg(d, 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) -- trimmed aggregates (on tdigest) EXPLAIN (COSTS OFF) SELECT tdigest_digest_sum(tdigest(v, 100), 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) -- trimming nothing has to reproduce the exact sum SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_digest_sum(tdigest(v, 100), 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_digest_avg(tdigest(v, 100), 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_digest_avg(tdigest(v, 100), 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_digest_sum(tdigest(d), 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_digest_sum(tdigest(d), 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_digest_avg(tdigest(d), 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_digest_avg(tdigest(d), 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) -- casting a digest to text must not force a serial plan EXPLAIN (COSTS OFF) SELECT count(*) FROM test_parallel_2 WHERE length(d::text) > 5; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 Filter: (length((d)::text) > 5) (6 rows) DROP TABLE test_parallel; DROP TABLE test_parallel_2; tvondra-tdigest-c0af713/test/expected/parallel_query_0.out000066400000000000000000000400621525543331000240120ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('extra_float_digits', '0', false); PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; -- the user-facing functions must all be parallel safe, otherwise queries -- using them lose parallelism (the aggregate support functions take or -- return "internal" and cannot be called from SQL, so they are exempt) SELECT p.proname, p.proparallel FROM pg_proc p JOIN pg_depend d ON (d.objid = p.oid AND d.deptype = 'e') JOIN pg_extension e ON (e.oid = d.refobjid) WHERE e.extname = 'tdigest' AND p.proargtypes::oid[] && ARRAY['internal'::regtype]::oid[] IS NOT TRUE AND p.prorettype <> 'internal'::regtype AND p.proparallel <> 's' ORDER BY 1; proname | proparallel ---------+------------- (0 rows) -- test parallel query CREATE TABLE test_parallel (v double precision, c int, d int); INSERT INTO test_parallel SELECT 1000 * random(), 1 + mod(i,7), mod(i,113) FROM generate_series(1,100000) s(i); ANALYZE test_parallel; CREATE TABLE test_parallel_2 (d tdigest); INSERT INTO test_parallel_2 SELECT tdigest(v, 100) FROM test_parallel GROUP BY d; ANALYZE test_parallel_2; -- individual values EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, 100, 0.95) AS b FROM test_parallel) foo; QUERY PLAN ------------------------------------------------------------------------------ Subquery Scan on foo -> Finalize Aggregate InitPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel test_parallel_1 -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (11 rows) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, 100, 0.95) AS b FROM test_parallel) foo; ?column? | ?column? ----------+---------- 0.95 | t (1 row) EXPLAIN (COSTS OFF) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(v, 100, 950) AS b FROM test_parallel) foo; QUERY PLAN ------------------------------------------------------------ Subquery Scan on foo -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (6 rows) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(v, 100, 950) AS b FROM test_parallel) foo; ?column? | ?column? ----------+---------- 950 | t (1 row) EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(d, 0.95) AS b FROM test_parallel_2) foo; QUERY PLAN -------------------------------------------------------------- Subquery Scan on foo -> Finalize Aggregate InitPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (11 rows) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(d, 0.95) AS b FROM test_parallel_2) foo; ?column? | ?column? ----------+---------- 0.95 | t (1 row) EXPLAIN (COSTS OFF) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(d, 950) AS b FROM test_parallel_2) foo; QUERY PLAN -------------------------------------------------------------- Subquery Scan on foo -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (6 rows) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(d, 950) AS b FROM test_parallel_2) foo; ?column? | ?column? ----------+---------- 950 | t (1 row) -- array of percentiles / values EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel) foo; QUERY PLAN ------------------------------------------------------------------------------ Subquery Scan on foo -> ProjectSet InitPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel test_parallel_1 -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (12 rows) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel) foo; p | ?column? ------+---------- 0.0 | t 0.95 | t 0.99 | t 1.0 | t (4 rows) EXPLAIN (COSTS OFF) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(v, 100, ARRAY[950, 990])) AS b FROM test_parallel) foo; QUERY PLAN -------------------------------------------------------------------------------------- Subquery Scan on foo -> ProjectSet InitPlan 2 -> Aggregate -> Function Scan on unnest f SubPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel test_parallel_1 -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (15 rows) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(v, 100, ARRAY[950, 990])) AS b FROM test_parallel) foo; p | ?column? -----+---------- 950 | t 990 | t (2 rows) EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(d, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel_2) foo; QUERY PLAN -------------------------------------------------------------------- Subquery Scan on foo -> ProjectSet InitPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (12 rows) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(d, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel_2) foo; p | ?column? ------+---------- 0.0 | t 0.95 | t 0.99 | t 1.0 | t (4 rows) EXPLAIN (COSTS OFF) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(d, ARRAY[950, 990])) AS b FROM test_parallel_2) foo; QUERY PLAN ---------------------------------------------------------------------- Subquery Scan on foo -> ProjectSet InitPlan 2 -> Aggregate -> Function Scan on unnest f SubPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (15 rows) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(d, ARRAY[950, 990])) AS b FROM test_parallel_2) foo; p | ?column? -----+---------- 950 | t 990 | t (2 rows) -- trimmed aggregates EXPLAIN (COSTS OFF) SELECT tdigest_sum(v, 100, 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) -- trimming nothing has to reproduce the exact sum SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_sum(v, 100, 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_avg(v, 100, 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_avg(v, 100, 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_sum(d, 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_sum(d, 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_avg(d, 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_avg(d, 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) -- trimmed aggregates (on tdigest) EXPLAIN (COSTS OFF) SELECT tdigest_digest_sum(tdigest(v, 100), 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) -- trimming nothing has to reproduce the exact sum SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_digest_sum(tdigest(v, 100), 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_digest_avg(tdigest(v, 100), 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_digest_avg(tdigest(v, 100), 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_digest_sum(tdigest(d), 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_digest_sum(tdigest(d), 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_digest_avg(tdigest(d), 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_digest_avg(tdigest(d), 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) -- casting a digest to text must not force a serial plan EXPLAIN (COSTS OFF) SELECT count(*) FROM test_parallel_2 WHERE length(d::text) > 5; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 Filter: (length((d)::text) > 5) (6 rows) DROP TABLE test_parallel; DROP TABLE test_parallel_2; tvondra-tdigest-c0af713/test/expected/parallel_query_1.out000066400000000000000000000402101525543331000240060ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('extra_float_digits', '0', false); PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; -- the user-facing functions must all be parallel safe, otherwise queries -- using them lose parallelism (the aggregate support functions take or -- return "internal" and cannot be called from SQL, so they are exempt) SELECT p.proname, p.proparallel FROM pg_proc p JOIN pg_depend d ON (d.objid = p.oid AND d.deptype = 'e') JOIN pg_extension e ON (e.oid = d.refobjid) WHERE e.extname = 'tdigest' AND p.proargtypes::oid[] && ARRAY['internal'::regtype]::oid[] IS NOT TRUE AND p.prorettype <> 'internal'::regtype AND p.proparallel <> 's' ORDER BY 1; proname | proparallel ---------+------------- (0 rows) -- test parallel query CREATE TABLE test_parallel (v double precision, c int, d int); INSERT INTO test_parallel SELECT 1000 * random(), 1 + mod(i,7), mod(i,113) FROM generate_series(1,100000) s(i); ANALYZE test_parallel; CREATE TABLE test_parallel_2 (d tdigest); INSERT INTO test_parallel_2 SELECT tdigest(v, 100) FROM test_parallel GROUP BY d; ANALYZE test_parallel_2; -- individual values EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, 100, 0.95) AS b FROM test_parallel) foo; QUERY PLAN ------------------------------------------------------------------------ Subquery Scan on foo CTE x -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel test_parallel_1 -> Finalize Aggregate InitPlan 2 (returns $2) -> CTE Scan on x -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (13 rows) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, 100, 0.95) AS b FROM test_parallel) foo; ?column? | ?column? ----------+---------- 0.95 | t (1 row) EXPLAIN (COSTS OFF) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(v, 100, 950) AS b FROM test_parallel) foo; QUERY PLAN ------------------------------------------------------------ Subquery Scan on foo -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (6 rows) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(v, 100, 950) AS b FROM test_parallel) foo; ?column? | ?column? ----------+---------- 950 | t (1 row) EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(d, 0.95) AS b FROM test_parallel_2) foo; QUERY PLAN -------------------------------------------------------------- Subquery Scan on foo CTE x -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel -> Finalize Aggregate InitPlan 2 (returns $2) -> CTE Scan on x -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (13 rows) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(d, 0.95) AS b FROM test_parallel_2) foo; ?column? | ?column? ----------+---------- 0.95 | t (1 row) EXPLAIN (COSTS OFF) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(d, 950) AS b FROM test_parallel_2) foo; QUERY PLAN -------------------------------------------------------------- Subquery Scan on foo -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (6 rows) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(d, 950) AS b FROM test_parallel_2) foo; ?column? | ?column? ----------+---------- 950 | t (1 row) -- array of percentiles / values EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel) foo; QUERY PLAN ------------------------------------------------------------------------ Subquery Scan on foo CTE x -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel test_parallel_1 -> ProjectSet InitPlan 2 (returns $2) -> CTE Scan on x -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (14 rows) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel) foo; p | ?column? ------+---------- 0.0 | t 0.95 | t 0.99 | t 1.0 | t (4 rows) EXPLAIN (COSTS OFF) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(v, 100, ARRAY[950, 990])) AS b FROM test_parallel) foo; QUERY PLAN -------------------------------------------------------------------------------- Subquery Scan on foo CTE x -> Aggregate -> Function Scan on unnest f SubPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel test_parallel_1 -> ProjectSet InitPlan 3 (returns $3) -> CTE Scan on x -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (17 rows) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(v, 100, ARRAY[950, 990])) AS b FROM test_parallel) foo; p | ?column? -----+---------- 950 | t 990 | t (2 rows) EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(d, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel_2) foo; QUERY PLAN -------------------------------------------------------------------- Subquery Scan on foo CTE x -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel -> ProjectSet InitPlan 2 (returns $2) -> CTE Scan on x -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (14 rows) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(d, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel_2) foo; p | ?column? ------+---------- 0.0 | t 0.95 | t 0.99 | t 1.0 | t (4 rows) EXPLAIN (COSTS OFF) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(d, ARRAY[950, 990])) AS b FROM test_parallel_2) foo; QUERY PLAN -------------------------------------------------------------------- Subquery Scan on foo CTE x -> Aggregate -> Function Scan on unnest f SubPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel -> ProjectSet InitPlan 3 (returns $3) -> CTE Scan on x -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (17 rows) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(d, ARRAY[950, 990])) AS b FROM test_parallel_2) foo; p | ?column? -----+---------- 950 | t 990 | t (2 rows) -- trimmed aggregates EXPLAIN (COSTS OFF) SELECT tdigest_sum(v, 100, 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) -- trimming nothing has to reproduce the exact sum SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_sum(v, 100, 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_avg(v, 100, 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_avg(v, 100, 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_sum(d, 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_sum(d, 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_avg(d, 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_avg(d, 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) -- trimmed aggregates (on tdigest) EXPLAIN (COSTS OFF) SELECT tdigest_digest_sum(tdigest(v, 100), 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) -- trimming nothing has to reproduce the exact sum SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_digest_sum(tdigest(v, 100), 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_digest_avg(tdigest(v, 100), 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_digest_avg(tdigest(v, 100), 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_digest_sum(tdigest(d), 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_digest_sum(tdigest(d), 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_digest_avg(tdigest(d), 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_digest_avg(tdigest(d), 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) -- casting a digest to text must not force a serial plan EXPLAIN (COSTS OFF) SELECT count(*) FROM test_parallel_2 WHERE length(d::text) > 5; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 Filter: (length((d)::text) > 5) (6 rows) DROP TABLE test_parallel; DROP TABLE test_parallel_2; tvondra-tdigest-c0af713/test/expected/parallel_query_2.out000066400000000000000000000374221525543331000240220ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('extra_float_digits', '0', false); PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; -- the user-facing functions must all be parallel safe, otherwise queries -- using them lose parallelism (the aggregate support functions take or -- return "internal" and cannot be called from SQL, so they are exempt) SELECT p.proname, p.proparallel FROM pg_proc p JOIN pg_depend d ON (d.objid = p.oid AND d.deptype = 'e') JOIN pg_extension e ON (e.oid = d.refobjid) WHERE e.extname = 'tdigest' AND p.proargtypes::oid[] && ARRAY['internal'::regtype]::oid[] IS NOT TRUE AND p.prorettype <> 'internal'::regtype AND p.proparallel <> 's' ORDER BY 1; proname | proparallel ---------+------------- (0 rows) -- test parallel query CREATE TABLE test_parallel (v double precision, c int, d int); INSERT INTO test_parallel SELECT 1000 * random(), 1 + mod(i,7), mod(i,113) FROM generate_series(1,100000) s(i); ANALYZE test_parallel; CREATE TABLE test_parallel_2 (d tdigest); INSERT INTO test_parallel_2 SELECT tdigest(v, 100) FROM test_parallel GROUP BY d; ANALYZE test_parallel_2; -- individual values EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, 100, 0.95) AS b FROM test_parallel) foo; QUERY PLAN ------------------------------------------------ Subquery Scan on foo CTE x -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel test_parallel_1 -> Aggregate InitPlan 2 (returns $2) -> CTE Scan on x -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel (12 rows) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, 100, 0.95) AS b FROM test_parallel) foo; ?column? | ?column? ----------+---------- 0.95 | t (1 row) EXPLAIN (COSTS OFF) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(v, 100, 950) AS b FROM test_parallel) foo; QUERY PLAN ------------------------------------------------ Subquery Scan on foo -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (6 rows) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(v, 100, 950) AS b FROM test_parallel) foo; ?column? | ?column? ----------+---------- 950 | t (1 row) EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(d, 0.95) AS b FROM test_parallel_2) foo; QUERY PLAN -------------------------------------------- Subquery Scan on foo CTE x -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel -> Aggregate InitPlan 2 (returns $2) -> CTE Scan on x -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel_2 (12 rows) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(d, 0.95) AS b FROM test_parallel_2) foo; ?column? | ?column? ----------+---------- 0.95 | t (1 row) EXPLAIN (COSTS OFF) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(d, 950) AS b FROM test_parallel_2) foo; QUERY PLAN ------------------------------------------------- Subquery Scan on foo -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (6 rows) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(d, 950) AS b FROM test_parallel_2) foo; ?column? | ?column? ----------+---------- 950 | t (1 row) -- array of percentiles / values EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel) foo; QUERY PLAN ------------------------------------------------------ Subquery Scan on foo CTE x -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel test_parallel_1 -> ProjectSet InitPlan 2 (returns $2) -> CTE Scan on x -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (14 rows) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel) foo; p | ?column? ------+---------- 0.0 | t 0.95 | t 0.99 | t 1.0 | t (4 rows) EXPLAIN (COSTS OFF) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(v, 100, ARRAY[950, 990])) AS b FROM test_parallel) foo; QUERY PLAN -------------------------------------------------------- Subquery Scan on foo CTE x -> Aggregate -> Function Scan on unnest f SubPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel test_parallel_1 -> ProjectSet InitPlan 3 (returns $3) -> CTE Scan on x -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (17 rows) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(v, 100, ARRAY[950, 990])) AS b FROM test_parallel) foo; p | ?column? -----+---------- 950 | t 990 | t (2 rows) EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(d, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel_2) foo; QUERY PLAN ------------------------------------------------------- Subquery Scan on foo CTE x -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel -> ProjectSet InitPlan 2 (returns $2) -> CTE Scan on x -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (14 rows) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(d, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel_2) foo; p | ?column? ------+---------- 0.0 | t 0.95 | t 0.99 | t 1.0 | t (4 rows) EXPLAIN (COSTS OFF) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(d, ARRAY[950, 990])) AS b FROM test_parallel_2) foo; QUERY PLAN ------------------------------------------------------- Subquery Scan on foo CTE x -> Aggregate -> Function Scan on unnest f SubPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel -> ProjectSet InitPlan 3 (returns $3) -> CTE Scan on x -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (17 rows) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(d, ARRAY[950, 990])) AS b FROM test_parallel_2) foo; p | ?column? -----+---------- 950 | t 990 | t (2 rows) -- trimmed aggregates EXPLAIN (COSTS OFF) SELECT tdigest_sum(v, 100, 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) -- trimming nothing has to reproduce the exact sum SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_sum(v, 100, 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_avg(v, 100, 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_avg(v, 100, 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_sum(d, 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_sum(d, 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_avg(d, 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_avg(d, 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) -- trimmed aggregates (on tdigest) EXPLAIN (COSTS OFF) SELECT tdigest_digest_sum(tdigest(v, 100), 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) -- trimming nothing has to reproduce the exact sum SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_digest_sum(tdigest(v, 100), 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_digest_avg(tdigest(v, 100), 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_digest_avg(tdigest(v, 100), 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_digest_sum(tdigest(d), 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_digest_sum(tdigest(d), 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_digest_avg(tdigest(d), 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_digest_avg(tdigest(d), 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) -- casting a digest to text must not force a serial plan EXPLAIN (COSTS OFF) SELECT count(*) FROM test_parallel_2 WHERE length(d::text) > 5; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 Filter: (length((d)::text) > 5) (6 rows) DROP TABLE test_parallel; DROP TABLE test_parallel_2; tvondra-tdigest-c0af713/test/expected/parallel_query_3.out000066400000000000000000000365331525543331000240250ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('extra_float_digits', '0', false); PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; -- the user-facing functions must all be parallel safe, otherwise queries -- using them lose parallelism (the aggregate support functions take or -- return "internal" and cannot be called from SQL, so they are exempt) SELECT p.proname, p.proparallel FROM pg_proc p JOIN pg_depend d ON (d.objid = p.oid AND d.deptype = 'e') JOIN pg_extension e ON (e.oid = d.refobjid) WHERE e.extname = 'tdigest' AND p.proargtypes::oid[] && ARRAY['internal'::regtype]::oid[] IS NOT TRUE AND p.prorettype <> 'internal'::regtype AND p.proparallel <> 's' ORDER BY 1; proname | proparallel ---------+------------- (0 rows) -- test parallel query CREATE TABLE test_parallel (v double precision, c int, d int); INSERT INTO test_parallel SELECT 1000 * random(), 1 + mod(i,7), mod(i,113) FROM generate_series(1,100000) s(i); ANALYZE test_parallel; CREATE TABLE test_parallel_2 (d tdigest); INSERT INTO test_parallel_2 SELECT tdigest(v, 100) FROM test_parallel GROUP BY d; ANALYZE test_parallel_2; -- individual values EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, 100, 0.95) AS b FROM test_parallel) foo; QUERY PLAN ------------------------------------------------ Subquery Scan on foo CTE x -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel test_parallel_1 -> Aggregate InitPlan 2 (returns $1) -> CTE Scan on x -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel (12 rows) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, 100, 0.95) AS b FROM test_parallel) foo; ?column? | ?column? ----------+---------- 0.95 | t (1 row) EXPLAIN (COSTS OFF) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(v, 100, 950) AS b FROM test_parallel) foo; QUERY PLAN ------------------------------------------------ Subquery Scan on foo -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (6 rows) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(v, 100, 950) AS b FROM test_parallel) foo; ?column? | ?column? ----------+---------- 950 | t (1 row) EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(d, 0.95) AS b FROM test_parallel_2) foo; QUERY PLAN -------------------------------------------- Subquery Scan on foo CTE x -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel -> Aggregate InitPlan 2 (returns $1) -> CTE Scan on x -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel_2 (12 rows) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(d, 0.95) AS b FROM test_parallel_2) foo; ?column? | ?column? ----------+---------- 0.95 | t (1 row) EXPLAIN (COSTS OFF) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(d, 950) AS b FROM test_parallel_2) foo; QUERY PLAN ------------------------------------------------- Subquery Scan on foo -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (6 rows) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(d, 950) AS b FROM test_parallel_2) foo; ?column? | ?column? ----------+---------- 950 | t (1 row) -- array of percentiles / values EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel) foo; QUERY PLAN ------------------------------------------------ Subquery Scan on foo CTE x -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel test_parallel_1 -> Aggregate InitPlan 2 (returns $1) -> CTE Scan on x -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel (12 rows) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel) foo; p | ?column? ------+---------- 0.0 | t 0.95 | t 0.99 | t 1.0 | t (4 rows) EXPLAIN (COSTS OFF) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(v, 100, ARRAY[950, 990])) AS b FROM test_parallel) foo; QUERY PLAN -------------------------------------------------------- Subquery Scan on foo CTE x -> Aggregate -> Function Scan on unnest f SubPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel test_parallel_1 -> Aggregate InitPlan 3 (returns $2) -> CTE Scan on x -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel (15 rows) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(v, 100, ARRAY[950, 990])) AS b FROM test_parallel) foo; p | ?column? -----+---------- 950 | t 990 | t (2 rows) EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(d, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel_2) foo; QUERY PLAN -------------------------------------------- Subquery Scan on foo CTE x -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel -> Aggregate InitPlan 2 (returns $1) -> CTE Scan on x -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel_2 (12 rows) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(d, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel_2) foo; p | ?column? ------+---------- 0.0 | t 0.95 | t 0.99 | t 1.0 | t (4 rows) EXPLAIN (COSTS OFF) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(d, ARRAY[950, 990])) AS b FROM test_parallel_2) foo; QUERY PLAN ---------------------------------------------------- Subquery Scan on foo CTE x -> Aggregate -> Function Scan on unnest f SubPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel -> Aggregate InitPlan 3 (returns $2) -> CTE Scan on x -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel_2 (15 rows) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(d, ARRAY[950, 990])) AS b FROM test_parallel_2) foo; p | ?column? -----+---------- 950 | t 990 | t (2 rows) -- trimmed aggregates EXPLAIN (COSTS OFF) SELECT tdigest_sum(v, 100, 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) -- trimming nothing has to reproduce the exact sum SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_sum(v, 100, 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_avg(v, 100, 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_avg(v, 100, 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_sum(d, 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_sum(d, 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_avg(d, 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_avg(d, 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) -- trimmed aggregates (on tdigest) EXPLAIN (COSTS OFF) SELECT tdigest_digest_sum(tdigest(v, 100), 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) -- trimming nothing has to reproduce the exact sum SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_digest_sum(tdigest(v, 100), 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_digest_avg(tdigest(v, 100), 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_digest_avg(tdigest(v, 100), 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_digest_sum(tdigest(d), 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_digest_sum(tdigest(d), 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_digest_avg(tdigest(d), 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_digest_avg(tdigest(d), 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) -- casting a digest to text must not force a serial plan EXPLAIN (COSTS OFF) SELECT count(*) FROM test_parallel_2 WHERE length(d::text) > 5; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 Filter: (length((d)::text) > 5) (6 rows) DROP TABLE test_parallel; DROP TABLE test_parallel_2; tvondra-tdigest-c0af713/test/expected/parallel_query_4.out000066400000000000000000000401321525543331000240140ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('extra_float_digits', '0', false); PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; -- the user-facing functions must all be parallel safe, otherwise queries -- using them lose parallelism (the aggregate support functions take or -- return "internal" and cannot be called from SQL, so they are exempt) SELECT p.proname, p.proparallel FROM pg_proc p JOIN pg_depend d ON (d.objid = p.oid AND d.deptype = 'e') JOIN pg_extension e ON (e.oid = d.refobjid) WHERE e.extname = 'tdigest' AND p.proargtypes::oid[] && ARRAY['internal'::regtype]::oid[] IS NOT TRUE AND p.prorettype <> 'internal'::regtype AND p.proparallel <> 's' ORDER BY 1; proname | proparallel ---------+------------- (0 rows) -- test parallel query CREATE TABLE test_parallel (v double precision, c int, d int); INSERT INTO test_parallel SELECT 1000 * random(), 1 + mod(i,7), mod(i,113) FROM generate_series(1,100000) s(i); ANALYZE test_parallel; CREATE TABLE test_parallel_2 (d tdigest); INSERT INTO test_parallel_2 SELECT tdigest(v, 100) FROM test_parallel GROUP BY d; ANALYZE test_parallel_2; -- individual values EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, 100, 0.95) AS b FROM test_parallel) foo; QUERY PLAN ------------------------------------------------------------------------------ Subquery Scan on foo -> Finalize Aggregate InitPlan expr_1 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel test_parallel_1 -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (11 rows) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, 100, 0.95) AS b FROM test_parallel) foo; ?column? | ?column? ----------+---------- 0.95 | t (1 row) EXPLAIN (COSTS OFF) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(v, 100, 950) AS b FROM test_parallel) foo; QUERY PLAN ------------------------------------------------------------ Subquery Scan on foo -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (6 rows) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(v, 100, 950) AS b FROM test_parallel) foo; ?column? | ?column? ----------+---------- 950 | t (1 row) EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(d, 0.95) AS b FROM test_parallel_2) foo; QUERY PLAN -------------------------------------------------------------- Subquery Scan on foo -> Finalize Aggregate InitPlan expr_1 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (11 rows) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(d, 0.95) AS b FROM test_parallel_2) foo; ?column? | ?column? ----------+---------- 0.95 | t (1 row) EXPLAIN (COSTS OFF) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(d, 950) AS b FROM test_parallel_2) foo; QUERY PLAN -------------------------------------------------------------- Subquery Scan on foo -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (6 rows) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(d, 950) AS b FROM test_parallel_2) foo; ?column? | ?column? ----------+---------- 950 | t (1 row) -- array of percentiles / values EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel) foo; QUERY PLAN ------------------------------------------------------------------------------ Subquery Scan on foo -> ProjectSet InitPlan expr_1 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel test_parallel_1 -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (12 rows) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel) foo; p | ?column? ------+---------- 0.0 | t 0.95 | t 0.99 | t 1.0 | t (4 rows) EXPLAIN (COSTS OFF) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(v, 100, ARRAY[950, 990])) AS b FROM test_parallel) foo; QUERY PLAN -------------------------------------------------------------------------------------- Subquery Scan on foo -> ProjectSet InitPlan expr_1 -> Aggregate -> Function Scan on unnest f SubPlan expr_2 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel test_parallel_1 -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (15 rows) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(v, 100, ARRAY[950, 990])) AS b FROM test_parallel) foo; p | ?column? -----+---------- 950 | t 990 | t (2 rows) EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(d, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel_2) foo; QUERY PLAN -------------------------------------------------------------------- Subquery Scan on foo -> ProjectSet InitPlan expr_1 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (12 rows) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(d, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel_2) foo; p | ?column? ------+---------- 0.0 | t 0.95 | t 0.99 | t 1.0 | t (4 rows) EXPLAIN (COSTS OFF) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(d, ARRAY[950, 990])) AS b FROM test_parallel_2) foo; QUERY PLAN ---------------------------------------------------------------------- Subquery Scan on foo -> ProjectSet InitPlan expr_1 -> Aggregate -> Function Scan on unnest f SubPlan expr_2 -> Aggregate -> Gather Workers Planned: 2 -> Parallel Seq Scan on test_parallel -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (15 rows) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(d, ARRAY[950, 990])) AS b FROM test_parallel_2) foo; p | ?column? -----+---------- 950 | t 990 | t (2 rows) -- trimmed aggregates EXPLAIN (COSTS OFF) SELECT tdigest_sum(v, 100, 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) -- trimming nothing has to reproduce the exact sum SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_sum(v, 100, 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_avg(v, 100, 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_avg(v, 100, 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_sum(d, 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_sum(d, 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_avg(d, 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_avg(d, 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) -- trimmed aggregates (on tdigest) EXPLAIN (COSTS OFF) SELECT tdigest_digest_sum(tdigest(v, 100), 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) -- trimming nothing has to reproduce the exact sum SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_digest_sum(tdigest(v, 100), 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_digest_avg(tdigest(v, 100), 0.05, 0.95) FROM test_parallel; QUERY PLAN ------------------------------------------------------ Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_digest_avg(tdigest(v, 100), 0.0, 1.0) AS b FROM test_parallel) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_digest_sum(tdigest(d), 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_digest_sum(tdigest(d), 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) EXPLAIN (COSTS OFF) SELECT tdigest_digest_avg(tdigest(d), 0.05, 0.95) FROM test_parallel_2; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 (5 rows) SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_digest_avg(tdigest(d), 0.0, 1.0) AS b FROM test_parallel_2) foo; ?column? ---------- t (1 row) -- casting a digest to text must not force a serial plan EXPLAIN (COSTS OFF) SELECT count(*) FROM test_parallel_2 WHERE length(d::text) > 5; QUERY PLAN -------------------------------------------------------- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_parallel_2 Filter: (length((d)::text) > 5) (6 rows) DROP TABLE test_parallel; DROP TABLE test_parallel_2; tvondra-tdigest-c0af713/test/expected/parse_checks.out000066400000000000000000000140311525543331000232010ustar00rootroot00000000000000-- Regression test for the input function tdigest_in(), which parses the -- textual representation of a t-digest. -- -- The input is user supplied, so it has to be validated carefully - both -- the syntax of the value and the sanity of the parsed data. The queries -- exercise all the error checks in the input function, one by one. \set VERBOSITY terse -- a valid value, to make sure the checks don't reject correct input SELECT 'flags 1 count 3 compression 10 centroids 2 (1, 1) (2, 2)'::tdigest; tdigest ---------------------------------------------------------- flags 1 count 3 compression 10 centroids 2 (1, 1) (2, 2) (1 row) -- -- syntax errors detected while parsing the individual fields -- -- unexpected space (the value must not start with whitespace) SELECT ' flags 1 count 3 compression 10 centroids 1 (1, 3)'::tdigest; ERROR: failed to parse t-digest value, unexpected space at character 8 -- unexpected space (only a single space may separate the fields) SELECT 'flags 1 count 3 compression 10 centroids 1 (1, 3)'::tdigest; ERROR: failed to parse t-digest value, unexpected space at character 8 -- missing space (the fields have to be separated) SELECT 'flags 1count 3 compression 10 centroids 1 (1, 3)'::tdigest; ERROR: failed to parse t-digest value, missing space at character 8 -- unexpected keyword SELECT 'flgs 1 count 3 compression 10 centroids 1 (1, 3)'::tdigest; ERROR: failed to parse t-digest value, expected "flags" at character 8 -- integer field that is not a number at all SELECT 'flags x count 3 compression 10 centroids 1 (1, 3)'::tdigest; ERROR: failed to parse flags of a t-digest at character 8 -- integer field that does not fit into bigint SELECT 'flags 1 count 99999999999999999999 compression 10 centroids 1 (1, 3)'::tdigest; ERROR: count of a t-digest is out of range for bigint at character 8 -- integer field that fits into bigint but not into integer SELECT 'flags 1 count 3 compression 3000000000 centroids 1 (1, 3)'::tdigest; ERROR: compression of a t-digest is out of range for integer at character 8 -- floating point field that is not a number at all SELECT 'flags 1 count 3 compression 10 centroids 1 (x, 3)'::tdigest; ERROR: failed to parse mean of a centroid of a t-digest at character 8 -- floating point field that does not fit into double precision SELECT 'flags 1 count 3 compression 10 centroids 1 (1e400, 3)'::tdigest; ERROR: mean of a centroid of a t-digest is out of range for double precision at character 8 -- complete underflow must not silently turn a nonzero input into zero SELECT 'flags 1 count 3 compression 10 centroids 1 (1e-400, 3)'::tdigest; ERROR: mean of a centroid of a t-digest is out of range for double precision at character 8 SELECT 'flags 1 count 3 compression 10 centroids 1 (-1e-400, 3)'::tdigest; ERROR: mean of a centroid of a t-digest is out of range for double precision at character 8 SELECT 'flags 0 count 3 compression 10 centroids 1 (1e-400, 3)'::tdigest; ERROR: mean of a centroid of a t-digest is out of range for double precision at character 8 SELECT 'flags 0 count 3 compression 10 centroids 1 (-1e-400, 3)'::tdigest; ERROR: mean of a centroid of a t-digest is out of range for double precision at character 8 -- -- sanity checks on the parsed header -- -- unknown flags SELECT 'flags 2 count 3 compression 10 centroids 1 (1, 3)'::tdigest; ERROR: invalid flags for t-digest at character 8 -- compression out of the supported range SELECT 'flags 1 count 3 compression 9 centroids 1 (1, 3)'::tdigest; ERROR: compression for t-digest must be in [10, 10000] at character 8 SELECT 'flags 1 count 3 compression 10001 centroids 1 (1, 3)'::tdigest; ERROR: compression for t-digest must be in [10, 10000] at character 8 -- non-positive total count SELECT 'flags 1 count 0 compression 10 centroids 1 (1, 1)'::tdigest; ERROR: count value for the t-digest must be positive at character 8 -- non-positive number of centroids SELECT 'flags 1 count 3 compression 10 centroids 0'::tdigest; ERROR: number of centroids for the t-digest must be positive at character 8 -- more centroids than fit into the buffer (10 * compression) SELECT 'flags 1 count 3 compression 10 centroids 101'::tdigest; ERROR: number of centroids for the t-digest exceeds buffer size at character 8 -- -- sanity checks on the parsed centroids -- -- centroid mean that is not a valid number SELECT 'flags 1 count 3 compression 10 centroids 1 (NaN, 3)'::tdigest; ERROR: mean value for all centroids in a t-digest must be valid at character 8 SELECT 'flags 1 count 3 compression 10 centroids 1 (Infinity, 3)'::tdigest; ERROR: mean value for all centroids in a t-digest must be valid at character 8 -- non-positive centroid count SELECT 'flags 1 count 3 compression 10 centroids 1 (1, 0)'::tdigest; ERROR: count value for all centroids in a t-digest must be positive at character 8 -- centroid count exceeding the total count SELECT 'flags 1 count 3 compression 10 centroids 1 (1, 4)'::tdigest; ERROR: count value of a centroid exceeds total count at character 8 -- centroids not sorted by mean SELECT 'flags 1 count 3 compression 10 centroids 2 (2, 1) (1, 2)'::tdigest; tdigest ---------------------------------------------------------- flags 1 count 3 compression 10 centroids 2 (2, 1) (1, 2) (1 row) -- centroid counts overflowing the total count SELECT 'flags 1 count 9223372036854775807 compression 10 centroids 2 (1, 9223372036854775807) (2, 9223372036854775807)'::tdigest; ERROR: tdigest count overflow at character 8 -- -- sanity checks on the value as a whole -- -- more centroids than the header says SELECT 'flags 1 count 1 compression 10 centroids 1 (1, 1) (2, 1)'::tdigest; ERROR: input t-digest value too long at character 8 -- fewer centroids than the header says SELECT 'flags 1 count 3 compression 10 centroids 3 (1, 1)'::tdigest; ERROR: input t-digest value too short at character 8 -- centroid counts not adding up to the total count SELECT 'flags 1 count 3 compression 10 centroids 1 (1, 1)'::tdigest; ERROR: total count does not match the data (1 != 3) at character 8 tvondra-tdigest-c0af713/test/expected/percentile_of_extreme.out000066400000000000000000000154501525543331000251240ustar00rootroot00000000000000-- tdigest_percentile_of() used to interpolate between two centroids by -- computing the slope of the line connecting their means: -- -- m = (curr->mean - prev->mean) / (curr->count / 2.0 + prev->count / 2.0); -- x = (value - prev->mean) / m; -- -- Both the difference of the means and the slope could leave the float8 -- range even though every input was an ordinary finite number: -- -- * (curr->mean - prev->mean) overflows to infinity when the means are -- large and have opposite signs. The slope is then infinity, and x is -- either 0 (silently wrong) or NaN. -- -- * the slope underflows to zero when the means are tiny (subnormal) and -- the counts are huge. x is then infinity, and so is the result. -- -- Both quantile functions now use convex combinations. The inverse function -- computes the relative position between means without dividing by a slope, -- and halves the means and probe when their difference would overflow. -- -- For finite probes the result must be a probability in [0, 1]. Power-of-two -- rescaling without input precision loss should agree within rounding error. \set VERBOSITY terse SET extra_float_digits = 0; CREATE TABLE tdigest_extreme_digests (id int, descr text, d tdigest, probes double precision[]); INSERT INTO tdigest_extreme_digests VALUES (1, 'means of opposite sign, the difference overflows', 'flags 1 count 2 compression 10000 centroids 2 (-1.7e308, 1) (1.7e308, 1)', ARRAY[-1.7e308, -1e308, -1e307, 0, 1e307, 1e308, 1.7e308]), (2, 'the same, with uneven counts', 'flags 1 count 10 compression 10000 centroids 2 (-1.5e308, 7) (1.5e308, 3)', ARRAY[-1.5e308, -1e308, 0, 1e308, 1.5e308]), (3, 'the same, with a centroid in the middle', 'flags 1 count 3 compression 10000 centroids 3 (-1.7e308, 1) (0, 1) (1.7e308, 1)', ARRAY[-1.7e308, -1e308, -1e-300, 0, 1e-300, 1e308, 1.7e308]), (4, 'subnormal means and huge counts, the slope underflows', 'flags 1 count 9223372036854775806 compression 10000 centroids 2 (1e-320, 4611686018427387903) (2e-320, 4611686018427387903)', ARRAY[1e-320, 1.2e-320, 1.5e-320, 1.8e-320, 2e-320]), (5, 'almost adjacent means and huge counts', 'flags 1 count 9223372036854775806 compression 10000 centroids 2 (1, 4611686018427387903) (1.0000000000000004, 4611686018427387903)', ARRAY[1, 1.0000000000000002, 1.0000000000000004]), (6, 'large means and huge counts', 'flags 1 count 4611686018427387903 compression 10000 centroids 2 (-1e308, 2305843009213693951) (1e308, 2305843009213693952)', ARRAY[-1e308, -5e307, 0, 5e307, 1e308]), (7, 'ordinary means, for comparison', 'flags 1 count 10 compression 10000 centroids 2 (-1.5, 7) (1.5, 3)', ARRAY[-1.5, -1, 0, 1, 1.5]); -- Every result has to be a probability. The old implementation returned NaN -- when the difference overflowed or Infinity when the slope underflowed. WITH q AS ( SELECT id, descr, probes, tdigest_percentile_of(d, probes) AS res FROM tdigest_extreme_digests GROUP BY id, descr, probes ) SELECT id, descr, v AS value, r AS percentile_of FROM q, unnest(q.probes, q.res) WITH ORDINALITY AS x(v, r, ord) WHERE NOT (r BETWEEN 0 AND 1) ORDER BY id, ord; id | descr | value | percentile_of ----+-------+-------+--------------- (0 rows) -- And the result still has to be a non-decreasing function of the value. WITH q AS ( SELECT id, descr, probes, tdigest_percentile_of(d, probes) AS res FROM tdigest_extreme_digests GROUP BY id, descr, probes ), u AS ( SELECT id, descr, ord, v, r FROM q, unnest(q.probes, q.res) WITH ORDINALITY AS x(v, r, ord) ), s AS ( SELECT id, descr, v, r, lag(v) OVER w AS prev_v, lag(r) OVER w AS prev_r FROM u WINDOW w AS (PARTITION BY id ORDER BY ord) ) SELECT id, descr, count(*) AS decreasing_steps FROM s WHERE r < prev_r GROUP BY id, descr ORDER BY id; id | descr | decreasing_steps ----+-------+------------------ (0 rows) -- This is reachable without writing the digest by hand - two ordinary finite -- values are enough to build a digest whose centroid means are far enough -- apart. WITH q AS ( SELECT tdigest_percentile_of(v, 10000, ARRAY[-1e307, 0.0, 1e307]::double precision[]) AS res FROM (VALUES (-1.7e308::double precision), (1.7e308::double precision)) t(v) ) SELECT bool_and(r BETWEEN 0 AND 1) AS all_probabilities FROM q, unnest(q.res) AS r; all_probabilities ------------------- t (1 row) WITH q AS ( SELECT tdigest_percentile_of(v, c, 10000, ARRAY[-1e307, 0.0, 1e307]::double precision[]) AS res FROM (VALUES (-1.7e308::double precision, 7::bigint), (1.7e308::double precision, 3::bigint)) t(v, c) ) SELECT bool_and(r BETWEEN 0 AND 1) AS all_probabilities FROM q, unnest(q.res) AS r; all_probabilities ------------------- t (1 row) -- The percentile only depends on where the value sits between the two -- centroid means, so multiplying the means and the probed values by the same -- power of two must not change the result in any way. 2^1022 pushes the -- difference of the means (4) just past the float8 range, while every single -- mean still fits comfortably. The comparison allows for a rounding error of -- a couple of ULPs, the difference we are looking for here is 0.0625. WITH s(k) AS ( VALUES (1::double precision), (2::double precision ^ 1022) ), r AS ( SELECT s.k, tdigest_percentile_of(t.v * s.k, t.c, 10000, ARRAY[-3*s.k, -2*s.k, -1*s.k, 0*s.k, 1*s.k]) AS q FROM s, (VALUES (-3.0::double precision, 3::bigint), (1.0::double precision, 1::bigint)) t(v, c) GROUP BY s.k ), z AS ( SELECT a.q AS unscaled, b.q AS scaled FROM r a, r b WHERE a.k = 1 AND b.k <> 1 ) SELECT bool_and(abs(u - s) <= 1e-15) AS scale_invariant FROM z, unnest(z.unscaled, z.scaled) AS x(u, s); scale_invariant ----------------- t (1 row) DROP TABLE tdigest_extreme_digests; -- The calculated percentile has to be between the means, and not outside -- due to rounding, etc. All the centroids have mean 123.456, so all quantiles -- can be just 123.456 too (without the clamping we'd get 123.45600000000002). SELECT tdigest_percentile('flags 1 count 1152921504606846978 compression 100 centroids 3 (123.456, 1) (123.456, 1152921504606846976) (123.456, 1)'::tdigest, 0.1); tdigest_percentile -------------------- 123.456 (1 row) -- Similar issue in the other direction - rounding errors can yield values -- outside the valid [0.0, 1.0] range close to the 1.0 boundary, and need to -- be correctly clamped. For example, this might return 1.0000000000000002. SELECT tdigest_percentile_of('flags 1 count 6381118572735248714 compression 10 centroids 3 (0.0, 5752940093788208662) (1.0, 628178478947040051) (2.0, 1)'::tdigest, 2.0); tdigest_percentile_of ----------------------- 1 (1 row) tvondra-tdigest-c0af713/test/expected/percentile_of_monotonic.out000066400000000000000000000172511525543331000254610ustar00rootroot00000000000000-- tdigest_percentile_of() evaluates the CDF of the digest, so for a given -- digest the result has to be a non-decreasing function of the value, and it -- has to fall into the [0, 1] range for finite probes. This guards a former -- integer-division bug that violated both properties. -- -- tdigest_compute_quantiles_of() interpolates between two centroids, and to -- do that it first moves from the boundary of the previous centroid to its -- mean, which sits in the middle of the centroid. It used to do this with: -- -- count -= (prev->count / 2); -- -- That was an int64 division, even though "count" is a double and the very -- next statement halves the same value as (prev->count / 2.0). For centroids -- with an odd count the two halves therefore disagreed by 0.5 items, and the -- interpolated CDF ended up shifted by 0.5/count with respect to the exact -- value calculated in the (value == curr->mean) branch just above. -- -- The result was a saw-tooth: the CDF jumped up by 0.5/count right after the -- mean of every odd-sized centroid, and dropped back at the next centroid. For -- the (0, 7) (100, 3) digest used below: -- -- value before expected -- 0 0.35 0.35 -- 0.0001 0.4000005 0.3500005 -- 50 0.65 0.6 -- 99.9999 0.8999995 0.8499995 -- 100 0.85 0.85 -- -- i.e. the CDF at 99.9999 was higher than the CDF at 100. Both divisions -- now use floating point, preserving monotonicity. \set VERBOSITY terse SET extra_float_digits = 0; CREATE TABLE tdigest_monotonic_digests (id int, descr text, d tdigest); -- Hand-built digests. All of them use centroid means from the [0, 100] range, -- so that a single set of probes works for all of them. What matters are the -- centroid counts - the old discontinuity showed up at the mean of every centroid -- with an odd count. INSERT INTO tdigest_monotonic_digests VALUES (1, 'two centroids, odd + odd', 'flags 1 count 10 compression 10000 centroids 2 (0, 7) (100, 3)'), (2, 'two centroids, odd + even', 'flags 1 count 11 compression 10000 centroids 2 (0, 7) (100, 4)'), (3, 'two centroids, even + odd', 'flags 1 count 11 compression 10000 centroids 2 (0, 8) (100, 3)'), (4, 'two centroids, even + even', 'flags 1 count 4 compression 10000 centroids 2 (0, 2) (100, 2)'), (5, 'two centroids, single item each', 'flags 1 count 2 compression 10000 centroids 2 (0, 1) (100, 1)'), (6, 'three centroids, single item each', 'flags 1 count 3 compression 10000 centroids 3 (0, 1) (50, 1) (100, 1)'), (7, 'three centroids, mixed counts', 'flags 1 count 12 compression 10000 centroids 3 (0, 3) (50, 4) (100, 5)'), (8, 'a single centroid', 'flags 1 count 10 compression 10000 centroids 1 (50, 10)'), (9, 'repeated means', 'flags 1 count 6 compression 10000 centroids 4 (0, 1) (50, 2) (50, 2) (100, 1)'); -- And a couple of digests built by the aggregate from actual data. The first -- one retains singleton centroids after final compaction, while the other two -- merge input values. INSERT INTO tdigest_monotonic_digests SELECT 10, 'built from 51 distinct values, no centroid merging', tdigest((2 * i)::double precision, 10000) FROM generate_series(0, 50) s(i); INSERT INTO tdigest_monotonic_digests SELECT 11, 'built from 10000 rows, compression 10', tdigest((i % 101)::double precision, 10) FROM generate_series(1, 10000) s(i); INSERT INTO tdigest_monotonic_digests SELECT 12, 'built from 10000 rows, compression 100', tdigest((i % 101)::double precision, 100) FROM generate_series(1, 10000) s(i); -- The values to probe the digests with: a coarse grid reaching outside the -- range of the digests, and then the centroid means used above with values -- immediately below and above them. The interpolation switches from one pair -- of centroids to the next exactly at the means, so that is where it is -- discontinuous. CREATE TABLE tdigest_monotonic_probes (v double precision); INSERT INTO tdigest_monotonic_probes SELECT DISTINCT v FROM ( SELECT i / 2.0::double precision FROM generate_series(-20, 220) s(i) UNION ALL SELECT m + o FROM generate_series(0, 100, 2) s(m), unnest(ARRAY[-1e-6, -1e-9, 0, 1e-9, 1e-6]::double precision[]) o ) s(v); -- The CDF must never decrease. Report one row per digest that breaks it, so -- that a failure stays readable. WITH q AS ( SELECT g.id, g.descr, p.arr, tdigest_percentile_of(g.d, p.arr) AS res FROM tdigest_monotonic_digests g, (SELECT array_agg(v ORDER BY v) AS arr FROM tdigest_monotonic_probes) p GROUP BY g.id, g.descr, p.arr ), u AS ( SELECT id, descr, ord, v, r FROM q, unnest(q.arr, q.res) WITH ORDINALITY AS x(v, r, ord) ), s AS ( SELECT id, descr, v, r, lag(v) OVER w AS prev_v, lag(r) OVER w AS prev_r FROM u WINDOW w AS (PARTITION BY id ORDER BY ord) ) SELECT id, descr, count(*) AS decreasing_steps, max(prev_r - r) AS max_drop FROM s WHERE r < prev_r GROUP BY id, descr ORDER BY id; id | descr | decreasing_steps | max_drop ----+-------+------------------+---------- (0 rows) -- And the result must always be a probability. WITH q AS ( SELECT g.id, g.descr, p.arr, tdigest_percentile_of(g.d, p.arr) AS res FROM tdigest_monotonic_digests g, (SELECT array_agg(v ORDER BY v) AS arr FROM tdigest_monotonic_probes) p GROUP BY g.id, g.descr, p.arr ) SELECT id, descr, count(*) AS out_of_range FROM q, unnest(q.res) AS r WHERE NOT (r BETWEEN 0 AND 1) GROUP BY id, descr ORDER BY id; id | descr | out_of_range ----+-------+-------------- (0 rows) -- The concrete case from the comment at the top, spelled out. The CDF has to -- grow (or at least stay the same) on each of the four steps. SELECT r[1] <= r[2] AS "0 -> 0.0001", r[2] <= r[3] AS "0.0001 -> 50", r[3] <= r[4] AS "50 -> 99.9999", r[4] <= r[5] AS "99.9999 -> 100" FROM (SELECT tdigest_percentile_of('flags 1 count 10 compression 10000 centroids 2 (0, 7) (100, 3)'::tdigest, ARRAY[0.0, 0.0001, 50.0, 99.9999, 100.0]::double precision[]) AS r) x; 0 -> 0.0001 | 0.0001 -> 50 | 50 -> 99.9999 | 99.9999 -> 100 -------------+--------------+---------------+---------------- t | t | t | t (1 row) -- The same thing through the two aggregates building the digest from data, -- to show this is not an artifact of a hand-written digest. Three items with -- the value 0 and a single one with the value 100. SELECT r[1] <= r[2] AS "0 -> 0.0001", r[2] <= r[3] AS "0.0001 -> 50", r[3] <= r[4] AS "50 -> 99.9999", r[4] <= r[5] AS "99.9999 -> 100" FROM (SELECT tdigest_percentile_of(v, 10000, ARRAY[0.0, 0.0001, 50.0, 99.9999, 100.0]::double precision[]) AS r FROM (VALUES (0.0::double precision), (0.0::double precision), (0.0::double precision), (100.0::double precision)) t(v)) x; 0 -> 0.0001 | 0.0001 -> 50 | 50 -> 99.9999 | 99.9999 -> 100 -------------+--------------+---------------+---------------- t | t | t | t (1 row) SELECT r[1] <= r[2] AS "0 -> 0.0001", r[2] <= r[3] AS "0.0001 -> 50", r[3] <= r[4] AS "50 -> 99.9999", r[4] <= r[5] AS "99.9999 -> 100" FROM (SELECT tdigest_percentile_of(v, c, 10000, ARRAY[0.0, 0.0001, 50.0, 99.9999, 100.0]::double precision[]) AS r FROM (VALUES (0.0::double precision, 3::bigint), (100.0::double precision, 1::bigint)) t(v, c)) x; 0 -> 0.0001 | 0.0001 -> 50 | 50 -> 99.9999 | 99.9999 -> 100 -------------+--------------+---------------+---------------- t | t | t | t (1 row) DROP TABLE tdigest_monotonic_digests; DROP TABLE tdigest_monotonic_probes; tvondra-tdigest-c0af713/test/expected/percentile_roundtrip.out000066400000000000000000000133261525543331000250150ustar00rootroot00000000000000-- tdigest_percentile() and tdigest_percentile_of() are inverse functions - -- one interpolates a value from a percentile, the other a percentile from a -- value - so composing them has to be an identity: -- -- tdigest_percentile_of(d, tdigest_percentile(d, p)) = p -- -- This only holds for percentiles that are actually interpolated. Both -- functions place the mean of a centroid in the middle of it, so below -- (first centroid count / 2) items and above (last centroid count / 2) items -- there is nothing to interpolate between and the result is clamped to the -- first/last mean. Each digest below therefore comes with the range of -- percentiles that are inside the interpolated part. -- -- The identity does not hold at the moment, because the two functions do not -- agree on where the mean of a centroid is. tdigest_compute_quantiles_of() -- computes the offset of the mean with an int64 division -- -- count -= (prev->count / 2); -- -- (note "count" is a double, and the very next statement uses / 2.0), so for -- centroids with an odd count it is off by half an item, i.e. the round trip -- is off by 0.5/count. On the ten item digest below that is a whole 0.05: -- -- p tdigest_percentile round trip expected -- 0.1 1.5 0.15 0.1 -- 0.5 5.5 0.55 0.5 -- 0.9 9.5 0.95 0.9 \set VERBOSITY terse SET extra_float_digits = 0; CREATE TABLE tdigest_roundtrip_digests (id int, descr text, d tdigest, p_low double precision, p_high double precision); INSERT INTO tdigest_roundtrip_digests VALUES (1, 'ten centroids, one item each', 'flags 1 count 10 compression 10000 centroids 10 (1, 1) (2, 1) (3, 1) (4, 1) (5, 1) (6, 1) (7, 1) (8, 1) (9, 1) (10, 1)', 0.05, 0.95), (2, 'two centroids, odd + odd', 'flags 1 count 10 compression 10000 centroids 2 (0, 7) (100, 3)', 0.35, 0.85), (3, 'three centroids, mixed counts', 'flags 1 count 12 compression 10000 centroids 3 (0, 3) (50, 4) (100, 5)', 0.125, 0.79), (4, 'a large centroid in the middle', 'flags 1 count 22 compression 10000 centroids 3 (0, 1) (10, 20) (20, 1)', 0.05, 0.95), (5, 'negative and positive means', 'flags 1 count 8 compression 10000 centroids 4 (-100, 2) (-1, 2) (1, 2) (100, 2)', 0.125, 0.875); -- And digests built by the aggregate from actual data. The first one retains -- singleton centroids after final compaction, while the second merges values. INSERT INTO tdigest_roundtrip_digests SELECT 6, 'built from 1000 distinct values, no centroid merging', tdigest(i::double precision, 10000), 0.005, 0.995 FROM generate_series(1, 1000) s(i); INSERT INTO tdigest_roundtrip_digests SELECT 7, 'built from 10000 rows, compression 100', tdigest((i % 1000)::double precision, 100), 0.05, 0.95 FROM generate_series(1, 10000) s(i); CREATE TABLE tdigest_roundtrip_percentiles (p double precision); INSERT INTO tdigest_roundtrip_percentiles SELECT i / 100.0 FROM generate_series(1, 99) s(i); -- Report one row per digest that fails the round trip, so that a failure -- stays readable. WITH v AS ( SELECT g.id, g.descr, p.p, tdigest_percentile(g.d, p.p) AS value FROM tdigest_roundtrip_digests g, tdigest_roundtrip_percentiles p WHERE p.p >= g.p_low AND p.p <= g.p_high GROUP BY g.id, g.descr, p.p ), rt AS ( SELECT v.id, v.descr, v.p, v.value, tdigest_percentile_of(g.d, v.value) AS back FROM v, tdigest_roundtrip_digests g WHERE g.id = v.id GROUP BY v.id, v.descr, v.p, v.value ) SELECT id, descr, count(*) AS failures, max(abs(back - p)) AS max_error FROM rt WHERE abs(back - p) > 1e-12 GROUP BY id, descr ORDER BY id; id | descr | failures | max_error ----+-------+----------+----------- (0 rows) -- The three concrete percentiles from the comment at the top. WITH v AS ( SELECT p, tdigest_percentile('flags 1 count 10 compression 10000 centroids 10 (1, 1) (2, 1) (3, 1) (4, 1) (5, 1) (6, 1) (7, 1) (8, 1) (9, 1) (10, 1)'::tdigest, p) AS value FROM unnest(ARRAY[0.1, 0.5, 0.9]::double precision[]) AS p GROUP BY p ) SELECT p, value, tdigest_percentile_of('flags 1 count 10 compression 10000 centroids 10 (1, 1) (2, 1) (3, 1) (4, 1) (5, 1) (6, 1) (7, 1) (8, 1) (9, 1) (10, 1)'::tdigest, value) AS round_trip FROM v GROUP BY p, value ORDER BY p; p | value | round_trip -----+-------+------------ 0.1 | 1.5 | 0.1 0.5 | 5.5 | 0.5 0.9 | 9.5 | 0.9 (3 rows) -- The same identity has to hold for the aggregates building the digest from -- data. Ten items, so the interpolated range is [0.05, 0.95]. WITH v AS ( SELECT p, tdigest_percentile(i::double precision, 10000, p) AS value FROM generate_series(1, 10) s(i), unnest(ARRAY[0.1, 0.5, 0.9]::double precision[]) AS p GROUP BY p ) SELECT v.p, v.value, tdigest_percentile_of(i::double precision, 10000, v.value) AS round_trip FROM v, generate_series(1, 10) s(i) GROUP BY v.p, v.value ORDER BY v.p; p | value | round_trip -----+-------+------------ 0.1 | 1.5 | 0.1 0.5 | 5.5 | 0.5 0.9 | 9.5 | 0.9 (3 rows) WITH v AS ( SELECT p, tdigest_percentile(i::double precision, 2::bigint, 10000, p) AS value FROM generate_series(1, 5) s(i), unnest(ARRAY[0.2, 0.5, 0.8]::double precision[]) AS p GROUP BY p ) SELECT v.p, v.value, tdigest_percentile_of(i::double precision, 2::bigint, 10000, v.value) AS round_trip FROM v, generate_series(1, 5) s(i) GROUP BY v.p, v.value ORDER BY v.p; p | value | round_trip -----+-------+------------ 0.2 | 1.5 | 0.2 0.5 | 3 | 0.5 0.8 | 4.5 | 0.8 (3 rows) DROP TABLE tdigest_roundtrip_digests; DROP TABLE tdigest_roundtrip_percentiles; tvondra-tdigest-c0af713/test/expected/random.out000066400000000000000000001170101525543331000220300ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; ----------------------------------------------------------- -- nice data set with ordered (asc) / evenly-spaced data -- ----------------------------------------------------------- -- compression 10 WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(1,10000) s(i)) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(1,10000) s(i)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 10, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- compression 100 WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(1,10000) s(i)) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(1,10000) s(i)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 100, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- compression 1000 WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(1,10000) s(i)) SELECT p, abs(a - b) < 0.001, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.001 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(1,10000) s(i)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 1000, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) ------------------------------------------------------------ -- nice data set with ordered (desc) / evenly-spaced data -- ------------------------------------------------------------ -- compression 10 WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(10000,1,-1) s(i)) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(10000,1,-1) s(i)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 10, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- compression 100 WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(10000,1,-1) s(i)) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(10000,1,-1) s(i)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 100, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- compression 1000 WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(10000,1,-1) s(i)) SELECT p, abs(a - b) < 0.001, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.001 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(10000,1,-1) s(i)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 1000, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) ---------------------------------------------------- -- nice data set with random / evenly-spaced data -- ---------------------------------------------------- -- compression 10 WITH data AS (SELECT i / 10000.0 AS x FROM (SELECT generate_series(1,10000) AS i, prng(10000, 49979693) AS x ORDER BY x) foo) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT i / 10000.0 AS x FROM (SELECT generate_series(1,10000) AS i, prng(10000, 49979693) AS x ORDER BY x) foo), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 10, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- compression 100 WITH data AS (SELECT i / 10000.0 AS x FROM (SELECT generate_series(1,10000) AS i, prng(10000, 49979693) AS x ORDER BY x) foo) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT i / 10000.0 AS x FROM (SELECT generate_series(1,10000) AS i, prng(10000, 49979693) AS x ORDER BY x) foo), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 100, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- compression 1000 WITH data AS (SELECT i / 10000.0 AS x FROM (SELECT generate_series(1,10000) AS i, prng(10000, 49979693) AS x ORDER BY x) foo) SELECT p, abs(a - b) < 0.001, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.001 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT i / 10000.0 AS x FROM (SELECT generate_series(1,10000) AS i, prng(10000, 49979693) AS x ORDER BY x) foo), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 1000, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) ---------------------------------------------- -- nice data set with random data (uniform) -- ---------------------------------------------- -- compression 10 WITH data AS (SELECT x FROM prng(10000) s(x)) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT x FROM prng(10000) s(x)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 10, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- compression 100 WITH data AS (SELECT x FROM prng(10000) s(x)) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT x FROM prng(10000) s(x)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 100, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- compression 1000 WITH data AS (SELECT x FROM prng(10000) s(x)) SELECT p, abs(a - b) < 0.001, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.001 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT x FROM prng(10000) s(x)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 1000, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -------------------------------------------------- -- nice data set with random data (skewed sqrt) -- -------------------------------------------------- -- compression 10 WITH data AS (SELECT sqrt(z) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT sqrt(z) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 10, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- compression 100 WITH data AS (SELECT sqrt(z) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT sqrt(z) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 100, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- compression 1000 WITH data AS (SELECT sqrt(z) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.001, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.001 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT sqrt(z) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 1000, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) ------------------------------------------------------- -- nice data set with random data (skewed sqrt+sqrt) -- ------------------------------------------------------- -- compression 10 WITH data AS (SELECT sqrt(sqrt(z)) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT sqrt(sqrt(z)) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 10, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- compression 100 WITH data AS (SELECT sqrt(sqrt(z)) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT sqrt(sqrt(z)) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 100, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- compression 1000 WITH data AS (SELECT sqrt(sqrt(z)) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.001, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.001 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT sqrt(sqrt(z)) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 1000, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) ------------------------------------------------- -- nice data set with random data (skewed pow) -- ------------------------------------------------- -- compression 10 WITH data AS (SELECT pow(z, 2) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT pow(z, 2) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 10, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- compression 100 WITH data AS (SELECT pow(z, 2) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.005, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.005 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT pow(z, 2) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 100, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- compression 1000 WITH data AS (SELECT pow(z, 2) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.001, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.001 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT pow(z, 2) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 1000, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) ----------------------------------------------------- -- nice data set with random data (skewed pow+pow) -- ----------------------------------------------------- -- compression 10 WITH data AS (SELECT pow(z, 4) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT pow(z, 4) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 10, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- compression 100 WITH data AS (SELECT pow(z, 4) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT pow(z, 4) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 100, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- compression 1000 WITH data AS (SELECT pow(z, 4) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.001, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.001 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT pow(z, 4) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 1000, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) ---------------------------------------------------------- -- fourth power of data from a truncated normal distribution -- ---------------------------------------------------------- -- compression 10 WITH data AS (SELECT pow(z, 4) AS x FROM random_normal(10000) s(z)) SELECT p, abs(a - b) < 0.025, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.025 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT pow(z, 4) AS x FROM random_normal(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 10, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- compression 100 WITH data AS (SELECT pow(z, 4) AS x FROM random_normal(10000) s(z)) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT pow(z, 4) AS x FROM random_normal(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 100, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- compression 1000 WITH data AS (SELECT pow(z, 4) AS x FROM random_normal(10000) s(z)) SELECT p, abs(a - b) < 0.001, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.001 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- make sure the resulting percentiles are in the right order WITH data AS (SELECT pow(z, 4) AS x FROM random_normal(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 1000, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; p | a | b ---+---+--- (0 rows) -- some basic tests to verify transforming from and to text works -- compression 10 WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(1,10000) s(i)), intermediate AS (SELECT tdigest(x, 10)::text AS intermediate_x FROM data), tdigest_parsed AS (SELECT tdigest_percentile(intermediate_x::tdigest, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS a FROM intermediate), pg_percentile AS (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) AS b FROM data) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM tdigest_parsed, pg_percentile ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- verify we can store tdigest in a summary table CREATE TABLE intermediate_tdigest (grouping int, summary tdigest); WITH data AS (SELECT row_number() OVER () AS i, pow(z, 4) AS x FROM random_normal(10000) s(z)) INSERT INTO intermediate_tdigest SELECT i % 10 AS grouping, tdigest(x, 100) AS summary FROM data GROUP BY i % 10; WITH data AS (SELECT pow(z, 4) AS x FROM random_normal(10000) s(z)), intermediate AS (SELECT tdigest_percentile(summary, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS a FROM intermediate_tdigest), pg_percentile AS (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) AS b FROM data) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM intermediate, pg_percentile ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) DROP TABLE intermediate_tdigest; -- verify 'extreme' percentiles for the dataset would not read out of bounds on the centroids WITH data AS (SELECT x FROM generate_series(1,10) AS x) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; p | ?column? | err ------+----------+----- 0.01 | t | 0.99 | t | (2 rows) -- check that the computed percentiles don't decrease for higher p values -- first test on a tiny t-digest with all centroids having count = 1 WITH -- percentiles to compute perc AS (SELECT array_agg((i / 100.0)::double precision) AS percentiles FROM generate_series(1,99) s(i)), -- input data (just 15 points) input_data AS (select i::double precision AS val FROM generate_series(1,15) s(i)) SELECT * FROM ( SELECT p, v AS v1, lag(v, 1) OVER (ORDER BY p) v2 FROM ( SELECT unnest(perc.percentiles) p, unnest(tdigest_percentile(input_data.val, 100, perc.percentiles)) v FROM perc, input_data GROUP BY perc.percentiles ) foo ) bar where v2 > v1; p | v1 | v2 ---+----+---- (0 rows) tvondra-tdigest-c0af713/test/expected/recv_checks.out000066400000000000000000000156211525543331000230340ustar00rootroot00000000000000-- Regression test for the receive function tdigest_recv(), which parses the -- binary representation of a t-digest. -- -- The input is user supplied, so it has to be validated just like the text -- input. The queries exercise all the error checks in the receive function, -- one by one. -- -- Test through a binary COPY, which goes through the receive function. We -- build the input with the send functions of the individual fields. \set VERBOSITY terse CREATE TABLE tdigest_dst (s tdigest); -- a valid value, to make sure the checks don't reject correct input COPY (SELECT int4send(1) -- flags || int8send(3::bigint) -- count || int4send(10) -- compression || int4send(2) -- ncentroids || float8send(1) || int8send(1::bigint) || float8send(2) || int8send(2::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); -- -- sanity checks on the header -- -- unknown flags COPY (SELECT int4send(2) || int8send(1::bigint) || int4send(10) || int4send(1) || float8send(1) || int8send(1::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); ERROR: unsupported t-digest on-disk format -- compression out of the supported range COPY (SELECT int4send(1) || int8send(1::bigint) || int4send(9) || int4send(1) || float8send(1) || int8send(1::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); ERROR: compression for t-digest must be in [10, 10000] COPY (SELECT int4send(1) || int8send(1::bigint) || int4send(10001) || int4send(1) || float8send(1) || int8send(1::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); ERROR: compression for t-digest must be in [10, 10000] -- non-positive total count COPY (SELECT int4send(1) || int8send(0::bigint) || int4send(10) || int4send(1) || float8send(1) || int8send(1::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); ERROR: count value for the t-digest must be positive -- non-positive number of centroids COPY (SELECT int4send(1) || int8send(1::bigint) || int4send(10) || int4send(0)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); ERROR: number of centroids for the t-digest must be positive -- more centroids than fit into the buffer (10 * compression) COPY (SELECT int4send(1) || int8send(1::bigint) || int4send(10) || int4send(101)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); ERROR: number of centroids for the t-digest exceeds buffer size -- -- sanity checks on the centroids -- -- centroid mean that is not a valid number COPY (SELECT int4send(1) || int8send(1::bigint) || int4send(10) || int4send(1) || float8send('NaN') || int8send(1::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); ERROR: mean value for all centroids in a t-digest must be valid COPY (SELECT int4send(1) || int8send(1::bigint) || int4send(10) || int4send(1) || float8send('Infinity') || int8send(1::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); ERROR: mean value for all centroids in a t-digest must be valid -- non-positive centroid count COPY (SELECT int4send(1) || int8send(1::bigint) || int4send(10) || int4send(1) || float8send(1) || int8send(0::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); ERROR: count value for all centroids in a t-digest must be positive -- centroid count exceeding the total count COPY (SELECT int4send(1) || int8send(1::bigint) || int4send(10) || int4send(1) || float8send(1) || int8send(2::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); ERROR: count value of a centroid exceeds total count -- centroids not sorted by mean COPY (SELECT int4send(1) || int8send(3::bigint) || int4send(10) || int4send(2) || float8send(2) || int8send(2::bigint) || float8send(1) || int8send(1::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); -- centroids not sorted by mean, in the old on-disk format storing sums -- instead of means (the sums are sorted, but the means are not) COPY (SELECT int4send(0) || int8send(3::bigint) || int4send(10) || int4send(2) || float8send(4) || int8send(1::bigint) || float8send(6) || int8send(2::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); -- centroid counts overflowing the total count COPY (SELECT int4send(1) || int8send(9223372036854775807::bigint) || int4send(10) || int4send(2) || float8send(1) || int8send(9223372036854775807::bigint) || float8send(2) || int8send(9223372036854775807::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); ERROR: tdigest count overflow -- -- sanity checks on the value as a whole -- -- centroid counts not adding up to the total count COPY (SELECT int4send(1) || int8send(3::bigint) || int4send(10) || int4send(1) || float8send(1) || int8send(1::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); ERROR: total count does not match the data (1 != 3) -- only the valid value should have made it into the table SELECT s FROM tdigest_dst; s ---------------------------------------------------------- flags 1 count 3 compression 10 centroids 2 (1, 1) (2, 2) flags 1 count 3 compression 10 centroids 2 (2, 2) (1, 1) flags 1 count 3 compression 10 centroids 2 (4, 1) (3, 2) (3 rows) DROP TABLE tdigest_dst; tvondra-tdigest-c0af713/test/expected/trimmed_agg_overflow.out000066400000000000000000000074271525543331000247640ustar00rootroot00000000000000-- Use terse verbosity, so that the expected output does not depend on the -- error context, which differs between PostgreSQL versions. \set VERBOSITY terse -- Reproducer: tdigest_trimmed_agg() used to convert the trimmed range from -- double to int64 without checking whether the rounded bounds still fit. -- -- This one does not crash, it just gives wrong answers. -- -- The unsafe boundary calculation converted expressions such as -- -- ceil(count * high) -- -- directly to int64. count is int64, but multiplication is done in double, with -- a 53-bit mantissa, so once the total count gets close enough to 2^63 the -- product rounds up to exactly 2^63, which is not representable as int64. The -- conversion back to int64 is undefined behaviour; on x86-64 it yields -- INT64_MIN. count_high then ends up hugely negative, every centroid falls -- outside [count_low, count_high], nothing is accumulated, and the function -- returns NULL as if the digest were empty. -- -- The threshold is sharp - the last total count that still works is -- 9223372036854775295, one more and the answer turns into NULL: -- -- total count tdigest_digest_sum -- 9223372036854775295 1.844674407370955e+19 correct -- 9223372036854775296 NULL wrong -- 9223372036854775807 NULL wrong -- -- This affects tdigest_digest_sum() and tdigest_digest_avg() as well as the -- tdigest_sum() and tdigest_avg() aggregates, i.e. anything going through -- tdigest_trimmed_agg(). -- the last value that still produces the right answer SELECT tdigest_digest_sum('flags 1 count 9223372036854775295 compression 10 centroids 1 (2, 9223372036854775295)'::tdigest); tdigest_digest_sum ----------------------- 1.844674407370955e+19 (1 row) SELECT tdigest_digest_avg('flags 1 count 9223372036854775295 compression 10 centroids 1 (2, 9223372036854775295)'::tdigest); tdigest_digest_avg -------------------- 2 (1 row) -- one more, and the result silently becomes NULL SELECT tdigest_digest_sum('flags 1 count 9223372036854775296 compression 10 centroids 1 (2, 9223372036854775296)'::tdigest); tdigest_digest_sum ------------------------ 1.8446744073709552e+19 (1 row) SELECT tdigest_digest_avg('flags 1 count 9223372036854775296 compression 10 centroids 1 (2, 9223372036854775296)'::tdigest); tdigest_digest_avg -------------------- 2 (1 row) -- INT64_MAX, same thing SELECT tdigest_digest_sum('flags 1 count 9223372036854775807 compression 10 centroids 1 (2, 9223372036854775807)'::tdigest); tdigest_digest_sum ------------------------ 1.8446744073709552e+19 (1 row) SELECT tdigest_digest_avg('flags 1 count 9223372036854775807 compression 10 centroids 1 (2, 9223372036854775807)'::tdigest); tdigest_digest_avg -------------------- 2 (1 row) -- side by side, so the discontinuity is easy to see SELECT n AS total_count, tdigest_digest_sum(('flags 1 count ' || n || ' compression 10 centroids 1 (2, ' || n || ')')::tdigest) AS sum FROM (VALUES (9223372036854775295::bigint), (9223372036854775296::bigint), (9223372036854775807::bigint)) v(n); total_count | sum ---------------------+------------------------ 9223372036854775295 | 1.844674407370955e+19 9223372036854775296 | 1.8446744073709552e+19 9223372036854775807 | 1.8446744073709552e+19 (3 rows) SELECT n AS total_count, tdigest_digest_avg(('flags 1 count ' || n || ' compression 10 centroids 1 (2, ' || n || ')')::tdigest) AS avg FROM (VALUES (9223372036854775295::bigint), (9223372036854775296::bigint), (9223372036854775807::bigint)) v(n); total_count | avg ---------------------+----- 9223372036854775295 | 2 9223372036854775296 | 2 9223372036854775807 | 2 (3 rows) tvondra-tdigest-c0af713/test/expected/trimmed_agg_overflow_1.out000066400000000000000000000073761525543331000252070ustar00rootroot00000000000000-- Use terse verbosity, so that the expected output does not depend on the -- error context, which differs between PostgreSQL versions. \set VERBOSITY terse -- Reproducer: tdigest_trimmed_agg() used to convert the trimmed range from -- double to int64 without checking whether the rounded bounds still fit. -- -- This one does not crash, it just gives wrong answers. -- -- The unsafe boundary calculation converted expressions such as -- -- ceil(count * high) -- -- directly to int64. count is int64, but multiplication is done in double, with -- a 53-bit mantissa, so once the total count gets close enough to 2^63 the -- product rounds up to exactly 2^63, which is not representable as int64. The -- conversion back to int64 is undefined behaviour; on x86-64 it yields -- INT64_MIN. count_high then ends up hugely negative, every centroid falls -- outside [count_low, count_high], nothing is accumulated, and the function -- returns NULL as if the digest were empty. -- -- The threshold is sharp - the last total count that still works is -- 9223372036854775295, one more and the answer turns into NULL: -- -- total count tdigest_digest_sum -- 9223372036854775295 1.844674407370955e+19 correct -- 9223372036854775296 NULL wrong -- 9223372036854775807 NULL wrong -- -- This affects tdigest_digest_sum() and tdigest_digest_avg() as well as the -- tdigest_sum() and tdigest_avg() aggregates, i.e. anything going through -- tdigest_trimmed_agg(). -- the last value that still produces the right answer SELECT tdigest_digest_sum('flags 1 count 9223372036854775295 compression 10 centroids 1 (2, 9223372036854775295)'::tdigest); tdigest_digest_sum ---------------------- 1.84467440737095e+19 (1 row) SELECT tdigest_digest_avg('flags 1 count 9223372036854775295 compression 10 centroids 1 (2, 9223372036854775295)'::tdigest); tdigest_digest_avg -------------------- 2 (1 row) -- one more, and the result silently becomes NULL SELECT tdigest_digest_sum('flags 1 count 9223372036854775296 compression 10 centroids 1 (2, 9223372036854775296)'::tdigest); tdigest_digest_sum ---------------------- 1.84467440737096e+19 (1 row) SELECT tdigest_digest_avg('flags 1 count 9223372036854775296 compression 10 centroids 1 (2, 9223372036854775296)'::tdigest); tdigest_digest_avg -------------------- 2 (1 row) -- INT64_MAX, same thing SELECT tdigest_digest_sum('flags 1 count 9223372036854775807 compression 10 centroids 1 (2, 9223372036854775807)'::tdigest); tdigest_digest_sum ---------------------- 1.84467440737096e+19 (1 row) SELECT tdigest_digest_avg('flags 1 count 9223372036854775807 compression 10 centroids 1 (2, 9223372036854775807)'::tdigest); tdigest_digest_avg -------------------- 2 (1 row) -- side by side, so the discontinuity is easy to see SELECT n AS total_count, tdigest_digest_sum(('flags 1 count ' || n || ' compression 10 centroids 1 (2, ' || n || ')')::tdigest) AS sum FROM (VALUES (9223372036854775295::bigint), (9223372036854775296::bigint), (9223372036854775807::bigint)) v(n); total_count | sum ---------------------+---------------------- 9223372036854775295 | 1.84467440737095e+19 9223372036854775296 | 1.84467440737096e+19 9223372036854775807 | 1.84467440737096e+19 (3 rows) SELECT n AS total_count, tdigest_digest_avg(('flags 1 count ' || n || ' compression 10 centroids 1 (2, ' || n || ')')::tdigest) AS avg FROM (VALUES (9223372036854775295::bigint), (9223372036854775296::bigint), (9223372036854775807::bigint)) v(n); total_count | avg ---------------------+----- 9223372036854775295 | 2 9223372036854775296 | 2 9223372036854775807 | 2 (3 rows) tvondra-tdigest-c0af713/test/expected/trimmed_aggregates.out000066400000000000000000000461061525543331000244110ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('extra_float_digits', '0', false); PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; -- check trimmed mean (from raw data) -- we compare the result to a range, to deal with the randomness WITH data AS (SELECT random() AS r FROM generate_series(1,10000) AS x) SELECT tdigest_avg(data.r, 50, 0.1, 0.9) between 0.45 and 0.55 AS mean_10_90, tdigest_avg(data.r, 50, 0.25, 0.75) between 0.45 and 0.55 AS mean_25_75, tdigest_avg(data.r, 50, 0.0, 0.5) between 0.2 and 0.3 AS mean_0_50, tdigest_avg(data.r, 50, 0.5, 1.0) between 0.7 and 0.8 AS mean_50_100 FROM data; mean_10_90 | mean_25_75 | mean_0_50 | mean_50_100 ------------+------------+-----------+------------- t | t | t | t (1 row) WITH data AS (SELECT random() AS r, (1 + (3 * random())::int) AS c FROM generate_series(1,10000) AS x) SELECT tdigest_avg(data.r, data.c, 100, 0.1, 0.9) between 0.45 and 0.55 AS mean_10_90, tdigest_avg(data.r, data.c, 100, 0.25, 0.75) between 0.45 and 0.55 AS mean_25_75, tdigest_avg(data.r, data.c, 100, 0.0, 0.5) between 0.2 and 0.3 AS mean_0_50, tdigest_avg(data.r, data.c, 100, 0.5, 1.0) between 0.7 and 0.8 AS mean_50_100 FROM data; mean_10_90 | mean_25_75 | mean_0_50 | mean_50_100 ------------+------------+-----------+------------- t | t | t | t (1 row) -- check trimmed mean (from precalculated tdigest) -- we compare the result to a range, to deal with the randomness WITH data AS (SELECT tdigest(random(), 50) AS d FROM generate_series(1,10000) AS x) SELECT tdigest_avg(data.d, 0.1, 0.9) between 0.45 and 0.55 AS mean_10_90, tdigest_avg(data.d, 0.25, 0.75) between 0.45 and 0.55 AS mean_25_75, tdigest_avg(data.d, 0.0, 0.5) between 0.2 and 0.3 AS mean_0_50, tdigest_avg(data.d, 0.5, 1.0) between 0.7 and 0.8 AS mean_50_100 FROM data; mean_10_90 | mean_25_75 | mean_0_50 | mean_50_100 ------------+------------+-----------+------------- t | t | t | t (1 row) -- check trimmed sum (from raw data) -- we compare the result to a range, to deal with the randomness WITH data AS (SELECT random() AS r FROM generate_series(1,10000) AS x) SELECT tdigest_sum(data.r, 50, 0.1, 0.9) between 8000 * 0.45 and 8000 * 0.55 AS sum_10_90, tdigest_sum(data.r, 50, 0.25, 0.75) between 5000 * 0.45 and 5000 * 0.55 AS sum_25_75, tdigest_sum(data.r, 50, 0.0, 0.5) between 5000 * 0.2 and 5000 * 0.3 AS sum_0_50, tdigest_sum(data.r, 50, 0.5, 1.0) between 5000 * 0.7 and 5000 * 0.8 AS sum_50_100 FROM data; sum_10_90 | sum_25_75 | sum_0_50 | sum_50_100 -----------+-----------+----------+------------ t | t | t | t (1 row) WITH data AS (SELECT random() AS r, (1 + (3 * random())::int) AS c FROM generate_series(1,10000) AS x) SELECT tdigest_sum(data.r, data.c, 100, 0.1, 0.9) between 20000 * 0.45 and 20000 * 0.55 AS sum_10_90, tdigest_sum(data.r, data.c, 100, 0.25, 0.75) between 12500 * 0.45 and 12500 * 0.55 AS sum_25_75, tdigest_sum(data.r, data.c, 100, 0.0, 0.5) between 12500 * 0.2 and 12500 * 0.3 AS sum_0_50, tdigest_sum(data.r, data.c, 100, 0.5, 1.0) between 12500 * 0.7 and 12500 * 0.8 AS sum_50_100 FROM data; sum_10_90 | sum_25_75 | sum_0_50 | sum_50_100 -----------+-----------+----------+------------ t | t | t | t (1 row) -- check trimmed sum (from precalculated tdigest) -- we compare the result to a range, to deal with the randomness WITH data AS (SELECT tdigest(random(), 50) AS d FROM generate_series(1,10000) AS x) SELECT tdigest_sum(data.d, 0.1, 0.9) between 8000 * 0.45 and 8000 * 0.55 AS sum_10_90, tdigest_sum(data.d, 0.25, 0.75) between 5000 * 0.45 and 5000 * 0.55 AS sum_25_75, tdigest_sum(data.d, 0.0, 0.5) between 5000 * 0.2 and 5000 * 0.3 AS sum_0_50, tdigest_sum(data.d, 0.5, 1.0) between 5000 * 0.7 and 5000 * 0.8 AS sum_50_100 FROM data; sum_10_90 | sum_25_75 | sum_0_50 | sum_50_100 -----------+-----------+----------+------------ t | t | t | t (1 row) WITH data AS (SELECT tdigest(random(), 50) AS d FROM generate_series(1,10000) AS x) SELECT tdigest_digest_sum(data.d, 0.05, 0.95) between 9000 * 0.45 and 9000 * 0.55 AS sum_05_95, tdigest_digest_avg(data.d, 0.05, 0.95) between 0.45 and 0.55 AS mean_05_95 FROM data; sum_05_95 | mean_05_95 -----------+------------ t | t (1 row) -- With no compaction while accumulating input, final compaction starts from -- the same sorted centroids for both ascending and descending input. SELECT tdigest_sum(i, 10000, 0.0, 0.5) FROM generate_series(1, 10000) s(i); tdigest_sum ------------- 12502500 (1 row) SELECT tdigest_sum(i, 10000, 0.0, 0.5) from generate_series(10000, 1, -1) s(i); tdigest_sum ------------- 12502500 (1 row) SELECT tdigest_sum(i, 10000, 0.0, 0.25) FROM generate_series(1, 10000) s(i); tdigest_sum ------------- 3126250 (1 row) SELECT tdigest_sum(i, 10000, 0.0, 0.25) from generate_series(10000, 1, -1) s(i); tdigest_sum ------------- 3126250 (1 row) SELECT tdigest_sum(i, 10000, 0.1, 0.2) FROM generate_series(1, 10000) s(i); tdigest_sum ------------- 1500500 (1 row) SELECT tdigest_sum(i, 10000, 0.1, 0.2) from generate_series(10000, 1, -1) s(i); tdigest_sum ------------- 1500500 (1 row) SELECT tdigest_sum(i, 10000, 0.5, 1.0) FROM generate_series(1, 10000) s(i); tdigest_sum ------------- 37502500 (1 row) SELECT tdigest_sum(i, 10000, 0.5, 1.0) from generate_series(10000, 1, -1) s(i); tdigest_sum ------------- 37502500 (1 row) SELECT tdigest_sum(i, 10000, 0.75, 1.0) FROM generate_series(1, 10000) s(i); tdigest_sum ------------- 21876250 (1 row) SELECT tdigest_sum(i, 10000, 0.75, 1.0) from generate_series(10000, 1, -1) s(i); tdigest_sum ------------- 21876250 (1 row) SELECT tdigest_sum(i, 10000, 0.75, 0.9) FROM generate_series(1, 10000) s(i); tdigest_sum ------------- 12375750 (1 row) SELECT tdigest_sum(i, 10000, 0.75, 0.9) from generate_series(10000, 1, -1) s(i); tdigest_sum ------------- 12375750 (1 row) -- the same thing with calculating a t-digest first (but make sure all -- centroids have count 1, to make it exactly the same) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1, 1500) s(i)) SELECT tdigest_sum(d, 0.0, 0.5) from tmp; tdigest_sum ------------- 281625 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1500, 1, -1) s(i)) SELECT tdigest_sum(d, 0.0, 0.5) from tmp; tdigest_sum ------------- 281625 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1, 1500) s(i)) SELECT tdigest_sum(d, 0.0, 0.25) from tmp; tdigest_sum ------------- 70500 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1500, 1, -1) s(i)) SELECT tdigest_sum(d, 0.0, 0.25) from tmp; tdigest_sum ------------- 70500 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1, 1500) s(i)) SELECT tdigest_sum(d, 0.1, 0.2) from tmp; tdigest_sum ------------- 33825 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1500, 1, -1) s(i)) SELECT tdigest_sum(d, 0.1, 0.2) from tmp; tdigest_sum ------------- 33825 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1, 1500) s(i)) SELECT tdigest_sum(d, 0.5, 1.0) from tmp; tdigest_sum ------------- 844125 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1500, 1, -1) s(i)) SELECT tdigest_sum(d, 0.5, 1.0) from tmp; tdigest_sum ------------- 844125 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1, 1500) s(i)) SELECT tdigest_sum(d, 0.75, 1.0) from tmp; tdigest_sum ------------- 492375 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1500, 1, -1) s(i)) SELECT tdigest_sum(d, 0.75, 1.0) from tmp; tdigest_sum ------------- 492375 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1, 1500) s(i)) SELECT tdigest_sum(d, 0.75, 0.9) from tmp; tdigest_sum ------------- 278550 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1500, 1, -1) s(i)) SELECT tdigest_sum(d, 0.75, 0.9) from tmp; tdigest_sum ------------- 278550 (1 row) -- simple sanity checks of high/low thresholds -- correct low/high ranges SELECT tdigest_sum(1.0, 100, 0.0, 1.0); tdigest_sum ------------- 1 (1 row) SELECT tdigest_avg(1.0, 100, 0.0, 1.0); tdigest_avg ------------- 1 (1 row) SELECT tdigest_sum(1.0, 100, 0.25, 0.75); tdigest_sum ------------- 1 (1 row) SELECT tdigest_avg(1.0, 100, 0.25, 0.75); tdigest_avg ------------- 1 (1 row) SELECT tdigest_sum(1.0, 100, 0.5, 0.5); tdigest_sum ------------- 1 (1 row) SELECT tdigest_avg(1.0, 100, 0.5, 0.5); tdigest_avg ------------- 1 (1 row) -- inverted range SELECT tdigest_sum(1.0, 100, 0.75, 0.25); ERROR: invalid low/high percentile values 0.750000/0.250000, should be low <= high SELECT tdigest_avg(1.0, 100, 0.75, 0.25); ERROR: invalid low/high percentile values 0.750000/0.250000, should be low <= high -- negative low threshold SELECT tdigest_sum(1.0, 100, -1.0, 1.0); ERROR: invalid low percentile value -1.000000, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, -1.0, 1.0); ERROR: invalid low percentile value -1.000000, should be in [0.0, 1.0] -- bogus high threshold SELECT tdigest_sum(1.0, 100, -1.0, 1.0); ERROR: invalid low percentile value -1.000000, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, -1.0, 1.0); ERROR: invalid low percentile value -1.000000, should be in [0.0, 1.0] -- infinity in low threshold SELECT tdigest_sum(1.0, 100, '-infinity'::double precision, 1.0); ERROR: invalid low percentile value -Infinity, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, '-infinity'::double precision, 1.0); ERROR: invalid low percentile value -Infinity, should be in [0.0, 1.0] SELECT tdigest_sum(1.0, 100, 'infinity'::double precision, 1.0); ERROR: invalid low percentile value Infinity, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, 'infinity'::double precision, 1.0); ERROR: invalid low percentile value Infinity, should be in [0.0, 1.0] -- infinity in high threshold SELECT tdigest_sum(1.0, 100, 0.0, '-infinity'::double precision); ERROR: invalid high percentile value -Infinity, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, 0.0, '-infinity'::double precision); ERROR: invalid high percentile value -Infinity, should be in [0.0, 1.0] SELECT tdigest_sum(1.0, 100, 0.0, 'infinity'::double precision); ERROR: invalid high percentile value Infinity, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, 0.0, 'infinity'::double precision); ERROR: invalid high percentile value Infinity, should be in [0.0, 1.0] -- infinity in both thresholds SELECT tdigest_sum(1.0, 100, '-infinity'::double precision, '-infinity'::double precision); ERROR: invalid low percentile value -Infinity, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, '-infinity'::double precision, '-infinity'::double precision); ERROR: invalid low percentile value -Infinity, should be in [0.0, 1.0] SELECT tdigest_sum(1.0, 100, '-infinity'::double precision, 'infinity'::double precision); ERROR: invalid low percentile value -Infinity, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, '-infinity'::double precision, 'infinity'::double precision); ERROR: invalid low percentile value -Infinity, should be in [0.0, 1.0] SELECT tdigest_sum(1.0, 100, 'infinity'::double precision, '-infinity'::double precision); ERROR: invalid low percentile value Infinity, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, 'infinity'::double precision, '-infinity'::double precision); ERROR: invalid low percentile value Infinity, should be in [0.0, 1.0] SELECT tdigest_sum(1.0, 100, 'infinity'::double precision, 'infinity'::double precision); ERROR: invalid low percentile value Infinity, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, 'infinity'::double precision, 'infinity'::double precision); ERROR: invalid low percentile value Infinity, should be in [0.0, 1.0] -- NaN in thresholds SELECT tdigest_sum(1.0, 100, 'NaN'::double precision, 1.0); ERROR: invalid low percentile value NaN, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, 'NaN'::double precision, 1.0); ERROR: invalid low percentile value NaN, should be in [0.0, 1.0] SELECT tdigest_sum(1.0, 100, 0.0, 'NaN'::double precision); ERROR: invalid high percentile value NaN, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, 0.0, 'NaN'::double precision); ERROR: invalid high percentile value NaN, should be in [0.0, 1.0] SELECT tdigest_sum(1.0, 100, 'NaN'::double precision, 'NaN'::double precision); ERROR: invalid low percentile value NaN, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, 'NaN'::double precision, 'NaN'::double precision); ERROR: invalid low percentile value NaN, should be in [0.0, 1.0] -- NULL in thresholds SELECT tdigest_sum(1.0, 100, NULL::double precision, 1.0); ERROR: trim thresholds must not be NULL SELECT tdigest_avg(1.0, 100, NULL::double precision, 1.0); ERROR: trim thresholds must not be NULL SELECT tdigest_sum(1.0, 100, 0.0, NULL::double precision); ERROR: trim thresholds must not be NULL SELECT tdigest_avg(1.0, 100, 0.0, NULL::double precision); ERROR: trim thresholds must not be NULL SELECT tdigest_sum(1.0, 100, NULL::double precision, NULL::double precision); ERROR: trim thresholds must not be NULL SELECT tdigest_avg(1.0, 100, NULL::double precision, NULL::double precision); ERROR: trim thresholds must not be NULL -- check tdigest_trimmed_agg() calculates and aggregates the right range -- of centroids to process, especially when [count_low, count_high) falls -- into a single centroid -- -- A digest holding a single centroid, i.e. 100 items that all have the value -- 10. The trimmed sum then has to be exactly (10 * number of items in the -- range) for every range, which makes the expected values easy to derive -- -- count_low = floor(100 * lo) and count_high = ceil(100 * hi). SELECT lo, hi, tdigest_digest_sum(d, lo, hi) AS trimmed_sum, 10 * (ceil(100::double precision * hi) - floor(100::double precision * lo)) AS expected_sum, tdigest_digest_avg(d, lo, hi) AS trimmed_avg FROM (SELECT 'flags 1 count 100 compression 10 centroids 1 (10, 100)'::tdigest) t(d), (VALUES (0.0::double precision, 1.0::double precision), (0.0::double precision, 0.5::double precision), (0.5::double precision, 1.0::double precision), (0.4::double precision, 0.5::double precision), (0.1::double precision, 0.2::double precision), (0.25::double precision, 0.75::double precision)) v(lo, hi) ORDER BY lo, hi; lo | hi | trimmed_sum | expected_sum | trimmed_avg ------+------+-------------+--------------+------------- 0 | 0.5 | 500 | 500 | 10 0 | 1 | 1000 | 1000 | 10 0.1 | 0.2 | 100 | 100 | 10 0.25 | 0.75 | 500 | 500 | 10 0.4 | 0.5 | 100 | 100 | 10 0.5 | 1 | 500 | 500 | 10 (6 rows) -- The same through the aggregates, to cover tdigest_trimmed_sum() and -- tdigest_trimmed_avg() too. All the input values are the same, so the digest -- is a single centroid no matter how the compaction merges the centroids. SELECT tdigest(10.0::double precision, 10) AS digest FROM generate_series(1, 100); digest ------------------------------------------------------------------------------------------------------------------------- flags 1 count 100 compression 10 centroids 9 (10, 1) (10, 1) (10, 4) (10, 13) (10, 42) (10, 28) (10, 8) (10, 2) (10, 1) (1 row) SELECT tdigest_sum(10.0::double precision, 10, 0.4, 0.5) AS agg_sum, tdigest_avg(10.0::double precision, 10, 0.4, 0.5) AS agg_avg FROM generate_series(1, 100); agg_sum | agg_avg ---------+--------- 100 | 10 (1 row) -- Now the same thing, but with more than one centroid, covering a range -- inside the middle centroid, a range straddling a centroid boundary, a -- range inside the last centroid, and the untrimmed case. SELECT lo, hi, tdigest_digest_sum(d, lo, hi) AS trimmed_sum, tdigest_digest_avg(d, lo, hi) AS trimmed_avg FROM (SELECT 'flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest) t(d), (VALUES (0.4::float8, 0.5::float8), (0.35::float8, 0.45::float8), (0.45::float8, 0.55::float8), (0.3::float8, 0.6::float8), (0.6::float8, 0.9::float8), (0.0::float8, 1.0::float8)) v(lo, hi) ORDER BY lo, hi; lo | hi | trimmed_sum | trimmed_avg ------+------+-------------+------------------ 0 | 1 | 650 | 21.6666666666667 0.3 | 0.6 | 200 | 22.2222222222222 0.35 | 0.45 | 80 | 20 0.4 | 0.5 | 60 | 20 0.45 | 0.55 | 100 | 25 0.6 | 0.9 | 270 | 30 (6 rows) -- Exhaustive cross-check of every trimmed range against a brute-force -- expansion of the same digest into individual items. Expected to return no -- rows; any row is a range where the trimmed sum disagrees with the sum of -- the items the range is supposed to cover. WITH digest(d) AS ( SELECT 'flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest ), -- the individual items of the digest, with their 0-based index items(idx, val) AS ( SELECT c.first_idx + g - 1, c.mean FROM (VALUES (0, 10.0::float8, 10), (10, 20.0::float8, 5), (15, 30.0::float8, 15)) c(first_idx, mean, cnt), generate_series(1, c.cnt) g ), ranges(lo, hi) AS ( SELECT (a / 20.0)::float8, (b / 20.0)::float8 FROM generate_series(0, 20) a, generate_series(0, 20) b WHERE a < b ) SELECT r.lo, r.hi, tdigest_digest_sum(digest.d, r.lo, r.hi) AS trimmed_sum, (SELECT sum(i.val) FROM items i WHERE i.idx >= floor(30::float8 * r.lo) AND i.idx < ceil(30::float8 * r.hi)) AS expected_sum FROM ranges r, digest WHERE tdigest_digest_sum(digest.d, r.lo, r.hi) IS DISTINCT FROM (SELECT sum(i.val) FROM items i WHERE i.idx >= floor(30::float8 * r.lo) AND i.idx < ceil(30::float8 * r.hi)) ORDER BY r.lo, r.hi; lo | hi | trimmed_sum | expected_sum ----+----+-------------+-------------- (0 rows) tvondra-tdigest-c0af713/test/expected/trimmed_aggregates_1.out000066400000000000000000000457661525543331000246440ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('extra_float_digits', '0', false); PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; -- check trimmed mean (from raw data) -- we compare the result to a range, to deal with the randomness WITH data AS (SELECT random() AS r FROM generate_series(1,10000) AS x) SELECT tdigest_avg(data.r, 50, 0.1, 0.9) between 0.45 and 0.55 AS mean_10_90, tdigest_avg(data.r, 50, 0.25, 0.75) between 0.45 and 0.55 AS mean_25_75, tdigest_avg(data.r, 50, 0.0, 0.5) between 0.2 and 0.3 AS mean_0_50, tdigest_avg(data.r, 50, 0.5, 1.0) between 0.7 and 0.8 AS mean_50_100 FROM data; mean_10_90 | mean_25_75 | mean_0_50 | mean_50_100 ------------+------------+-----------+------------- t | t | t | t (1 row) WITH data AS (SELECT random() AS r, (1 + (3 * random())::int) AS c FROM generate_series(1,10000) AS x) SELECT tdigest_avg(data.r, data.c, 100, 0.1, 0.9) between 0.45 and 0.55 AS mean_10_90, tdigest_avg(data.r, data.c, 100, 0.25, 0.75) between 0.45 and 0.55 AS mean_25_75, tdigest_avg(data.r, data.c, 100, 0.0, 0.5) between 0.2 and 0.3 AS mean_0_50, tdigest_avg(data.r, data.c, 100, 0.5, 1.0) between 0.7 and 0.8 AS mean_50_100 FROM data; mean_10_90 | mean_25_75 | mean_0_50 | mean_50_100 ------------+------------+-----------+------------- t | t | t | t (1 row) -- check trimmed mean (from precalculated tdigest) -- we compare the result to a range, to deal with the randomness WITH data AS (SELECT tdigest(random(), 50) AS d FROM generate_series(1,10000) AS x) SELECT tdigest_avg(data.d, 0.1, 0.9) between 0.45 and 0.55 AS mean_10_90, tdigest_avg(data.d, 0.25, 0.75) between 0.45 and 0.55 AS mean_25_75, tdigest_avg(data.d, 0.0, 0.5) between 0.2 and 0.3 AS mean_0_50, tdigest_avg(data.d, 0.5, 1.0) between 0.7 and 0.8 AS mean_50_100 FROM data; mean_10_90 | mean_25_75 | mean_0_50 | mean_50_100 ------------+------------+-----------+------------- t | t | t | t (1 row) -- check trimmed sum (from raw data) -- we compare the result to a range, to deal with the randomness WITH data AS (SELECT random() AS r FROM generate_series(1,10000) AS x) SELECT tdigest_sum(data.r, 50, 0.1, 0.9) between 8000 * 0.45 and 8000 * 0.55 AS sum_10_90, tdigest_sum(data.r, 50, 0.25, 0.75) between 5000 * 0.45 and 5000 * 0.55 AS sum_25_75, tdigest_sum(data.r, 50, 0.0, 0.5) between 5000 * 0.2 and 5000 * 0.3 AS sum_0_50, tdigest_sum(data.r, 50, 0.5, 1.0) between 5000 * 0.7 and 5000 * 0.8 AS sum_50_100 FROM data; sum_10_90 | sum_25_75 | sum_0_50 | sum_50_100 -----------+-----------+----------+------------ t | t | t | t (1 row) WITH data AS (SELECT random() AS r, (1 + (3 * random())::int) AS c FROM generate_series(1,10000) AS x) SELECT tdigest_sum(data.r, data.c, 100, 0.1, 0.9) between 20000 * 0.45 and 20000 * 0.55 AS sum_10_90, tdigest_sum(data.r, data.c, 100, 0.25, 0.75) between 12500 * 0.45 and 12500 * 0.55 AS sum_25_75, tdigest_sum(data.r, data.c, 100, 0.0, 0.5) between 12500 * 0.2 and 12500 * 0.3 AS sum_0_50, tdigest_sum(data.r, data.c, 100, 0.5, 1.0) between 12500 * 0.7 and 12500 * 0.8 AS sum_50_100 FROM data; sum_10_90 | sum_25_75 | sum_0_50 | sum_50_100 -----------+-----------+----------+------------ t | t | t | t (1 row) -- check trimmed sum (from precalculated tdigest) -- we compare the result to a range, to deal with the randomness WITH data AS (SELECT tdigest(random(), 50) AS d FROM generate_series(1,10000) AS x) SELECT tdigest_sum(data.d, 0.1, 0.9) between 8000 * 0.45 and 8000 * 0.55 AS sum_10_90, tdigest_sum(data.d, 0.25, 0.75) between 5000 * 0.45 and 5000 * 0.55 AS sum_25_75, tdigest_sum(data.d, 0.0, 0.5) between 5000 * 0.2 and 5000 * 0.3 AS sum_0_50, tdigest_sum(data.d, 0.5, 1.0) between 5000 * 0.7 and 5000 * 0.8 AS sum_50_100 FROM data; sum_10_90 | sum_25_75 | sum_0_50 | sum_50_100 -----------+-----------+----------+------------ t | t | t | t (1 row) WITH data AS (SELECT tdigest(random(), 50) AS d FROM generate_series(1,10000) AS x) SELECT tdigest_digest_sum(data.d, 0.05, 0.95) between 9000 * 0.45 and 9000 * 0.55 AS sum_05_95, tdigest_digest_avg(data.d, 0.05, 0.95) between 0.45 and 0.55 AS mean_05_95 FROM data; sum_05_95 | mean_05_95 -----------+------------ t | t (1 row) -- With no compaction while accumulating input, final compaction starts from -- the same sorted centroids for both ascending and descending input. SELECT tdigest_sum(i, 10000, 0.0, 0.5) FROM generate_series(1, 10000) s(i); tdigest_sum ------------- 12502500 (1 row) SELECT tdigest_sum(i, 10000, 0.0, 0.5) from generate_series(10000, 1, -1) s(i); tdigest_sum ------------- 12502500 (1 row) SELECT tdigest_sum(i, 10000, 0.0, 0.25) FROM generate_series(1, 10000) s(i); tdigest_sum ------------- 3126250 (1 row) SELECT tdigest_sum(i, 10000, 0.0, 0.25) from generate_series(10000, 1, -1) s(i); tdigest_sum ------------- 3126250 (1 row) SELECT tdigest_sum(i, 10000, 0.1, 0.2) FROM generate_series(1, 10000) s(i); tdigest_sum ------------- 1500500 (1 row) SELECT tdigest_sum(i, 10000, 0.1, 0.2) from generate_series(10000, 1, -1) s(i); tdigest_sum ------------- 1500500 (1 row) SELECT tdigest_sum(i, 10000, 0.5, 1.0) FROM generate_series(1, 10000) s(i); tdigest_sum ------------- 37502500 (1 row) SELECT tdigest_sum(i, 10000, 0.5, 1.0) from generate_series(10000, 1, -1) s(i); tdigest_sum ------------- 37502500 (1 row) SELECT tdigest_sum(i, 10000, 0.75, 1.0) FROM generate_series(1, 10000) s(i); tdigest_sum ------------- 21876250 (1 row) SELECT tdigest_sum(i, 10000, 0.75, 1.0) from generate_series(10000, 1, -1) s(i); tdigest_sum ------------- 21876250 (1 row) SELECT tdigest_sum(i, 10000, 0.75, 0.9) FROM generate_series(1, 10000) s(i); tdigest_sum ------------- 12375750 (1 row) SELECT tdigest_sum(i, 10000, 0.75, 0.9) from generate_series(10000, 1, -1) s(i); tdigest_sum ------------- 12375750 (1 row) -- the same thing with calculating a t-digest first (but make sure all -- centroids have count 1, to make it exactly the same) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1, 1500) s(i)) SELECT tdigest_sum(d, 0.0, 0.5) from tmp; tdigest_sum ------------- 281625 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1500, 1, -1) s(i)) SELECT tdigest_sum(d, 0.0, 0.5) from tmp; tdigest_sum ------------- 281625 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1, 1500) s(i)) SELECT tdigest_sum(d, 0.0, 0.25) from tmp; tdigest_sum ------------- 70500 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1500, 1, -1) s(i)) SELECT tdigest_sum(d, 0.0, 0.25) from tmp; tdigest_sum ------------- 70500 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1, 1500) s(i)) SELECT tdigest_sum(d, 0.1, 0.2) from tmp; tdigest_sum ------------- 33825 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1500, 1, -1) s(i)) SELECT tdigest_sum(d, 0.1, 0.2) from tmp; tdigest_sum ------------- 33825 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1, 1500) s(i)) SELECT tdigest_sum(d, 0.5, 1.0) from tmp; tdigest_sum ------------- 844125 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1500, 1, -1) s(i)) SELECT tdigest_sum(d, 0.5, 1.0) from tmp; tdigest_sum ------------- 844125 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1, 1500) s(i)) SELECT tdigest_sum(d, 0.75, 1.0) from tmp; tdigest_sum ------------- 492375 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1500, 1, -1) s(i)) SELECT tdigest_sum(d, 0.75, 1.0) from tmp; tdigest_sum ------------- 492375 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1, 1500) s(i)) SELECT tdigest_sum(d, 0.75, 0.9) from tmp; tdigest_sum ------------- 278550 (1 row) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1500, 1, -1) s(i)) SELECT tdigest_sum(d, 0.75, 0.9) from tmp; tdigest_sum ------------- 278550 (1 row) -- simple sanity checks of high/low thresholds -- correct low/high ranges SELECT tdigest_sum(1.0, 100, 0.0, 1.0); tdigest_sum ------------- 1 (1 row) SELECT tdigest_avg(1.0, 100, 0.0, 1.0); tdigest_avg ------------- 1 (1 row) SELECT tdigest_sum(1.0, 100, 0.25, 0.75); tdigest_sum ------------- 1 (1 row) SELECT tdigest_avg(1.0, 100, 0.25, 0.75); tdigest_avg ------------- 1 (1 row) SELECT tdigest_sum(1.0, 100, 0.5, 0.5); tdigest_sum ------------- 1 (1 row) SELECT tdigest_avg(1.0, 100, 0.5, 0.5); tdigest_avg ------------- 1 (1 row) -- inverted range SELECT tdigest_sum(1.0, 100, 0.75, 0.25); ERROR: invalid low/high percentile values 0.750000/0.250000, should be low <= high SELECT tdigest_avg(1.0, 100, 0.75, 0.25); ERROR: invalid low/high percentile values 0.750000/0.250000, should be low <= high -- negative low threshold SELECT tdigest_sum(1.0, 100, -1.0, 1.0); ERROR: invalid low percentile value -1.000000, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, -1.0, 1.0); ERROR: invalid low percentile value -1.000000, should be in [0.0, 1.0] -- bogus high threshold SELECT tdigest_sum(1.0, 100, -1.0, 1.0); ERROR: invalid low percentile value -1.000000, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, -1.0, 1.0); ERROR: invalid low percentile value -1.000000, should be in [0.0, 1.0] -- infinity in low threshold SELECT tdigest_sum(1.0, 100, '-infinity'::double precision, 1.0); ERROR: invalid low percentile value -inf, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, '-infinity'::double precision, 1.0); ERROR: invalid low percentile value -inf, should be in [0.0, 1.0] SELECT tdigest_sum(1.0, 100, 'infinity'::double precision, 1.0); ERROR: invalid low percentile value inf, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, 'infinity'::double precision, 1.0); ERROR: invalid low percentile value inf, should be in [0.0, 1.0] -- infinity in high threshold SELECT tdigest_sum(1.0, 100, 0.0, '-infinity'::double precision); ERROR: invalid high percentile value -inf, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, 0.0, '-infinity'::double precision); ERROR: invalid high percentile value -inf, should be in [0.0, 1.0] SELECT tdigest_sum(1.0, 100, 0.0, 'infinity'::double precision); ERROR: invalid high percentile value inf, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, 0.0, 'infinity'::double precision); ERROR: invalid high percentile value inf, should be in [0.0, 1.0] -- infinity in both thresholds SELECT tdigest_sum(1.0, 100, '-infinity'::double precision, '-infinity'::double precision); ERROR: invalid low percentile value -inf, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, '-infinity'::double precision, '-infinity'::double precision); ERROR: invalid low percentile value -inf, should be in [0.0, 1.0] SELECT tdigest_sum(1.0, 100, '-infinity'::double precision, 'infinity'::double precision); ERROR: invalid low percentile value -inf, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, '-infinity'::double precision, 'infinity'::double precision); ERROR: invalid low percentile value -inf, should be in [0.0, 1.0] SELECT tdigest_sum(1.0, 100, 'infinity'::double precision, '-infinity'::double precision); ERROR: invalid low percentile value inf, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, 'infinity'::double precision, '-infinity'::double precision); ERROR: invalid low percentile value inf, should be in [0.0, 1.0] SELECT tdigest_sum(1.0, 100, 'infinity'::double precision, 'infinity'::double precision); ERROR: invalid low percentile value inf, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, 'infinity'::double precision, 'infinity'::double precision); ERROR: invalid low percentile value inf, should be in [0.0, 1.0] -- NaN in thresholds SELECT tdigest_sum(1.0, 100, 'NaN'::double precision, 1.0); ERROR: invalid low percentile value nan, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, 'NaN'::double precision, 1.0); ERROR: invalid low percentile value nan, should be in [0.0, 1.0] SELECT tdigest_sum(1.0, 100, 0.0, 'NaN'::double precision); ERROR: invalid high percentile value nan, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, 0.0, 'NaN'::double precision); ERROR: invalid high percentile value nan, should be in [0.0, 1.0] SELECT tdigest_sum(1.0, 100, 'NaN'::double precision, 'NaN'::double precision); ERROR: invalid low percentile value nan, should be in [0.0, 1.0] SELECT tdigest_avg(1.0, 100, 'NaN'::double precision, 'NaN'::double precision); ERROR: invalid low percentile value nan, should be in [0.0, 1.0] -- NULL in thresholds SELECT tdigest_sum(1.0, 100, NULL::double precision, 1.0); ERROR: trim thresholds must not be NULL SELECT tdigest_avg(1.0, 100, NULL::double precision, 1.0); ERROR: trim thresholds must not be NULL SELECT tdigest_sum(1.0, 100, 0.0, NULL::double precision); ERROR: trim thresholds must not be NULL SELECT tdigest_avg(1.0, 100, 0.0, NULL::double precision); ERROR: trim thresholds must not be NULL SELECT tdigest_sum(1.0, 100, NULL::double precision, NULL::double precision); ERROR: trim thresholds must not be NULL SELECT tdigest_avg(1.0, 100, NULL::double precision, NULL::double precision); ERROR: trim thresholds must not be NULL -- check tdigest_trimmed_agg() calculates and aggregates the right range -- of centroids to process, especially when [count_low, count_high) falls -- into a single centroid -- -- A digest holding a single centroid, i.e. 100 items that all have the value -- 10. The trimmed sum then has to be exactly (10 * number of items in the -- range) for every range, which makes the expected values easy to derive -- -- count_low = floor(100 * lo) and count_high = ceil(100 * hi). SELECT lo, hi, tdigest_digest_sum(d, lo, hi) AS trimmed_sum, 10 * (ceil(100::double precision * hi) - floor(100::double precision * lo)) AS expected_sum, tdigest_digest_avg(d, lo, hi) AS trimmed_avg FROM (SELECT 'flags 1 count 100 compression 10 centroids 1 (10, 100)'::tdigest) t(d), (VALUES (0.0::double precision, 1.0::double precision), (0.0::double precision, 0.5::double precision), (0.5::double precision, 1.0::double precision), (0.4::double precision, 0.5::double precision), (0.1::double precision, 0.2::double precision), (0.25::double precision, 0.75::double precision)) v(lo, hi) ORDER BY lo, hi; lo | hi | trimmed_sum | expected_sum | trimmed_avg ------+------+-------------+--------------+------------- 0 | 0.5 | 500 | 500 | 10 0 | 1 | 1000 | 1000 | 10 0.1 | 0.2 | 100 | 100 | 10 0.25 | 0.75 | 500 | 500 | 10 0.4 | 0.5 | 100 | 100 | 10 0.5 | 1 | 500 | 500 | 10 (6 rows) -- The same through the aggregates, to cover tdigest_trimmed_sum() and -- tdigest_trimmed_avg() too. All the input values are the same, so the digest -- is a single centroid no matter how the compaction merges the centroids. SELECT tdigest(10.0::double precision, 10) AS digest FROM generate_series(1, 100); digest ------------------------------------------------------------------------------------------------------------------------- flags 1 count 100 compression 10 centroids 9 (10, 1) (10, 1) (10, 4) (10, 13) (10, 42) (10, 28) (10, 8) (10, 2) (10, 1) (1 row) SELECT tdigest_sum(10.0::double precision, 10, 0.4, 0.5) AS agg_sum, tdigest_avg(10.0::double precision, 10, 0.4, 0.5) AS agg_avg FROM generate_series(1, 100); agg_sum | agg_avg ---------+--------- 100 | 10 (1 row) -- Now the same thing, but with more than one centroid, covering a range -- inside the middle centroid, a range straddling a centroid boundary, a -- range inside the last centroid, and the untrimmed case. SELECT lo, hi, tdigest_digest_sum(d, lo, hi) AS trimmed_sum, tdigest_digest_avg(d, lo, hi) AS trimmed_avg FROM (SELECT 'flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest) t(d), (VALUES (0.4::float8, 0.5::float8), (0.35::float8, 0.45::float8), (0.45::float8, 0.55::float8), (0.3::float8, 0.6::float8), (0.6::float8, 0.9::float8), (0.0::float8, 1.0::float8)) v(lo, hi) ORDER BY lo, hi; lo | hi | trimmed_sum | trimmed_avg ------+------+-------------+------------------ 0 | 1 | 650 | 21.6666666666667 0.3 | 0.6 | 200 | 22.2222222222222 0.35 | 0.45 | 80 | 20 0.4 | 0.5 | 60 | 20 0.45 | 0.55 | 100 | 25 0.6 | 0.9 | 270 | 30 (6 rows) -- Exhaustive cross-check of every trimmed range against a brute-force -- expansion of the same digest into individual items. Expected to return no -- rows; any row is a range where the trimmed sum disagrees with the sum of -- the items the range is supposed to cover. WITH digest(d) AS ( SELECT 'flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest ), -- the individual items of the digest, with their 0-based index items(idx, val) AS ( SELECT c.first_idx + g - 1, c.mean FROM (VALUES (0, 10.0::float8, 10), (10, 20.0::float8, 5), (15, 30.0::float8, 15)) c(first_idx, mean, cnt), generate_series(1, c.cnt) g ), ranges(lo, hi) AS ( SELECT (a / 20.0)::float8, (b / 20.0)::float8 FROM generate_series(0, 20) a, generate_series(0, 20) b WHERE a < b ) SELECT r.lo, r.hi, tdigest_digest_sum(digest.d, r.lo, r.hi) AS trimmed_sum, (SELECT sum(i.val) FROM items i WHERE i.idx >= floor(30::float8 * r.lo) AND i.idx < ceil(30::float8 * r.hi)) AS expected_sum FROM ranges r, digest WHERE tdigest_digest_sum(digest.d, r.lo, r.hi) IS DISTINCT FROM (SELECT sum(i.val) FROM items i WHERE i.idx >= floor(30::float8 * r.lo) AND i.idx < ceil(30::float8 * r.hi)) ORDER BY r.lo, r.hi; lo | hi | trimmed_sum | expected_sum ----+----+-------------+-------------- (0 rows) tvondra-tdigest-c0af713/test/expected/trimmed_count_checks.out000066400000000000000000000022211525543331000247360ustar00rootroot00000000000000-- Regression test for the count check in tdigest_add_double_count_trimmed, -- the transition function of the trimmed aggregates accepting an explicit -- number of occurrences of a value. -- -- A non-NULL count must be positive. A NULL count means one occurrence -- of the input value. \set VERBOSITY terse -- zero count SELECT tdigest_avg(v, 0::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0)) AS t(v); ERROR: invalid count value 0, must be a positive value SELECT tdigest_sum(v, 0::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0)) AS t(v); ERROR: invalid count value 0, must be a positive value -- negative count SELECT tdigest_avg(v, -1::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0)) AS t(v); ERROR: invalid count value -1, must be a positive value SELECT tdigest_sum(v, -1::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0)) AS t(v); ERROR: invalid count value -1, must be a positive value -- the smallest valid count has to work SELECT tdigest_avg(v, 1::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0)) AS t(v); tdigest_avg ------------- 1 (1 row) SELECT tdigest_sum(v, 1::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0)) AS t(v); tdigest_sum ------------- 1 (1 row) tvondra-tdigest-c0af713/test/expected/trimmed_null_values.out000066400000000000000000000031531525543331000246240ustar00rootroot00000000000000-- Regression test for the NULL handling in the transition functions of the -- trimmed aggregates. -- -- Neither of the transition functions is strict, so NULL rows do reach them -- and have to be ignored. Each function is exercised twice: once with all -- input rows NULL (so the state stays NULL and the transition function -- returns NULL), and once with a NULL row following a non-NULL one (so the -- transition function returns the existing state unmodified). -- tdigest_add_double_trimmed SELECT tdigest_avg(v, 100, 0.0, 1.0) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); tdigest_avg ------------- (1 row) SELECT tdigest_avg(v, 100, 0.0, 1.0) FROM (VALUES (1.0), (NULL)) AS t(v); tdigest_avg ------------- 1 (1 row) SELECT tdigest_sum(v, 100, 0.0, 1.0) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); tdigest_sum ------------- (1 row) SELECT tdigest_sum(v, 100, 0.0, 1.0) FROM (VALUES (1.0), (NULL)) AS t(v); tdigest_sum ------------- 1 (1 row) -- tdigest_add_double_count_trimmed SELECT tdigest_avg(v, 1::bigint, 100, 0.0, 1.0) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); tdigest_avg ------------- (1 row) SELECT tdigest_avg(v, 1::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0), (NULL)) AS t(v); tdigest_avg ------------- 1 (1 row) SELECT tdigest_sum(v, 1::bigint, 100, 0.0, 1.0) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); tdigest_sum ------------- (1 row) SELECT tdigest_sum(v, 1::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0), (NULL)) AS t(v); tdigest_sum ------------- 1 (1 row) tvondra-tdigest-c0af713/test/expected/trimmed_sum_overflow.out000066400000000000000000000135451525543331000250300ustar00rootroot00000000000000-- Use terse verbosity, so that the expected output does not depend on the -- error context, which differs between PostgreSQL versions. \set VERBOSITY terse -- Reproducer: tdigest_trimmed_agg() accumulated the trimmed sum by adding -- (mean * count) for all the centroids in the range, and the trimmed average -- was then calculated by dividing that sum by the number of items. -- -- But the sum may overflow to infinity even when the average is perfectly -- representable. The average of 1e307 values is 1e307, yet the sum of a -- hundred of them is not a float8. The average was then reported as -- Infinity, which is simply a wrong answer - the average of a digest is -- always within the range of the centroid means, so it can never overflow. -- -- And once one partial sum saturates to +Infinity and another one to -- -Infinity, the accumulator turns into NaN, so a digest with large values -- of both signs reported NaN for both the sum and the average. -- -- The sum, unlike the average, may legitimately exceed the float8 range. -- Report that as an error, just like the regular float8 arithmetic does, -- instead of silently returning infinity. -- -- query before after -- tdigest_digest_avg((1e307, 100)) Infinity 1e+307 -- tdigest_digest_sum((1e307, 100)) Infinity ERROR -- tdigest_digest_avg((-1e308,2) (1e308,2)) NaN 0 -- tdigest_digest_sum((-1e308,2) (1e308,2)) NaN 0 -- -- This affects tdigest_digest_sum() and tdigest_digest_avg() as well as the -- tdigest_sum() and tdigest_avg() aggregates, i.e. anything going through -- tdigest_trimmed_agg(). -- The average is representable, the sum is not. The average has to be -- 1e307, the sum has to be reported as an overflow. SELECT tdigest_digest_avg('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest); tdigest_digest_avg -------------------- 1e+307 (1 row) SELECT tdigest_digest_sum('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest); ERROR: value out of range: overflow -- Trimming does not change that, as long as enough items remain. SELECT tdigest_digest_avg('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest, 0.25, 0.75); tdigest_digest_avg -------------------- 1e+307 (1 row) SELECT tdigest_digest_sum('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest, 0.25, 0.75); ERROR: value out of range: overflow -- Trim it down to a single item, and even the sum fits again. SELECT tdigest_digest_avg('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest, 0.5, 0.51); tdigest_digest_avg -------------------- 1e+307 (1 row) SELECT tdigest_digest_sum('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest, 0.5, 0.51); tdigest_digest_sum -------------------- 1e+307 (1 row) -- Large values of both signs, cancelling out exactly. Both the average and -- the sum are 0, the accumulator used to end up as NaN. SELECT tdigest_digest_avg('flags 1 count 4 compression 10 centroids 2 (-1e308, 2) (1e308, 2)'::tdigest); tdigest_digest_avg -------------------- 0 (1 row) SELECT tdigest_digest_sum('flags 1 count 4 compression 10 centroids 2 (-1e308, 2) (1e308, 2)'::tdigest); tdigest_digest_sum -------------------- 0 (1 row) -- The same, but with the values cancelling only partially. SELECT tdigest_digest_avg('flags 1 count 4 compression 10 centroids 2 (-1e308, 1) (1e308, 3)'::tdigest); tdigest_digest_avg -------------------- 5e+307 (1 row) SELECT tdigest_digest_sum('flags 1 count 4 compression 10 centroids 2 (-1e308, 1) (1e308, 3)'::tdigest); ERROR: value out of range: overflow -- Trim the negative values away, and only the overflowing part remains. SELECT tdigest_digest_avg('flags 1 count 4 compression 10 centroids 2 (-1e308, 2) (1e308, 2)'::tdigest, 0.5, 1.0); tdigest_digest_avg -------------------- 1e+308 (1 row) SELECT tdigest_digest_sum('flags 1 count 4 compression 10 centroids 2 (-1e308, 2) (1e308, 2)'::tdigest, 0.5, 1.0); ERROR: value out of range: overflow -- The trimmed aggregates on the value/count API go through a different code -- path (the aggregate final functions), so check those too. SELECT tdigest_avg(1e307::float8, 200::bigint, 10, 0.0, 1.0); tdigest_avg ------------------------ 9.999999999999999e+306 (1 row) SELECT tdigest_sum(1e307::float8, 200::bigint, 10, 0.0, 1.0); ERROR: value out of range: overflow -- And the plain value API. SELECT tdigest_avg(v, 10, 0.0, 1.0) FROM (VALUES (1e308::float8), (1e308::float8)) t(v); tdigest_avg ------------- 1e+308 (1 row) SELECT tdigest_sum(v, 10, 0.0, 1.0) FROM (VALUES (1e308::float8), (1e308::float8)) t(v); ERROR: value out of range: overflow SELECT tdigest_avg(v, 10, 0.0, 1.0) FROM (VALUES (-1e308::float8), (1e308::float8)) t(v); tdigest_avg ------------- 0 (1 row) SELECT tdigest_sum(v, 10, 0.0, 1.0) FROM (VALUES (-1e308::float8), (1e308::float8)) t(v); tdigest_sum ------------- 0 (1 row) -- Ordinary values are not affected in any way, and an empty range still -- returns NULL. SELECT tdigest_digest_avg('flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest, 0.1, 0.9); tdigest_digest_avg -------------------- 22.083333333333336 (1 row) SELECT tdigest_digest_sum('flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest, 0.1, 0.9); tdigest_digest_sum -------------------- 530 (1 row) SELECT tdigest_digest_avg('flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest, 0.5, 0.5); tdigest_digest_avg -------------------- (1 row) SELECT tdigest_digest_sum('flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest, 0.5, 0.5); tdigest_digest_sum -------------------- (1 row) tvondra-tdigest-c0af713/test/expected/trimmed_sum_overflow_0.out000066400000000000000000000135041525543331000252420ustar00rootroot00000000000000-- Use terse verbosity, so that the expected output does not depend on the -- error context, which differs between PostgreSQL versions. \set VERBOSITY terse -- Reproducer: tdigest_trimmed_agg() accumulated the trimmed sum by adding -- (mean * count) for all the centroids in the range, and the trimmed average -- was then calculated by dividing that sum by the number of items. -- -- But the sum may overflow to infinity even when the average is perfectly -- representable. The average of 1e307 values is 1e307, yet the sum of a -- hundred of them is not a float8. The average was then reported as -- Infinity, which is simply a wrong answer - the average of a digest is -- always within the range of the centroid means, so it can never overflow. -- -- And once one partial sum saturates to +Infinity and another one to -- -Infinity, the accumulator turns into NaN, so a digest with large values -- of both signs reported NaN for both the sum and the average. -- -- The sum, unlike the average, may legitimately exceed the float8 range. -- Report that as an error, just like the regular float8 arithmetic does, -- instead of silently returning infinity. -- -- query before after -- tdigest_digest_avg((1e307, 100)) Infinity 1e+307 -- tdigest_digest_sum((1e307, 100)) Infinity ERROR -- tdigest_digest_avg((-1e308,2) (1e308,2)) NaN 0 -- tdigest_digest_sum((-1e308,2) (1e308,2)) NaN 0 -- -- This affects tdigest_digest_sum() and tdigest_digest_avg() as well as the -- tdigest_sum() and tdigest_avg() aggregates, i.e. anything going through -- tdigest_trimmed_agg(). -- The average is representable, the sum is not. The average has to be -- 1e307, the sum has to be reported as an overflow. SELECT tdigest_digest_avg('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest); tdigest_digest_avg -------------------- 1e+307 (1 row) SELECT tdigest_digest_sum('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest); ERROR: value out of range: overflow -- Trimming does not change that, as long as enough items remain. SELECT tdigest_digest_avg('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest, 0.25, 0.75); tdigest_digest_avg -------------------- 1e+307 (1 row) SELECT tdigest_digest_sum('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest, 0.25, 0.75); ERROR: value out of range: overflow -- Trim it down to a single item, and even the sum fits again. SELECT tdigest_digest_avg('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest, 0.5, 0.51); tdigest_digest_avg -------------------- 1e+307 (1 row) SELECT tdigest_digest_sum('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest, 0.5, 0.51); tdigest_digest_sum -------------------- 1e+307 (1 row) -- Large values of both signs, cancelling out exactly. Both the average and -- the sum are 0, the accumulator used to end up as NaN. SELECT tdigest_digest_avg('flags 1 count 4 compression 10 centroids 2 (-1e308, 2) (1e308, 2)'::tdigest); tdigest_digest_avg -------------------- 0 (1 row) SELECT tdigest_digest_sum('flags 1 count 4 compression 10 centroids 2 (-1e308, 2) (1e308, 2)'::tdigest); tdigest_digest_sum -------------------- 0 (1 row) -- The same, but with the values cancelling only partially. SELECT tdigest_digest_avg('flags 1 count 4 compression 10 centroids 2 (-1e308, 1) (1e308, 3)'::tdigest); tdigest_digest_avg -------------------- 5e+307 (1 row) SELECT tdigest_digest_sum('flags 1 count 4 compression 10 centroids 2 (-1e308, 1) (1e308, 3)'::tdigest); ERROR: value out of range: overflow -- Trim the negative values away, and only the overflowing part remains. SELECT tdigest_digest_avg('flags 1 count 4 compression 10 centroids 2 (-1e308, 2) (1e308, 2)'::tdigest, 0.5, 1.0); tdigest_digest_avg -------------------- 1e+308 (1 row) SELECT tdigest_digest_sum('flags 1 count 4 compression 10 centroids 2 (-1e308, 2) (1e308, 2)'::tdigest, 0.5, 1.0); ERROR: value out of range: overflow -- The trimmed aggregates on the value/count API go through a different code -- path (the aggregate final functions), so check those too. SELECT tdigest_avg(1e307::float8, 200::bigint, 10, 0.0, 1.0); tdigest_avg ------------- 1e+307 (1 row) SELECT tdigest_sum(1e307::float8, 200::bigint, 10, 0.0, 1.0); ERROR: value out of range: overflow -- And the plain value API. SELECT tdigest_avg(v, 10, 0.0, 1.0) FROM (VALUES (1e308::float8), (1e308::float8)) t(v); tdigest_avg ------------- 1e+308 (1 row) SELECT tdigest_sum(v, 10, 0.0, 1.0) FROM (VALUES (1e308::float8), (1e308::float8)) t(v); ERROR: value out of range: overflow SELECT tdigest_avg(v, 10, 0.0, 1.0) FROM (VALUES (-1e308::float8), (1e308::float8)) t(v); tdigest_avg ------------- 0 (1 row) SELECT tdigest_sum(v, 10, 0.0, 1.0) FROM (VALUES (-1e308::float8), (1e308::float8)) t(v); tdigest_sum ------------- 0 (1 row) -- Ordinary values are not affected in any way, and an empty range still -- returns NULL. SELECT tdigest_digest_avg('flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest, 0.1, 0.9); tdigest_digest_avg -------------------- 22.0833333333333 (1 row) SELECT tdigest_digest_sum('flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest, 0.1, 0.9); tdigest_digest_sum -------------------- 530 (1 row) SELECT tdigest_digest_avg('flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest, 0.5, 0.5); tdigest_digest_avg -------------------- (1 row) SELECT tdigest_digest_sum('flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest, 0.5, 0.5); tdigest_digest_sum -------------------- (1 row) tvondra-tdigest-c0af713/test/expected/trimmed_sum_overflow_1.out000066400000000000000000000135041525543331000252430ustar00rootroot00000000000000-- Use terse verbosity, so that the expected output does not depend on the -- error context, which differs between PostgreSQL versions. \set VERBOSITY terse -- Reproducer: tdigest_trimmed_agg() accumulated the trimmed sum by adding -- (mean * count) for all the centroids in the range, and the trimmed average -- was then calculated by dividing that sum by the number of items. -- -- But the sum may overflow to infinity even when the average is perfectly -- representable. The average of 1e307 values is 1e307, yet the sum of a -- hundred of them is not a float8. The average was then reported as -- Infinity, which is simply a wrong answer - the average of a digest is -- always within the range of the centroid means, so it can never overflow. -- -- And once one partial sum saturates to +Infinity and another one to -- -Infinity, the accumulator turns into NaN, so a digest with large values -- of both signs reported NaN for both the sum and the average. -- -- The sum, unlike the average, may legitimately exceed the float8 range. -- Report that as an error, just like the regular float8 arithmetic does, -- instead of silently returning infinity. -- -- query before after -- tdigest_digest_avg((1e307, 100)) Infinity 1e+307 -- tdigest_digest_sum((1e307, 100)) Infinity ERROR -- tdigest_digest_avg((-1e308,2) (1e308,2)) NaN 0 -- tdigest_digest_sum((-1e308,2) (1e308,2)) NaN 0 -- -- This affects tdigest_digest_sum() and tdigest_digest_avg() as well as the -- tdigest_sum() and tdigest_avg() aggregates, i.e. anything going through -- tdigest_trimmed_agg(). -- The average is representable, the sum is not. The average has to be -- 1e307, the sum has to be reported as an overflow. SELECT tdigest_digest_avg('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest); tdigest_digest_avg -------------------- 1e+307 (1 row) SELECT tdigest_digest_sum('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest); ERROR: value out of range: overflow -- Trimming does not change that, as long as enough items remain. SELECT tdigest_digest_avg('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest, 0.25, 0.75); tdigest_digest_avg -------------------- 1e+307 (1 row) SELECT tdigest_digest_sum('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest, 0.25, 0.75); ERROR: value out of range: overflow -- Trim it down to a single item, and even the sum fits again. SELECT tdigest_digest_avg('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest, 0.5, 0.51); tdigest_digest_avg -------------------- 1e+307 (1 row) SELECT tdigest_digest_sum('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest, 0.5, 0.51); tdigest_digest_sum -------------------- 1e+307 (1 row) -- Large values of both signs, cancelling out exactly. Both the average and -- the sum are 0, the accumulator used to end up as NaN. SELECT tdigest_digest_avg('flags 1 count 4 compression 10 centroids 2 (-1e308, 2) (1e308, 2)'::tdigest); tdigest_digest_avg -------------------- 0 (1 row) SELECT tdigest_digest_sum('flags 1 count 4 compression 10 centroids 2 (-1e308, 2) (1e308, 2)'::tdigest); tdigest_digest_sum -------------------- 0 (1 row) -- The same, but with the values cancelling only partially. SELECT tdigest_digest_avg('flags 1 count 4 compression 10 centroids 2 (-1e308, 1) (1e308, 3)'::tdigest); tdigest_digest_avg -------------------- 5e+307 (1 row) SELECT tdigest_digest_sum('flags 1 count 4 compression 10 centroids 2 (-1e308, 1) (1e308, 3)'::tdigest); ERROR: value out of range: overflow -- Trim the negative values away, and only the overflowing part remains. SELECT tdigest_digest_avg('flags 1 count 4 compression 10 centroids 2 (-1e308, 2) (1e308, 2)'::tdigest, 0.5, 1.0); tdigest_digest_avg -------------------- 1e+308 (1 row) SELECT tdigest_digest_sum('flags 1 count 4 compression 10 centroids 2 (-1e308, 2) (1e308, 2)'::tdigest, 0.5, 1.0); ERROR: value out of range: overflow -- The trimmed aggregates on the value/count API go through a different code -- path (the aggregate final functions), so check those too. SELECT tdigest_avg(1e307::float8, 200::bigint, 10, 0.0, 1.0); tdigest_avg ------------- 1e+307 (1 row) SELECT tdigest_sum(1e307::float8, 200::bigint, 10, 0.0, 1.0); ERROR: value out of range: overflow -- And the plain value API. SELECT tdigest_avg(v, 10, 0.0, 1.0) FROM (VALUES (1e308::float8), (1e308::float8)) t(v); tdigest_avg ------------- 1e+308 (1 row) SELECT tdigest_sum(v, 10, 0.0, 1.0) FROM (VALUES (1e308::float8), (1e308::float8)) t(v); ERROR: value out of range: overflow SELECT tdigest_avg(v, 10, 0.0, 1.0) FROM (VALUES (-1e308::float8), (1e308::float8)) t(v); tdigest_avg ------------- 0 (1 row) SELECT tdigest_sum(v, 10, 0.0, 1.0) FROM (VALUES (-1e308::float8), (1e308::float8)) t(v); tdigest_sum ------------- 0 (1 row) -- Ordinary values are not affected in any way, and an empty range still -- returns NULL. SELECT tdigest_digest_avg('flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest, 0.1, 0.9); tdigest_digest_avg -------------------- 22.083333333333336 (1 row) SELECT tdigest_digest_sum('flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest, 0.1, 0.9); tdigest_digest_sum -------------------- 530 (1 row) SELECT tdigest_digest_avg('flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest, 0.5, 0.5); tdigest_digest_avg -------------------- (1 row) SELECT tdigest_digest_sum('flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest, 0.5, 0.5); tdigest_digest_sum -------------------- (1 row) tvondra-tdigest-c0af713/test/expected/unsorted_centroids.out000066400000000000000000000427111525543331000244720ustar00rootroot00000000000000-- The centroids of a t-digest are expected to be sorted by mean, but a digest -- with the centroids in an arbitrary order is a perfectly valid value - the -- incremental API can leave digests uncompacted and unsorted with compact=false, -- and those may get stored on disk, dumped, replicated, etc. The input functions -- therefore accept unsorted digests, and all the places that need the sorted -- order are expected to do the sort themselves. -- -- This checks that really is the case - for each digest we build a couple of -- variants with the centroids reordered (reversed, and then shuffled in a -- deterministic but arbitrary way), and then compare the results of -- -- tdigest_percentile() (both the scalar and the array variant) -- tdigest_percentile_of() (both the scalar and the array variant) -- tdigest_sum() -- tdigest_avg() -- tdigest_digest_sum() -- tdigest_digest_avg() -- -- to the results for the original (sorted) digest. The results have to match -- exactly, not just approximately - reordering the centroids does not lose any -- information, so the sort has to reconstruct exactly the same digest. \set VERBOSITY terse SET extra_float_digits = 0; -- Decompose a digest into individual centroids. The digest has no API to do -- this directly, so we go through the text representation. The functions set -- extra_float_digits, so that the means survive the round trip exactly. CREATE FUNCTION tdigest_unsorted_centroids(d tdigest) RETURNS TABLE (ord bigint, mean double precision, cnt bigint) LANGUAGE sql SET extra_float_digits = 3 AS $$ SELECT o, split_part(btrim(m[1], ' ()'), ',', 1)::double precision, split_part(btrim(m[1], ' ()'), ',', 2)::bigint FROM regexp_matches(d::text, ' \([^)]*\)', 'g') WITH ORDINALITY AS x(m, o); $$; -- Rebuild the digest with the centroids in a different order. Seed 0 simply -- reverses the centroids (so that the means are in a descending order), any -- other seed shuffles them - the order is arbitrary, but it's derived from -- md5() so it's stable and does not depend on the PRNG of the server. CREATE FUNCTION tdigest_unsorted_reorder(d tdigest, seed int) RETURNS tdigest LANGUAGE sql SET extra_float_digits = 3 AS $$ SELECT (rtrim(substring(d::text FROM '^[^(]*')) || coalesce(string_agg(m[1], '' ORDER BY CASE WHEN seed = 0 THEN -o ELSE 0 END, md5(seed::text || ':' || o::text)), ''))::tdigest FROM regexp_matches(d::text, ' \([^)]*\)', 'g') WITH ORDINALITY AS x(m, o); $$; -- Are the centroids of the digest sorted by mean? CREATE FUNCTION tdigest_unsorted_is_sorted(d tdigest) RETURNS boolean LANGUAGE sql AS $$ SELECT coalesce(bool_and(mean >= prev_mean), true) FROM (SELECT mean, lag(mean) OVER (ORDER BY ord) AS prev_mean FROM tdigest_unsorted_centroids(d)) x WHERE prev_mean IS NOT NULL; $$; -- The digests to check. A couple of hand-written ones first, so that we know -- exactly what centroids they consist of. CREATE TABLE tdigest_unsorted_digests (id int, descr text, d tdigest); INSERT INTO tdigest_unsorted_digests VALUES (1, 'two centroids', 'flags 1 count 10 compression 10000 centroids 2 (0, 7) (100, 3)'), (2, 'three centroids, mixed counts', 'flags 1 count 12 compression 10000 centroids 3 (0, 3) (50, 4) (100, 5)'), (3, 'negative and positive means', 'flags 1 count 8 compression 10000 centroids 4 (-100, 2) (-1, 2) (1, 2) (100, 2)'), (4, 'repeated means (the sort has to rebalance those)', 'flags 1 count 12 compression 10000 centroids 5 (0, 1) (50, 3) (50, 4) (50, 3) (100, 1)'), (5, 'a large centroid in the middle', 'flags 1 count 22 compression 10000 centroids 3 (0, 1) (10, 20) (20, 1)'), (6, 'a single centroid', 'flags 1 count 10 compression 10000 centroids 1 (50, 10)'); -- And then digests built by the aggregate from actual data. The first one -- retains singleton centroids after final compaction; the others merge input -- values with different compression settings. INSERT INTO tdigest_unsorted_digests SELECT 7, 'built from 1000 distinct values, no centroid merging', tdigest(i::double precision, 10000) FROM generate_series(1, 1000) s(i); INSERT INTO tdigest_unsorted_digests SELECT 8, 'built from 10000 rows, compression 100', tdigest((i % 1000)::double precision, 100) FROM generate_series(1, 10000) s(i); INSERT INTO tdigest_unsorted_digests SELECT 9, 'built from 10000 rows, compression 10', tdigest((i % 1000)::double precision, 10) FROM generate_series(1, 10000) s(i); INSERT INTO tdigest_unsorted_digests SELECT 10, 'built from 10000 random values, compression 100', tdigest(1000 * v, 100) FROM prng(10000) s(v); INSERT INTO tdigest_unsorted_digests SELECT 11, 'built from 10000 values with duplicities, compression 50', tdigest((10 * v)::int::double precision, 50) FROM prng(10000) s(v); -- All of them have to be sorted, otherwise the whole test is pointless. SELECT id, descr FROM tdigest_unsorted_digests WHERE NOT tdigest_unsorted_is_sorted(d) ORDER BY id; id | descr ----+------- (0 rows) -- Build the variants. Variant 0 is the original (sorted) digest, variant 1 has -- the centroids in the reverse order, and variants 2-6 have them shuffled. CREATE TABLE tdigest_unsorted_variants (id int, variant int, d tdigest); INSERT INTO tdigest_unsorted_variants SELECT id, 0, d FROM tdigest_unsorted_digests; INSERT INTO tdigest_unsorted_variants SELECT id, k + 1, tdigest_unsorted_reorder(d, k) FROM tdigest_unsorted_digests, generate_series(0, 5) s(k); -- The reordering must not change the contents of the digest in any way - the -- header has to be the same, and the centroids have to be the very same -- multiset, just in a different order. So both the "differs" columns have to -- be 0. The last column says how many of the variants really ended up with -- the centroids out of order - it's less than the number of variants for the -- digests with very few centroids (and it's 0 for the single centroid digest, -- which can't be unsorted at all). SELECT v.id, g.descr, count(*) AS variants, count(*) FILTER (WHERE substring(v.d::text FROM '^[^(]*') IS DISTINCT FROM substring(g.d::text FROM '^[^(]*')) AS header_differs, count(*) FILTER (WHERE EXISTS (SELECT mean, cnt FROM tdigest_unsorted_centroids(v.d) EXCEPT ALL SELECT mean, cnt FROM tdigest_unsorted_centroids(g.d)) OR EXISTS (SELECT mean, cnt FROM tdigest_unsorted_centroids(g.d) EXCEPT ALL SELECT mean, cnt FROM tdigest_unsorted_centroids(v.d))) AS centroids_differ, count(*) FILTER (WHERE NOT tdigest_unsorted_is_sorted(v.d)) AS actually_unsorted FROM tdigest_unsorted_variants v JOIN tdigest_unsorted_digests g ON (g.id = v.id) WHERE v.variant > 0 GROUP BY v.id, g.descr ORDER BY v.id; id | descr | variants | header_differs | centroids_differ | actually_unsorted ----+----------------------------------------------------------+----------+----------------+------------------+------------------- 1 | two centroids | 6 | 0 | 0 | 4 2 | three centroids, mixed counts | 6 | 0 | 0 | 5 3 | negative and positive means | 6 | 0 | 0 | 6 4 | repeated means (the sort has to rebalance those) | 6 | 0 | 0 | 6 5 | a large centroid in the middle | 6 | 0 | 0 | 5 6 | a single centroid | 6 | 0 | 0 | 0 7 | built from 1000 distinct values, no centroid merging | 6 | 0 | 0 | 6 8 | built from 10000 rows, compression 100 | 6 | 0 | 0 | 6 9 | built from 10000 rows, compression 10 | 6 | 0 | 0 | 6 10 | built from 10000 random values, compression 100 | 6 | 0 | 0 | 6 11 | built from 10000 values with duplicities, compression 50 | 6 | 0 | 0 | 6 (11 rows) -- The percentiles / values / trim ranges to probe the digests with. The digests -- cover fairly different ranges of values, but that does not matter - we only -- compare the results to each other, not to some expected value. CREATE TABLE tdigest_unsorted_percentiles (id int, p double precision); INSERT INTO tdigest_unsorted_percentiles SELECT i, i / 20.0 FROM generate_series(0, 20) s(i); CREATE TABLE tdigest_unsorted_values (id int, v double precision); INSERT INTO tdigest_unsorted_values (id, v) SELECT row_number() OVER (ORDER BY v), v FROM unnest(ARRAY[-1000, -100.5, -100, -50, -1, -0.5, 0, 0.5, 1, 10, 20, 49.5, 50, 99.5, 100, 250, 500, 750, 999, 1000, 5000]::double precision[]) AS v; CREATE TABLE tdigest_unsorted_trims (id int, low double precision, high double precision); INSERT INTO tdigest_unsorted_trims VALUES (1, 0.0, 1.0), (2, 0.1, 0.9), (3, 0.0, 0.5), (4, 0.5, 1.0), (5, 0.25, 0.75), (6, 0.05, 0.15), (7, 0.9, 0.95), (8, 0.33, 0.34); -- Evaluate all the functions for all the digests / variants / probes. CREATE TABLE tdigest_unsorted_results (func text, id int, variant int, probe int, res double precision); INSERT INTO tdigest_unsorted_results SELECT 'tdigest_percentile', v.id, v.variant, p.id, tdigest_percentile(v.d, p.p) FROM tdigest_unsorted_variants v, tdigest_unsorted_percentiles p GROUP BY v.id, v.variant, p.id, p.p; INSERT INTO tdigest_unsorted_results SELECT 'tdigest_percentile (array)', r.id, r.variant, o::int, x FROM (SELECT v.id, v.variant, tdigest_percentile(v.d, p.arr) AS res FROM tdigest_unsorted_variants v, (SELECT array_agg(p ORDER BY id) AS arr FROM tdigest_unsorted_percentiles) p GROUP BY v.id, v.variant, p.arr) r, unnest(r.res) WITH ORDINALITY AS u(x, o); INSERT INTO tdigest_unsorted_results SELECT 'tdigest_percentile_of', v.id, v.variant, p.id, tdigest_percentile_of(v.d, p.v) FROM tdigest_unsorted_variants v, tdigest_unsorted_values p GROUP BY v.id, v.variant, p.id, p.v; INSERT INTO tdigest_unsorted_results SELECT 'tdigest_percentile_of (array)', r.id, r.variant, o::int, x FROM (SELECT v.id, v.variant, tdigest_percentile_of(v.d, p.arr) AS res FROM tdigest_unsorted_variants v, (SELECT array_agg(v ORDER BY id) AS arr FROM tdigest_unsorted_values) p GROUP BY v.id, v.variant, p.arr) r, unnest(r.res) WITH ORDINALITY AS u(x, o); INSERT INTO tdigest_unsorted_results SELECT 'tdigest_sum', v.id, v.variant, t.id, tdigest_sum(v.d, t.low, t.high) FROM tdigest_unsorted_variants v, tdigest_unsorted_trims t GROUP BY v.id, v.variant, t.id, t.low, t.high; INSERT INTO tdigest_unsorted_results SELECT 'tdigest_avg', v.id, v.variant, t.id, tdigest_avg(v.d, t.low, t.high) FROM tdigest_unsorted_variants v, tdigest_unsorted_trims t GROUP BY v.id, v.variant, t.id, t.low, t.high; -- the same two, but as plain functions operating on a single digest INSERT INTO tdigest_unsorted_results SELECT 'tdigest_digest_sum', v.id, v.variant, t.id, tdigest_digest_sum(v.d, t.low, t.high) FROM tdigest_unsorted_variants v, tdigest_unsorted_trims t; INSERT INTO tdigest_unsorted_results SELECT 'tdigest_digest_avg', v.id, v.variant, t.id, tdigest_digest_avg(v.d, t.low, t.high) FROM tdigest_unsorted_variants v, tdigest_unsorted_trims t; -- How many results did we compare, and how many of them do not match the -- result for the sorted digest? The number of mismatches has to be 0. SELECT r.func, count(*) AS comparisons, count(*) FILTER (WHERE r.res IS DISTINCT FROM s.res) AS mismatches FROM tdigest_unsorted_results r JOIN tdigest_unsorted_results s ON (s.func = r.func AND s.id = r.id AND s.probe = r.probe AND s.variant = 0) WHERE r.variant > 0 GROUP BY r.func ORDER BY r.func; func | comparisons | mismatches -------------------------------+-------------+------------ tdigest_avg | 528 | 0 tdigest_digest_avg | 528 | 0 tdigest_digest_sum | 528 | 0 tdigest_percentile | 1386 | 0 tdigest_percentile (array) | 1386 | 0 tdigest_percentile_of | 1386 | 0 tdigest_percentile_of (array) | 1386 | 0 tdigest_sum | 528 | 0 (8 rows) -- And the details of the mismatches, so that a failure is readable. This has -- to produce no rows at all. SELECT r.func, r.id, g.descr, r.variant, r.probe, s.res AS sorted, r.res AS unsorted FROM tdigest_unsorted_results r JOIN tdigest_unsorted_results s ON (s.func = r.func AND s.id = r.id AND s.probe = r.probe AND s.variant = 0) JOIN tdigest_unsorted_digests g ON (g.id = r.id) WHERE r.variant > 0 AND r.res IS DISTINCT FROM s.res ORDER BY r.func, r.id, r.variant, r.probe; func | id | descr | variant | probe | sorted | unsorted ------+----+-------+---------+-------+--------+---------- (0 rows) -- Finally a couple of explicit checks on a small digest with the centroids in -- the reverse order, spelled out - just to make sure the comparison above is -- not comparing something trivially identical. SELECT tdigest_unsorted_reorder('flags 1 count 12 compression 10000 centroids 3 (0, 3) (50, 4) (100, 5)'::tdigest, 0); tdigest_unsorted_reorder ------------------------------------------------------------------------ flags 1 count 12 compression 10000 centroids 3 (100, 5) (50, 4) (0, 3) (1 row) WITH d(descr, d) AS ( VALUES ('sorted', 'flags 1 count 12 compression 10000 centroids 3 (0, 3) (50, 4) (100, 5)'::tdigest), ('reversed', 'flags 1 count 12 compression 10000 centroids 3 (100, 5) (50, 4) (0, 3)'::tdigest) ) SELECT descr, tdigest_percentile(d, 0.25) AS "percentile 0.25", tdigest_percentile(d, 0.75) AS "percentile 0.75", tdigest_percentile_of(d, 25.0) AS "percentile of 25", tdigest_percentile_of(d, 75.0) AS "percentile of 75", tdigest_sum(d, 0.1, 0.9) AS "sum 0.1-0.9", tdigest_avg(d, 0.1, 0.9) AS "avg 0.1-0.9" FROM d GROUP BY descr ORDER BY descr; descr | percentile 0.25 | percentile 0.75 | percentile of 25 | percentile of 75 | sum 0.1-0.9 | avg 0.1-0.9 ----------+------------------+------------------+-------------------+-------------------+-------------+------------- reversed | 21.4285714285714 | 94.4444444444444 | 0.270833333333333 | 0.604166666666667 | 600 | 60 sorted | 21.4285714285714 | 94.4444444444444 | 0.270833333333333 | 0.604166666666667 | 600 | 60 (2 rows) WITH d(descr, d) AS ( VALUES ('sorted', 'flags 1 count 12 compression 10000 centroids 3 (0, 3) (50, 4) (100, 5)'::tdigest), ('reversed', 'flags 1 count 12 compression 10000 centroids 3 (100, 5) (50, 4) (0, 3)'::tdigest) ) SELECT descr, tdigest_digest_sum(d, 0.1, 0.9) AS "digest_sum 0.1-0.9", tdigest_digest_avg(d, 0.1, 0.9) AS "digest_avg 0.1-0.9" FROM d ORDER BY descr; descr | digest_sum 0.1-0.9 | digest_avg 0.1-0.9 ----------+--------------------+-------------------- reversed | 600 | 60 sorted | 600 | 60 (2 rows) -- The old on-disk format, where the centroids store sums instead of means, is -- accepted with the centroids unsorted too. And the sums being sorted does not -- mean the means are - all three digests below hold the very same centroids -- (means 20, 50 and 200, with counts 5, 6 and 1). WITH d(descr, d) AS ( VALUES ('sorted by mean', 'flags 0 count 12 compression 10000 centroids 3 (100, 5) (300, 6) (200, 1)'::tdigest), ('reversed', 'flags 0 count 12 compression 10000 centroids 3 (200, 1) (300, 6) (100, 5)'::tdigest), ('sorted by sum, not by mean', 'flags 0 count 12 compression 10000 centroids 3 (100, 5) (200, 1) (300, 6)'::tdigest) ) SELECT descr, tdigest_digest_sum(d, 0.1, 0.9) AS "digest_sum 0.1-0.9", tdigest_digest_avg(d, 0.1, 0.9) AS "digest_avg 0.1-0.9" FROM d ORDER BY descr; descr | digest_sum 0.1-0.9 | digest_avg 0.1-0.9 ----------------------------+--------------------+-------------------- reversed | 380 | 38 sorted by mean | 380 | 38 sorted by sum, not by mean | 380 | 38 (3 rows) DROP TABLE tdigest_unsorted_results; DROP TABLE tdigest_unsorted_trims; DROP TABLE tdigest_unsorted_values; DROP TABLE tdigest_unsorted_percentiles; DROP TABLE tdigest_unsorted_variants; DROP TABLE tdigest_unsorted_digests; DROP FUNCTION tdigest_unsorted_is_sorted(tdigest); DROP FUNCTION tdigest_unsorted_reorder(tdigest, int); DROP FUNCTION tdigest_unsorted_centroids(tdigest); tvondra-tdigest-c0af713/test/expected/value_checks.out000066400000000000000000000060221525543331000232040ustar00rootroot00000000000000-- Unlike percentiles passed to tdigest_percentile(), non-finite hypothetical -- values passed to tdigest_percentile_of() are accepted and mapped to fixed -- results. Input values stored in the digest must still be finite. \set VERBOSITY terse -- hypothetical values map to NaN, 0, and 1, respectively SELECT tdigest_percentile_of(1.0::double precision, 10, 'NaN'::double precision); tdigest_percentile_of ----------------------- NaN (1 row) SELECT tdigest_percentile_of(1.0::double precision, 10, '-infinity'::double precision); tdigest_percentile_of ----------------------- 0 (1 row) SELECT tdigest_percentile_of(1.0::double precision, 10, 'infinity'::double precision); tdigest_percentile_of ----------------------- 1 (1 row) SELECT tdigest_percentile_of(1.0::double precision, 10, ARRAY['NaN', '-infinity', 'infinity']::double precision[]); tdigest_percentile_of ----------------------- {NaN,0,1} (1 row) SELECT tdigest_percentile_of('flags 1 count 3 compression 10 centroids 3 (1, 1) (2, 1) (3, 1)'::tdigest, 'NaN'::double precision); tdigest_percentile_of ----------------------- NaN (1 row) SELECT tdigest_percentile_of('flags 1 count 3 compression 10 centroids 3 (1, 1) (2, 1) (3, 1)'::tdigest, '-infinity'::double precision); tdigest_percentile_of ----------------------- 0 (1 row) SELECT tdigest_percentile_of('flags 1 count 3 compression 10 centroids 3 (1, 1) (2, 1) (3, 1)'::tdigest, 'infinity'::double precision); tdigest_percentile_of ----------------------- 1 (1 row) SELECT tdigest_percentile_of('flags 1 count 3 compression 10 centroids 3 (1, 1) (2, 1) (3, 1)'::tdigest, ARRAY['NaN', '-infinity', 'infinity']::double precision[]); tdigest_percentile_of ----------------------- {NaN,0,1} (1 row) -- the value/count API has the same handling of non-finite hypothetical values SELECT tdigest_percentile_of(1.0::double precision, 2::bigint, 10, 'NaN'::double precision); tdigest_percentile_of ----------------------- NaN (1 row) SELECT tdigest_percentile_of(1.0::double precision, 2::bigint, 10, '-infinity'::double precision); tdigest_percentile_of ----------------------- 0 (1 row) SELECT tdigest_percentile_of(1.0::double precision, 2::bigint, 10, 'infinity'::double precision); tdigest_percentile_of ----------------------- 1 (1 row) SELECT tdigest_percentile_of(1.0::double precision, 2::bigint, 10, ARRAY['NaN']::double precision[]); tdigest_percentile_of ----------------------- {NaN} (1 row) SELECT tdigest_percentile_of(1.0::double precision, 2::bigint, 10, ARRAY['-infinity']::double precision[]); tdigest_percentile_of ----------------------- {0} (1 row) SELECT tdigest_percentile_of(1.0::double precision, 2::bigint, 10, ARRAY['infinity']::double precision[]); tdigest_percentile_of ----------------------- {1} (1 row) tvondra-tdigest-c0af713/test/expected/value_count_api.out000066400000000000000000000272331525543331000237340ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('extra_float_digits', '0', false); PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; -- API select tdigest_percentile(value, count, 100, 0.95) from (values (47325940488,1), (15457695432,2), (6889790700,3), (4188763788,4), (2882932224,5), (2114815860,6), (1615194324,7), (2342114568,9), (1626471924,11), (1660755408,14), (1143728292,17), (1082582424,21), (911488284,26), (728863908,32), (654898692,40), (530198076,50), (417883440,62), (341452344,77), (274579584,95), (231921120,118), (184091820,146), (152469828,181), (125634972,224), (107059704,278), (88746120,345), (73135668,428), (61035756,531), (50683320,658), (42331824,816), (35234400,1012), (29341356,1255), (24290928,1556), (20284668,1929), (17215908,2391), (14737488,2964), (12692772,3674), (11220732,4555), (9787584,5647), (8148420,7000), (6918612,8678), (6015000,10758), (5480316,13336), (5443356,16532), (4535616,20494), (3962316,25406), (3914484,31495), (3828108,39043), (3583536,48400), (4104120,60000), (166024740,2147483647)) foo (count, value); tdigest_percentile -------------------- 33.3452131657954 (1 row) ---------------------------------------------- -- nice data set with random data (uniform) -- ---------------------------------------------- -- compression 10 WITH data AS (SELECT prng(1000) x, prng(1000, 29823218) cnt), data_expanded AS (SELECT x FROM (SELECT x, generate_series(1, (10 + 100 * cnt)::int) FROM data) foo ORDER BY random()) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) a FROM data_expanded) foo, (SELECT tdigest_percentile(x, (10 + 100 * cnt)::int, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) b FROM data) bar ) baz; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- compression 100 WITH data AS (SELECT prng(1000) x, prng(1000, 29823218) cnt), data_expanded AS (SELECT x FROM (SELECT x, generate_series(1, (10 + 100 * cnt)::int) FROM data) foo ORDER BY random()) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) a FROM data_expanded) foo, (SELECT tdigest_percentile(x, (10 + 100 * cnt)::int, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) b FROM data) bar ) baz; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- compression 1000 WITH data AS (SELECT prng(1000) x, prng(1000, 29823218) cnt), data_expanded AS (SELECT x FROM (SELECT x, generate_series(1, (10 + 100 * cnt)::int) FROM data) foo ORDER BY random()) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) a FROM data_expanded) foo, (SELECT tdigest_percentile(x, (10 + 100 * cnt)::int, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) b FROM data) bar ) baz; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- generate data CREATE TABLE test_value_count (v double precision, c int, d int); INSERT INTO test_value_count SELECT 1000 * random(), 1 + mod(i,7), mod(i,113) FROM generate_series(1,100000) s(i); ANALYZE test_value_count; -- API EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, c, 100, 0.95) AS b FROM test_value_count) foo; QUERY PLAN ------------------------------------------------------------------------------------------ Subquery Scan on foo -> Finalize Aggregate InitPlan 1 (returns $2) -> Aggregate -> Gather Workers Planned: 2 -> Nested Loop -> Parallel Seq Scan on test_value_count test_value_count_1 -> Function Scan on generate_series -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_value_count (13 rows) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, c, 100, 0.95) AS b FROM test_value_count) foo; ?column? | ?column? ----------+---------- 0.95 | t (1 row) EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percent_rank(950) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile_of(v, c, 100, 950) AS b FROM test_value_count) foo; QUERY PLAN ------------------------------------------------------------------------------------------ Subquery Scan on foo -> Finalize Aggregate InitPlan 1 (returns $2) -> Aggregate -> Gather Workers Planned: 2 -> Nested Loop -> Parallel Seq Scan on test_value_count test_value_count_1 -> Function Scan on generate_series -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_value_count (13 rows) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percent_rank(950) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile_of(v, c, 100, 950) AS b FROM test_value_count) foo; ?column? | ?column? ----------+---------- 950 | t (1 row) -- array of percentiles / values EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, c, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_value_count) foo; QUERY PLAN ------------------------------------------------------------------------------------------ Subquery Scan on foo -> ProjectSet InitPlan 1 (returns $2) -> Aggregate -> Gather Workers Planned: 2 -> Nested Loop -> Parallel Seq Scan on test_value_count test_value_count_1 -> Function Scan on generate_series -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_value_count (14 rows) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, c, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_value_count) foo; p | ?column? ------+---------- 0.0 | t 0.95 | t 0.99 | t 1.0 | t (4 rows) EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) AS p FROM d)) p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((select x.p from x)) AS a, unnest(tdigest_percentile_of(v, c, 100, ARRAY[950, 990])) AS b FROM test_value_count) foo; QUERY PLAN -------------------------------------------------------------------------------------------------- Subquery Scan on foo -> ProjectSet InitPlan 2 (returns $3) -> Aggregate -> Function Scan on unnest f SubPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Nested Loop -> Parallel Seq Scan on test_value_count test_value_count_1 -> Function Scan on generate_series -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_value_count (17 rows) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) AS p FROM d)) p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((select x.p from x)) AS a, unnest(tdigest_percentile_of(v, c, 100, ARRAY[950, 990])) AS b FROM test_value_count) foo; p | ?column? -----+---------- 950 | t 990 | t (2 rows) DROP TABLE test_value_count; tvondra-tdigest-c0af713/test/expected/value_count_api_0.out000066400000000000000000000271471525543331000241570ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('extra_float_digits', '0', false); PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; -- API select tdigest_percentile(value, count, 100, 0.95) from (values (47325940488,1), (15457695432,2), (6889790700,3), (4188763788,4), (2882932224,5), (2114815860,6), (1615194324,7), (2342114568,9), (1626471924,11), (1660755408,14), (1143728292,17), (1082582424,21), (911488284,26), (728863908,32), (654898692,40), (530198076,50), (417883440,62), (341452344,77), (274579584,95), (231921120,118), (184091820,146), (152469828,181), (125634972,224), (107059704,278), (88746120,345), (73135668,428), (61035756,531), (50683320,658), (42331824,816), (35234400,1012), (29341356,1255), (24290928,1556), (20284668,1929), (17215908,2391), (14737488,2964), (12692772,3674), (11220732,4555), (9787584,5647), (8148420,7000), (6918612,8678), (6015000,10758), (5480316,13336), (5443356,16532), (4535616,20494), (3962316,25406), (3914484,31495), (3828108,39043), (3583536,48400), (4104120,60000), (166024740,2147483647)) foo (count, value); tdigest_percentile -------------------- 33.3452131657954 (1 row) ---------------------------------------------- -- nice data set with random data (uniform) -- ---------------------------------------------- -- compression 10 WITH data AS (SELECT prng(1000) x, prng(1000, 29823218) cnt), data_expanded AS (SELECT x FROM (SELECT x, generate_series(1, (10 + 100 * cnt)::int) FROM data) foo ORDER BY random()) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) a FROM data_expanded) foo, (SELECT tdigest_percentile(x, (10 + 100 * cnt)::int, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) b FROM data) bar ) baz; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- compression 100 WITH data AS (SELECT prng(1000) x, prng(1000, 29823218) cnt), data_expanded AS (SELECT x FROM (SELECT x, generate_series(1, (10 + 100 * cnt)::int) FROM data) foo ORDER BY random()) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) a FROM data_expanded) foo, (SELECT tdigest_percentile(x, (10 + 100 * cnt)::int, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) b FROM data) bar ) baz; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- compression 1000 WITH data AS (SELECT prng(1000) x, prng(1000, 29823218) cnt), data_expanded AS (SELECT x FROM (SELECT x, generate_series(1, (10 + 100 * cnt)::int) FROM data) foo ORDER BY random()) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) a FROM data_expanded) foo, (SELECT tdigest_percentile(x, (10 + 100 * cnt)::int, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) b FROM data) bar ) baz; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- generate data CREATE TABLE test_value_count (v double precision, c int, d int); INSERT INTO test_value_count SELECT 1000 * random(), 1 + mod(i,7), mod(i,113) FROM generate_series(1,100000) s(i); ANALYZE test_value_count; -- API EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, c, 100, 0.95) AS b FROM test_value_count) foo; QUERY PLAN ------------------------------------------------------------------------------------------ Subquery Scan on foo -> Finalize Aggregate InitPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Nested Loop -> Parallel Seq Scan on test_value_count test_value_count_1 -> Function Scan on generate_series -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_value_count (13 rows) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, c, 100, 0.95) AS b FROM test_value_count) foo; ?column? | ?column? ----------+---------- 0.95 | t (1 row) EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percent_rank(950) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile_of(v, c, 100, 950) AS b FROM test_value_count) foo; QUERY PLAN ------------------------------------------------------------------------------------------ Subquery Scan on foo -> Finalize Aggregate InitPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Nested Loop -> Parallel Seq Scan on test_value_count test_value_count_1 -> Function Scan on generate_series -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_value_count (13 rows) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percent_rank(950) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile_of(v, c, 100, 950) AS b FROM test_value_count) foo; ?column? | ?column? ----------+---------- 950 | t (1 row) -- array of percentiles / values EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, c, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_value_count) foo; QUERY PLAN ------------------------------------------------------------------------------------------ Subquery Scan on foo -> ProjectSet InitPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Nested Loop -> Parallel Seq Scan on test_value_count test_value_count_1 -> Function Scan on generate_series -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_value_count (14 rows) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, c, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_value_count) foo; p | ?column? ------+---------- 0.0 | t 0.95 | t 0.99 | t 1.0 | t (4 rows) EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) AS p FROM d)) p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((select x.p from x)) AS a, unnest(tdigest_percentile_of(v, c, 100, ARRAY[950, 990])) AS b FROM test_value_count) foo; QUERY PLAN -------------------------------------------------------------------------------------------------- Subquery Scan on foo -> ProjectSet InitPlan 2 -> Aggregate -> Function Scan on unnest f SubPlan 1 -> Aggregate -> Gather Workers Planned: 2 -> Nested Loop -> Parallel Seq Scan on test_value_count test_value_count_1 -> Function Scan on generate_series -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_value_count (17 rows) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) AS p FROM d)) p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((select x.p from x)) AS a, unnest(tdigest_percentile_of(v, c, 100, ARRAY[950, 990])) AS b FROM test_value_count) foo; p | ?column? -----+---------- 950 | t 990 | t (2 rows) DROP TABLE test_value_count; tvondra-tdigest-c0af713/test/expected/value_count_api_1.out000066400000000000000000000270411525543331000241510ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('extra_float_digits', '0', false); PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; -- API select tdigest_percentile(value, count, 100, 0.95) from (values (47325940488,1), (15457695432,2), (6889790700,3), (4188763788,4), (2882932224,5), (2114815860,6), (1615194324,7), (2342114568,9), (1626471924,11), (1660755408,14), (1143728292,17), (1082582424,21), (911488284,26), (728863908,32), (654898692,40), (530198076,50), (417883440,62), (341452344,77), (274579584,95), (231921120,118), (184091820,146), (152469828,181), (125634972,224), (107059704,278), (88746120,345), (73135668,428), (61035756,531), (50683320,658), (42331824,816), (35234400,1012), (29341356,1255), (24290928,1556), (20284668,1929), (17215908,2391), (14737488,2964), (12692772,3674), (11220732,4555), (9787584,5647), (8148420,7000), (6918612,8678), (6015000,10758), (5480316,13336), (5443356,16532), (4535616,20494), (3962316,25406), (3914484,31495), (3828108,39043), (3583536,48400), (4104120,60000), (166024740,2147483647)) foo (count, value); tdigest_percentile -------------------- 33.3452131657954 (1 row) ---------------------------------------------- -- nice data set with random data (uniform) -- ---------------------------------------------- -- compression 10 WITH data AS (SELECT prng(1000) x, prng(1000, 29823218) cnt), data_expanded AS (SELECT x FROM (SELECT x, generate_series(1, (10 + 100 * cnt)::int) FROM data) foo ORDER BY random()) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) a FROM data_expanded) foo, (SELECT tdigest_percentile(x, (10 + 100 * cnt)::int, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) b FROM data) bar ) baz; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- compression 100 WITH data AS (SELECT prng(1000) x, prng(1000, 29823218) cnt), data_expanded AS (SELECT x FROM (SELECT x, generate_series(1, (10 + 100 * cnt)::int) FROM data) foo ORDER BY random()) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) a FROM data_expanded) foo, (SELECT tdigest_percentile(x, (10 + 100 * cnt)::int, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) b FROM data) bar ) baz; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- compression 1000 WITH data AS (SELECT prng(1000) x, prng(1000, 29823218) cnt), data_expanded AS (SELECT x FROM (SELECT x, generate_series(1, (10 + 100 * cnt)::int) FROM data) foo ORDER BY random()) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) a FROM data_expanded) foo, (SELECT tdigest_percentile(x, (10 + 100 * cnt)::int, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) b FROM data) bar ) baz; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- generate data CREATE TABLE test_value_count (v double precision, c int, d int); INSERT INTO test_value_count SELECT 1000 * random(), 1 + mod(i,7), mod(i,113) FROM generate_series(1,100000) s(i); ANALYZE test_value_count; -- API EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, c, 100, 0.95) AS b FROM test_value_count) foo; QUERY PLAN ------------------------------------------------------------------------------ Subquery Scan on foo CTE d -> Gather Workers Planned: 2 -> Nested Loop -> Parallel Seq Scan on test_value_count test_value_count_1 -> Function Scan on generate_series CTE x -> Aggregate -> CTE Scan on d -> Finalize Aggregate InitPlan 3 (returns $4) -> CTE Scan on x -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_value_count (17 rows) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, c, 100, 0.95) AS b FROM test_value_count) foo; ?column? | ?column? ----------+---------- 0.95 | t (1 row) EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percent_rank(950) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile_of(v, c, 100, 950) AS b FROM test_value_count) foo; QUERY PLAN ------------------------------------------------------------------------------ Subquery Scan on foo CTE d -> Gather Workers Planned: 2 -> Nested Loop -> Parallel Seq Scan on test_value_count test_value_count_1 -> Function Scan on generate_series CTE x -> Aggregate -> CTE Scan on d -> Finalize Aggregate InitPlan 3 (returns $4) -> CTE Scan on x -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_value_count (17 rows) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percent_rank(950) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile_of(v, c, 100, 950) AS b FROM test_value_count) foo; ?column? | ?column? ----------+---------- 950 | t (1 row) -- array of percentiles / values EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, c, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_value_count) foo; QUERY PLAN ------------------------------------------------------------------------------ Subquery Scan on foo CTE d -> Gather Workers Planned: 2 -> Nested Loop -> Parallel Seq Scan on test_value_count test_value_count_1 -> Function Scan on generate_series CTE x -> Aggregate -> CTE Scan on d -> ProjectSet InitPlan 3 (returns $4) -> CTE Scan on x -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_value_count (18 rows) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, c, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_value_count) foo; p | ?column? ------+---------- 0.0 | t 0.95 | t 0.99 | t 1.0 | t (4 rows) EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) AS p FROM d)) p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((select x.p from x)) AS a, unnest(tdigest_percentile_of(v, c, 100, ARRAY[950, 990])) AS b FROM test_value_count) foo; QUERY PLAN ------------------------------------------------------------------------------ Subquery Scan on foo CTE d -> Gather Workers Planned: 2 -> Nested Loop -> Parallel Seq Scan on test_value_count test_value_count_1 -> Function Scan on generate_series CTE x -> Aggregate -> Function Scan on unnest f SubPlan 2 -> Aggregate -> CTE Scan on d -> ProjectSet InitPlan 4 (returns $5) -> CTE Scan on x -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_value_count (21 rows) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) AS p FROM d)) p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((select x.p from x)) AS a, unnest(tdigest_percentile_of(v, c, 100, ARRAY[950, 990])) AS b FROM test_value_count) foo; p | ?column? -----+---------- 950 | t 990 | t (2 rows) DROP TABLE test_value_count; tvondra-tdigest-c0af713/test/expected/value_count_api_4.out000066400000000000000000000272001525543331000241510ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('extra_float_digits', '0', false); PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; -- API select tdigest_percentile(value, count, 100, 0.95) from (values (47325940488,1), (15457695432,2), (6889790700,3), (4188763788,4), (2882932224,5), (2114815860,6), (1615194324,7), (2342114568,9), (1626471924,11), (1660755408,14), (1143728292,17), (1082582424,21), (911488284,26), (728863908,32), (654898692,40), (530198076,50), (417883440,62), (341452344,77), (274579584,95), (231921120,118), (184091820,146), (152469828,181), (125634972,224), (107059704,278), (88746120,345), (73135668,428), (61035756,531), (50683320,658), (42331824,816), (35234400,1012), (29341356,1255), (24290928,1556), (20284668,1929), (17215908,2391), (14737488,2964), (12692772,3674), (11220732,4555), (9787584,5647), (8148420,7000), (6918612,8678), (6015000,10758), (5480316,13336), (5443356,16532), (4535616,20494), (3962316,25406), (3914484,31495), (3828108,39043), (3583536,48400), (4104120,60000), (166024740,2147483647)) foo (count, value); tdigest_percentile -------------------- 33.3452131657954 (1 row) ---------------------------------------------- -- nice data set with random data (uniform) -- ---------------------------------------------- -- compression 10 WITH data AS (SELECT prng(1000) x, prng(1000, 29823218) cnt), data_expanded AS (SELECT x FROM (SELECT x, generate_series(1, (10 + 100 * cnt)::int) FROM data) foo ORDER BY random()) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) a FROM data_expanded) foo, (SELECT tdigest_percentile(x, (10 + 100 * cnt)::int, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) b FROM data) bar ) baz; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- compression 100 WITH data AS (SELECT prng(1000) x, prng(1000, 29823218) cnt), data_expanded AS (SELECT x FROM (SELECT x, generate_series(1, (10 + 100 * cnt)::int) FROM data) foo ORDER BY random()) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) a FROM data_expanded) foo, (SELECT tdigest_percentile(x, (10 + 100 * cnt)::int, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) b FROM data) bar ) baz; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- compression 1000 WITH data AS (SELECT prng(1000) x, prng(1000, 29823218) cnt), data_expanded AS (SELECT x FROM (SELECT x, generate_series(1, (10 + 100 * cnt)::int) FROM data) foo ORDER BY random()) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) a FROM data_expanded) foo, (SELECT tdigest_percentile(x, (10 + 100 * cnt)::int, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) b FROM data) bar ) baz; p | ?column? | err ------+----------+----- 0.01 | t | 0.05 | t | 0.1 | t | 0.9 | t | 0.95 | t | 0.99 | t | (6 rows) -- generate data CREATE TABLE test_value_count (v double precision, c int, d int); INSERT INTO test_value_count SELECT 1000 * random(), 1 + mod(i,7), mod(i,113) FROM generate_series(1,100000) s(i); ANALYZE test_value_count; -- API EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, c, 100, 0.95) AS b FROM test_value_count) foo; QUERY PLAN ------------------------------------------------------------------------------------------ Subquery Scan on foo -> Finalize Aggregate InitPlan expr_1 -> Aggregate -> Gather Workers Planned: 2 -> Nested Loop -> Parallel Seq Scan on test_value_count test_value_count_1 -> Function Scan on generate_series -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_value_count (13 rows) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, c, 100, 0.95) AS b FROM test_value_count) foo; ?column? | ?column? ----------+---------- 0.95 | t (1 row) EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percent_rank(950) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile_of(v, c, 100, 950) AS b FROM test_value_count) foo; QUERY PLAN ------------------------------------------------------------------------------------------ Subquery Scan on foo -> Finalize Aggregate InitPlan expr_1 -> Aggregate -> Gather Workers Planned: 2 -> Nested Loop -> Parallel Seq Scan on test_value_count test_value_count_1 -> Function Scan on generate_series -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_value_count (13 rows) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percent_rank(950) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile_of(v, c, 100, 950) AS b FROM test_value_count) foo; ?column? | ?column? ----------+---------- 950 | t (1 row) -- array of percentiles / values EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, c, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_value_count) foo; QUERY PLAN ------------------------------------------------------------------------------------------ Subquery Scan on foo -> ProjectSet InitPlan expr_1 -> Aggregate -> Gather Workers Planned: 2 -> Nested Loop -> Parallel Seq Scan on test_value_count test_value_count_1 -> Function Scan on generate_series -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_value_count (14 rows) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, c, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_value_count) foo; p | ?column? ------+---------- 0.0 | t 0.95 | t 0.99 | t 1.0 | t (4 rows) EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) AS p FROM d)) p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((select x.p from x)) AS a, unnest(tdigest_percentile_of(v, c, 100, ARRAY[950, 990])) AS b FROM test_value_count) foo; QUERY PLAN -------------------------------------------------------------------------------------------------- Subquery Scan on foo -> ProjectSet InitPlan expr_1 -> Aggregate -> Function Scan on unnest f SubPlan expr_2 -> Aggregate -> Gather Workers Planned: 2 -> Nested Loop -> Parallel Seq Scan on test_value_count test_value_count_1 -> Function Scan on generate_series -> Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Seq Scan on test_value_count (17 rows) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) AS p FROM d)) p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((select x.p from x)) AS a, unnest(tdigest_percentile_of(v, c, 100, ARRAY[950, 990])) AS b FROM test_value_count) foo; p | ?column? -----+---------- 950 | t 990 | t (2 rows) DROP TABLE test_value_count; tvondra-tdigest-c0af713/test/expected/values_count_checks.out000066400000000000000000000020441525543331000245770ustar00rootroot00000000000000-- Regression test for the count check in tdigest_add_double_values_count(), -- the transition function of the tdigest_percentile_of() aggregates accepting -- an explicit number of occurrences of a value. -- -- A non-NULL count must be positive. A NULL count means one occurrence -- of the input value. \set VERBOSITY terse -- zero count SELECT tdigest_percentile_of(v, 0::bigint, 100, 1.0) FROM (VALUES (1.0)) AS t(v); ERROR: invalid count value 0, must be a positive value -- negative count SELECT tdigest_percentile_of(v, -1::bigint, 100, 1.0) FROM (VALUES (1.0)) AS t(v); ERROR: invalid count value -1, must be a positive value -- the smallest valid count has to work SELECT tdigest_percentile_of(v, 1::bigint, 100, 1.0) FROM (VALUES (1.0)) AS t(v); tdigest_percentile_of ----------------------- 0.5 (1 row) -- NULL count means a single occurrence, so it's valid too SELECT tdigest_percentile_of(v, NULL::bigint, 100, 1.0) FROM (VALUES (1.0)) AS t(v); tdigest_percentile_of ----------------------- 0.5 (1 row) tvondra-tdigest-c0af713/test/expected/values_count_generated.out000066400000000000000000000032111525543331000252720ustar00rootroot00000000000000-- Regression test for adding a value with a count exceeding BUFFER_SIZE in -- tdigest_add_double_values_count. -- -- Adding the values one by one would mean too many compactions, so counts -- larger than the buffer (BUFFER_SIZE = 10 * compression, i.e. 100 for the -- smallest compression) are added as pre-generated centroids instead. Both -- paths have to produce comparable results. -- a single distinct value, so the result is exact on both sides of the -- BUFFER_SIZE threshold SELECT tdigest_percentile_of(v, 100::bigint, 10, 1.0) FROM (VALUES (1.0)) AS t(v); tdigest_percentile_of ----------------------- 0.5 (1 row) SELECT tdigest_percentile_of(v, 101::bigint, 10, 1.0) FROM (VALUES (1.0)) AS t(v); tdigest_percentile_of ----------------------- 0.5 (1 row) SELECT tdigest_percentile_of(v, 1000000::bigint, 10, 1.0) FROM (VALUES (1.0)) AS t(v); tdigest_percentile_of ----------------------- 0.5 (1 row) -- two distinct values, with the counts at and just above the -- threshold SELECT tdigest_percentile_of(v, 100::bigint, 10, 2.0) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_percentile_of ----------------------- 0.775 (1 row) SELECT tdigest_percentile_of(v, 101::bigint, 10, 2.0) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_percentile_of ----------------------- 0.75 (1 row) -- a larger compression means a larger buffer, so the very same count now -- goes through the one-by-one path SELECT tdigest_percentile_of(v, 101::bigint, 100, 2.0) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_percentile_of ----------------------- 0.7846534653465347 (1 row) tvondra-tdigest-c0af713/test/expected/values_count_generated_0.out000066400000000000000000000032111525543331000255110ustar00rootroot00000000000000-- Regression test for adding a value with a count exceeding BUFFER_SIZE in -- tdigest_add_double_values_count. -- -- Adding the values one by one would mean too many compactions, so counts -- larger than the buffer (BUFFER_SIZE = 10 * compression, i.e. 100 for the -- smallest compression) are added as pre-generated centroids instead. Both -- paths have to produce comparable results. -- a single distinct value, so the result is exact on both sides of the -- BUFFER_SIZE threshold SELECT tdigest_percentile_of(v, 100::bigint, 10, 1.0) FROM (VALUES (1.0)) AS t(v); tdigest_percentile_of ----------------------- 0.5 (1 row) SELECT tdigest_percentile_of(v, 101::bigint, 10, 1.0) FROM (VALUES (1.0)) AS t(v); tdigest_percentile_of ----------------------- 0.5 (1 row) SELECT tdigest_percentile_of(v, 1000000::bigint, 10, 1.0) FROM (VALUES (1.0)) AS t(v); tdigest_percentile_of ----------------------- 0.5 (1 row) -- two distinct values, with the counts at and just above the -- threshold SELECT tdigest_percentile_of(v, 100::bigint, 10, 2.0) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_percentile_of ----------------------- 0.775 (1 row) SELECT tdigest_percentile_of(v, 101::bigint, 10, 2.0) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_percentile_of ----------------------- 0.75 (1 row) -- a larger compression means a larger buffer, so the very same count now -- goes through the one-by-one path SELECT tdigest_percentile_of(v, 101::bigint, 100, 2.0) FROM (VALUES (1.0), (2.0)) AS t(v); tdigest_percentile_of ----------------------- 0.784653465346535 (1 row) tvondra-tdigest-c0af713/test/expected/window_finalization.out000066400000000000000000000037421525543331000246340ustar00rootroot00000000000000-- Finalizing a prefix window must not change the state used for later -- frames. Compare all three builders with independent aggregation of the -- same ordered input. This is an equality check between two executions of -- tdigest, not a claim that its estimates equal exact percentiles. -- -- Before the fix, each builder disagrees on 62 of the 100 frames. -- After the fix, this script succeeds and leaves no database objects behind. SET max_parallel_workers_per_gather = 0; DO $$ DECLARE raw_mismatches bigint; weighted_mismatches bigint; merged_mismatches bigint; BEGIN WITH inputs AS ( SELECT i, tdigest_add(NULL::tdigest, i::double precision, 100) AS d FROM generate_series(1, 100) AS t(i) ), windows AS ( SELECT i, tdigest(i::double precision, 100) OVER w AS raw, tdigest(i::double precision, 1::bigint, 100) OVER w AS weighted, tdigest(d) OVER w AS merged FROM inputs WINDOW w AS (ORDER BY i ROWS UNBOUNDED PRECEDING) ), compared AS ( SELECT w.*, (SELECT tdigest(j::double precision, 100 ORDER BY j) FROM generate_series(1, w.i) AS t(j)) AS expected FROM windows w ) SELECT count(*) FILTER (WHERE tdigest_send(raw) IS DISTINCT FROM tdigest_send(expected)), count(*) FILTER (WHERE tdigest_send(weighted) IS DISTINCT FROM tdigest_send(expected)), count(*) FILTER (WHERE tdigest_send(merged) IS DISTINCT FROM tdigest_send(expected)) INTO raw_mismatches, weighted_mismatches, merged_mismatches FROM compared; IF raw_mismatches <> 0 OR weighted_mismatches <> 0 OR merged_mismatches <> 0 THEN RAISE EXCEPTION 'window finalization changed reusable state: raw %, weighted %, merged % mismatches', raw_mismatches, weighted_mismatches, merged_mismatches; END IF; END $$; tvondra-tdigest-c0af713/test/parallel_schedule000066400000000000000000000013261525543331000216130ustar00rootroot00000000000000test: init test: basic cast incremental conversions combine_crash count_overflow compact_no_progress generate_overflow trimmed_agg_overflow input_saturation value_checks trimmed_sum_overflow percentile_of_monotonic percentile_of_extreme percentile_roundtrip interpolation_monotonicity inverse_rank_boundaries add_overflow compression_checks null_values count_checks empty_input null_increment null_count trimmed_count_checks count_generated trimmed_null_values values_count_checks parse_checks recv_checks increment_compression array_count_checks values_count_generated unsorted_centroids null_arrays digest_is_valid window_finalization memory_usage test: copy parallel_query value_count_api trimmed_aggregates combine random tvondra-tdigest-c0af713/test/sql/000077500000000000000000000000001525543331000170155ustar00rootroot00000000000000tvondra-tdigest-c0af713/test/sql/add_overflow.sql000066400000000000000000000017451525543331000222200ustar00rootroot00000000000000-- Regression test for the count overflow in tdigest_add(), which adds a -- single value (a centroid with count 1) to the aggregate state. -- -- The total count is int64, so adding one more value to a t-digest with -- count INT64_MAX has to fail instead of wrapping around to a negative -- total. The incremental API is the simplest way to get a state with such -- a total count. \set VERBOSITY terse -- adding a single value overflows the total count SELECT tdigest_add('flags 1 count 9223372036854775807 compression 100 centroids 1 (1, 9223372036854775807)'::tdigest, 2.0); -- the same thing when adding an array of values SELECT tdigest_add('flags 1 count 9223372036854775807 compression 100 centroids 1 (1, 9223372036854775807)'::tdigest, ARRAY[2.0]); -- the last value that still fits must not fail, though SELECT tdigest_add('flags 1 count 9223372036854775806 compression 100 centroids 1 (1, 9223372036854775806)'::tdigest, 2.0); tvondra-tdigest-c0af713/test/sql/array_count_checks.sql000066400000000000000000000023171525543331000234070ustar00rootroot00000000000000-- Regression test for the count check in tdigest_add_double_array_count() -- and tdigest_add_double_array_values_count(), the transition functions of -- the aggregates accepting an explicit number of occurrences of a value and -- an array of percentiles (or values). -- -- A non-NULL count must be positive. A NULL count means one occurrence -- of the input value. \set VERBOSITY terse -- zero count SELECT tdigest_percentile(v, 0::bigint, 100, ARRAY[0.5]) FROM (VALUES (1.0)) AS t(v); SELECT tdigest_percentile_of(v, 0::bigint, 100, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v); -- negative count SELECT tdigest_percentile(v, -1::bigint, 100, ARRAY[0.5]) FROM (VALUES (1.0)) AS t(v); SELECT tdigest_percentile_of(v, -1::bigint, 100, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v); -- the smallest valid count has to work SELECT tdigest_percentile(v, 1::bigint, 100, ARRAY[0.5]) FROM (VALUES (1.0)) AS t(v); SELECT tdigest_percentile_of(v, 1::bigint, 100, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v); -- NULL count means a single occurrence, so it's valid too SELECT tdigest_percentile(v, NULL::bigint, 100, ARRAY[0.5]) FROM (VALUES (1.0)) AS t(v); SELECT tdigest_percentile_of(v, NULL::bigint, 100, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v); tvondra-tdigest-c0af713/test/sql/basic.sql000066400000000000000000000155771525543331000206360ustar00rootroot00000000000000-- validation of percentile values -- correct values SELECT tdigest_percentile(1.0, 10, 0.0); SELECT tdigest_percentile(1.0, 10, 0.1); SELECT tdigest_percentile(1.0, 10, 0.5); SELECT tdigest_percentile(1.0, 10, 0.9); SELECT tdigest_percentile(1.0, 10, 1.0); SELECT tdigest_percentile(1.0, 10, ARRAY[0.0, 0.1, 0.5, 0.9, 1.0]); -- out of range values SELECT tdigest_percentile(1.0, 10, -1.0); SELECT tdigest_percentile(1.0, 10, 1.5); SELECT tdigest_percentile(1.0, 10, ARRAY[0.0, -1.0, 0.5, 0.9, 1.0]); SELECT tdigest_percentile(1.0, 10, ARRAY[0.0, 0.1, 0.5, 0.9, 1.5]); -- NaN SELECT tdigest_percentile(1.0, 10, 'NaN'::double precision); SELECT tdigest_percentile(1.0, 10, ARRAY[0.0, 'NaN'::double precision, 0.5, 0.9, 1.0]); -- infinite values SELECT tdigest_percentile(1.0, 10, 'infinity'::double precision); SELECT tdigest_percentile(1.0, 10, '-infinity'::double precision); SELECT tdigest_percentile(1.0, 10, ARRAY[0.0, 'infinity'::double precision, 0.5, 0.9, 1.0]); SELECT tdigest_percentile(1.0, 10, ARRAY[0.0, '-infinity'::double precision, 0.5, 0.9, 1.0]); -- more tests SELECT tdigest_percentile(1.0::double precision, 10, 'NaN'::double precision); SELECT tdigest_percentile(1.0::double precision, 10, '-infinity'::double precision); SELECT tdigest_percentile(1.0::double precision, 10, 'infinity'::double precision); SELECT tdigest_percentile(1.0::double precision, 10, ARRAY[0.5, 'NaN']::double precision[]); SELECT tdigest_percentile(1.0::double precision, 10, ARRAY[0.5, '-infinity']::double precision[]); SELECT tdigest_percentile(1.0::double precision, 10, ARRAY[0.5, 'infinity']::double precision[]); SELECT tdigest_percentile('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, 'NaN'::double precision); SELECT tdigest_percentile('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, '-infinity'::double precision); SELECT tdigest_percentile('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, 'infinity'::double precision); SELECT tdigest_percentile('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, ARRAY[0.5, 'NaN']::double precision[]); SELECT tdigest_percentile('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, ARRAY[0.5, '-infinity']::double precision[]); SELECT tdigest_percentile('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, ARRAY[0.5, 'infinity']::double precision[]); -- trim thresholds are checked too SELECT tdigest_avg(1.0::double precision, 10, 'NaN'::double precision, 1.0); SELECT tdigest_avg(1.0::double precision, 10, '-infinity'::double precision, 1.0); SELECT tdigest_avg(1.0::double precision, 10, 'infinity'::double precision, 1.0); SELECT tdigest_digest_sum('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, 0.0, 'NaN'::double precision); SELECT tdigest_digest_sum('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, 0.0, '-infinity'::double precision); SELECT tdigest_digest_sum('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest, 0.0, 'infinity'::double precision); -- validation of input values -- input functions must reject NaN / infinity means in various places SELECT 'flags 1 count 1 compression 10 centroids 1 (NaN, 1)'::tdigest; SELECT 'flags 1 count 1 compression 10 centroids 1 (infinity, 1)'::tdigest; SELECT 'flags 1 count 1 compression 10 centroids 1 (-infinity, 1)'::tdigest; -- the aggregates building the digest have to reject NaN / infinity too SELECT tdigest('NaN'::float8, 10); SELECT tdigest('infinity'::float8, 10); SELECT tdigest('-infinity'::float8, 10); -- same for the value/count API SELECT tdigest('NaN'::float8, 50::bigint, 10); SELECT tdigest('NaN'::float8, 200::bigint, 10); SELECT tdigest('Infinity'::float8, 200::bigint, 10); SELECT tdigest('-Infinity'::float8, 200::bigint, 10); -- t-digest extreme enough for the percentile to fall to the right of the last centroid SELECT tdigest_percentile('flags 1 count 4611686018427388928 compression 10 centroids 3 (1, 4611686018427387904) (2, 512) (3, 512)'::tdigest, 0.9999999999999998::double precision); -- same digest, but the percentile falls in between the second and third centroid SELECT tdigest_percentile('flags 1 count 4611686018427388928 compression 10 centroids 3 (1, 4611686018427387904) (2, 512) (3, 512)'::tdigest, ARRAY[0.0, 0.5, 0.9999999999999996, 1.0]::double precision[]); -- the same digest with other compressions, to show the compression does not affect this -- the compaction leaves the digest alone for all compression values SELECT c AS compression, tdigest_percentile(('flags 1 count 4611686018427388928 compression ' || c || ' centroids 3 (1, 4611686018427387904) (2, 512) (3, 512)')::tdigest, 0.9999999999999998::double precision) FROM (VALUES (10), (100), (1000), (10000)) v(c) GROUP BY c ORDER BY c; -- make sure finite inputs do not produce infinite centroids SELECT tdigest_count(tdigest(v, 10)) FROM (SELECT (CASE WHEN i % 2 = 0 THEN 1e308 ELSE 1e307 END)::float8 AS v FROM generate_series(1,200) i) x; -- make sure finite inputs do not result in NaN means WITH x AS (SELECT (CASE i % 4 WHEN 0 THEN -1e308 WHEN 1 THEN -1e307 WHEN 2 THEN 1e307 ELSE 1e308 END)::float8 AS v FROM generate_series(1,400) i) SELECT tdigest_percentile(v,10,0.5) FROM x; -- extreme digest - make sure we can read the output we produced SELECT tdigest_count((SELECT tdigest(v, c, 10) FROM (VALUES (1e307::float8, 1000000::bigint), (1e308::float8, 1000000::bigint)) x(v, c))::text::tdigest); -- extreme digest - overflow/rounding issue in tdigest_compute_quantiles SELECT tdigest_percentile('flags 1 count 204745659738676291 compression 100 centroids 3 (5360.6421513780951, 96854112897858161) (5670.4376627340216, 36597233557612361) (5670.4376627340216, 71294313283205769)'::tdigest, 0.56241841621276667); -- extreme digest - overflow/rounding issue in tdigest_compute_quantiles_of SELECT tdigest_percentile_of('flags 1 count 467525031770889061 compression 100 centroids 5 (-714.32532319527991, 372151463885287745) (-649.22210009079686, 12983637447676689) (-613.86327584091737, 82389930437924625) (-554.7460856469786, 1) (-497.74083276256061, 1)'::tdigest, -517.61603495911652); -- most user-facing scalar functions should be strict, except for functions -- that need to accept NULL as initial digest value (e.g. for the incremental -- API, or when combining digests) SELECT p.proname, p.proisstrict FROM pg_proc p JOIN pg_depend d ON (d.objid = p.oid AND d.deptype = 'e') JOIN pg_extension e ON (e.oid = d.refobjid) WHERE e.extname = 'tdigest' AND p.proargtypes::oid[] && ARRAY['internal'::regtype]::oid[] IS NOT TRUE AND p.prorettype <> 'internal'::regtype AND p.proisstrict <> true AND NOT EXISTS (SELECT 1 FROM pg_aggregate WHERE aggfnoid = p.oid) ORDER BY 1; tvondra-tdigest-c0af713/test/sql/cast.sql000066400000000000000000000014311525543331000204670ustar00rootroot00000000000000-- test casting to json SELECT cast(tdigest(i / 1000.0, 10) as json) from generate_series(1,1000) s(i); SELECT cast(tdigest(i / 1000.0, 25) as json) from generate_series(1,1000) s(i); SELECT cast(tdigest(i / 1000.0, 100) as json) from generate_series(1,1000) s(i); -- test casting to double precision array SELECT array_agg(round(v::numeric,3)) FROM ( SELECT unnest(cast(tdigest(i / 1000.0, 10) as double precision[])) AS v from generate_series(1,1000) s(i) ) foo; SELECT array_agg(round(v::numeric,3)) FROM ( SELECT unnest(cast(tdigest(i / 1000.0, 25) as double precision[])) AS v from generate_series(1,1000) s(i) ) foo; SELECT array_agg(round(v::numeric,3)) FROM ( SELECT unnest(cast(tdigest(i / 1000.0, 100) as double precision[])) AS v from generate_series(1,1000) s(i) ) foo; tvondra-tdigest-c0af713/test/sql/combine.sql000066400000000000000000000066311525543331000211600ustar00rootroot00000000000000-- partitioned table: two partitions, each containing a t-digest with vastly -- different compression values CREATE TABLE digest_combine_test(k int, d tdigest) PARTITION BY LIST (k); CREATE TABLE digest_combine_test_1 PARTITION OF digest_combine_test FOR VALUES IN (1); CREATE TABLE digest_combine_test_2 PARTITION OF digest_combine_test FOR VALUES IN (2); DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '0', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '0', false); END IF; -- parallel_leader_participation and enable_partitionwise_aggregate -- were introduced in 11 IF v_version >= 11 THEN PERFORM set_config('parallel_leader_participation', 'off', false); PERFORM set_config('enable_partitionwise_aggregate', 'on', false); END IF; END; $$ LANGUAGE plpgsql; DO $$ DECLARE v_scale_1 INT; v_scale_2 INT; v_compress INT; v_rows INT; v_rec RECORD; BEGIN FOR v_scale_1 IN 1..3 LOOP FOR v_scale_2 IN 1..3 LOOP FOR r IN 1..5 LOOP TRUNCATE digest_combine_test; -- first random t-digest v_compress := pow(10, v_scale_1) + (random() * pow(10, (v_scale_1 + 1)))::int; v_compress := GREATEST(10, LEAST(v_compress, 10000)); v_rows := (random() * v_compress * 10); v_rows := GREATEST(100, LEAST(v_rows, 25000)); -- RAISE NOTICE '% %', v_compress, v_rows; INSERT INTO digest_combine_test SELECT 1, tdigest(v * random(), v_compress) FROM generate_series(1, v_rows) v; -- second random t-digest v_compress := pow(10, v_scale_2) + (random() * pow(10, (v_scale_2 + 1)))::int; v_compress := GREATEST(10, LEAST(v_compress, 10000)); v_rows := (random() * v_compress * 10); v_rows := GREATEST(100, LEAST(v_rows, 25000)); -- RAISE NOTICE '% %', v_compress, v_rows; INSERT INTO digest_combine_test SELECT 2, tdigest(v * random(), v_compress) FROM generate_series(1, v_rows) v; -- duplicate the data enough times to reliably trigger partitionwise aggregate INSERT INTO digest_combine_test SELECT * FROM digest_combine_test; INSERT INTO digest_combine_test SELECT * FROM digest_combine_test; INSERT INTO digest_combine_test SELECT * FROM digest_combine_test; INSERT INTO digest_combine_test SELECT * FROM digest_combine_test; ANALYZE digest_combine_test; -- FOR v_rec IN EXPLAIN SELECT tdigest(d) FROM digest_combine_test LOOP -- RAISE NOTICE '%', v_rec."QUERY PLAN"; -- END LOOP; -- tdigest_combine(d1, d2) PERFORM tdigest(d) FROM digest_combine_test; END LOOP; END LOOP; END LOOP; END; $$ LANGUAGE plpgsql; DROP TABLE digest_combine_test; tvondra-tdigest-c0af713/test/sql/combine_crash.sql000066400000000000000000000026601525543331000223360ustar00rootroot00000000000000-- partitioned table: two partitions, each containing a t-digest with vastly -- different compression values CREATE TABLE combine_crash_test(k int, d tdigest) PARTITION BY LIST (k); CREATE TABLE combine_crash_test_1 PARTITION OF combine_crash_test FOR VALUES IN (1); CREATE TABLE combine_crash_test_2 PARTITION OF combine_crash_test FOR VALUES IN (2); -- comp=10: BUFFER_SIZE=100 slots INSERT INTO combine_crash_test SELECT 1, tdigest(v::float8, 10) FROM generate_series(1,100) v; -- comp=10000: BUFFER_SIZE=100000 slots, with 10000 input values INSERT INTO combine_crash_test SELECT 2, tdigest(v::float8, 10000) FROM generate_series(1,10000) v; -- enough data to reliably trigger partitionwise aggregate INSERT INTO combine_crash_test SELECT * FROM combine_crash_test; INSERT INTO combine_crash_test SELECT * FROM combine_crash_test; INSERT INTO combine_crash_test SELECT * FROM combine_crash_test; ANALYZE combine_crash_test; -- force partitionwise parallel aggregate SET enable_partitionwise_aggregate = on; SET max_parallel_workers_per_gather = 2; SET parallel_leader_participation = off; SET parallel_setup_cost = 0; SET parallel_tuple_cost = 0; SET min_parallel_table_scan_size = 0; -- tdigest_combine(small, huge) EXPLAIN (COSTS OFF) SELECT tdigest(d) FROM combine_crash_test; -- stabilize the output (some randomness due to parallelism) SET extra_float_digits = 0; SELECT tdigest(d) FROM combine_crash_test; DROP TABLE combine_crash_test; tvondra-tdigest-c0af713/test/sql/compact_no_progress.sql000066400000000000000000000032761525543331000236140ustar00rootroot00000000000000-- Use terse verbosity, so that the expected output does not depend on the -- error context, which differs between PostgreSQL versions. \set VERBOSITY terse -- check tdigest_compact() makes progress even for extreme digests -- -- tdigest_compact() walks the sorted centroids and merges each one into the -- current output centroid while the merged count stays below the size limit -- derived from the quantile position. With the former subtraction-based -- calculation of (1-q), a centroid holding nearly all the weight could make -- q0 round to 1 for everything following it. The size limit then became zero -- and compaction returned with ncentroids unchanged, still at BUFFER_SIZE. -- The current code uses exact integer remainders for these factors, and -- raises an error if a compaction still leaves the buffer full. -- -- Note that nothing here overflows - all the counts are perfectly valid and -- the digests are well-formed. The old code assumed compaction always freed -- some space. These regressions check progress and preservation of the count. -- -- 1) small number of values SELECT tdigest_count(tdigest(d)) FROM ( SELECT ('flags 1 count ' || c || ' compression 10 centroids 1 (' || i || ', ' || c || ')')::tdigest AS d FROM generate_series(1, 500) s(i), LATERAL (SELECT CASE WHEN i = 1 THEN 9000000000000000000::bigint ELSE 1 END AS c) x ) q; -- 2) more values, requiring additional compactions SELECT tdigest_count(tdigest(d)) FROM ( SELECT ('flags 1 count ' || c || ' compression 10 centroids 1 (' || i || ', ' || c || ')')::tdigest AS d FROM generate_series(1, 2000) s(i), LATERAL (SELECT CASE WHEN i = 1 THEN 9000000000000000000::bigint ELSE 1 END AS c) x ) q; tvondra-tdigest-c0af713/test/sql/compression_checks.sql000066400000000000000000000013571525543331000234250ustar00rootroot00000000000000-- Regression test for check_compression(), which rejects compression -- values outside the [MIN_COMPRESSION, MAX_COMPRESSION] = [10, 10000] -- range accepted by the t-digest implementation. \set VERBOSITY terse -- compression below the minimum SELECT tdigest_percentile(v, 9, 0.5) FROM (VALUES (1.0)) AS t(v); -- compression above the maximum SELECT tdigest_percentile(v, 10001, 0.5) FROM (VALUES (1.0)) AS t(v); -- the incremental API checks the compression too SELECT tdigest_add(NULL::tdigest, 1.0, 9); SELECT tdigest_add(NULL::tdigest, ARRAY[1.0], 10001); -- both ends of the accepted range have to work SELECT tdigest_percentile(v, 10, 0.5) FROM (VALUES (1.0)) AS t(v); SELECT tdigest_percentile(v, 10000, 0.5) FROM (VALUES (1.0)) AS t(v); tvondra-tdigest-c0af713/test/sql/conversions.sql000066400000000000000000000017171525543331000221140ustar00rootroot00000000000000-- test input function, and conversion from old to new format SELECT 'flags 0 count 20 compression 10 centroids 8 (1000.000000, 1) (2000.000000, 1) (7000.000000, 2) (26000.000000, 4) (84000.000000, 7) (51000.000000, 3) (19000.000000, 1) (20000.000000, 1)'::tdigest; -- test input of invalid data -- negative count SELECT 'flags 0 count -20 compression 10 centroids 8 (1000.000000, 1) (2000.000000, 1) (7000.000000, 2) (26000.000000, 4) (84000.000000, 7) (51000.000000, 3) (19000.000000, 1) (20000.000000, 1)'::tdigest; -- mismatching count SELECT 'flags 0 count 21 compression 10 centroids 8 (1000.000000, 1) (2000.000000, 1) (7000.000000, 2) (26000.000000, 4) (84000.000000, 7) (51000.000000, 3) (19000.000000, 1) (20000.000000, 1)'::tdigest; -- unsorted centroids SELECT 'flags 0 count 20 compression 10 centroids 8 (1000.000000, 1) (2000.000000, 1) (1000.000000, 2) (26000.000000, 4) (84000.000000, 7) (51000.000000, 3) (19000.000000, 1) (20000.000000, 1)'::tdigest; tvondra-tdigest-c0af713/test/sql/copy.sql000066400000000000000000000034501525543331000205120ustar00rootroot00000000000000CREATE TABLE tdigest_src (id int, s tdigest); CREATE TABLE tdigest_dst (id int, s tdigest); -- generate 100 random sketches, export/import them in text/binary mode DO $$ DECLARE compression int; BEGIN FOR i IN 1..100 LOOP -- random compression value compression := 100 + random() * 1000; INSERT INTO tdigest_src SELECT i, tdigest(random(), compression) FROM generate_series(1, compression * 10) s(x); END LOOP; END; $$ LANGUAGE plpgsql; -- export in text and binary formats COPY tdigest_src TO '/tmp/tdigest_copy_text.out'; COPY tdigest_src TO '/tmp/tdigest_copy_binary.out' WITH (FORMAT BINARY); -- import in text and binary formats COPY tdigest_dst FROM '/tmp/tdigest_copy_text.out'; COPY tdigest_dst FROM '/tmp/tdigest_copy_binary.out' WITH (FORMAT BINARY); -- all imported values have to match the source SELECT COUNT(*) AS count_all, COUNT(CASE WHEN (src.s::text != dst.s::text) THEN 1 ELSE NULL END) AS count_mismatching FROM tdigest_src src JOIN tdigest_dst dst ON (src.id = dst.id); DROP TABLE tdigest_src; DROP TABLE tdigest_dst; -- tdigest_recv() has to accept digests with unsorted centroids -- -- Test through a binary COPY, which goes through the receive function. -- We build the input with the send functions of the individual fields. CREATE TABLE tdigest_dst (s tdigest); COPY (SELECT int4send(1) -- flags || int8send(3::bigint) -- count || int4send(10) -- compression || int4send(2) -- ncentroids || float8send(5) || int8send(2::bigint) || float8send(1) || int8send(1::bigint)) TO '/tmp/tdigest_recv_unsorted.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_unsorted.bin' WITH (FORMAT binary); DROP TABLE tdigest_dst; tvondra-tdigest-c0af713/test/sql/count_checks.sql000066400000000000000000000012551525543331000222110ustar00rootroot00000000000000-- Regression test for the count check in tdigest_add_double_count(), the -- transition function of the aggregates accepting an explicit number of -- occurrences of a value. -- -- A non-NULL count must be positive. A NULL count means one occurrence -- of the input value. \set VERBOSITY terse -- zero count SELECT tdigest(v, 0::bigint, 100) FROM (VALUES (1.0)) AS t(v); -- negative count SELECT tdigest(v, -1::bigint, 100) FROM (VALUES (1.0)) AS t(v); -- the smallest valid count has to work SELECT tdigest(v, 1::bigint, 100) FROM (VALUES (1.0)) AS t(v); -- NULL count means a single occurrence, so it's valid too SELECT tdigest(v, NULL::bigint, 100) FROM (VALUES (1.0)) AS t(v); tvondra-tdigest-c0af713/test/sql/count_generated.sql000066400000000000000000000025041525543331000227050ustar00rootroot00000000000000-- Regression test for adding a value with a count exceeding BUFFER_SIZE in -- tdigest_add_double_array_values_count(), the transition function for -- inverse percentiles of an array of hypothetical values. -- -- Adding the values one by one would mean too many compactions, so counts -- larger than the buffer (BUFFER_SIZE = 10 * compression, i.e. 100 for the -- smallest compression) are added as pre-generated centroids instead. Both -- paths have to produce comparable results. -- a single distinct value, so the result is exact on both sides of the -- BUFFER_SIZE threshold SELECT tdigest_percentile_of(v, 100::bigint, 10, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v); SELECT tdigest_percentile_of(v, 101::bigint, 10, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v); SELECT tdigest_percentile_of(v, 1000000::bigint, 10, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v); -- two distinct values, with the counts at and just above the -- threshold SELECT tdigest_percentile_of(v, 100::bigint, 10, ARRAY[1.0, 2.0]) FROM (VALUES (1.0), (2.0)) AS t(v); SELECT tdigest_percentile_of(v, 101::bigint, 10, ARRAY[1.0, 2.0]) FROM (VALUES (1.0), (2.0)) AS t(v); -- a larger compression means a larger buffer, so the very same count now -- goes through the one-by-one path SELECT tdigest_percentile_of(v, 101::bigint, 100, ARRAY[1.0, 2.0]) FROM (VALUES (1.0), (2.0)) AS t(v); tvondra-tdigest-c0af713/test/sql/count_overflow.sql000066400000000000000000000120331525543331000226100ustar00rootroot00000000000000-- Use terse verbosity, so that the expected output does not depend on the -- error context, which differs between PostgreSQL versions (particularly -- for the COPY ... FROM statements below). \set VERBOSITY terse CREATE TABLE tdigest_overflow_test (d tdigest); -- Regression test for the count overflow in tdigest_in() and tdigest_recv(). -- -- Both input functions sum the centroid counts and compare the result to the -- total count stored in the header. Both the centroid counts and the total -- are int64, so the sum may overflow - and if it happens to wrap around to -- exactly the value stored in the header, the "total count does not match" -- check passes and a corrupted t-digest gets accepted. -- -- Five centroids with count 2^62 are the smallest such case: the centroids -- weigh 5 * 2^62 in total, which is 2^62 modulo 2^64. So a digest claiming -- "count 2^62" passes the check, while the centroids actually weigh five -- times as much. -- tdigest_in() has to reject the wrapping value SELECT 'flags 1 count 4611686018427387904 compression 10 centroids 5 (1, 4611686018427387904) (2, 4611686018427387904) (3, 4611686018427387904) (4, 4611686018427387904) (5, 4611686018427387904)'::tdigest; -- tdigest_recv() has to reject it too. There's no way to call the receive -- function directly, so go through a binary COPY. A binary COPY file with a -- single bytea column is byte-for-byte the same as a file with a single -- tdigest column holding the same bytes, so we can build the input with the -- send functions of the individual header/centroid fields. COPY (SELECT int4send(1) -- flags || int8send(4611686018427387904::bigint) -- count || int4send(10) -- compression || int4send(5) -- ncentroids || float8send(1) || int8send(4611686018427387904::bigint) || float8send(2) || int8send(4611686018427387904::bigint) || float8send(3) || int8send(4611686018427387904::bigint) || float8send(4) || int8send(4611686018427387904::bigint) || float8send(5) || int8send(4611686018427387904::bigint)) TO '/tmp/tdigest_count_overflow.bin' WITH (FORMAT binary); COPY tdigest_overflow_test FROM '/tmp/tdigest_count_overflow.bin' WITH (FORMAT binary); -- a t-digest with a total count of exactly INT64_MAX is still valid, though, -- and has to be accepted by both input functions SELECT 'flags 1 count 9223372036854775807 compression 10 centroids 2 (1, 4611686018427387903) (2, 4611686018427387904)'::tdigest; COPY (SELECT int4send(1) || int8send(9223372036854775807::bigint) || int4send(10) || int4send(2) || float8send(1) || int8send(4611686018427387903::bigint) || float8send(2) || int8send(4611686018427387904::bigint)) TO '/tmp/tdigest_count_overflow.bin' WITH (FORMAT binary); COPY tdigest_overflow_test FROM '/tmp/tdigest_count_overflow.bin' WITH (FORMAT binary); SELECT * FROM tdigest_overflow_test; -- Regression test for the count overflow in tdigest_add() and tdigest_add_centroid(). -- A couple examples triggering overflows of total count when adding values -- or centroids to a digest (or aggregate state) WITH source(value) AS ( SELECT * FROM (VALUES ('flags 1 count 9223372036854775807 compression 10 centroids 1 (-1000, 9223372036854775807)'::tdigest), ('flags 1 count 1 compression 10 centroids 1 (-1000, 1)'::tdigest)) AS t(v) ) SELECT tdigest(value) FROM source; WITH source(value) AS ( SELECT * FROM (VALUES ('flags 1 count 4611686018427387904 compression 10 centroids 1 (-1000, 4611686018427387904)'::tdigest), ('flags 1 count 4611686018427387904 compression 10 centroids 1 (-1000, 4611686018427387904)'::tdigest)) AS t(v) ) SELECT tdigest(value) FROM source; WITH source(value) AS ( SELECT 'flags 1 count 5000000000000000000 compression 10 centroids 1 (-1000, 5000000000000000000)'::tdigest FROM generate_series(1, 2) ) SELECT tdigest(value) FROM source; -- the negative cases (no overflows, hitting INT64_MAX exactly) should still pass WITH source(value) AS ( SELECT * FROM (VALUES ('flags 1 count 9223372036854775806 compression 10 centroids 1 (-1000, 9223372036854775806)'::tdigest), ('flags 1 count 1 compression 10 centroids 1 (-1000, 1)'::tdigest)) AS t(v) ) SELECT tdigest(value) FROM source; WITH source(value) AS ( SELECT * FROM (VALUES ('flags 1 count 4611686018427387903 compression 10 centroids 1 (-1000, 4611686018427387903)'::tdigest), ('flags 1 count 4611686018427387904 compression 10 centroids 1 (-1000, 4611686018427387904)'::tdigest)) AS t(v) ) SELECT tdigest(value) FROM source; WITH source(value) AS ( SELECT 'flags 1 count 4611686018427387903 compression 10 centroids 1 (-1000, 4611686018427387903)'::tdigest FROM generate_series(1, 2) UNION ALL SELECT 'flags 1 count 1 compression 10 centroids 1 (-1000, 1)'::tdigest ) SELECT tdigest(value) FROM source; DROP TABLE tdigest_overflow_test; tvondra-tdigest-c0af713/test/sql/digest_is_valid.sql000066400000000000000000000250471525543331000226770ustar00rootroot00000000000000-- Regression test for tdigest_is_valid(), which performs the same checks as -- the input functions (tdigest_in and tdigest_recv), but on an existing value -- and without raising an error. -- -- Digests written by older versions of the extension (which did not have all -- of the checks) may be broken in various ways, and nothing re-validates the -- values when reading them back from disk. This function allows finding such -- digests. -- -- The input functions reject invalid values, so we can't use them to build -- the test data. Instead we build the on-disk representation by hand, and use -- a binary-coercible cast to turn it into a t-digest. Note the forged values -- must not be passed to anything except tdigest_is_valid() - the whole point -- is that they are broken, so e.g. printing them might read past the end of -- the value. \set VERBOSITY terse -- casts allowing us to inspect and forge the on-disk representation CREATE CAST (tdigest AS bytea) WITHOUT FUNCTION; CREATE CAST (bytea AS tdigest) WITHOUT FUNCTION; -- The fields of the on-disk representation use the host byte order, while -- the send functions produce network byte order (big endian), so we may need -- to reverse the bytes. Determine the host byte order by looking at the flags -- of a digest we know has flags set to 1. CREATE FUNCTION is_little_endian() RETURNS bool AS $$ SELECT substring('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest::bytea from 1 for 4) = '\x01000000'::bytea; $$ LANGUAGE sql IMMUTABLE; CREATE FUNCTION swap_bytes(b bytea) RETURNS bytea AS $$ SELECT CASE WHEN is_little_endian() THEN (SELECT string_agg(substring(b from i for 1), ''::bytea ORDER BY i DESC) FROM generate_series(1, length(b)) AS i) ELSE b END; $$ LANGUAGE sql IMMUTABLE; -- Transform values to big endian (if needed) CREATE FUNCTION be_int4(v int) RETURNS bytea AS $$ SELECT swap_bytes(int4send(v)); $$ LANGUAGE sql IMMUTABLE; CREATE FUNCTION be_int8(v bigint) RETURNS bytea AS $$ SELECT swap_bytes(int8send(v)); $$ LANGUAGE sql IMMUTABLE; CREATE FUNCTION be_float8(v double precision) RETURNS bytea AS $$ SELECT swap_bytes(float8send(v)); $$ LANGUAGE sql IMMUTABLE; -- the layout has to be (flags, count, compression, ncentroids) followed by -- the centroids, each stored as (mean, count) SELECT length('flags 1 count 3 compression 10 centroids 2 (1, 1) (2, 2)'::tdigest::bytea) AS bytes; -- a forged value has to be the same as the parsed one (sanity check of the -- forging, not of the validation) SELECT (be_int4(1) || be_int8(3::bigint) || be_int4(10) || be_int4(2) || be_float8(1) || be_int8(1::bigint) || be_float8(2) || be_int8(2::bigint))::tdigest; -- -- valid values, to make sure the checks don't reject correct digests -- SELECT tdigest_is_valid('flags 1 count 3 compression 10 centroids 2 (1, 1) (2, 2)'::tdigest); -- the old on-disk format, storing sums instead of means SELECT tdigest_is_valid('flags 0 count 3 compression 10 centroids 2 (1, 1) (4, 2)'::tdigest); -- centroids not sorted by mean (produced by the incremental API) SELECT tdigest_is_valid('flags 1 count 3 compression 10 centroids 2 (2, 2) (1, 1)'::tdigest); -- the extreme compression values SELECT tdigest_is_valid('flags 1 count 1 compression 10 centroids 1 (1, 1)'::tdigest); SELECT tdigest_is_valid('flags 1 count 1 compression 10000 centroids 1 (1, 1)'::tdigest); -- digests built by the extension itself SELECT tdigest_is_valid(tdigest(i::double precision, 10)) FROM generate_series(1, 1000) AS s(i); SELECT tdigest_is_valid(tdigest(i::double precision, 100)) FROM generate_series(1, 1000) AS s(i); SELECT tdigest_is_valid(tdigest_add(NULL::tdigest, ARRAY[1.0, 2.0, 3.0], 10, false)); SELECT tdigest_is_valid(tdigest_union(tdigest_add(NULL::tdigest, 1.0, 10), tdigest_add(NULL::tdigest, 2.0, 10))); -- -- sanity checks on the header -- -- unknown flags SELECT tdigest_is_valid((be_int4(2) || be_int8(1::bigint) || be_int4(10) || be_int4(1) || be_float8(1) || be_int8(1::bigint))::tdigest); -- compression out of the supported range SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(9) || be_int4(1) || be_float8(1) || be_int8(1::bigint))::tdigest); SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10001) || be_int4(1) || be_float8(1) || be_int8(1::bigint))::tdigest); -- non-positive total count SELECT tdigest_is_valid((be_int4(1) || be_int8(0::bigint) || be_int4(10) || be_int4(1) || be_float8(1) || be_int8(1::bigint))::tdigest); SELECT tdigest_is_valid((be_int4(1) || be_int8((-1)::bigint) || be_int4(10) || be_int4(1) || be_float8(1) || be_int8(1::bigint))::tdigest); -- non-positive number of centroids SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(0))::tdigest); SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(-1))::tdigest); -- more centroids than fit into the buffer (10 * compression) SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(101) || be_float8(1) || be_int8(1::bigint))::tdigest); -- -- sanity checks on the length of the value -- -- The input functions build the digest from the parsed header, so the length -- always matches. For an existing value we have to check it really is long -- enough for the centroids the header promises. -- -- value too short to even contain the header SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10))::tdigest); -- fewer centroids than the header says SELECT tdigest_is_valid((be_int4(1) || be_int8(3::bigint) || be_int4(10) || be_int4(2) || be_float8(1) || be_int8(1::bigint))::tdigest); -- more centroids than the header says SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(1) || be_float8(1) || be_int8(1::bigint) || be_float8(2) || be_int8(2::bigint))::tdigest); -- incomplete centroid at the end of the value SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(1) || be_float8(1))::tdigest); -- -- sanity checks on the centroids -- -- centroid mean that is not a valid number SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(1) || be_float8('NaN') || be_int8(1::bigint))::tdigest); SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(1) || be_float8('Infinity') || be_int8(1::bigint))::tdigest); SELECT tdigest_is_valid((be_int4(1) || be_int8(3::bigint) || be_int4(10) || be_int4(2) || be_float8(1) || be_int8(1::bigint) || be_float8('-Infinity') || be_int8(2::bigint))::tdigest); -- non-positive centroid count SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(1) || be_float8(1) || be_int8(0::bigint))::tdigest); SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(2) || be_float8(1) || be_int8(1::bigint) || be_float8(2) || be_int8((-1)::bigint))::tdigest); -- centroid count exceeding the total count SELECT tdigest_is_valid((be_int4(1) || be_int8(1::bigint) || be_int4(10) || be_int4(1) || be_float8(1) || be_int8(2::bigint))::tdigest); -- centroid counts overflowing the total count SELECT tdigest_is_valid((be_int4(1) || be_int8(9223372036854775807::bigint) || be_int4(10) || be_int4(2) || be_float8(1) || be_int8(9223372036854775807::bigint) || be_float8(2) || be_int8(9223372036854775807::bigint))::tdigest); -- -- sanity checks on the value as a whole -- -- centroid counts not adding up to the total count SELECT tdigest_is_valid((be_int4(1) || be_int8(3::bigint) || be_int4(10) || be_int4(1) || be_float8(1) || be_int8(1::bigint))::tdigest); SELECT tdigest_is_valid((be_int4(1) || be_int8(3::bigint) || be_int4(10) || be_int4(2) || be_float8(1) || be_int8(1::bigint) || be_float8(2) || be_int8(3::bigint))::tdigest); -- -- the checks have to work on values read back from a table too (which may -- use a different varlena header, or be TOASTed) -- CREATE TABLE tdigest_values (id int, d tdigest); INSERT INTO tdigest_values SELECT 1, 'flags 1 count 3 compression 10 centroids 2 (1, 1) (2, 2)'::tdigest; INSERT INTO tdigest_values SELECT 2, (be_int4(1) || be_int8(3::bigint) || be_int4(10) || be_int4(1) || be_float8(1) || be_int8(1::bigint))::tdigest; INSERT INTO tdigest_values SELECT 3, tdigest(i::double precision, 1000) FROM generate_series(1, 100000) AS s(i); SELECT id, tdigest_is_valid(d) FROM tdigest_values ORDER BY id; -- NULL input produces NULL SELECT tdigest_is_valid(NULL::tdigest) IS NULL; DROP TABLE tdigest_values; DROP FUNCTION be_float8(double precision); DROP FUNCTION be_int8(bigint); DROP FUNCTION be_int4(int); DROP FUNCTION swap_bytes(bytea); DROP FUNCTION is_little_endian(); DROP CAST (bytea AS tdigest); DROP CAST (tdigest AS bytea); tvondra-tdigest-c0af713/test/sql/empty_input.sql000066400000000000000000000015511525543331000221150ustar00rootroot00000000000000-- Regression test for the NULL handling in the final functions. -- -- None of the final functions is strict, so an aggregate over an empty -- input calls them with a NULL state, and they have to return NULL. Each -- final function is reached through the matching aggregate, with a WHERE -- clause filtering out all the rows. -- tdigest_percentiles SELECT tdigest_percentile(v, 100, 0.5) FROM (VALUES (1.0)) AS t(v) WHERE v > 2.0; -- tdigest_percentiles_of SELECT tdigest_percentile_of(v, 100, 1.0) FROM (VALUES (1.0)) AS t(v) WHERE v > 2.0; -- tdigest_digest SELECT tdigest(v, 100) FROM (VALUES (1.0)) AS t(v) WHERE v > 2.0; -- tdigest_array_percentiles SELECT tdigest_percentile(v, 100, ARRAY[0.5]) FROM (VALUES (1.0)) AS t(v) WHERE v > 2.0; -- tdigest_array_percentiles_of SELECT tdigest_percentile_of(v, 100, ARRAY[1.0]) FROM (VALUES (1.0)) AS t(v) WHERE v > 2.0; tvondra-tdigest-c0af713/test/sql/generate_overflow.sql000066400000000000000000000017621525543331000232610ustar00rootroot00000000000000-- Use terse verbosity, so that the expected output does not depend on the -- error context, which differs between PostgreSQL versions. \set VERBOSITY terse -- via the tdigest(value, count, compression) aggregate SELECT length(tdigest(1.0::float8, 9223372036854775807::bigint, 10000)::text); -- the count does not have to be anywhere near the int64 limit, a large enough -- count with a moderate compression is enough SELECT length(tdigest(1.0::float8, 1000000000000000000::bigint, 1000)::text); -- via the percentile aggregates on the value/count API SELECT tdigest_percentile(1.0::float8, 9223372036854775807::bigint, 10000, 0.5); -- via the trimmed aggregates on the value/count API -- -- The exact value depends on where exactly the centroid boundaries end up, -- so don't compare it to an exact value - it'd be a 1 ULP difference away -- from failing. It just has to be (close to) 1.0, not 0.9 or NULL. SELECT abs(tdigest_avg(1.0::float8, 9223372036854775807::bigint, 10000, 0.1, 0.9) - 1.0) < 1e-12; tvondra-tdigest-c0af713/test/sql/increment_compression.sql000066400000000000000000000023451525543331000241470ustar00rootroot00000000000000-- Regression test for the compression check in tdigest_add_double_increment() -- and tdigest_add_double_array_increment(), the functions of the incremental -- API. -- -- The compression is only optional when there already is a t-digest to add -- the value to, because that determines the compression. Without a digest -- there is nothing to derive the compression from, so it has to be supplied. \set VERBOSITY terse -- NULL digest and NULL compression (which is the default) SELECT tdigest_add(NULL::tdigest, 1.0::double precision); SELECT tdigest_add(NULL::tdigest, 1.0::double precision, NULL::int); SELECT tdigest_add(NULL::tdigest, ARRAY[1.0]::double precision[]); SELECT tdigest_add(NULL::tdigest, ARRAY[1.0]::double precision[], NULL::int); -- with the compression supplied it has to work SELECT tdigest_add(NULL::tdigest, 1.0::double precision, 100); SELECT tdigest_add(NULL::tdigest, ARRAY[1.0]::double precision[], 100); -- with an existing digest the compression is not needed SELECT tdigest_add('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest, 2.0::double precision); SELECT tdigest_add('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest, ARRAY[2.0]::double precision[]); tvondra-tdigest-c0af713/test/sql/incremental.sql000066400000000000000000000063651525543331000220510ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('extra_float_digits', '0', false); PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; -- test incremental API (adding values one by one) CREATE TABLE test_incremental (d tdigest); INSERT INTO test_incremental VALUES (NULL); -- check this produces the same result as building the tdigest at once, but we -- need to be careful about feeding the data in the same order, and we must -- not compact the t-digest after each increment DO LANGUAGE plpgsql $$ DECLARE r RECORD; BEGIN FOR r IN (SELECT i FROM generate_series(1,1000) s(i) ORDER BY md5(i::text)) LOOP UPDATE test_incremental SET d = tdigest_add(d, r.i, 100, false); END LOOP; END$$; -- compare the results, but do force a compaction of the incremental result WITH x AS (SELECT i FROM generate_series(1,1000) s(i) ORDER BY md5(i::text)) SELECT (SELECT tdigest(d)::text FROM test_incremental) = (SELECT tdigest(x.i, 100)::text FROM x) AS match; -- now try the same thing with bulk incremental update (using arrays) TRUNCATE test_incremental; INSERT INTO test_incremental VALUES (NULL); DO LANGUAGE plpgsql $$ DECLARE r RECORD; BEGIN FOR r IN (SELECT a, array_agg(i::double precision) AS v FROM (SELECT mod(i,5) AS a, i FROM generate_series(1,1000) s(i) ORDER BY mod(i,5), md5(i::text)) foo GROUP BY a ORDER BY a) LOOP UPDATE test_incremental SET d = tdigest_add(d, r.v, 100, false); END LOOP; END$$; -- compare the results, but do force a compaction of the incremental result WITH x AS (SELECT mod(i,5) AS a, i::double precision AS d FROM generate_series(1,1000) s(i) ORDER BY mod(i,5), i) SELECT (SELECT tdigest(d)::text FROM test_incremental) = (SELECT tdigest(x.d, 100)::text FROM x); -- now try the same thing with bulk incremental update (using t-digests) TRUNCATE test_incremental; INSERT INTO test_incremental VALUES (NULL); DO LANGUAGE plpgsql $$ DECLARE r RECORD; BEGIN FOR r IN (SELECT a, tdigest(i,100) AS d FROM (SELECT mod(i,5) AS a, i FROM generate_series(1,1000) s(i) ORDER BY mod(i,5), md5(i::text)) foo GROUP BY a ORDER BY a) LOOP UPDATE test_incremental SET d = tdigest_union(d, r.d, false); END LOOP; END$$; -- compare the results, but do force a compaction of the incremental result WITH x AS (SELECT a, tdigest(i,100) AS d FROM (SELECT mod(i,5) AS a, i FROM generate_series(1,1000) s(i) ORDER BY mod(i,5), md5(i::text)) foo GROUP BY a ORDER BY a) SELECT (SELECT tdigest(d)::text FROM test_incremental) = (SELECT tdigest(x.d)::text FROM x); DROP TABLE test_incremental; tvondra-tdigest-c0af713/test/sql/init.sql000066400000000000000000000035571525543331000205130ustar00rootroot00000000000000\set ECHO none -- disable the notices for the create script (shell types etc.) SET client_min_messages = 'WARNING'; CREATE EXTENSION tdigest VERSION '1.0.0'; ALTER EXTENSION tdigest UPDATE TO '1.4.7'; SET client_min_messages = 'NOTICE'; SET extra_float_digits = 0; \set ECHO all -- Set-returning function implementing a simple deterministic PRNG CREATE OR REPLACE FUNCTION prng(nrows int, seed int = 23982, p1 bigint = 16807, p2 bigint = 0, n bigint = 2147483647) RETURNS SETOF double precision AS $$ DECLARE val INT := seed; BEGIN FOR i IN 1..nrows LOOP val := (val * p1 + p2) % n; RETURN NEXT (val::double precision / n); END LOOP; RETURN; END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION random_normal(nrows int, mean double precision = 0.5, stddev double precision = 0.1, minval double precision = 0.0, maxval double precision = 1.0, seed int = 23982, p1 bigint = 16807, p2 bigint = 0, n bigint = 2147483647) RETURNS SETOF double precision AS $$ DECLARE v BIGINT := seed; x DOUBLE PRECISION; y DOUBLE PRECISION; s DOUBLE PRECISION; r INT := nrows; BEGIN WHILE true LOOP -- random x v := (v * p1 + p2) % n; x := 2 * v / n::double precision - 1.0; -- random y v := (v * p1 + p2) % n; y := 2 * v / n::double precision - 1.0; s := x^2 + y^2; IF s != 0.0 AND s < 1.0 THEN s = sqrt(-2 * ln(s) / s); x := mean + stddev * s * x; IF x >= minval AND x <= maxval THEN RETURN NEXT x; r := r - 1; END IF; EXIT WHEN r = 0; y := mean + stddev * s * y; IF y >= minval AND y <= maxval THEN RETURN NEXT y; r := r - 1; END IF; EXIT WHEN r = 0; END IF; END LOOP; END; $$ LANGUAGE plpgsql; tvondra-tdigest-c0af713/test/sql/input_saturation.sql000066400000000000000000000055421525543331000231540ustar00rootroot00000000000000-- Reproducer: tdigest_in() accepts out-of-range numbers, because sscanf() has -- no way to report a range error. -- -- This one does not crash either, it silently accepts input it should reject -- and stores something different from what was written. -- -- tdigest_in() parses the whole value with sscanf() (tdigest.c around lines -- 2646 and 2694) and only checks that the parsed values make sense relative to -- each other. But sscanf() has undefined behaviour when the number does not -- fit the target type (C99 7.21.6.2p10), and glibc's implementation does not -- report it: -- -- %lld saturates: "99999999999999999999" -> INT64_MAX -- "-99999999999999999999" -> INT64_MIN -- %d wraps: "2147483648" -> -2147483648 -- "4294967296" -> 0 -- "4294967306" -> 10 -- "99999999999" -> 1215752191 -- -- The count overflow checks added in d030f1f run on the parsed (already -- mangled) values, so they do not catch any of this. \set VERBOSITY terse -- 1) the counts saturate to INT64_MAX. Both the total and the centroid count -- saturate to the same value, so they still agree with each other and the -- digest is accepted. Note the output does not match the input. SELECT 'flags 1 count 99999999999999999999 compression 10 centroids 1 (1, 99999999999999999999)'::tdigest; -- 2) worse, the compression is parsed with %d and wraps modulo 2^32. 4294967306 -- is 2^32 + 10, so this is accepted and stored as compression 10 - well inside -- the allowed range, so the range check that follows sees nothing wrong. SELECT 'flags 1 count 3 compression 4294967306 centroids 1 (1, 3)'::tdigest; -- and 2^33 + 10 gives the same thing SELECT 'flags 1 count 3 compression 8589934602 centroids 1 (1, 3)'::tdigest; -- this means the text output of tdigest_in() is not the text that was fed to -- it, i.e. the type is not round-trip stable and a dump/restore of such a -- value produces a different digest SELECT 'flags 1 count 3 compression 4294967306 centroids 1 (1, 3)'::tdigest::text = 'flags 1 count 3 compression 4294967306 centroids 1 (1.000000, 3)' AS round_trips; -- 3) values that happen to wrap to something outside the allowed range are -- still rejected, but by the range check and with a misleading message - the -- parse itself never fails SELECT 'flags 1 count 3 compression 4294967296 centroids 1 (1, 3)'::tdigest; SELECT 'flags 1 count 3 compression 2147483648 centroids 1 (1, 3)'::tdigest; -- 4) negative counts saturate to INT64_MIN, and are then caught by the -- "count value for the t-digest must be positive" check SELECT 'flags 1 count -99999999999999999999 compression 10 centroids 1 (1, 3)'::tdigest; -- subnormal means should be accepted (and treated as 0.0) SELECT 'flags 1 count 2 compression 10 centroids 2 (1e-320, 1) (2e-320, 1)'::tdigest; tvondra-tdigest-c0af713/test/sql/interpolation_monotonicity.sql000066400000000000000000000125341525543331000252450ustar00rootroot00000000000000\set ECHO none SET max_parallel_workers_per_gather = 0; DO $$ DECLARE d tdigest; percentiles double precision[]; probes double precision[]; results double precision[]; raw_results double precision[]; weighted_results double precision[]; scalar_result double precision; bounds double precision[]; i integer; BEGIN SELECT array_agg(n / 10000.0::double precision ORDER BY n) INTO percentiles FROM generate_series(0, 10000) AS s(n); FOREACH d IN ARRAY ARRAY[ 'flags 1 count 3 compression 10000 centroids 2 (1, 1) (1.000000000000001, 2)'::tdigest, 'flags 1 count 3 compression 10000 centroids 2 (-1.000000000000001, 2) (-1, 1)'::tdigest, 'flags 1 count 2 compression 10000 centroids 2 (-18014398509481984, 1) (-1, 1)'::tdigest, 'flags 1 count 2 compression 10000 centroids 2 (-1.7e308, 1) (-5e-324, 1)'::tdigest, 'flags 1 count 2 compression 10000 centroids 2 (-1.7e308, 1) (1.7e308, 1)'::tdigest, 'flags 1 count 2 compression 10000 centroids 2 (5e-324, 1) (1e-323, 1)'::tdigest ] LOOP SELECT tdigest_percentile(d, percentiles) INTO results; bounds := tdigest_double_array(d); IF array_length(results, 1) IS DISTINCT FROM array_length(percentiles, 1) OR EXISTS ( SELECT 1 FROM ( SELECT v, lag(v) OVER (ORDER BY ord) AS prev FROM unnest(results) WITH ORDINALITY AS u(v, ord) ) AS steps WHERE v IS NULL OR v < prev OR NOT (v BETWEEN bounds[5] AND bounds[array_length(bounds, 1) - 1]) ) THEN RAISE EXCEPTION 'Nonmonotone or out-of-range percentiles for %', d; END IF; RAISE NOTICE 'Percentiles are monotone and in range'; FOREACH i IN ARRAY ARRAY[1, 2018, 2019, 5001, 10001] LOOP SELECT tdigest_percentile(d, percentiles[i]) INTO scalar_result; IF scalar_result IS DISTINCT FROM results[i] THEN RAISE EXCEPTION 'Scalar and array percentiles disagree'; END IF; END LOOP; RAISE NOTICE 'Scalar and array percentiles agree'; END LOOP; SELECT tdigest_percentile(v, 10000, percentiles), tdigest_percentile(v, 1::bigint, 10000, percentiles) INTO raw_results, weighted_results FROM (VALUES (1::double precision), (1.000000000000001::double precision), (1.000000000000001::double precision)) AS t(v); FOREACH results SLICE 1 IN ARRAY ARRAY[raw_results, weighted_results] LOOP IF EXISTS ( SELECT 1 FROM ( SELECT v, lag(v) OVER (ORDER BY ord) AS prev FROM unnest(results) WITH ORDINALITY AS u(v, ord) ) AS steps WHERE v IS NULL OR v < prev ) THEN RAISE EXCEPTION 'Nonmonotone raw or weighted percentiles'; END IF; RAISE NOTICE 'Raw and weighted percentiles are monotone'; END LOOP; d := 'flags 1 count 4503599627370498 compression 10000 centroids 3 (0, 4503599627370496) (1, 1) (2, 1)'::tdigest; SELECT array_agg(1 + n / 10000.0::double precision ORDER BY n) INTO probes FROM generate_series(1, 9999) AS s(n); SELECT tdigest_percentile_of(d, probes) INTO results; IF EXISTS ( SELECT 1 FROM ( SELECT v, lag(v) OVER (ORDER BY ord) AS prev FROM unnest(results) WITH ORDINALITY AS u(v, ord) ) AS steps WHERE v IS NULL OR v < prev OR NOT (v BETWEEN 0 AND 1) ) THEN RAISE EXCEPTION 'Nonmonotone inverse interpolation'; END IF; RAISE NOTICE 'Inverse interpolation is monotone'; FOREACH i IN ARRAY ARRAY[1, 1874, 1875, 9999] LOOP SELECT tdigest_percentile_of(d, probes[i]) INTO scalar_result; IF scalar_result IS DISTINCT FROM results[i] THEN RAISE EXCEPTION 'Scalar and array inverse percentiles disagree'; END IF; END LOOP; RAISE NOTICE 'Scalar and array inverse percentiles agree'; END $$; \set ECHO all -- At p = 0.75 - 2^-53, q = 1 - 2^-52 and the result rounds to -5. -- Subtracting nearly equal magnitudes used to return -4 instead. SELECT tdigest_percentile(v, 10000, 0.7499999999999999::double precision) AS raw_scalar, tdigest_percentile(v, 10000, ARRAY[0.7499999999999999]::double precision[]) AS raw_array, tdigest_percentile(v, 1::bigint, 10000, 0.7499999999999999::double precision) AS weighted_scalar, tdigest_percentile(v, 1::bigint, 10000, ARRAY[0.7499999999999999]::double precision[]) AS weighted_array FROM (VALUES (-18014398509481984::double precision), (-1::double precision)) AS input(v); -- Also cover stored digests and count rounding that makes q exactly 1. -- The endpoint case used to return -4, inside the bracket but not equal to -3. WITH inputs(label, d, p) AS ( VALUES ( 'precision', 'flags 1 count 2 compression 10000 centroids 2 (-18014398509481984, 1) (-1, 1)'::tdigest, 0.7499999999999999::double precision ), ( 'endpoint', 'flags 1 count 9007199254743615 compression 10000 centroids 3 (-10000000000000000, 9007199254740839) (-3, 290) (-1, 2486)'::tdigest, 0.9999999999997077::double precision ) ) SELECT label, tdigest_percentile(d, p) AS scalar_result, tdigest_percentile(d, ARRAY[p]) AS array_result FROM inputs GROUP BY label ORDER BY label; tvondra-tdigest-c0af713/test/sql/inverse_rank_boundaries.sql000066400000000000000000000070741525543331000244470ustar00rootroot00000000000000\set ECHO none SET max_parallel_workers_per_gather = 0; DO $$ DECLARE d tdigest; probes double precision[] := ARRAY[ -1.0, 0.0, 0.9999999999999999, 1.0, 1.0000000000000002, 1.9999999999999998, 2.0, 3.0 ]; results double precision[]; scalar_result double precision; i integer; BEGIN FOREACH d IN ARRAY ARRAY[ 'flags 1 count 18014398509481990 compression 10000 centroids 3 (0, 1) (1, 18014398509481987) (2, 2)'::tdigest, 'flags 1 count 19140298416324609 compression 10000 centroids 3 (0, 1) (1, 18014398509481981) (2, 1125899906842627)'::tdigest, 'flags 1 count 22517998136852481 compression 10000 centroids 3 (0, 1) (1, 18014398509481981) (2, 4503599627370499)'::tdigest, 'flags 1 count 9223372036854775807 compression 10000 centroids 3 (0, 9223372036854775805) (1, 1) (2, 1)'::tdigest ] LOOP SELECT tdigest_percentile_of(d, probes) INTO results; IF array_length(results, 1) IS DISTINCT FROM array_length(probes, 1) OR EXISTS ( SELECT 1 FROM ( SELECT v, lag(v) OVER (ORDER BY ord) AS prev FROM unnest(results) WITH ORDINALITY AS u(v, ord) ) AS steps WHERE v IS NULL OR v < prev OR NOT (v BETWEEN 0 AND 1) ) THEN RAISE EXCEPTION 'Inverse ranks decrease at a centroid boundary: %', results; END IF; RAISE NOTICE 'Inverse ranks increase at a centroid boundary'; FOR i IN 1..array_length(probes, 1) LOOP SELECT tdigest_percentile_of(d, probes[i]) INTO scalar_result; IF scalar_result IS DISTINCT FROM results[i] THEN RAISE EXCEPTION 'Scalar and array inverse percentiles disagree'; END IF; END LOOP; RAISE NOTICE 'Scalar and array inverse percentiles agree'; END LOOP; -- Exact equality still counts half of the entire equal-mean run. SELECT tdigest_percentile_of( 'flags 1 count 18014398509481990 compression 10000 centroids 3 (1, 1) (1, 18014398509481987) (1, 2)'::tdigest, 1.0) INTO scalar_result; IF scalar_result IS DISTINCT FROM 0.5::double precision THEN RAISE EXCEPTION 'Equal-mean ranks lost their half-weight convention'; END IF; RAISE NOTICE 'Equal-mean ranks follow the half-weight convention'; END $$; \set ECHO all SET extra_float_digits = 2; -- The CDF used to decrease from 0.375 to 0.37499999999999994 just above 1. -- Both equal-mean centroids survive compaction at this compression. WITH ranks AS ( SELECT tdigest_percentile_of(d, 1::double precision) AS at_mean, tdigest_percentile_of(d, 1.0000000000000002::double precision) AS above_mean, tdigest_percentile_of(d, ARRAY[1, 1.0000000000000002]::double precision[]) AS array_ranks FROM (VALUES ( 'flags 1 count 36028797018963981 compression 10000 centroids 4 (0, 9007199254740995) (1, 2) (1, 9007199254740993) (10, 18014398509481991)'::tdigest )) AS input(d) ) SELECT at_mean, above_mean, array_ranks, at_mean <= above_mean AS monotone, array_ranks = ARRAY[at_mean, above_mean] AS scalar_array_agree FROM ranks; -- Preserve half-items without rounding the integer part and half separately. -- In particular, (double) (18014398509481987 / 2) + 0.5 double-rounds. WITH counts(n) AS ( VALUES (1::bigint), (3), (18014398509481987), (9223372036854775807) ) SELECT n, tdigest_percentile_of( format('flags 1 count %s compression 10000 centroids 1 (1, %s)', n, n)::tdigest, 1::double precision) AS rank FROM counts GROUP BY n ORDER BY n; tvondra-tdigest-c0af713/test/sql/memory_usage.sql000066400000000000000000000030361525543331000222340ustar00rootroot00000000000000-- check the number of batches and amount of memory used by the HashAgg node CREATE FUNCTION check_memory_usage(text) RETURNS TABLE (nbatches INT, memorykb INT) LANGUAGE plpgsql AS $$ DECLARE ln TEXT; tmp TEXT[]; found_match BOOL := false; BEGIN FOR LN IN EXECUTE format('EXPLAIN ANALYZE %s', $1) LOOP IF NOT found_match THEN tmp := regexp_match(ln, '.*Batches: (\d*) .* Memory Usage: (\d*)')kB; IF tmp IS NOT NULL THEN found_match := true; RETURN query SELECT tmp[1]::int, tmp[2]::int; END IF; END IF; END LOOP; END; $$; -- make sure we use hash aggregate, with a single batch SET max_parallel_workers_per_gather = 0; SET work_mem = '256MB'; SET enable_hashagg = on; SET enable_sort = off; -- tiny table with just 256 rows CREATE TABLE tdigest_memory AS SELECT i % 128 AS g, i::float8 AS v FROM generate_series(1, 256) AS s(i); ANALYZE tdigest_memory; -- check we're actually using hash aggregate EXPLAIN (COSTS OFF) SELECT g, tdigest_count(tdigest(v, 10000)) AS count FROM tdigest_memory GROUP BY g; -- Hash aggregation holds all 128 groups at once: single batch should be enough, -- and we compare the memory to 200kB, in case older releases or some platforms -- happen to use a bit more memory etc. SELECT (nbatches = 1) AS batches_ok, (memorykb < 200) AS memory_ok FROM check_memory_usage('SELECT g, tdigest_count(tdigest(v, 10000)) AS count FROM tdigest_memory GROUP BY g'); DROP TABLE tdigest_memory; DROP FUNCTION check_memory_usage; tvondra-tdigest-c0af713/test/sql/null_arrays.sql000066400000000000000000000120031525543331000220650ustar00rootroot00000000000000-- Non-STRICT transition functions must reject NULL percentile or hypothetical -- value arguments when a non-NULL input creates the aggregate state. NULL -- input values themselves are skipped instead. \set VERBOSITY terse -- The requested percentile/value is only read while the aggregate state is -- still NULL, so the first non-NULL input is enough to exercise each check. -- tdigest_add_double(internal, double precision, int, double precision) SELECT tdigest_percentile(v, 100, NULL::double precision) FROM (VALUES (1.0::double precision), (2.0)) s(v); -- tdigest_add_double_array(internal, double precision, int, double precision[]) SELECT tdigest_percentile(v, 100, NULL::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); -- tdigest_add_double_count(internal, double precision, bigint, int, double precision) SELECT tdigest_percentile(v, 2, 100, NULL::double precision) FROM (VALUES (1.0::double precision), (2.0)) s(v); -- tdigest_add_double_array_count(internal, double precision, bigint, int, double precision[]) SELECT tdigest_percentile(v, 2, 100, NULL::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); -- tdigest_add_digest(internal, tdigest, double precision) SELECT tdigest_percentile(d, NULL::double precision) FROM (SELECT tdigest(v, 100) AS d FROM (VALUES (1.0::double precision), (2.0)) s(v)) t; -- tdigest_add_digest_array(internal, tdigest, double precision[]) SELECT tdigest_percentile(d, NULL::double precision[]) FROM (SELECT tdigest(v, 100) AS d FROM (VALUES (1.0::double precision), (2.0)) s(v)) t; -- tdigest_add_double_values(internal, double precision, int, double precision) SELECT tdigest_percentile_of(v, 100, NULL::double precision) FROM (VALUES (1.0::double precision), (2.0)) s(v); -- tdigest_add_double_array_values(internal, double precision, int, double precision[]) SELECT tdigest_percentile_of(v, 100, NULL::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); -- tdigest_add_double_values_count(internal, double precision, bigint, int, double precision) SELECT tdigest_percentile_of(v, 2, 100, NULL::double precision) FROM (VALUES (1.0::double precision), (2.0)) s(v); -- tdigest_add_double_array_values_count(internal, double precision, bigint, int, double precision[]) SELECT tdigest_percentile_of(v, 2, 100, NULL::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); -- tdigest_add_digest_values(internal, tdigest, double precision) SELECT tdigest_percentile_of(d, NULL::double precision) FROM (SELECT tdigest(v, 100) AS d FROM (VALUES (1.0::double precision), (2.0)) s(v)) t; -- tdigest_add_digest_array_values(internal, tdigest, double precision[]) SELECT tdigest_percentile_of(d, NULL::double precision[]) FROM (SELECT tdigest(v, 100) AS d FROM (VALUES (1.0::double precision), (2.0)) s(v)) t; -- A NULL array on a later row is harmless (the state already exists), and -- tdigest_add_double_array_increment() guards the same call correctly. Both -- work before and after the fix. SELECT tdigest_percentile(v, 100, CASE WHEN v = 1.0 THEN ARRAY[0.5] END) FROM (VALUES (1.0::double precision), (2.0)) s(v); SELECT tdigest_count(tdigest_add(NULL::tdigest, NULL::double precision[], 100)); -- The arrays however should not be empty {}. -- tdigest_add_double_array(internal, double precision, int, double precision[]) SELECT tdigest_percentile(v, 100, '{}'::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); -- tdigest_add_double_array_count(internal, double precision, bigint, int, double precision[]) SELECT tdigest_percentile(v, 2, 100, '{}'::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); -- tdigest_add_digest_array(internal, tdigest, double precision[]) SELECT tdigest_percentile(d, '{}'::double precision[]) FROM (SELECT tdigest(v, 100) AS d FROM (VALUES (1.0::double precision), (2.0)) s(v)) t; -- tdigest_add_double_array_values(internal, double precision, int, double precision[]) SELECT tdigest_percentile_of(v, 100, '{}'::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); -- tdigest_add_double_array_values_count(internal, double precision, bigint, int, double precision[]) SELECT tdigest_percentile_of(v, 2, 100, '{}'::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); -- tdigest_add_digest_array_values(internal, tdigest, double precision[]) SELECT tdigest_percentile_of(d, '{}'::double precision[]) FROM (SELECT tdigest(v, 100) AS d FROM (VALUES (1.0::double precision), (2.0)) s(v)) t; -- a NULL element inside an array is rejected too, and the message has to name -- the argument the array was passed as, not always a percentile SELECT tdigest_percentile(v, 100, ARRAY[0.5, NULL]::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); SELECT tdigest_percentile_of(v, 100, ARRAY[1.0, NULL]::double precision[]) FROM (VALUES (1.0::double precision), (2.0)) s(v); SELECT tdigest_count(tdigest_add(NULL::tdigest, ARRAY[1.0, NULL]::double precision[], 100)); tvondra-tdigest-c0af713/test/sql/null_count.sql000066400000000000000000000032631525543331000217240ustar00rootroot00000000000000-- Regression test for a NULL "count" argument in the transition functions -- accepting an explicit number of occurrences of a value. -- -- The count argument is not required to be non-NULL, and a NULL count is -- interpreted as a single occurrence of the value - exactly like the -- variants without the count argument. -- tdigest_add_double_values_count SELECT tdigest_percentile_of(v, NULL::bigint, 100, 1.0) FROM (VALUES (1.0), (2.0)) AS t(v); SELECT tdigest_percentile_of(v, 1::bigint, 100, 1.0) FROM (VALUES (1.0), (2.0)) AS t(v); -- tdigest_add_double_array_count SELECT tdigest_percentile(v, NULL::bigint, 100, ARRAY[0.5]) FROM (VALUES (1.0), (2.0)) AS t(v); SELECT tdigest_percentile(v, 1::bigint, 100, ARRAY[0.5]) FROM (VALUES (1.0), (2.0)) AS t(v); -- tdigest_add_double_array_values_count SELECT tdigest_percentile_of(v, NULL::bigint, 100, ARRAY[1.0]) FROM (VALUES (1.0), (2.0)) AS t(v); SELECT tdigest_percentile_of(v, 1::bigint, 100, ARRAY[1.0]) FROM (VALUES (1.0), (2.0)) AS t(v); -- tdigest_add_double_count_trimmed SELECT tdigest_avg(v, NULL::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0), (2.0)) AS t(v); SELECT tdigest_avg(v, 1::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0), (2.0)) AS t(v); -- tdigest_add_digest_trimmed has no count argument - the counts come from -- the centroids of the digest - so the closest equivalent is a NULL digest, -- which contributes no values at all SELECT tdigest_avg(d, 0.0, 1.0) FROM (VALUES (NULL::tdigest), (NULL)) AS t(d); SELECT tdigest_avg(d, 0.0, 1.0) FROM (VALUES ('flags 1 count 2 compression 100 centroids 2 (1, 1) (2, 1)'::tdigest), (NULL), ('flags 1 count 2 compression 100 centroids 2 (3, 1) (4, 1)'::tdigest)) AS t(d); tvondra-tdigest-c0af713/test/sql/null_increment.sql000066400000000000000000000035621525543331000225620ustar00rootroot00000000000000-- Regression test for the NULL handling in the incremental API. -- -- None of the incremental functions is strict, so NULL arguments do reach -- them. Adding a NULL value (or merging a NULL digest) is a no-op, and -- returns the digest passed in - possibly NULL, if that was NULL too. -- tdigest_add_double_increment SELECT tdigest_add(NULL::tdigest, NULL::double precision); SELECT tdigest_add('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest, NULL::double precision); SELECT tdigest_add(NULL::tdigest, 1.0::double precision, 100); -- tdigest_add_double_array_increment SELECT tdigest_add(NULL::tdigest, NULL::double precision[]); SELECT tdigest_add('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest, NULL::double precision[]); SELECT tdigest_add(NULL::tdigest, ARRAY[1.0]::double precision[], 100); -- tdigest_union_double_increment SELECT tdigest_union(NULL::tdigest, NULL::tdigest); SELECT tdigest_union('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest, NULL::tdigest); SELECT tdigest_union(NULL::tdigest, 'flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest); -- The compact flag is used on every code path that builds a result, so unlike -- the value/digest arguments it is not something that can be skipped. Passing -- NULL explicitly has to be rejected rather than silently taken as false. SELECT tdigest_add(NULL::tdigest, 1.0::double precision, 100, NULL); SELECT tdigest_add('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest, 1.0::double precision, NULL, NULL); SELECT tdigest_add(NULL::tdigest, ARRAY[1.0]::double precision[], 100, NULL); SELECT tdigest_add('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest, ARRAY[1.0]::double precision[], NULL, NULL); SELECT tdigest_union(NULL::tdigest, NULL::tdigest, NULL); SELECT tdigest_union('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest, NULL::tdigest, NULL); tvondra-tdigest-c0af713/test/sql/null_values.sql000066400000000000000000000061731525543331000220760ustar00rootroot00000000000000-- Regression test for the NULL handling in the transition functions. -- -- None of the transition functions is strict, so NULL rows do reach them -- and have to be ignored. Each function is exercised twice: once with all -- input rows NULL (so the state stays NULL and the transition function -- returns NULL), and once with a NULL row following a non-NULL one (so the -- transition function returns the existing state unmodified). -- tdigest_add_double SELECT tdigest_percentile(v, 100, 0.5) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); SELECT tdigest_percentile(v, 100, 0.5) FROM (VALUES (1.0), (NULL)) AS t(v); -- tdigest_add_double_count SELECT tdigest(v, 1::bigint, 100) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); SELECT tdigest(v, 1::bigint, 100) FROM (VALUES (1.0), (NULL)) AS t(v); -- tdigest_add_double_values SELECT tdigest_percentile_of(v, 100, 1.0) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); SELECT tdigest_percentile_of(v, 100, 1.0) FROM (VALUES (1.0), (NULL)) AS t(v); -- tdigest_add_double_values_count SELECT tdigest_percentile_of(v, 1::bigint, 100, 1.0) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); SELECT tdigest_percentile_of(v, 1::bigint, 100, 1.0) FROM (VALUES (1.0), (NULL)) AS t(v); -- tdigest_add_digest SELECT tdigest_percentile(d, 0.5) FROM (VALUES (NULL::tdigest), (NULL)) AS t(d); SELECT tdigest_percentile(d, 0.5) FROM (VALUES ('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest), (NULL)) AS t(d); -- tdigest_add_digest_values SELECT tdigest_percentile_of(d, 1.0) FROM (VALUES (NULL::tdigest), (NULL)) AS t(d); SELECT tdigest_percentile_of(d, 1.0) FROM (VALUES ('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest), (NULL)) AS t(d); -- tdigest_add_double_array SELECT tdigest_percentile(v, 100, ARRAY[0.5]) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); SELECT tdigest_percentile(v, 100, ARRAY[0.5]) FROM (VALUES (1.0), (NULL)) AS t(v); -- tdigest_add_double_array_count SELECT tdigest_percentile(v, 1::bigint, 100, ARRAY[0.5]) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); SELECT tdigest_percentile(v, 1::bigint, 100, ARRAY[0.5]) FROM (VALUES (1.0), (NULL)) AS t(v); -- tdigest_add_double_array_values SELECT tdigest_percentile_of(v, 100, ARRAY[1.0]) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); SELECT tdigest_percentile_of(v, 100, ARRAY[1.0]) FROM (VALUES (1.0), (NULL)) AS t(v); -- tdigest_add_double_array_values_count SELECT tdigest_percentile_of(v, 1::bigint, 100, ARRAY[1.0]) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); SELECT tdigest_percentile_of(v, 1::bigint, 100, ARRAY[1.0]) FROM (VALUES (1.0), (NULL)) AS t(v); -- tdigest_add_digest_array SELECT tdigest_percentile(d, ARRAY[0.5]) FROM (VALUES (NULL::tdigest), (NULL)) AS t(d); SELECT tdigest_percentile(d, ARRAY[0.5]) FROM (VALUES ('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest), (NULL)) AS t(d); -- tdigest_add_digest_array_values SELECT tdigest_percentile_of(d, ARRAY[1.0]) FROM (VALUES (NULL::tdigest), (NULL)) AS t(d); SELECT tdigest_percentile_of(d, ARRAY[1.0]) FROM (VALUES ('flags 1 count 1 compression 100 centroids 1 (1, 1)'::tdigest), (NULL)) AS t(d); tvondra-tdigest-c0af713/test/sql/parallel_query.sql000066400000000000000000000210061525543331000225560ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('extra_float_digits', '0', false); PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; -- the user-facing functions must all be parallel safe, otherwise queries -- using them lose parallelism (the aggregate support functions take or -- return "internal" and cannot be called from SQL, so they are exempt) SELECT p.proname, p.proparallel FROM pg_proc p JOIN pg_depend d ON (d.objid = p.oid AND d.deptype = 'e') JOIN pg_extension e ON (e.oid = d.refobjid) WHERE e.extname = 'tdigest' AND p.proargtypes::oid[] && ARRAY['internal'::regtype]::oid[] IS NOT TRUE AND p.prorettype <> 'internal'::regtype AND p.proparallel <> 's' ORDER BY 1; -- test parallel query CREATE TABLE test_parallel (v double precision, c int, d int); INSERT INTO test_parallel SELECT 1000 * random(), 1 + mod(i,7), mod(i,113) FROM generate_series(1,100000) s(i); ANALYZE test_parallel; CREATE TABLE test_parallel_2 (d tdigest); INSERT INTO test_parallel_2 SELECT tdigest(v, 100) FROM test_parallel GROUP BY d; ANALYZE test_parallel_2; -- individual values EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, 100, 0.95) AS b FROM test_parallel) foo; WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, 100, 0.95) AS b FROM test_parallel) foo; EXPLAIN (COSTS OFF) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(v, 100, 950) AS b FROM test_parallel) foo; SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(v, 100, 950) AS b FROM test_parallel) foo; EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(d, 0.95) AS b FROM test_parallel_2) foo; WITH x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(d, 0.95) AS b FROM test_parallel_2) foo; EXPLAIN (COSTS OFF) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(d, 950) AS b FROM test_parallel_2) foo; SELECT 950, abs(a - b) < 0.01 FROM ( SELECT 0.95 AS a, tdigest_percentile_of(d, 950) AS b FROM test_parallel_2) foo; -- array of percentiles / values EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel) foo; WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel) foo; EXPLAIN (COSTS OFF) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(v, 100, ARRAY[950, 990])) AS b FROM test_parallel) foo; WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(v, 100, ARRAY[950, 990])) AS b FROM test_parallel) foo; EXPLAIN (COSTS OFF) WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(d, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel_2) foo; WITH x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM test_parallel) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(d, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_parallel_2) foo; EXPLAIN (COSTS OFF) WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(d, ARRAY[950, 990])) AS b FROM test_parallel_2) foo; WITH x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) FROM test_parallel)) AS p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile_of(d, ARRAY[950, 990])) AS b FROM test_parallel_2) foo; -- trimmed aggregates EXPLAIN (COSTS OFF) SELECT tdigest_sum(v, 100, 0.05, 0.95) FROM test_parallel; -- trimming nothing has to reproduce the exact sum SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_sum(v, 100, 0.0, 1.0) AS b FROM test_parallel) foo; EXPLAIN (COSTS OFF) SELECT tdigest_avg(v, 100, 0.05, 0.95) FROM test_parallel; SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_avg(v, 100, 0.0, 1.0) AS b FROM test_parallel) foo; EXPLAIN (COSTS OFF) SELECT tdigest_sum(d, 0.05, 0.95) FROM test_parallel_2; SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_sum(d, 0.0, 1.0) AS b FROM test_parallel_2) foo; EXPLAIN (COSTS OFF) SELECT tdigest_avg(d, 0.05, 0.95) FROM test_parallel_2; SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_avg(d, 0.0, 1.0) AS b FROM test_parallel_2) foo; -- trimmed aggregates (on tdigest) EXPLAIN (COSTS OFF) SELECT tdigest_digest_sum(tdigest(v, 100), 0.05, 0.95) FROM test_parallel; -- trimming nothing has to reproduce the exact sum SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_digest_sum(tdigest(v, 100), 0.0, 1.0) AS b FROM test_parallel) foo; EXPLAIN (COSTS OFF) SELECT tdigest_digest_avg(tdigest(v, 100), 0.05, 0.95) FROM test_parallel; SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_digest_avg(tdigest(v, 100), 0.0, 1.0) AS b FROM test_parallel) foo; EXPLAIN (COSTS OFF) SELECT tdigest_digest_sum(tdigest(d), 0.05, 0.95) FROM test_parallel_2; SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT sum(v) FROM test_parallel) AS a, tdigest_digest_sum(tdigest(d), 0.0, 1.0) AS b FROM test_parallel_2) foo; EXPLAIN (COSTS OFF) SELECT tdigest_digest_avg(tdigest(d), 0.05, 0.95) FROM test_parallel_2; SELECT abs(a - b) / a < 0.01 FROM ( SELECT (SELECT avg(v) FROM test_parallel) AS a, tdigest_digest_avg(tdigest(d), 0.0, 1.0) AS b FROM test_parallel_2) foo; -- casting a digest to text must not force a serial plan EXPLAIN (COSTS OFF) SELECT count(*) FROM test_parallel_2 WHERE length(d::text) > 5; DROP TABLE test_parallel; DROP TABLE test_parallel_2; tvondra-tdigest-c0af713/test/sql/parse_checks.sql000066400000000000000000000074101525543331000221720ustar00rootroot00000000000000-- Regression test for the input function tdigest_in(), which parses the -- textual representation of a t-digest. -- -- The input is user supplied, so it has to be validated carefully - both -- the syntax of the value and the sanity of the parsed data. The queries -- exercise all the error checks in the input function, one by one. \set VERBOSITY terse -- a valid value, to make sure the checks don't reject correct input SELECT 'flags 1 count 3 compression 10 centroids 2 (1, 1) (2, 2)'::tdigest; -- -- syntax errors detected while parsing the individual fields -- -- unexpected space (the value must not start with whitespace) SELECT ' flags 1 count 3 compression 10 centroids 1 (1, 3)'::tdigest; -- unexpected space (only a single space may separate the fields) SELECT 'flags 1 count 3 compression 10 centroids 1 (1, 3)'::tdigest; -- missing space (the fields have to be separated) SELECT 'flags 1count 3 compression 10 centroids 1 (1, 3)'::tdigest; -- unexpected keyword SELECT 'flgs 1 count 3 compression 10 centroids 1 (1, 3)'::tdigest; -- integer field that is not a number at all SELECT 'flags x count 3 compression 10 centroids 1 (1, 3)'::tdigest; -- integer field that does not fit into bigint SELECT 'flags 1 count 99999999999999999999 compression 10 centroids 1 (1, 3)'::tdigest; -- integer field that fits into bigint but not into integer SELECT 'flags 1 count 3 compression 3000000000 centroids 1 (1, 3)'::tdigest; -- floating point field that is not a number at all SELECT 'flags 1 count 3 compression 10 centroids 1 (x, 3)'::tdigest; -- floating point field that does not fit into double precision SELECT 'flags 1 count 3 compression 10 centroids 1 (1e400, 3)'::tdigest; -- complete underflow must not silently turn a nonzero input into zero SELECT 'flags 1 count 3 compression 10 centroids 1 (1e-400, 3)'::tdigest; SELECT 'flags 1 count 3 compression 10 centroids 1 (-1e-400, 3)'::tdigest; SELECT 'flags 0 count 3 compression 10 centroids 1 (1e-400, 3)'::tdigest; SELECT 'flags 0 count 3 compression 10 centroids 1 (-1e-400, 3)'::tdigest; -- -- sanity checks on the parsed header -- -- unknown flags SELECT 'flags 2 count 3 compression 10 centroids 1 (1, 3)'::tdigest; -- compression out of the supported range SELECT 'flags 1 count 3 compression 9 centroids 1 (1, 3)'::tdigest; SELECT 'flags 1 count 3 compression 10001 centroids 1 (1, 3)'::tdigest; -- non-positive total count SELECT 'flags 1 count 0 compression 10 centroids 1 (1, 1)'::tdigest; -- non-positive number of centroids SELECT 'flags 1 count 3 compression 10 centroids 0'::tdigest; -- more centroids than fit into the buffer (10 * compression) SELECT 'flags 1 count 3 compression 10 centroids 101'::tdigest; -- -- sanity checks on the parsed centroids -- -- centroid mean that is not a valid number SELECT 'flags 1 count 3 compression 10 centroids 1 (NaN, 3)'::tdigest; SELECT 'flags 1 count 3 compression 10 centroids 1 (Infinity, 3)'::tdigest; -- non-positive centroid count SELECT 'flags 1 count 3 compression 10 centroids 1 (1, 0)'::tdigest; -- centroid count exceeding the total count SELECT 'flags 1 count 3 compression 10 centroids 1 (1, 4)'::tdigest; -- centroids not sorted by mean SELECT 'flags 1 count 3 compression 10 centroids 2 (2, 1) (1, 2)'::tdigest; -- centroid counts overflowing the total count SELECT 'flags 1 count 9223372036854775807 compression 10 centroids 2 (1, 9223372036854775807) (2, 9223372036854775807)'::tdigest; -- -- sanity checks on the value as a whole -- -- more centroids than the header says SELECT 'flags 1 count 1 compression 10 centroids 1 (1, 1) (2, 1)'::tdigest; -- fewer centroids than the header says SELECT 'flags 1 count 3 compression 10 centroids 3 (1, 1)'::tdigest; -- centroid counts not adding up to the total count SELECT 'flags 1 count 3 compression 10 centroids 1 (1, 1)'::tdigest; tvondra-tdigest-c0af713/test/sql/percentile_of_extreme.sql000066400000000000000000000145471525543331000241200ustar00rootroot00000000000000-- tdigest_percentile_of() used to interpolate between two centroids by -- computing the slope of the line connecting their means: -- -- m = (curr->mean - prev->mean) / (curr->count / 2.0 + prev->count / 2.0); -- x = (value - prev->mean) / m; -- -- Both the difference of the means and the slope could leave the float8 -- range even though every input was an ordinary finite number: -- -- * (curr->mean - prev->mean) overflows to infinity when the means are -- large and have opposite signs. The slope is then infinity, and x is -- either 0 (silently wrong) or NaN. -- -- * the slope underflows to zero when the means are tiny (subnormal) and -- the counts are huge. x is then infinity, and so is the result. -- -- Both quantile functions now use convex combinations. The inverse function -- computes the relative position between means without dividing by a slope, -- and halves the means and probe when their difference would overflow. -- -- For finite probes the result must be a probability in [0, 1]. Power-of-two -- rescaling without input precision loss should agree within rounding error. \set VERBOSITY terse SET extra_float_digits = 0; CREATE TABLE tdigest_extreme_digests (id int, descr text, d tdigest, probes double precision[]); INSERT INTO tdigest_extreme_digests VALUES (1, 'means of opposite sign, the difference overflows', 'flags 1 count 2 compression 10000 centroids 2 (-1.7e308, 1) (1.7e308, 1)', ARRAY[-1.7e308, -1e308, -1e307, 0, 1e307, 1e308, 1.7e308]), (2, 'the same, with uneven counts', 'flags 1 count 10 compression 10000 centroids 2 (-1.5e308, 7) (1.5e308, 3)', ARRAY[-1.5e308, -1e308, 0, 1e308, 1.5e308]), (3, 'the same, with a centroid in the middle', 'flags 1 count 3 compression 10000 centroids 3 (-1.7e308, 1) (0, 1) (1.7e308, 1)', ARRAY[-1.7e308, -1e308, -1e-300, 0, 1e-300, 1e308, 1.7e308]), (4, 'subnormal means and huge counts, the slope underflows', 'flags 1 count 9223372036854775806 compression 10000 centroids 2 (1e-320, 4611686018427387903) (2e-320, 4611686018427387903)', ARRAY[1e-320, 1.2e-320, 1.5e-320, 1.8e-320, 2e-320]), (5, 'almost adjacent means and huge counts', 'flags 1 count 9223372036854775806 compression 10000 centroids 2 (1, 4611686018427387903) (1.0000000000000004, 4611686018427387903)', ARRAY[1, 1.0000000000000002, 1.0000000000000004]), (6, 'large means and huge counts', 'flags 1 count 4611686018427387903 compression 10000 centroids 2 (-1e308, 2305843009213693951) (1e308, 2305843009213693952)', ARRAY[-1e308, -5e307, 0, 5e307, 1e308]), (7, 'ordinary means, for comparison', 'flags 1 count 10 compression 10000 centroids 2 (-1.5, 7) (1.5, 3)', ARRAY[-1.5, -1, 0, 1, 1.5]); -- Every result has to be a probability. The old implementation returned NaN -- when the difference overflowed or Infinity when the slope underflowed. WITH q AS ( SELECT id, descr, probes, tdigest_percentile_of(d, probes) AS res FROM tdigest_extreme_digests GROUP BY id, descr, probes ) SELECT id, descr, v AS value, r AS percentile_of FROM q, unnest(q.probes, q.res) WITH ORDINALITY AS x(v, r, ord) WHERE NOT (r BETWEEN 0 AND 1) ORDER BY id, ord; -- And the result still has to be a non-decreasing function of the value. WITH q AS ( SELECT id, descr, probes, tdigest_percentile_of(d, probes) AS res FROM tdigest_extreme_digests GROUP BY id, descr, probes ), u AS ( SELECT id, descr, ord, v, r FROM q, unnest(q.probes, q.res) WITH ORDINALITY AS x(v, r, ord) ), s AS ( SELECT id, descr, v, r, lag(v) OVER w AS prev_v, lag(r) OVER w AS prev_r FROM u WINDOW w AS (PARTITION BY id ORDER BY ord) ) SELECT id, descr, count(*) AS decreasing_steps FROM s WHERE r < prev_r GROUP BY id, descr ORDER BY id; -- This is reachable without writing the digest by hand - two ordinary finite -- values are enough to build a digest whose centroid means are far enough -- apart. WITH q AS ( SELECT tdigest_percentile_of(v, 10000, ARRAY[-1e307, 0.0, 1e307]::double precision[]) AS res FROM (VALUES (-1.7e308::double precision), (1.7e308::double precision)) t(v) ) SELECT bool_and(r BETWEEN 0 AND 1) AS all_probabilities FROM q, unnest(q.res) AS r; WITH q AS ( SELECT tdigest_percentile_of(v, c, 10000, ARRAY[-1e307, 0.0, 1e307]::double precision[]) AS res FROM (VALUES (-1.7e308::double precision, 7::bigint), (1.7e308::double precision, 3::bigint)) t(v, c) ) SELECT bool_and(r BETWEEN 0 AND 1) AS all_probabilities FROM q, unnest(q.res) AS r; -- The percentile only depends on where the value sits between the two -- centroid means, so multiplying the means and the probed values by the same -- power of two must not change the result in any way. 2^1022 pushes the -- difference of the means (4) just past the float8 range, while every single -- mean still fits comfortably. The comparison allows for a rounding error of -- a couple of ULPs, the difference we are looking for here is 0.0625. WITH s(k) AS ( VALUES (1::double precision), (2::double precision ^ 1022) ), r AS ( SELECT s.k, tdigest_percentile_of(t.v * s.k, t.c, 10000, ARRAY[-3*s.k, -2*s.k, -1*s.k, 0*s.k, 1*s.k]) AS q FROM s, (VALUES (-3.0::double precision, 3::bigint), (1.0::double precision, 1::bigint)) t(v, c) GROUP BY s.k ), z AS ( SELECT a.q AS unscaled, b.q AS scaled FROM r a, r b WHERE a.k = 1 AND b.k <> 1 ) SELECT bool_and(abs(u - s) <= 1e-15) AS scale_invariant FROM z, unnest(z.unscaled, z.scaled) AS x(u, s); DROP TABLE tdigest_extreme_digests; -- The calculated percentile has to be between the means, and not outside -- due to rounding, etc. All the centroids have mean 123.456, so all quantiles -- can be just 123.456 too (without the clamping we'd get 123.45600000000002). SELECT tdigest_percentile('flags 1 count 1152921504606846978 compression 100 centroids 3 (123.456, 1) (123.456, 1152921504606846976) (123.456, 1)'::tdigest, 0.1); -- Similar issue in the other direction - rounding errors can yield values -- outside the valid [0.0, 1.0] range close to the 1.0 boundary, and need to -- be correctly clamped. For example, this might return 1.0000000000000002. SELECT tdigest_percentile_of('flags 1 count 6381118572735248714 compression 10 centroids 3 (0.0, 5752940093788208662) (1.0, 628178478947040051) (2.0, 1)'::tdigest, 2.0); tvondra-tdigest-c0af713/test/sql/percentile_of_monotonic.sql000066400000000000000000000157671525543331000244610ustar00rootroot00000000000000-- tdigest_percentile_of() evaluates the CDF of the digest, so for a given -- digest the result has to be a non-decreasing function of the value, and it -- has to fall into the [0, 1] range for finite probes. This guards a former -- integer-division bug that violated both properties. -- -- tdigest_compute_quantiles_of() interpolates between two centroids, and to -- do that it first moves from the boundary of the previous centroid to its -- mean, which sits in the middle of the centroid. It used to do this with: -- -- count -= (prev->count / 2); -- -- That was an int64 division, even though "count" is a double and the very -- next statement halves the same value as (prev->count / 2.0). For centroids -- with an odd count the two halves therefore disagreed by 0.5 items, and the -- interpolated CDF ended up shifted by 0.5/count with respect to the exact -- value calculated in the (value == curr->mean) branch just above. -- -- The result was a saw-tooth: the CDF jumped up by 0.5/count right after the -- mean of every odd-sized centroid, and dropped back at the next centroid. For -- the (0, 7) (100, 3) digest used below: -- -- value before expected -- 0 0.35 0.35 -- 0.0001 0.4000005 0.3500005 -- 50 0.65 0.6 -- 99.9999 0.8999995 0.8499995 -- 100 0.85 0.85 -- -- i.e. the CDF at 99.9999 was higher than the CDF at 100. Both divisions -- now use floating point, preserving monotonicity. \set VERBOSITY terse SET extra_float_digits = 0; CREATE TABLE tdigest_monotonic_digests (id int, descr text, d tdigest); -- Hand-built digests. All of them use centroid means from the [0, 100] range, -- so that a single set of probes works for all of them. What matters are the -- centroid counts - the old discontinuity showed up at the mean of every centroid -- with an odd count. INSERT INTO tdigest_monotonic_digests VALUES (1, 'two centroids, odd + odd', 'flags 1 count 10 compression 10000 centroids 2 (0, 7) (100, 3)'), (2, 'two centroids, odd + even', 'flags 1 count 11 compression 10000 centroids 2 (0, 7) (100, 4)'), (3, 'two centroids, even + odd', 'flags 1 count 11 compression 10000 centroids 2 (0, 8) (100, 3)'), (4, 'two centroids, even + even', 'flags 1 count 4 compression 10000 centroids 2 (0, 2) (100, 2)'), (5, 'two centroids, single item each', 'flags 1 count 2 compression 10000 centroids 2 (0, 1) (100, 1)'), (6, 'three centroids, single item each', 'flags 1 count 3 compression 10000 centroids 3 (0, 1) (50, 1) (100, 1)'), (7, 'three centroids, mixed counts', 'flags 1 count 12 compression 10000 centroids 3 (0, 3) (50, 4) (100, 5)'), (8, 'a single centroid', 'flags 1 count 10 compression 10000 centroids 1 (50, 10)'), (9, 'repeated means', 'flags 1 count 6 compression 10000 centroids 4 (0, 1) (50, 2) (50, 2) (100, 1)'); -- And a couple of digests built by the aggregate from actual data. The first -- one retains singleton centroids after final compaction, while the other two -- merge input values. INSERT INTO tdigest_monotonic_digests SELECT 10, 'built from 51 distinct values, no centroid merging', tdigest((2 * i)::double precision, 10000) FROM generate_series(0, 50) s(i); INSERT INTO tdigest_monotonic_digests SELECT 11, 'built from 10000 rows, compression 10', tdigest((i % 101)::double precision, 10) FROM generate_series(1, 10000) s(i); INSERT INTO tdigest_monotonic_digests SELECT 12, 'built from 10000 rows, compression 100', tdigest((i % 101)::double precision, 100) FROM generate_series(1, 10000) s(i); -- The values to probe the digests with: a coarse grid reaching outside the -- range of the digests, and then the centroid means used above with values -- immediately below and above them. The interpolation switches from one pair -- of centroids to the next exactly at the means, so that is where it is -- discontinuous. CREATE TABLE tdigest_monotonic_probes (v double precision); INSERT INTO tdigest_monotonic_probes SELECT DISTINCT v FROM ( SELECT i / 2.0::double precision FROM generate_series(-20, 220) s(i) UNION ALL SELECT m + o FROM generate_series(0, 100, 2) s(m), unnest(ARRAY[-1e-6, -1e-9, 0, 1e-9, 1e-6]::double precision[]) o ) s(v); -- The CDF must never decrease. Report one row per digest that breaks it, so -- that a failure stays readable. WITH q AS ( SELECT g.id, g.descr, p.arr, tdigest_percentile_of(g.d, p.arr) AS res FROM tdigest_monotonic_digests g, (SELECT array_agg(v ORDER BY v) AS arr FROM tdigest_monotonic_probes) p GROUP BY g.id, g.descr, p.arr ), u AS ( SELECT id, descr, ord, v, r FROM q, unnest(q.arr, q.res) WITH ORDINALITY AS x(v, r, ord) ), s AS ( SELECT id, descr, v, r, lag(v) OVER w AS prev_v, lag(r) OVER w AS prev_r FROM u WINDOW w AS (PARTITION BY id ORDER BY ord) ) SELECT id, descr, count(*) AS decreasing_steps, max(prev_r - r) AS max_drop FROM s WHERE r < prev_r GROUP BY id, descr ORDER BY id; -- And the result must always be a probability. WITH q AS ( SELECT g.id, g.descr, p.arr, tdigest_percentile_of(g.d, p.arr) AS res FROM tdigest_monotonic_digests g, (SELECT array_agg(v ORDER BY v) AS arr FROM tdigest_monotonic_probes) p GROUP BY g.id, g.descr, p.arr ) SELECT id, descr, count(*) AS out_of_range FROM q, unnest(q.res) AS r WHERE NOT (r BETWEEN 0 AND 1) GROUP BY id, descr ORDER BY id; -- The concrete case from the comment at the top, spelled out. The CDF has to -- grow (or at least stay the same) on each of the four steps. SELECT r[1] <= r[2] AS "0 -> 0.0001", r[2] <= r[3] AS "0.0001 -> 50", r[3] <= r[4] AS "50 -> 99.9999", r[4] <= r[5] AS "99.9999 -> 100" FROM (SELECT tdigest_percentile_of('flags 1 count 10 compression 10000 centroids 2 (0, 7) (100, 3)'::tdigest, ARRAY[0.0, 0.0001, 50.0, 99.9999, 100.0]::double precision[]) AS r) x; -- The same thing through the two aggregates building the digest from data, -- to show this is not an artifact of a hand-written digest. Three items with -- the value 0 and a single one with the value 100. SELECT r[1] <= r[2] AS "0 -> 0.0001", r[2] <= r[3] AS "0.0001 -> 50", r[3] <= r[4] AS "50 -> 99.9999", r[4] <= r[5] AS "99.9999 -> 100" FROM (SELECT tdigest_percentile_of(v, 10000, ARRAY[0.0, 0.0001, 50.0, 99.9999, 100.0]::double precision[]) AS r FROM (VALUES (0.0::double precision), (0.0::double precision), (0.0::double precision), (100.0::double precision)) t(v)) x; SELECT r[1] <= r[2] AS "0 -> 0.0001", r[2] <= r[3] AS "0.0001 -> 50", r[3] <= r[4] AS "50 -> 99.9999", r[4] <= r[5] AS "99.9999 -> 100" FROM (SELECT tdigest_percentile_of(v, c, 10000, ARRAY[0.0, 0.0001, 50.0, 99.9999, 100.0]::double precision[]) AS r FROM (VALUES (0.0::double precision, 3::bigint), (100.0::double precision, 1::bigint)) t(v, c)) x; DROP TABLE tdigest_monotonic_digests; DROP TABLE tdigest_monotonic_probes; tvondra-tdigest-c0af713/test/sql/percentile_roundtrip.sql000066400000000000000000000123471525543331000240050ustar00rootroot00000000000000-- tdigest_percentile() and tdigest_percentile_of() are inverse functions - -- one interpolates a value from a percentile, the other a percentile from a -- value - so composing them has to be an identity: -- -- tdigest_percentile_of(d, tdigest_percentile(d, p)) = p -- -- This only holds for percentiles that are actually interpolated. Both -- functions place the mean of a centroid in the middle of it, so below -- (first centroid count / 2) items and above (last centroid count / 2) items -- there is nothing to interpolate between and the result is clamped to the -- first/last mean. Each digest below therefore comes with the range of -- percentiles that are inside the interpolated part. -- -- The identity does not hold at the moment, because the two functions do not -- agree on where the mean of a centroid is. tdigest_compute_quantiles_of() -- computes the offset of the mean with an int64 division -- -- count -= (prev->count / 2); -- -- (note "count" is a double, and the very next statement uses / 2.0), so for -- centroids with an odd count it is off by half an item, i.e. the round trip -- is off by 0.5/count. On the ten item digest below that is a whole 0.05: -- -- p tdigest_percentile round trip expected -- 0.1 1.5 0.15 0.1 -- 0.5 5.5 0.55 0.5 -- 0.9 9.5 0.95 0.9 \set VERBOSITY terse SET extra_float_digits = 0; CREATE TABLE tdigest_roundtrip_digests (id int, descr text, d tdigest, p_low double precision, p_high double precision); INSERT INTO tdigest_roundtrip_digests VALUES (1, 'ten centroids, one item each', 'flags 1 count 10 compression 10000 centroids 10 (1, 1) (2, 1) (3, 1) (4, 1) (5, 1) (6, 1) (7, 1) (8, 1) (9, 1) (10, 1)', 0.05, 0.95), (2, 'two centroids, odd + odd', 'flags 1 count 10 compression 10000 centroids 2 (0, 7) (100, 3)', 0.35, 0.85), (3, 'three centroids, mixed counts', 'flags 1 count 12 compression 10000 centroids 3 (0, 3) (50, 4) (100, 5)', 0.125, 0.79), (4, 'a large centroid in the middle', 'flags 1 count 22 compression 10000 centroids 3 (0, 1) (10, 20) (20, 1)', 0.05, 0.95), (5, 'negative and positive means', 'flags 1 count 8 compression 10000 centroids 4 (-100, 2) (-1, 2) (1, 2) (100, 2)', 0.125, 0.875); -- And digests built by the aggregate from actual data. The first one retains -- singleton centroids after final compaction, while the second merges values. INSERT INTO tdigest_roundtrip_digests SELECT 6, 'built from 1000 distinct values, no centroid merging', tdigest(i::double precision, 10000), 0.005, 0.995 FROM generate_series(1, 1000) s(i); INSERT INTO tdigest_roundtrip_digests SELECT 7, 'built from 10000 rows, compression 100', tdigest((i % 1000)::double precision, 100), 0.05, 0.95 FROM generate_series(1, 10000) s(i); CREATE TABLE tdigest_roundtrip_percentiles (p double precision); INSERT INTO tdigest_roundtrip_percentiles SELECT i / 100.0 FROM generate_series(1, 99) s(i); -- Report one row per digest that fails the round trip, so that a failure -- stays readable. WITH v AS ( SELECT g.id, g.descr, p.p, tdigest_percentile(g.d, p.p) AS value FROM tdigest_roundtrip_digests g, tdigest_roundtrip_percentiles p WHERE p.p >= g.p_low AND p.p <= g.p_high GROUP BY g.id, g.descr, p.p ), rt AS ( SELECT v.id, v.descr, v.p, v.value, tdigest_percentile_of(g.d, v.value) AS back FROM v, tdigest_roundtrip_digests g WHERE g.id = v.id GROUP BY v.id, v.descr, v.p, v.value ) SELECT id, descr, count(*) AS failures, max(abs(back - p)) AS max_error FROM rt WHERE abs(back - p) > 1e-12 GROUP BY id, descr ORDER BY id; -- The three concrete percentiles from the comment at the top. WITH v AS ( SELECT p, tdigest_percentile('flags 1 count 10 compression 10000 centroids 10 (1, 1) (2, 1) (3, 1) (4, 1) (5, 1) (6, 1) (7, 1) (8, 1) (9, 1) (10, 1)'::tdigest, p) AS value FROM unnest(ARRAY[0.1, 0.5, 0.9]::double precision[]) AS p GROUP BY p ) SELECT p, value, tdigest_percentile_of('flags 1 count 10 compression 10000 centroids 10 (1, 1) (2, 1) (3, 1) (4, 1) (5, 1) (6, 1) (7, 1) (8, 1) (9, 1) (10, 1)'::tdigest, value) AS round_trip FROM v GROUP BY p, value ORDER BY p; -- The same identity has to hold for the aggregates building the digest from -- data. Ten items, so the interpolated range is [0.05, 0.95]. WITH v AS ( SELECT p, tdigest_percentile(i::double precision, 10000, p) AS value FROM generate_series(1, 10) s(i), unnest(ARRAY[0.1, 0.5, 0.9]::double precision[]) AS p GROUP BY p ) SELECT v.p, v.value, tdigest_percentile_of(i::double precision, 10000, v.value) AS round_trip FROM v, generate_series(1, 10) s(i) GROUP BY v.p, v.value ORDER BY v.p; WITH v AS ( SELECT p, tdigest_percentile(i::double precision, 2::bigint, 10000, p) AS value FROM generate_series(1, 5) s(i), unnest(ARRAY[0.2, 0.5, 0.8]::double precision[]) AS p GROUP BY p ) SELECT v.p, v.value, tdigest_percentile_of(i::double precision, 2::bigint, 10000, v.value) AS round_trip FROM v, generate_series(1, 5) s(i) GROUP BY v.p, v.value ORDER BY v.p; DROP TABLE tdigest_roundtrip_digests; DROP TABLE tdigest_roundtrip_percentiles; tvondra-tdigest-c0af713/test/sql/random.sql000066400000000000000000001017701525543331000210240ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; ----------------------------------------------------------- -- nice data set with ordered (asc) / evenly-spaced data -- ----------------------------------------------------------- -- compression 10 WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(1,10000) s(i)) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(1,10000) s(i)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 10, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- compression 100 WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(1,10000) s(i)) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(1,10000) s(i)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 100, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- compression 1000 WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(1,10000) s(i)) SELECT p, abs(a - b) < 0.001, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.001 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(1,10000) s(i)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 1000, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; ------------------------------------------------------------ -- nice data set with ordered (desc) / evenly-spaced data -- ------------------------------------------------------------ -- compression 10 WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(10000,1,-1) s(i)) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(10000,1,-1) s(i)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 10, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- compression 100 WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(10000,1,-1) s(i)) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(10000,1,-1) s(i)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 100, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- compression 1000 WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(10000,1,-1) s(i)) SELECT p, abs(a - b) < 0.001, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.001 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(10000,1,-1) s(i)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 1000, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; ---------------------------------------------------- -- nice data set with random / evenly-spaced data -- ---------------------------------------------------- -- compression 10 WITH data AS (SELECT i / 10000.0 AS x FROM (SELECT generate_series(1,10000) AS i, prng(10000, 49979693) AS x ORDER BY x) foo) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT i / 10000.0 AS x FROM (SELECT generate_series(1,10000) AS i, prng(10000, 49979693) AS x ORDER BY x) foo), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 10, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- compression 100 WITH data AS (SELECT i / 10000.0 AS x FROM (SELECT generate_series(1,10000) AS i, prng(10000, 49979693) AS x ORDER BY x) foo) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT i / 10000.0 AS x FROM (SELECT generate_series(1,10000) AS i, prng(10000, 49979693) AS x ORDER BY x) foo), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 100, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- compression 1000 WITH data AS (SELECT i / 10000.0 AS x FROM (SELECT generate_series(1,10000) AS i, prng(10000, 49979693) AS x ORDER BY x) foo) SELECT p, abs(a - b) < 0.001, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.001 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT i / 10000.0 AS x FROM (SELECT generate_series(1,10000) AS i, prng(10000, 49979693) AS x ORDER BY x) foo), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 1000, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; ---------------------------------------------- -- nice data set with random data (uniform) -- ---------------------------------------------- -- compression 10 WITH data AS (SELECT x FROM prng(10000) s(x)) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT x FROM prng(10000) s(x)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 10, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- compression 100 WITH data AS (SELECT x FROM prng(10000) s(x)) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT x FROM prng(10000) s(x)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 100, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- compression 1000 WITH data AS (SELECT x FROM prng(10000) s(x)) SELECT p, abs(a - b) < 0.001, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.001 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT x FROM prng(10000) s(x)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 1000, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -------------------------------------------------- -- nice data set with random data (skewed sqrt) -- -------------------------------------------------- -- compression 10 WITH data AS (SELECT sqrt(z) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT sqrt(z) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 10, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- compression 100 WITH data AS (SELECT sqrt(z) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT sqrt(z) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 100, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- compression 1000 WITH data AS (SELECT sqrt(z) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.001, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.001 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT sqrt(z) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 1000, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; ------------------------------------------------------- -- nice data set with random data (skewed sqrt+sqrt) -- ------------------------------------------------------- -- compression 10 WITH data AS (SELECT sqrt(sqrt(z)) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT sqrt(sqrt(z)) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 10, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- compression 100 WITH data AS (SELECT sqrt(sqrt(z)) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT sqrt(sqrt(z)) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 100, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- compression 1000 WITH data AS (SELECT sqrt(sqrt(z)) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.001, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.001 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT sqrt(sqrt(z)) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 1000, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; ------------------------------------------------- -- nice data set with random data (skewed pow) -- ------------------------------------------------- -- compression 10 WITH data AS (SELECT pow(z, 2) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT pow(z, 2) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 10, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- compression 100 WITH data AS (SELECT pow(z, 2) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.005, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.005 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT pow(z, 2) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 100, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- compression 1000 WITH data AS (SELECT pow(z, 2) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.001, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.001 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT pow(z, 2) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 1000, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; ----------------------------------------------------- -- nice data set with random data (skewed pow+pow) -- ----------------------------------------------------- -- compression 10 WITH data AS (SELECT pow(z, 4) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT pow(z, 4) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 10, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- compression 100 WITH data AS (SELECT pow(z, 4) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT pow(z, 4) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 100, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- compression 1000 WITH data AS (SELECT pow(z, 4) AS x FROM prng(10000) s(z)) SELECT p, abs(a - b) < 0.001, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.001 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT pow(z, 4) AS x FROM prng(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 1000, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; ---------------------------------------------------------- -- fourth power of data from a truncated normal distribution -- ---------------------------------------------------------- -- compression 10 WITH data AS (SELECT pow(z, 4) AS x FROM random_normal(10000) s(z)) SELECT p, abs(a - b) < 0.025, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.025 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT pow(z, 4) AS x FROM random_normal(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 10, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- compression 100 WITH data AS (SELECT pow(z, 4) AS x FROM random_normal(10000) s(z)) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT pow(z, 4) AS x FROM random_normal(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 100, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- compression 1000 WITH data AS (SELECT pow(z, 4) AS x FROM random_normal(10000) s(z)) SELECT p, abs(a - b) < 0.001, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.001 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(tdigest_percentile(x, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- make sure the resulting percentiles are in the right order WITH data AS (SELECT pow(z, 4) AS x FROM random_normal(10000) s(z)), perc AS (SELECT array_agg((i/100.0)::double precision) AS p FROM generate_series(1,99) s(i)) SELECT * FROM ( SELECT p, a, LAG(a) OVER (ORDER BY p) AS b FROM ( SELECT unnest((SELECT p FROM perc)) AS p, unnest(tdigest_percentile(x, 1000, (SELECT p FROM perc))) AS a FROM data ) foo ) bar WHERE a <= b; -- some basic tests to verify transforming from and to text works -- compression 10 WITH data AS (SELECT i / 10000.0 AS x FROM generate_series(1,10000) s(i)), intermediate AS (SELECT tdigest(x, 10)::text AS intermediate_x FROM data), tdigest_parsed AS (SELECT tdigest_percentile(intermediate_x::tdigest, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS a FROM intermediate), pg_percentile AS (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) AS b FROM data) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM tdigest_parsed, pg_percentile ) foo; -- verify we can store tdigest in a summary table CREATE TABLE intermediate_tdigest (grouping int, summary tdigest); WITH data AS (SELECT row_number() OVER () AS i, pow(z, 4) AS x FROM random_normal(10000) s(z)) INSERT INTO intermediate_tdigest SELECT i % 10 AS grouping, tdigest(x, 100) AS summary FROM data GROUP BY i % 10; WITH data AS (SELECT pow(z, 4) AS x FROM random_normal(10000) s(z)), intermediate AS (SELECT tdigest_percentile(summary, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS a FROM intermediate_tdigest), pg_percentile AS (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) AS b FROM data) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.01 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM intermediate, pg_percentile ) foo; DROP TABLE intermediate_tdigest; -- verify 'extreme' percentiles for the dataset would not read out of bounds on the centroids WITH data AS (SELECT x FROM generate_series(1,10) AS x) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.99]) AS p, unnest(tdigest_percentile(x, 10, ARRAY[0.01, 0.99])) AS a, unnest(percentile_cont(ARRAY[0.01, 0.99]) WITHIN GROUP (ORDER BY x)) AS b FROM data ) foo; -- check that the computed percentiles don't decrease for higher p values -- first test on a tiny t-digest with all centroids having count = 1 WITH -- percentiles to compute perc AS (SELECT array_agg((i / 100.0)::double precision) AS percentiles FROM generate_series(1,99) s(i)), -- input data (just 15 points) input_data AS (select i::double precision AS val FROM generate_series(1,15) s(i)) SELECT * FROM ( SELECT p, v AS v1, lag(v, 1) OVER (ORDER BY p) v2 FROM ( SELECT unnest(perc.percentiles) p, unnest(tdigest_percentile(input_data.val, 100, perc.percentiles)) v FROM perc, input_data GROUP BY perc.percentiles ) foo ) bar where v2 > v1; tvondra-tdigest-c0af713/test/sql/recv_checks.sql000066400000000000000000000137461525543331000220300ustar00rootroot00000000000000-- Regression test for the receive function tdigest_recv(), which parses the -- binary representation of a t-digest. -- -- The input is user supplied, so it has to be validated just like the text -- input. The queries exercise all the error checks in the receive function, -- one by one. -- -- Test through a binary COPY, which goes through the receive function. We -- build the input with the send functions of the individual fields. \set VERBOSITY terse CREATE TABLE tdigest_dst (s tdigest); -- a valid value, to make sure the checks don't reject correct input COPY (SELECT int4send(1) -- flags || int8send(3::bigint) -- count || int4send(10) -- compression || int4send(2) -- ncentroids || float8send(1) || int8send(1::bigint) || float8send(2) || int8send(2::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); -- -- sanity checks on the header -- -- unknown flags COPY (SELECT int4send(2) || int8send(1::bigint) || int4send(10) || int4send(1) || float8send(1) || int8send(1::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); -- compression out of the supported range COPY (SELECT int4send(1) || int8send(1::bigint) || int4send(9) || int4send(1) || float8send(1) || int8send(1::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY (SELECT int4send(1) || int8send(1::bigint) || int4send(10001) || int4send(1) || float8send(1) || int8send(1::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); -- non-positive total count COPY (SELECT int4send(1) || int8send(0::bigint) || int4send(10) || int4send(1) || float8send(1) || int8send(1::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); -- non-positive number of centroids COPY (SELECT int4send(1) || int8send(1::bigint) || int4send(10) || int4send(0)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); -- more centroids than fit into the buffer (10 * compression) COPY (SELECT int4send(1) || int8send(1::bigint) || int4send(10) || int4send(101)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); -- -- sanity checks on the centroids -- -- centroid mean that is not a valid number COPY (SELECT int4send(1) || int8send(1::bigint) || int4send(10) || int4send(1) || float8send('NaN') || int8send(1::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY (SELECT int4send(1) || int8send(1::bigint) || int4send(10) || int4send(1) || float8send('Infinity') || int8send(1::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); -- non-positive centroid count COPY (SELECT int4send(1) || int8send(1::bigint) || int4send(10) || int4send(1) || float8send(1) || int8send(0::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); -- centroid count exceeding the total count COPY (SELECT int4send(1) || int8send(1::bigint) || int4send(10) || int4send(1) || float8send(1) || int8send(2::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); -- centroids not sorted by mean COPY (SELECT int4send(1) || int8send(3::bigint) || int4send(10) || int4send(2) || float8send(2) || int8send(2::bigint) || float8send(1) || int8send(1::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); -- centroids not sorted by mean, in the old on-disk format storing sums -- instead of means (the sums are sorted, but the means are not) COPY (SELECT int4send(0) || int8send(3::bigint) || int4send(10) || int4send(2) || float8send(4) || int8send(1::bigint) || float8send(6) || int8send(2::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); -- centroid counts overflowing the total count COPY (SELECT int4send(1) || int8send(9223372036854775807::bigint) || int4send(10) || int4send(2) || float8send(1) || int8send(9223372036854775807::bigint) || float8send(2) || int8send(9223372036854775807::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); -- -- sanity checks on the value as a whole -- -- centroid counts not adding up to the total count COPY (SELECT int4send(1) || int8send(3::bigint) || int4send(10) || int4send(1) || float8send(1) || int8send(1::bigint)) TO '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); COPY tdigest_dst FROM '/tmp/tdigest_recv_checks.bin' WITH (FORMAT binary); -- only the valid value should have made it into the table SELECT s FROM tdigest_dst; DROP TABLE tdigest_dst; tvondra-tdigest-c0af713/test/sql/trimmed_agg_overflow.sql000066400000000000000000000057151525543331000237500ustar00rootroot00000000000000-- Use terse verbosity, so that the expected output does not depend on the -- error context, which differs between PostgreSQL versions. \set VERBOSITY terse -- Reproducer: tdigest_trimmed_agg() used to convert the trimmed range from -- double to int64 without checking whether the rounded bounds still fit. -- -- This one does not crash, it just gives wrong answers. -- -- The unsafe boundary calculation converted expressions such as -- -- ceil(count * high) -- -- directly to int64. count is int64, but multiplication is done in double, with -- a 53-bit mantissa, so once the total count gets close enough to 2^63 the -- product rounds up to exactly 2^63, which is not representable as int64. The -- conversion back to int64 is undefined behaviour; on x86-64 it yields -- INT64_MIN. count_high then ends up hugely negative, every centroid falls -- outside [count_low, count_high], nothing is accumulated, and the function -- returns NULL as if the digest were empty. -- -- The threshold is sharp - the last total count that still works is -- 9223372036854775295, one more and the answer turns into NULL: -- -- total count tdigest_digest_sum -- 9223372036854775295 1.844674407370955e+19 correct -- 9223372036854775296 NULL wrong -- 9223372036854775807 NULL wrong -- -- This affects tdigest_digest_sum() and tdigest_digest_avg() as well as the -- tdigest_sum() and tdigest_avg() aggregates, i.e. anything going through -- tdigest_trimmed_agg(). -- the last value that still produces the right answer SELECT tdigest_digest_sum('flags 1 count 9223372036854775295 compression 10 centroids 1 (2, 9223372036854775295)'::tdigest); SELECT tdigest_digest_avg('flags 1 count 9223372036854775295 compression 10 centroids 1 (2, 9223372036854775295)'::tdigest); -- one more, and the result silently becomes NULL SELECT tdigest_digest_sum('flags 1 count 9223372036854775296 compression 10 centroids 1 (2, 9223372036854775296)'::tdigest); SELECT tdigest_digest_avg('flags 1 count 9223372036854775296 compression 10 centroids 1 (2, 9223372036854775296)'::tdigest); -- INT64_MAX, same thing SELECT tdigest_digest_sum('flags 1 count 9223372036854775807 compression 10 centroids 1 (2, 9223372036854775807)'::tdigest); SELECT tdigest_digest_avg('flags 1 count 9223372036854775807 compression 10 centroids 1 (2, 9223372036854775807)'::tdigest); -- side by side, so the discontinuity is easy to see SELECT n AS total_count, tdigest_digest_sum(('flags 1 count ' || n || ' compression 10 centroids 1 (2, ' || n || ')')::tdigest) AS sum FROM (VALUES (9223372036854775295::bigint), (9223372036854775296::bigint), (9223372036854775807::bigint)) v(n); SELECT n AS total_count, tdigest_digest_avg(('flags 1 count ' || n || ' compression 10 centroids 1 (2, ' || n || ')')::tdigest) AS avg FROM (VALUES (9223372036854775295::bigint), (9223372036854775296::bigint), (9223372036854775807::bigint)) v(n); tvondra-tdigest-c0af713/test/sql/trimmed_aggregates.sql000066400000000000000000000322611525543331000233740ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('extra_float_digits', '0', false); PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; -- check trimmed mean (from raw data) -- we compare the result to a range, to deal with the randomness WITH data AS (SELECT random() AS r FROM generate_series(1,10000) AS x) SELECT tdigest_avg(data.r, 50, 0.1, 0.9) between 0.45 and 0.55 AS mean_10_90, tdigest_avg(data.r, 50, 0.25, 0.75) between 0.45 and 0.55 AS mean_25_75, tdigest_avg(data.r, 50, 0.0, 0.5) between 0.2 and 0.3 AS mean_0_50, tdigest_avg(data.r, 50, 0.5, 1.0) between 0.7 and 0.8 AS mean_50_100 FROM data; WITH data AS (SELECT random() AS r, (1 + (3 * random())::int) AS c FROM generate_series(1,10000) AS x) SELECT tdigest_avg(data.r, data.c, 100, 0.1, 0.9) between 0.45 and 0.55 AS mean_10_90, tdigest_avg(data.r, data.c, 100, 0.25, 0.75) between 0.45 and 0.55 AS mean_25_75, tdigest_avg(data.r, data.c, 100, 0.0, 0.5) between 0.2 and 0.3 AS mean_0_50, tdigest_avg(data.r, data.c, 100, 0.5, 1.0) between 0.7 and 0.8 AS mean_50_100 FROM data; -- check trimmed mean (from precalculated tdigest) -- we compare the result to a range, to deal with the randomness WITH data AS (SELECT tdigest(random(), 50) AS d FROM generate_series(1,10000) AS x) SELECT tdigest_avg(data.d, 0.1, 0.9) between 0.45 and 0.55 AS mean_10_90, tdigest_avg(data.d, 0.25, 0.75) between 0.45 and 0.55 AS mean_25_75, tdigest_avg(data.d, 0.0, 0.5) between 0.2 and 0.3 AS mean_0_50, tdigest_avg(data.d, 0.5, 1.0) between 0.7 and 0.8 AS mean_50_100 FROM data; -- check trimmed sum (from raw data) -- we compare the result to a range, to deal with the randomness WITH data AS (SELECT random() AS r FROM generate_series(1,10000) AS x) SELECT tdigest_sum(data.r, 50, 0.1, 0.9) between 8000 * 0.45 and 8000 * 0.55 AS sum_10_90, tdigest_sum(data.r, 50, 0.25, 0.75) between 5000 * 0.45 and 5000 * 0.55 AS sum_25_75, tdigest_sum(data.r, 50, 0.0, 0.5) between 5000 * 0.2 and 5000 * 0.3 AS sum_0_50, tdigest_sum(data.r, 50, 0.5, 1.0) between 5000 * 0.7 and 5000 * 0.8 AS sum_50_100 FROM data; WITH data AS (SELECT random() AS r, (1 + (3 * random())::int) AS c FROM generate_series(1,10000) AS x) SELECT tdigest_sum(data.r, data.c, 100, 0.1, 0.9) between 20000 * 0.45 and 20000 * 0.55 AS sum_10_90, tdigest_sum(data.r, data.c, 100, 0.25, 0.75) between 12500 * 0.45 and 12500 * 0.55 AS sum_25_75, tdigest_sum(data.r, data.c, 100, 0.0, 0.5) between 12500 * 0.2 and 12500 * 0.3 AS sum_0_50, tdigest_sum(data.r, data.c, 100, 0.5, 1.0) between 12500 * 0.7 and 12500 * 0.8 AS sum_50_100 FROM data; -- check trimmed sum (from precalculated tdigest) -- we compare the result to a range, to deal with the randomness WITH data AS (SELECT tdigest(random(), 50) AS d FROM generate_series(1,10000) AS x) SELECT tdigest_sum(data.d, 0.1, 0.9) between 8000 * 0.45 and 8000 * 0.55 AS sum_10_90, tdigest_sum(data.d, 0.25, 0.75) between 5000 * 0.45 and 5000 * 0.55 AS sum_25_75, tdigest_sum(data.d, 0.0, 0.5) between 5000 * 0.2 and 5000 * 0.3 AS sum_0_50, tdigest_sum(data.d, 0.5, 1.0) between 5000 * 0.7 and 5000 * 0.8 AS sum_50_100 FROM data; WITH data AS (SELECT tdigest(random(), 50) AS d FROM generate_series(1,10000) AS x) SELECT tdigest_digest_sum(data.d, 0.05, 0.95) between 9000 * 0.45 and 9000 * 0.55 AS sum_05_95, tdigest_digest_avg(data.d, 0.05, 0.95) between 0.45 and 0.55 AS mean_05_95 FROM data; -- With no compaction while accumulating input, final compaction starts from -- the same sorted centroids for both ascending and descending input. SELECT tdigest_sum(i, 10000, 0.0, 0.5) FROM generate_series(1, 10000) s(i); SELECT tdigest_sum(i, 10000, 0.0, 0.5) from generate_series(10000, 1, -1) s(i); SELECT tdigest_sum(i, 10000, 0.0, 0.25) FROM generate_series(1, 10000) s(i); SELECT tdigest_sum(i, 10000, 0.0, 0.25) from generate_series(10000, 1, -1) s(i); SELECT tdigest_sum(i, 10000, 0.1, 0.2) FROM generate_series(1, 10000) s(i); SELECT tdigest_sum(i, 10000, 0.1, 0.2) from generate_series(10000, 1, -1) s(i); SELECT tdigest_sum(i, 10000, 0.5, 1.0) FROM generate_series(1, 10000) s(i); SELECT tdigest_sum(i, 10000, 0.5, 1.0) from generate_series(10000, 1, -1) s(i); SELECT tdigest_sum(i, 10000, 0.75, 1.0) FROM generate_series(1, 10000) s(i); SELECT tdigest_sum(i, 10000, 0.75, 1.0) from generate_series(10000, 1, -1) s(i); SELECT tdigest_sum(i, 10000, 0.75, 0.9) FROM generate_series(1, 10000) s(i); SELECT tdigest_sum(i, 10000, 0.75, 0.9) from generate_series(10000, 1, -1) s(i); -- the same thing with calculating a t-digest first (but make sure all -- centroids have count 1, to make it exactly the same) WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1, 1500) s(i)) SELECT tdigest_sum(d, 0.0, 0.5) from tmp; WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1500, 1, -1) s(i)) SELECT tdigest_sum(d, 0.0, 0.5) from tmp; WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1, 1500) s(i)) SELECT tdigest_sum(d, 0.0, 0.25) from tmp; WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1500, 1, -1) s(i)) SELECT tdigest_sum(d, 0.0, 0.25) from tmp; WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1, 1500) s(i)) SELECT tdigest_sum(d, 0.1, 0.2) from tmp; WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1500, 1, -1) s(i)) SELECT tdigest_sum(d, 0.1, 0.2) from tmp; WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1, 1500) s(i)) SELECT tdigest_sum(d, 0.5, 1.0) from tmp; WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1500, 1, -1) s(i)) SELECT tdigest_sum(d, 0.5, 1.0) from tmp; WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1, 1500) s(i)) SELECT tdigest_sum(d, 0.75, 1.0) from tmp; WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1500, 1, -1) s(i)) SELECT tdigest_sum(d, 0.75, 1.0) from tmp; WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1, 1500) s(i)) SELECT tdigest_sum(d, 0.75, 0.9) from tmp; WITH tmp AS (SELECT tdigest(i, 10000) AS d FROM generate_series(1500, 1, -1) s(i)) SELECT tdigest_sum(d, 0.75, 0.9) from tmp; -- simple sanity checks of high/low thresholds -- correct low/high ranges SELECT tdigest_sum(1.0, 100, 0.0, 1.0); SELECT tdigest_avg(1.0, 100, 0.0, 1.0); SELECT tdigest_sum(1.0, 100, 0.25, 0.75); SELECT tdigest_avg(1.0, 100, 0.25, 0.75); SELECT tdigest_sum(1.0, 100, 0.5, 0.5); SELECT tdigest_avg(1.0, 100, 0.5, 0.5); -- inverted range SELECT tdigest_sum(1.0, 100, 0.75, 0.25); SELECT tdigest_avg(1.0, 100, 0.75, 0.25); -- negative low threshold SELECT tdigest_sum(1.0, 100, -1.0, 1.0); SELECT tdigest_avg(1.0, 100, -1.0, 1.0); -- bogus high threshold SELECT tdigest_sum(1.0, 100, -1.0, 1.0); SELECT tdigest_avg(1.0, 100, -1.0, 1.0); -- infinity in low threshold SELECT tdigest_sum(1.0, 100, '-infinity'::double precision, 1.0); SELECT tdigest_avg(1.0, 100, '-infinity'::double precision, 1.0); SELECT tdigest_sum(1.0, 100, 'infinity'::double precision, 1.0); SELECT tdigest_avg(1.0, 100, 'infinity'::double precision, 1.0); -- infinity in high threshold SELECT tdigest_sum(1.0, 100, 0.0, '-infinity'::double precision); SELECT tdigest_avg(1.0, 100, 0.0, '-infinity'::double precision); SELECT tdigest_sum(1.0, 100, 0.0, 'infinity'::double precision); SELECT tdigest_avg(1.0, 100, 0.0, 'infinity'::double precision); -- infinity in both thresholds SELECT tdigest_sum(1.0, 100, '-infinity'::double precision, '-infinity'::double precision); SELECT tdigest_avg(1.0, 100, '-infinity'::double precision, '-infinity'::double precision); SELECT tdigest_sum(1.0, 100, '-infinity'::double precision, 'infinity'::double precision); SELECT tdigest_avg(1.0, 100, '-infinity'::double precision, 'infinity'::double precision); SELECT tdigest_sum(1.0, 100, 'infinity'::double precision, '-infinity'::double precision); SELECT tdigest_avg(1.0, 100, 'infinity'::double precision, '-infinity'::double precision); SELECT tdigest_sum(1.0, 100, 'infinity'::double precision, 'infinity'::double precision); SELECT tdigest_avg(1.0, 100, 'infinity'::double precision, 'infinity'::double precision); -- NaN in thresholds SELECT tdigest_sum(1.0, 100, 'NaN'::double precision, 1.0); SELECT tdigest_avg(1.0, 100, 'NaN'::double precision, 1.0); SELECT tdigest_sum(1.0, 100, 0.0, 'NaN'::double precision); SELECT tdigest_avg(1.0, 100, 0.0, 'NaN'::double precision); SELECT tdigest_sum(1.0, 100, 'NaN'::double precision, 'NaN'::double precision); SELECT tdigest_avg(1.0, 100, 'NaN'::double precision, 'NaN'::double precision); -- NULL in thresholds SELECT tdigest_sum(1.0, 100, NULL::double precision, 1.0); SELECT tdigest_avg(1.0, 100, NULL::double precision, 1.0); SELECT tdigest_sum(1.0, 100, 0.0, NULL::double precision); SELECT tdigest_avg(1.0, 100, 0.0, NULL::double precision); SELECT tdigest_sum(1.0, 100, NULL::double precision, NULL::double precision); SELECT tdigest_avg(1.0, 100, NULL::double precision, NULL::double precision); -- check tdigest_trimmed_agg() calculates and aggregates the right range -- of centroids to process, especially when [count_low, count_high) falls -- into a single centroid -- -- A digest holding a single centroid, i.e. 100 items that all have the value -- 10. The trimmed sum then has to be exactly (10 * number of items in the -- range) for every range, which makes the expected values easy to derive -- -- count_low = floor(100 * lo) and count_high = ceil(100 * hi). SELECT lo, hi, tdigest_digest_sum(d, lo, hi) AS trimmed_sum, 10 * (ceil(100::double precision * hi) - floor(100::double precision * lo)) AS expected_sum, tdigest_digest_avg(d, lo, hi) AS trimmed_avg FROM (SELECT 'flags 1 count 100 compression 10 centroids 1 (10, 100)'::tdigest) t(d), (VALUES (0.0::double precision, 1.0::double precision), (0.0::double precision, 0.5::double precision), (0.5::double precision, 1.0::double precision), (0.4::double precision, 0.5::double precision), (0.1::double precision, 0.2::double precision), (0.25::double precision, 0.75::double precision)) v(lo, hi) ORDER BY lo, hi; -- The same through the aggregates, to cover tdigest_trimmed_sum() and -- tdigest_trimmed_avg() too. All the input values are the same, so the digest -- is a single centroid no matter how the compaction merges the centroids. SELECT tdigest(10.0::double precision, 10) AS digest FROM generate_series(1, 100); SELECT tdigest_sum(10.0::double precision, 10, 0.4, 0.5) AS agg_sum, tdigest_avg(10.0::double precision, 10, 0.4, 0.5) AS agg_avg FROM generate_series(1, 100); -- Now the same thing, but with more than one centroid, covering a range -- inside the middle centroid, a range straddling a centroid boundary, a -- range inside the last centroid, and the untrimmed case. SELECT lo, hi, tdigest_digest_sum(d, lo, hi) AS trimmed_sum, tdigest_digest_avg(d, lo, hi) AS trimmed_avg FROM (SELECT 'flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest) t(d), (VALUES (0.4::float8, 0.5::float8), (0.35::float8, 0.45::float8), (0.45::float8, 0.55::float8), (0.3::float8, 0.6::float8), (0.6::float8, 0.9::float8), (0.0::float8, 1.0::float8)) v(lo, hi) ORDER BY lo, hi; -- Exhaustive cross-check of every trimmed range against a brute-force -- expansion of the same digest into individual items. Expected to return no -- rows; any row is a range where the trimmed sum disagrees with the sum of -- the items the range is supposed to cover. WITH digest(d) AS ( SELECT 'flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest ), -- the individual items of the digest, with their 0-based index items(idx, val) AS ( SELECT c.first_idx + g - 1, c.mean FROM (VALUES (0, 10.0::float8, 10), (10, 20.0::float8, 5), (15, 30.0::float8, 15)) c(first_idx, mean, cnt), generate_series(1, c.cnt) g ), ranges(lo, hi) AS ( SELECT (a / 20.0)::float8, (b / 20.0)::float8 FROM generate_series(0, 20) a, generate_series(0, 20) b WHERE a < b ) SELECT r.lo, r.hi, tdigest_digest_sum(digest.d, r.lo, r.hi) AS trimmed_sum, (SELECT sum(i.val) FROM items i WHERE i.idx >= floor(30::float8 * r.lo) AND i.idx < ceil(30::float8 * r.hi)) AS expected_sum FROM ranges r, digest WHERE tdigest_digest_sum(digest.d, r.lo, r.hi) IS DISTINCT FROM (SELECT sum(i.val) FROM items i WHERE i.idx >= floor(30::float8 * r.lo) AND i.idx < ceil(30::float8 * r.hi)) ORDER BY r.lo, r.hi; tvondra-tdigest-c0af713/test/sql/trimmed_count_checks.sql000066400000000000000000000015171525543331000237330ustar00rootroot00000000000000-- Regression test for the count check in tdigest_add_double_count_trimmed, -- the transition function of the trimmed aggregates accepting an explicit -- number of occurrences of a value. -- -- A non-NULL count must be positive. A NULL count means one occurrence -- of the input value. \set VERBOSITY terse -- zero count SELECT tdigest_avg(v, 0::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0)) AS t(v); SELECT tdigest_sum(v, 0::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0)) AS t(v); -- negative count SELECT tdigest_avg(v, -1::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0)) AS t(v); SELECT tdigest_sum(v, -1::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0)) AS t(v); -- the smallest valid count has to work SELECT tdigest_avg(v, 1::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0)) AS t(v); SELECT tdigest_sum(v, 1::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0)) AS t(v); tvondra-tdigest-c0af713/test/sql/trimmed_null_values.sql000066400000000000000000000023351525543331000236130ustar00rootroot00000000000000-- Regression test for the NULL handling in the transition functions of the -- trimmed aggregates. -- -- Neither of the transition functions is strict, so NULL rows do reach them -- and have to be ignored. Each function is exercised twice: once with all -- input rows NULL (so the state stays NULL and the transition function -- returns NULL), and once with a NULL row following a non-NULL one (so the -- transition function returns the existing state unmodified). -- tdigest_add_double_trimmed SELECT tdigest_avg(v, 100, 0.0, 1.0) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); SELECT tdigest_avg(v, 100, 0.0, 1.0) FROM (VALUES (1.0), (NULL)) AS t(v); SELECT tdigest_sum(v, 100, 0.0, 1.0) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); SELECT tdigest_sum(v, 100, 0.0, 1.0) FROM (VALUES (1.0), (NULL)) AS t(v); -- tdigest_add_double_count_trimmed SELECT tdigest_avg(v, 1::bigint, 100, 0.0, 1.0) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); SELECT tdigest_avg(v, 1::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0), (NULL)) AS t(v); SELECT tdigest_sum(v, 1::bigint, 100, 0.0, 1.0) FROM (VALUES (NULL::double precision), (NULL)) AS t(v); SELECT tdigest_sum(v, 1::bigint, 100, 0.0, 1.0) FROM (VALUES (1.0), (NULL)) AS t(v); tvondra-tdigest-c0af713/test/sql/trimmed_sum_overflow.sql000066400000000000000000000111251525543331000240060ustar00rootroot00000000000000-- Use terse verbosity, so that the expected output does not depend on the -- error context, which differs between PostgreSQL versions. \set VERBOSITY terse -- Reproducer: tdigest_trimmed_agg() accumulated the trimmed sum by adding -- (mean * count) for all the centroids in the range, and the trimmed average -- was then calculated by dividing that sum by the number of items. -- -- But the sum may overflow to infinity even when the average is perfectly -- representable. The average of 1e307 values is 1e307, yet the sum of a -- hundred of them is not a float8. The average was then reported as -- Infinity, which is simply a wrong answer - the average of a digest is -- always within the range of the centroid means, so it can never overflow. -- -- And once one partial sum saturates to +Infinity and another one to -- -Infinity, the accumulator turns into NaN, so a digest with large values -- of both signs reported NaN for both the sum and the average. -- -- The sum, unlike the average, may legitimately exceed the float8 range. -- Report that as an error, just like the regular float8 arithmetic does, -- instead of silently returning infinity. -- -- query before after -- tdigest_digest_avg((1e307, 100)) Infinity 1e+307 -- tdigest_digest_sum((1e307, 100)) Infinity ERROR -- tdigest_digest_avg((-1e308,2) (1e308,2)) NaN 0 -- tdigest_digest_sum((-1e308,2) (1e308,2)) NaN 0 -- -- This affects tdigest_digest_sum() and tdigest_digest_avg() as well as the -- tdigest_sum() and tdigest_avg() aggregates, i.e. anything going through -- tdigest_trimmed_agg(). -- The average is representable, the sum is not. The average has to be -- 1e307, the sum has to be reported as an overflow. SELECT tdigest_digest_avg('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest); SELECT tdigest_digest_sum('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest); -- Trimming does not change that, as long as enough items remain. SELECT tdigest_digest_avg('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest, 0.25, 0.75); SELECT tdigest_digest_sum('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest, 0.25, 0.75); -- Trim it down to a single item, and even the sum fits again. SELECT tdigest_digest_avg('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest, 0.5, 0.51); SELECT tdigest_digest_sum('flags 1 count 100 compression 10 centroids 1 (1e307, 100)'::tdigest, 0.5, 0.51); -- Large values of both signs, cancelling out exactly. Both the average and -- the sum are 0, the accumulator used to end up as NaN. SELECT tdigest_digest_avg('flags 1 count 4 compression 10 centroids 2 (-1e308, 2) (1e308, 2)'::tdigest); SELECT tdigest_digest_sum('flags 1 count 4 compression 10 centroids 2 (-1e308, 2) (1e308, 2)'::tdigest); -- The same, but with the values cancelling only partially. SELECT tdigest_digest_avg('flags 1 count 4 compression 10 centroids 2 (-1e308, 1) (1e308, 3)'::tdigest); SELECT tdigest_digest_sum('flags 1 count 4 compression 10 centroids 2 (-1e308, 1) (1e308, 3)'::tdigest); -- Trim the negative values away, and only the overflowing part remains. SELECT tdigest_digest_avg('flags 1 count 4 compression 10 centroids 2 (-1e308, 2) (1e308, 2)'::tdigest, 0.5, 1.0); SELECT tdigest_digest_sum('flags 1 count 4 compression 10 centroids 2 (-1e308, 2) (1e308, 2)'::tdigest, 0.5, 1.0); -- The trimmed aggregates on the value/count API go through a different code -- path (the aggregate final functions), so check those too. SELECT tdigest_avg(1e307::float8, 200::bigint, 10, 0.0, 1.0); SELECT tdigest_sum(1e307::float8, 200::bigint, 10, 0.0, 1.0); -- And the plain value API. SELECT tdigest_avg(v, 10, 0.0, 1.0) FROM (VALUES (1e308::float8), (1e308::float8)) t(v); SELECT tdigest_sum(v, 10, 0.0, 1.0) FROM (VALUES (1e308::float8), (1e308::float8)) t(v); SELECT tdigest_avg(v, 10, 0.0, 1.0) FROM (VALUES (-1e308::float8), (1e308::float8)) t(v); SELECT tdigest_sum(v, 10, 0.0, 1.0) FROM (VALUES (-1e308::float8), (1e308::float8)) t(v); -- Ordinary values are not affected in any way, and an empty range still -- returns NULL. SELECT tdigest_digest_avg('flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest, 0.1, 0.9); SELECT tdigest_digest_sum('flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest, 0.1, 0.9); SELECT tdigest_digest_avg('flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest, 0.5, 0.5); SELECT tdigest_digest_sum('flags 1 count 30 compression 10 centroids 3 (10, 10) (20, 5) (30, 15)'::tdigest, 0.5, 0.5); tvondra-tdigest-c0af713/test/sql/unsorted_centroids.sql000066400000000000000000000335321525543331000234610ustar00rootroot00000000000000-- The centroids of a t-digest are expected to be sorted by mean, but a digest -- with the centroids in an arbitrary order is a perfectly valid value - the -- incremental API can leave digests uncompacted and unsorted with compact=false, -- and those may get stored on disk, dumped, replicated, etc. The input functions -- therefore accept unsorted digests, and all the places that need the sorted -- order are expected to do the sort themselves. -- -- This checks that really is the case - for each digest we build a couple of -- variants with the centroids reordered (reversed, and then shuffled in a -- deterministic but arbitrary way), and then compare the results of -- -- tdigest_percentile() (both the scalar and the array variant) -- tdigest_percentile_of() (both the scalar and the array variant) -- tdigest_sum() -- tdigest_avg() -- tdigest_digest_sum() -- tdigest_digest_avg() -- -- to the results for the original (sorted) digest. The results have to match -- exactly, not just approximately - reordering the centroids does not lose any -- information, so the sort has to reconstruct exactly the same digest. \set VERBOSITY terse SET extra_float_digits = 0; -- Decompose a digest into individual centroids. The digest has no API to do -- this directly, so we go through the text representation. The functions set -- extra_float_digits, so that the means survive the round trip exactly. CREATE FUNCTION tdigest_unsorted_centroids(d tdigest) RETURNS TABLE (ord bigint, mean double precision, cnt bigint) LANGUAGE sql SET extra_float_digits = 3 AS $$ SELECT o, split_part(btrim(m[1], ' ()'), ',', 1)::double precision, split_part(btrim(m[1], ' ()'), ',', 2)::bigint FROM regexp_matches(d::text, ' \([^)]*\)', 'g') WITH ORDINALITY AS x(m, o); $$; -- Rebuild the digest with the centroids in a different order. Seed 0 simply -- reverses the centroids (so that the means are in a descending order), any -- other seed shuffles them - the order is arbitrary, but it's derived from -- md5() so it's stable and does not depend on the PRNG of the server. CREATE FUNCTION tdigest_unsorted_reorder(d tdigest, seed int) RETURNS tdigest LANGUAGE sql SET extra_float_digits = 3 AS $$ SELECT (rtrim(substring(d::text FROM '^[^(]*')) || coalesce(string_agg(m[1], '' ORDER BY CASE WHEN seed = 0 THEN -o ELSE 0 END, md5(seed::text || ':' || o::text)), ''))::tdigest FROM regexp_matches(d::text, ' \([^)]*\)', 'g') WITH ORDINALITY AS x(m, o); $$; -- Are the centroids of the digest sorted by mean? CREATE FUNCTION tdigest_unsorted_is_sorted(d tdigest) RETURNS boolean LANGUAGE sql AS $$ SELECT coalesce(bool_and(mean >= prev_mean), true) FROM (SELECT mean, lag(mean) OVER (ORDER BY ord) AS prev_mean FROM tdigest_unsorted_centroids(d)) x WHERE prev_mean IS NOT NULL; $$; -- The digests to check. A couple of hand-written ones first, so that we know -- exactly what centroids they consist of. CREATE TABLE tdigest_unsorted_digests (id int, descr text, d tdigest); INSERT INTO tdigest_unsorted_digests VALUES (1, 'two centroids', 'flags 1 count 10 compression 10000 centroids 2 (0, 7) (100, 3)'), (2, 'three centroids, mixed counts', 'flags 1 count 12 compression 10000 centroids 3 (0, 3) (50, 4) (100, 5)'), (3, 'negative and positive means', 'flags 1 count 8 compression 10000 centroids 4 (-100, 2) (-1, 2) (1, 2) (100, 2)'), (4, 'repeated means (the sort has to rebalance those)', 'flags 1 count 12 compression 10000 centroids 5 (0, 1) (50, 3) (50, 4) (50, 3) (100, 1)'), (5, 'a large centroid in the middle', 'flags 1 count 22 compression 10000 centroids 3 (0, 1) (10, 20) (20, 1)'), (6, 'a single centroid', 'flags 1 count 10 compression 10000 centroids 1 (50, 10)'); -- And then digests built by the aggregate from actual data. The first one -- retains singleton centroids after final compaction; the others merge input -- values with different compression settings. INSERT INTO tdigest_unsorted_digests SELECT 7, 'built from 1000 distinct values, no centroid merging', tdigest(i::double precision, 10000) FROM generate_series(1, 1000) s(i); INSERT INTO tdigest_unsorted_digests SELECT 8, 'built from 10000 rows, compression 100', tdigest((i % 1000)::double precision, 100) FROM generate_series(1, 10000) s(i); INSERT INTO tdigest_unsorted_digests SELECT 9, 'built from 10000 rows, compression 10', tdigest((i % 1000)::double precision, 10) FROM generate_series(1, 10000) s(i); INSERT INTO tdigest_unsorted_digests SELECT 10, 'built from 10000 random values, compression 100', tdigest(1000 * v, 100) FROM prng(10000) s(v); INSERT INTO tdigest_unsorted_digests SELECT 11, 'built from 10000 values with duplicities, compression 50', tdigest((10 * v)::int::double precision, 50) FROM prng(10000) s(v); -- All of them have to be sorted, otherwise the whole test is pointless. SELECT id, descr FROM tdigest_unsorted_digests WHERE NOT tdigest_unsorted_is_sorted(d) ORDER BY id; -- Build the variants. Variant 0 is the original (sorted) digest, variant 1 has -- the centroids in the reverse order, and variants 2-6 have them shuffled. CREATE TABLE tdigest_unsorted_variants (id int, variant int, d tdigest); INSERT INTO tdigest_unsorted_variants SELECT id, 0, d FROM tdigest_unsorted_digests; INSERT INTO tdigest_unsorted_variants SELECT id, k + 1, tdigest_unsorted_reorder(d, k) FROM tdigest_unsorted_digests, generate_series(0, 5) s(k); -- The reordering must not change the contents of the digest in any way - the -- header has to be the same, and the centroids have to be the very same -- multiset, just in a different order. So both the "differs" columns have to -- be 0. The last column says how many of the variants really ended up with -- the centroids out of order - it's less than the number of variants for the -- digests with very few centroids (and it's 0 for the single centroid digest, -- which can't be unsorted at all). SELECT v.id, g.descr, count(*) AS variants, count(*) FILTER (WHERE substring(v.d::text FROM '^[^(]*') IS DISTINCT FROM substring(g.d::text FROM '^[^(]*')) AS header_differs, count(*) FILTER (WHERE EXISTS (SELECT mean, cnt FROM tdigest_unsorted_centroids(v.d) EXCEPT ALL SELECT mean, cnt FROM tdigest_unsorted_centroids(g.d)) OR EXISTS (SELECT mean, cnt FROM tdigest_unsorted_centroids(g.d) EXCEPT ALL SELECT mean, cnt FROM tdigest_unsorted_centroids(v.d))) AS centroids_differ, count(*) FILTER (WHERE NOT tdigest_unsorted_is_sorted(v.d)) AS actually_unsorted FROM tdigest_unsorted_variants v JOIN tdigest_unsorted_digests g ON (g.id = v.id) WHERE v.variant > 0 GROUP BY v.id, g.descr ORDER BY v.id; -- The percentiles / values / trim ranges to probe the digests with. The digests -- cover fairly different ranges of values, but that does not matter - we only -- compare the results to each other, not to some expected value. CREATE TABLE tdigest_unsorted_percentiles (id int, p double precision); INSERT INTO tdigest_unsorted_percentiles SELECT i, i / 20.0 FROM generate_series(0, 20) s(i); CREATE TABLE tdigest_unsorted_values (id int, v double precision); INSERT INTO tdigest_unsorted_values (id, v) SELECT row_number() OVER (ORDER BY v), v FROM unnest(ARRAY[-1000, -100.5, -100, -50, -1, -0.5, 0, 0.5, 1, 10, 20, 49.5, 50, 99.5, 100, 250, 500, 750, 999, 1000, 5000]::double precision[]) AS v; CREATE TABLE tdigest_unsorted_trims (id int, low double precision, high double precision); INSERT INTO tdigest_unsorted_trims VALUES (1, 0.0, 1.0), (2, 0.1, 0.9), (3, 0.0, 0.5), (4, 0.5, 1.0), (5, 0.25, 0.75), (6, 0.05, 0.15), (7, 0.9, 0.95), (8, 0.33, 0.34); -- Evaluate all the functions for all the digests / variants / probes. CREATE TABLE tdigest_unsorted_results (func text, id int, variant int, probe int, res double precision); INSERT INTO tdigest_unsorted_results SELECT 'tdigest_percentile', v.id, v.variant, p.id, tdigest_percentile(v.d, p.p) FROM tdigest_unsorted_variants v, tdigest_unsorted_percentiles p GROUP BY v.id, v.variant, p.id, p.p; INSERT INTO tdigest_unsorted_results SELECT 'tdigest_percentile (array)', r.id, r.variant, o::int, x FROM (SELECT v.id, v.variant, tdigest_percentile(v.d, p.arr) AS res FROM tdigest_unsorted_variants v, (SELECT array_agg(p ORDER BY id) AS arr FROM tdigest_unsorted_percentiles) p GROUP BY v.id, v.variant, p.arr) r, unnest(r.res) WITH ORDINALITY AS u(x, o); INSERT INTO tdigest_unsorted_results SELECT 'tdigest_percentile_of', v.id, v.variant, p.id, tdigest_percentile_of(v.d, p.v) FROM tdigest_unsorted_variants v, tdigest_unsorted_values p GROUP BY v.id, v.variant, p.id, p.v; INSERT INTO tdigest_unsorted_results SELECT 'tdigest_percentile_of (array)', r.id, r.variant, o::int, x FROM (SELECT v.id, v.variant, tdigest_percentile_of(v.d, p.arr) AS res FROM tdigest_unsorted_variants v, (SELECT array_agg(v ORDER BY id) AS arr FROM tdigest_unsorted_values) p GROUP BY v.id, v.variant, p.arr) r, unnest(r.res) WITH ORDINALITY AS u(x, o); INSERT INTO tdigest_unsorted_results SELECT 'tdigest_sum', v.id, v.variant, t.id, tdigest_sum(v.d, t.low, t.high) FROM tdigest_unsorted_variants v, tdigest_unsorted_trims t GROUP BY v.id, v.variant, t.id, t.low, t.high; INSERT INTO tdigest_unsorted_results SELECT 'tdigest_avg', v.id, v.variant, t.id, tdigest_avg(v.d, t.low, t.high) FROM tdigest_unsorted_variants v, tdigest_unsorted_trims t GROUP BY v.id, v.variant, t.id, t.low, t.high; -- the same two, but as plain functions operating on a single digest INSERT INTO tdigest_unsorted_results SELECT 'tdigest_digest_sum', v.id, v.variant, t.id, tdigest_digest_sum(v.d, t.low, t.high) FROM tdigest_unsorted_variants v, tdigest_unsorted_trims t; INSERT INTO tdigest_unsorted_results SELECT 'tdigest_digest_avg', v.id, v.variant, t.id, tdigest_digest_avg(v.d, t.low, t.high) FROM tdigest_unsorted_variants v, tdigest_unsorted_trims t; -- How many results did we compare, and how many of them do not match the -- result for the sorted digest? The number of mismatches has to be 0. SELECT r.func, count(*) AS comparisons, count(*) FILTER (WHERE r.res IS DISTINCT FROM s.res) AS mismatches FROM tdigest_unsorted_results r JOIN tdigest_unsorted_results s ON (s.func = r.func AND s.id = r.id AND s.probe = r.probe AND s.variant = 0) WHERE r.variant > 0 GROUP BY r.func ORDER BY r.func; -- And the details of the mismatches, so that a failure is readable. This has -- to produce no rows at all. SELECT r.func, r.id, g.descr, r.variant, r.probe, s.res AS sorted, r.res AS unsorted FROM tdigest_unsorted_results r JOIN tdigest_unsorted_results s ON (s.func = r.func AND s.id = r.id AND s.probe = r.probe AND s.variant = 0) JOIN tdigest_unsorted_digests g ON (g.id = r.id) WHERE r.variant > 0 AND r.res IS DISTINCT FROM s.res ORDER BY r.func, r.id, r.variant, r.probe; -- Finally a couple of explicit checks on a small digest with the centroids in -- the reverse order, spelled out - just to make sure the comparison above is -- not comparing something trivially identical. SELECT tdigest_unsorted_reorder('flags 1 count 12 compression 10000 centroids 3 (0, 3) (50, 4) (100, 5)'::tdigest, 0); WITH d(descr, d) AS ( VALUES ('sorted', 'flags 1 count 12 compression 10000 centroids 3 (0, 3) (50, 4) (100, 5)'::tdigest), ('reversed', 'flags 1 count 12 compression 10000 centroids 3 (100, 5) (50, 4) (0, 3)'::tdigest) ) SELECT descr, tdigest_percentile(d, 0.25) AS "percentile 0.25", tdigest_percentile(d, 0.75) AS "percentile 0.75", tdigest_percentile_of(d, 25.0) AS "percentile of 25", tdigest_percentile_of(d, 75.0) AS "percentile of 75", tdigest_sum(d, 0.1, 0.9) AS "sum 0.1-0.9", tdigest_avg(d, 0.1, 0.9) AS "avg 0.1-0.9" FROM d GROUP BY descr ORDER BY descr; WITH d(descr, d) AS ( VALUES ('sorted', 'flags 1 count 12 compression 10000 centroids 3 (0, 3) (50, 4) (100, 5)'::tdigest), ('reversed', 'flags 1 count 12 compression 10000 centroids 3 (100, 5) (50, 4) (0, 3)'::tdigest) ) SELECT descr, tdigest_digest_sum(d, 0.1, 0.9) AS "digest_sum 0.1-0.9", tdigest_digest_avg(d, 0.1, 0.9) AS "digest_avg 0.1-0.9" FROM d ORDER BY descr; -- The old on-disk format, where the centroids store sums instead of means, is -- accepted with the centroids unsorted too. And the sums being sorted does not -- mean the means are - all three digests below hold the very same centroids -- (means 20, 50 and 200, with counts 5, 6 and 1). WITH d(descr, d) AS ( VALUES ('sorted by mean', 'flags 0 count 12 compression 10000 centroids 3 (100, 5) (300, 6) (200, 1)'::tdigest), ('reversed', 'flags 0 count 12 compression 10000 centroids 3 (200, 1) (300, 6) (100, 5)'::tdigest), ('sorted by sum, not by mean', 'flags 0 count 12 compression 10000 centroids 3 (100, 5) (200, 1) (300, 6)'::tdigest) ) SELECT descr, tdigest_digest_sum(d, 0.1, 0.9) AS "digest_sum 0.1-0.9", tdigest_digest_avg(d, 0.1, 0.9) AS "digest_avg 0.1-0.9" FROM d ORDER BY descr; DROP TABLE tdigest_unsorted_results; DROP TABLE tdigest_unsorted_trims; DROP TABLE tdigest_unsorted_values; DROP TABLE tdigest_unsorted_percentiles; DROP TABLE tdigest_unsorted_variants; DROP TABLE tdigest_unsorted_digests; DROP FUNCTION tdigest_unsorted_is_sorted(tdigest); DROP FUNCTION tdigest_unsorted_reorder(tdigest, int); DROP FUNCTION tdigest_unsorted_centroids(tdigest); tvondra-tdigest-c0af713/test/sql/value_checks.sql000066400000000000000000000040041525543331000221700ustar00rootroot00000000000000-- Unlike percentiles passed to tdigest_percentile(), non-finite hypothetical -- values passed to tdigest_percentile_of() are accepted and mapped to fixed -- results. Input values stored in the digest must still be finite. \set VERBOSITY terse -- hypothetical values map to NaN, 0, and 1, respectively SELECT tdigest_percentile_of(1.0::double precision, 10, 'NaN'::double precision); SELECT tdigest_percentile_of(1.0::double precision, 10, '-infinity'::double precision); SELECT tdigest_percentile_of(1.0::double precision, 10, 'infinity'::double precision); SELECT tdigest_percentile_of(1.0::double precision, 10, ARRAY['NaN', '-infinity', 'infinity']::double precision[]); SELECT tdigest_percentile_of('flags 1 count 3 compression 10 centroids 3 (1, 1) (2, 1) (3, 1)'::tdigest, 'NaN'::double precision); SELECT tdigest_percentile_of('flags 1 count 3 compression 10 centroids 3 (1, 1) (2, 1) (3, 1)'::tdigest, '-infinity'::double precision); SELECT tdigest_percentile_of('flags 1 count 3 compression 10 centroids 3 (1, 1) (2, 1) (3, 1)'::tdigest, 'infinity'::double precision); SELECT tdigest_percentile_of('flags 1 count 3 compression 10 centroids 3 (1, 1) (2, 1) (3, 1)'::tdigest, ARRAY['NaN', '-infinity', 'infinity']::double precision[]); -- the value/count API has the same handling of non-finite hypothetical values SELECT tdigest_percentile_of(1.0::double precision, 2::bigint, 10, 'NaN'::double precision); SELECT tdigest_percentile_of(1.0::double precision, 2::bigint, 10, '-infinity'::double precision); SELECT tdigest_percentile_of(1.0::double precision, 2::bigint, 10, 'infinity'::double precision); SELECT tdigest_percentile_of(1.0::double precision, 2::bigint, 10, ARRAY['NaN']::double precision[]); SELECT tdigest_percentile_of(1.0::double precision, 2::bigint, 10, ARRAY['-infinity']::double precision[]); SELECT tdigest_percentile_of(1.0::double precision, 2::bigint, 10, ARRAY['infinity']::double precision[]); tvondra-tdigest-c0af713/test/sql/value_count_api.sql000066400000000000000000000173071525543331000227230ustar00rootroot00000000000000DO $$ DECLARE v_version numeric; BEGIN SELECT substring(setting from '\d+')::numeric INTO v_version FROM pg_settings WHERE name = 'server_version'; -- GUCs common to all versions PERFORM set_config('extra_float_digits', '0', false); PERFORM set_config('parallel_setup_cost', '0', false); PERFORM set_config('parallel_tuple_cost', '0', false); PERFORM set_config('max_parallel_workers_per_gather', '2', false); -- 9.6 used a different GUC name for relation size IF v_version < 10 THEN PERFORM set_config('min_parallel_relation_size', '1kB', false); ELSE PERFORM set_config('min_parallel_table_scan_size', '1kB', false); END IF; -- on 14 and later, disable Memoize nodes to make EXPLAIN more consistent IF v_version >= 14 THEN PERFORM set_config('enable_memoize', 'off', false); END IF; END; $$ LANGUAGE plpgsql; -- API select tdigest_percentile(value, count, 100, 0.95) from (values (47325940488,1), (15457695432,2), (6889790700,3), (4188763788,4), (2882932224,5), (2114815860,6), (1615194324,7), (2342114568,9), (1626471924,11), (1660755408,14), (1143728292,17), (1082582424,21), (911488284,26), (728863908,32), (654898692,40), (530198076,50), (417883440,62), (341452344,77), (274579584,95), (231921120,118), (184091820,146), (152469828,181), (125634972,224), (107059704,278), (88746120,345), (73135668,428), (61035756,531), (50683320,658), (42331824,816), (35234400,1012), (29341356,1255), (24290928,1556), (20284668,1929), (17215908,2391), (14737488,2964), (12692772,3674), (11220732,4555), (9787584,5647), (8148420,7000), (6918612,8678), (6015000,10758), (5480316,13336), (5443356,16532), (4535616,20494), (3962316,25406), (3914484,31495), (3828108,39043), (3583536,48400), (4104120,60000), (166024740,2147483647)) foo (count, value); ---------------------------------------------- -- nice data set with random data (uniform) -- ---------------------------------------------- -- compression 10 WITH data AS (SELECT prng(1000) x, prng(1000, 29823218) cnt), data_expanded AS (SELECT x FROM (SELECT x, generate_series(1, (10 + 100 * cnt)::int) FROM data) foo ORDER BY random()) SELECT p, abs(a - b) < 0.1, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) a FROM data_expanded) foo, (SELECT tdigest_percentile(x, (10 + 100 * cnt)::int, 10, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) b FROM data) bar ) baz; -- compression 100 WITH data AS (SELECT prng(1000) x, prng(1000, 29823218) cnt), data_expanded AS (SELECT x FROM (SELECT x, generate_series(1, (10 + 100 * cnt)::int) FROM data) foo ORDER BY random()) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) a FROM data_expanded) foo, (SELECT tdigest_percentile(x, (10 + 100 * cnt)::int, 100, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) b FROM data) bar ) baz; -- compression 1000 WITH data AS (SELECT prng(1000) x, prng(1000, 29823218) cnt), data_expanded AS (SELECT x FROM (SELECT x, generate_series(1, (10 + 100 * cnt)::int) FROM data) foo ORDER BY random()) SELECT p, abs(a - b) < 0.01, -- absolute error tolerance for this data set (CASE WHEN abs(a - b) < 0.1 THEN NULL ELSE (a - b) END) AS err FROM ( SELECT unnest(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) AS p, unnest(a) AS a, unnest(b) AS b FROM (SELECT percentile_cont(ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY x) a FROM data_expanded) foo, (SELECT tdigest_percentile(x, (10 + 100 * cnt)::int, 1000, ARRAY[0.01, 0.05, 0.1, 0.9, 0.95, 0.99]) b FROM data) bar ) baz; -- generate data CREATE TABLE test_value_count (v double precision, c int, d int); INSERT INTO test_value_count SELECT 1000 * random(), 1 + mod(i,7), mod(i,113) FROM generate_series(1,100000) s(i); ANALYZE test_value_count; -- API EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, c, 100, 0.95) AS b FROM test_value_count) foo; WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(0.95) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 0.95, abs(a - b) / 1000 < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile(v, c, 100, 0.95) AS b FROM test_value_count) foo; EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percent_rank(950) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile_of(v, c, 100, 950) AS b FROM test_value_count) foo; WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percent_rank(950) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT 950, abs(a - b) < 0.01 FROM ( SELECT (SELECT p FROM x) AS a, tdigest_percentile_of(v, c, 100, 950) AS b FROM test_value_count) foo; -- array of percentiles / values EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, c, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_value_count) foo; WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT percentile_disc(ARRAY[0.0, 0.95, 0.99, 1.0]) WITHIN GROUP (ORDER BY v) AS p FROM d) SELECT p, abs(a - b) / 1000 < 0.01 FROM ( SELECT unnest(ARRAY[0.0, 0.95, 0.99, 1.0]) p, unnest((SELECT p FROM x)) AS a, unnest(tdigest_percentile(v, c, 100, ARRAY[0.0, 0.95, 0.99, 1.0])) AS b FROM test_value_count) foo; EXPLAIN (COSTS OFF) WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) AS p FROM d)) p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((select x.p from x)) AS a, unnest(tdigest_percentile_of(v, c, 100, ARRAY[950, 990])) AS b FROM test_value_count) foo; WITH d AS (SELECT test_value_count.* FROM test_value_count, LATERAL generate_series(1,test_value_count.c)), x AS (SELECT array_agg((SELECT percent_rank(f) WITHIN GROUP (ORDER BY v) AS p FROM d)) p FROM unnest(ARRAY[950, 990]) f) SELECT p, abs(a - b) < 0.01 FROM ( SELECT unnest(ARRAY[950, 990]) AS p, unnest((select x.p from x)) AS a, unnest(tdigest_percentile_of(v, c, 100, ARRAY[950, 990])) AS b FROM test_value_count) foo; DROP TABLE test_value_count; tvondra-tdigest-c0af713/test/sql/values_count_checks.sql000066400000000000000000000014301525543331000235630ustar00rootroot00000000000000-- Regression test for the count check in tdigest_add_double_values_count(), -- the transition function of the tdigest_percentile_of() aggregates accepting -- an explicit number of occurrences of a value. -- -- A non-NULL count must be positive. A NULL count means one occurrence -- of the input value. \set VERBOSITY terse -- zero count SELECT tdigest_percentile_of(v, 0::bigint, 100, 1.0) FROM (VALUES (1.0)) AS t(v); -- negative count SELECT tdigest_percentile_of(v, -1::bigint, 100, 1.0) FROM (VALUES (1.0)) AS t(v); -- the smallest valid count has to work SELECT tdigest_percentile_of(v, 1::bigint, 100, 1.0) FROM (VALUES (1.0)) AS t(v); -- NULL count means a single occurrence, so it's valid too SELECT tdigest_percentile_of(v, NULL::bigint, 100, 1.0) FROM (VALUES (1.0)) AS t(v); tvondra-tdigest-c0af713/test/sql/values_count_generated.sql000066400000000000000000000022541525543331000242660ustar00rootroot00000000000000-- Regression test for adding a value with a count exceeding BUFFER_SIZE in -- tdigest_add_double_values_count. -- -- Adding the values one by one would mean too many compactions, so counts -- larger than the buffer (BUFFER_SIZE = 10 * compression, i.e. 100 for the -- smallest compression) are added as pre-generated centroids instead. Both -- paths have to produce comparable results. -- a single distinct value, so the result is exact on both sides of the -- BUFFER_SIZE threshold SELECT tdigest_percentile_of(v, 100::bigint, 10, 1.0) FROM (VALUES (1.0)) AS t(v); SELECT tdigest_percentile_of(v, 101::bigint, 10, 1.0) FROM (VALUES (1.0)) AS t(v); SELECT tdigest_percentile_of(v, 1000000::bigint, 10, 1.0) FROM (VALUES (1.0)) AS t(v); -- two distinct values, with the counts at and just above the -- threshold SELECT tdigest_percentile_of(v, 100::bigint, 10, 2.0) FROM (VALUES (1.0), (2.0)) AS t(v); SELECT tdigest_percentile_of(v, 101::bigint, 10, 2.0) FROM (VALUES (1.0), (2.0)) AS t(v); -- a larger compression means a larger buffer, so the very same count now -- goes through the one-by-one path SELECT tdigest_percentile_of(v, 101::bigint, 100, 2.0) FROM (VALUES (1.0), (2.0)) AS t(v); tvondra-tdigest-c0af713/test/sql/window_finalization.sql000066400000000000000000000037441525543331000236240ustar00rootroot00000000000000-- Finalizing a prefix window must not change the state used for later -- frames. Compare all three builders with independent aggregation of the -- same ordered input. This is an equality check between two executions of -- tdigest, not a claim that its estimates equal exact percentiles. -- -- Before the fix, each builder disagrees on 62 of the 100 frames. -- After the fix, this script succeeds and leaves no database objects behind. SET max_parallel_workers_per_gather = 0; DO $$ DECLARE raw_mismatches bigint; weighted_mismatches bigint; merged_mismatches bigint; BEGIN WITH inputs AS ( SELECT i, tdigest_add(NULL::tdigest, i::double precision, 100) AS d FROM generate_series(1, 100) AS t(i) ), windows AS ( SELECT i, tdigest(i::double precision, 100) OVER w AS raw, tdigest(i::double precision, 1::bigint, 100) OVER w AS weighted, tdigest(d) OVER w AS merged FROM inputs WINDOW w AS (ORDER BY i ROWS UNBOUNDED PRECEDING) ), compared AS ( SELECT w.*, (SELECT tdigest(j::double precision, 100 ORDER BY j) FROM generate_series(1, w.i) AS t(j)) AS expected FROM windows w ) SELECT count(*) FILTER (WHERE tdigest_send(raw) IS DISTINCT FROM tdigest_send(expected)), count(*) FILTER (WHERE tdigest_send(weighted) IS DISTINCT FROM tdigest_send(expected)), count(*) FILTER (WHERE tdigest_send(merged) IS DISTINCT FROM tdigest_send(expected)) INTO raw_mismatches, weighted_mismatches, merged_mismatches FROM compared; IF raw_mismatches <> 0 OR weighted_mismatches <> 0 OR merged_mismatches <> 0 THEN RAISE EXCEPTION 'window finalization changed reusable state: raw %, weighted %, merged % mismatches', raw_mismatches, weighted_mismatches, merged_mismatches; END IF; END $$;