pax_global_header00006660000000000000000000000064152642255270014524gustar00rootroot0000000000000052 comment=20b57e2e2eff884bfcf3ede3c095762e614c51d1 fabriziomello-pg_stat_log-20b57e2/000077500000000000000000000000001526422552700172125ustar00rootroot00000000000000fabriziomello-pg_stat_log-20b57e2/.clang-format000066400000000000000000000103351526422552700215670ustar00rootroot00000000000000--- Language: Cpp # BasedOnStyle: LLVM AccessModifierOffset: -2 AlignAfterOpenBracket: Align AlignArrayOfStructures: Left AlignConsecutiveAssignments: true AlignConsecutiveDeclarations: true AlignConsecutiveMacros: true AlignEscapedNewlines: Right AlignOperands: true AlignTrailingComments: true AllowAllParametersOfDeclarationOnNextLine: true AllowShortBlocksOnASingleLine: false AllowShortCaseLabelsOnASingleLine: false AllowShortFunctionsOnASingleLine: All AllowShortIfStatementsOnASingleLine: false AllowShortLoopsOnASingleLine: false # AlwaysBreakAfterDefinitionReturnType: None # option is deprecated AlwaysBreakAfterReturnType: AllDefinitions AlwaysBreakBeforeMultilineStrings: false AlwaysBreakTemplateDeclarations: MultiLine BinPackArguments: false BinPackParameters: true BraceWrapping: AfterCaseLabel: true AfterClass: true AfterControlStatement: true AfterEnum: true AfterFunction: true AfterNamespace: true AfterObjCDeclaration: true AfterStruct: true AfterUnion: true AfterExternBlock: true BeforeCatch: true BeforeElse: true IndentBraces: false SplitEmptyFunction: true SplitEmptyRecord: true SplitEmptyNamespace: true BreakBeforeBinaryOperators: None BreakBeforeBraces: Custom BreakBeforeInheritanceComma: false # N/A C++ BreakInheritanceList: BeforeColon BreakBeforeTernaryOperators: false BreakConstructorInitializersBeforeComma: false BreakConstructorInitializers: BeforeColon BreakAfterJavaFieldAnnotations: false # N/A Java BreakStringLiterals: true ColumnLimit: 100 #CommentPragmas: '^ TS Pragma:' #For future proofing CompactNamespaces: false # N/A c++ ConstructorInitializerAllOnOneLineOrOnePerLine: false # N/A C++ ConstructorInitializerIndentWidth: 40 # N/A C++ ContinuationIndentWidth: 4 Cpp11BracedListStyle: true # see catalog.c array struct assigns for an example DerivePointerAlignment: false # always use Right DisableFormat: false # haha # ExperimentalAutoDetectBinPacking: false #the docs say not to have this in config file FixNamespaceComments: true # N/A C++ ForEachMacros: - foreach - forboth - for_each_cell - for_both_cell - forthree IncludeBlocks: Preserve # separate include blocks will not be merged IncludeCategories: # we want to ensure c.h and postgres.h appear first - Regex: '.*' Priority: 1 - Regex: '^' Priority: -1 - Regex: '^' Priority: -1 IncludeIsMainRegex: '' # filename_ will be seen as the primary include IndentCaseLabels: true IndentPPDirectives: None # do not indent preprocessor directives after the '#' IndentWidth: 4 IndentWrappedFunctionNames: false # we do not indent the function name in the declaration JavaScriptQuotes: Double # N/A js JavaScriptWrapImports: true # N/A js KeepEmptyLinesAtTheStartOfBlocks: false MacroBlockBegin: '' # regex of macros that behave like '{' MacroBlockEnd: '' # regex of macros that behave like '}' MaxEmptyLinesToKeep: 1 NamespaceIndentation: None # N/A c++ ObjCBinPackProtocolList: Auto # N/A objC ObjCBlockIndentWidth: 2 # N/A objC ObjCSpaceAfterProperty: false # N/A objC ObjCSpaceBeforeProtocolList: true # N/A objC PenaltyBreakAssignment: 2 PenaltyBreakBeforeFirstCallParameter: 10000 PenaltyBreakComment: 300 PenaltyBreakFirstLessLess: 120 PenaltyBreakString: 1000 PenaltyBreakTemplateDeclaration: 10 PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 60 PointerAlignment: Right # as in char *foo; ReflowComments: true # break up long comments into multiple lines SortIncludes: false # keep includes in the same order as we write them SortUsingDeclarations: false # N/A c++ SpaceAfterCStyleCast: true SpaceAfterTemplateKeyword: false # N/A c++ SpaceBeforeAssignmentOperators: true SpaceBeforeCpp11BracedList: false SpaceBeforeCtorInitializerColon: true # N/A c++ SpaceBeforeInheritanceColon: false # N/A c++ SpaceBeforeParens: ControlStatements SpaceBeforeRangeBasedForLoopColon: true # N/A C++ SpaceInEmptyParentheses: false SpacesBeforeTrailingComments: 1 SpacesInAngles: false # N/A c++ SpacesInContainerLiterals: true # N/A c++ SpacesInCStyleCastParentheses: false SpacesInParentheses: false SpacesInSquareBrackets: false Standard: Cpp11 TabWidth: 4 UseTab: Never ... fabriziomello-pg_stat_log-20b57e2/.github/000077500000000000000000000000001526422552700205525ustar00rootroot00000000000000fabriziomello-pg_stat_log-20b57e2/.github/workflows/000077500000000000000000000000001526422552700226075ustar00rootroot00000000000000fabriziomello-pg_stat_log-20b57e2/.github/workflows/code_style.yml000066400000000000000000000013041526422552700254620ustar00rootroot00000000000000name: Code style on: push: branches: [main] paths: - '**.c' - '**.h' - '.clang-format' pull_request: branches: [main] paths: - '**.c' - '**.h' - '.clang-format' jobs: clang-format: name: Check code formatting runs-on: ubuntu-latest steps: - name: Checkout source uses: actions/checkout@v5 - name: Check trailing whitespace if: always() run: | find . -type f -regex '.*\.\(c\|h\)$' -exec perl -pi -e 's/[ \t]+$//' {} + git diff --exit-code - name: Check code formatting if: always() run: | clang-format -i pg_stat_log.c git diff --exit-code fabriziomello-pg_stat_log-20b57e2/.github/workflows/installcheck.yml000066400000000000000000000011721526422552700257770ustar00rootroot00000000000000name: CI on: push: branches: [main] paths-ignore: - '**.md' - 'LICENSE' - '.gitignore' pull_request: branches: [main] paths-ignore: - '**.md' - 'LICENSE' - '.gitignore' jobs: test: runs-on: ubuntu-latest container: pgxn/pgxn-tools strategy: matrix: pg: [18, 19] steps: - name: Check out the repo uses: actions/checkout@v5 - name: Start PostgreSQL ${{ matrix.pg }} run: pg-start ${{ matrix.pg }} - name: Build and test run: | chown -R postgres:postgres . gosu postgres pg-build-test fabriziomello-pg_stat_log-20b57e2/.github/workflows/upstream_anchors.yml000066400000000000000000000031441526422552700267110ustar00rootroot00000000000000name: Upstream anchors on: push: branches: [main] paths: - 'pg_stat_log.c' - 'check-upstream-anchors.pl' - '.github/workflows/upstream_anchors.yml' pull_request: branches: [main] paths: - 'pg_stat_log.c' - 'check-upstream-anchors.pl' - '.github/workflows/upstream_anchors.yml' schedule: - cron: '0 6 * * 1' workflow_dispatch: jobs: anchors: name: PostgreSQL ${{ matrix.ref }} runs-on: ubuntu-latest # master drifts independently of this repo; only the weekly run fails on it continue-on-error: ${{ matrix.ref == 'master' && github.event_name != 'schedule' }} strategy: fail-fast: false matrix: ref: [REL_18_STABLE, REL_19_STABLE, master] steps: - name: Check out the repo uses: actions/checkout@v5 - name: Check out PostgreSQL ${{ matrix.ref }} uses: actions/checkout@v5 with: repository: postgres/postgres ref: ${{ matrix.ref }} path: postgres fetch-depth: 1 sparse-checkout-cone-mode: false sparse-checkout: | /configure.ac /src/backend/libpq/auth.c /src/backend/tcop/postgres.c /src/backend/utils/error/elog.c /src/backend/utils/init/miscinit.c /src/backend/utils/init/postinit.c /src/backend/utils/misc/guc.c /src/backend/utils/misc/guc_parameters.dat /src/backend/utils/misc/guc_tables.c /src/include/utils/guc.h - name: Check upstream anchors run: perl check-upstream-anchors.pl postgres fabriziomello-pg_stat_log-20b57e2/.gitignore000066400000000000000000000006011526422552700211770ustar00rootroot00000000000000# Global excludes across all subdirectories *.o *.obj *.bc *.so *.so.[0-9] *.so.[0-9].[0-9] *.so.[0-9].[0-9][0-9] *.dylib *.dll *.exp *.a *.mo *.pot objfiles.txt .deps/ *.gcno *.gcda *.gcov *.gcov.out lcov*.info coverage/ coverage-html-stamp *.vcproj *.vcxproj win32ver.rc *.exe lib*dll.def lib*.pc # Extension specific excludes /results/ /tmp*/ /log/ /.idea/ pg_stat_log_errcodes.h fabriziomello-pg_stat_log-20b57e2/LICENSE000066400000000000000000000017601526422552700202230ustar00rootroot00000000000000PostgreSQL License Copyright (c) 2026, Fabrízio de Royes Mello Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies. IN NO EVENT SHALL FABRÍZIO DE ROYES MELLO BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF FABRÍZIO DE ROYES MELLO HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. FABRÍZIO DE ROYES MELLO SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND FABRÍZIO DE ROYES MELLO HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. fabriziomello-pg_stat_log-20b57e2/Makefile000066400000000000000000000017531526422552700206600ustar00rootroot00000000000000MODULE_big = pg_stat_log EXTENSION = pg_stat_log DATA = pg_stat_log--0.1.sql pg_stat_log--0.1--0.3.sql OBJS = pg_stat_log.o REGRESS_OPTS = --temp-instance=tmp_check --temp-config=pg_stat_log.conf REGRESS = pg_stat_log pg_stat_log_upgrade TAP_TESTS = 1 PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) # Check for the errcodes.txt on the standard Postgres distribution ERRCODES_TXT = $(shell $(PG_CONFIG) --sharedir)/errcodes.txt # Generate the errcode-name lookup header from errcodes.txt. pg_stat_log_errcodes.h: generate-errcode-names.pl @if [ -f "$(ERRCODES_FILE)" ]; then \ $(PERL) $< --outfile $@ $(ERRCODES_FILE); \ elif [ -f "$(ERRCODES_TXT)" ]; then \ $(PERL) $< --outfile $@ $(ERRCODES_TXT); \ else \ echo "ERROR: cannot find errcodes.txt; set ERRCODES_FILE=/path/to/errcodes.txt" >&2; \ exit 1; \ fi pg_stat_log.o: pg_stat_log_errcodes.h format: clang-format -i pg_stat_log.c clean: clean-errcodes clean-errcodes: rm -f pg_stat_log_errcodes.h fabriziomello-pg_stat_log-20b57e2/README.md000066400000000000000000000476641526422552700205120ustar00rootroot00000000000000# pg_stat_log [![CI](https://github.com/fabriziomello/pg_stat_log/actions/workflows/installcheck.yml/badge.svg)](https://github.com/fabriziomello/pg_stat_log/actions/workflows/installcheck.yml) Cumulative statistics about PostgreSQL log messages, built on the [Custom Cumulative Stats](https://wiki.postgresql.org/wiki/CustomCumulativeStats) API introduced in PostgreSQL 18. ## Overview `pg_stat_log` hooks into PostgreSQL's `emit_log_hook` to count log messages grouped by: - **backend type** -- client backend, autovacuum worker, checkpointer, etc. - **database** -- which database the message originated from - **user** -- which role was active - **error level** -- WARNING, ERROR, FATAL, PANIC, etc. - **SQLSTATE code** -- the 5-character error code and its human-readable name - **query identifier** -- optionally, the statement that logged the message; see [Query identifiers](#query-identifiers) Counters are exposed through the `pg_stat_log` view and the underlying `pg_stat_log_data()` function. Statistics persist across clean restarts and are discarded after crash recovery, following standard PostgreSQL cumulative stats semantics. The records written by `log_connections` and `log_disconnections` are counted under their own SQLSTATEs; see [Connection lifecycle records](#connection-lifecycle-records). The extension uses custom stats kind ID **28**, registered on the [PostgreSQL wiki](https://wiki.postgresql.org/wiki/CustomCumulativeStats). ## Requirements - PostgreSQL 18 or later - Must be loaded via `shared_preload_libraries` ## Installation Build and install using PGXS: ```bash make make install ``` If the build cannot locate `errcodes.txt` automatically, point it to the PostgreSQL source tree: ```bash make ERRCODES_FILE=/path/to/postgresql/src/backend/utils/errcodes.txt ``` Then configure PostgreSQL to load the extension: ``` # postgresql.conf shared_preload_libraries = 'pg_stat_log' ``` Restart the server and create the extension: ```sql CREATE EXTENSION pg_stat_log; ``` ## Upgrading from 0.1 or 0.2 Version 0.3 adds the `query_id` column and counts [connection lifecycle records](#connection-lifecycle-records) under their own SQLSTATEs. Releases 0.1 and 0.2 both install the extension's SQL version 0.1 (`pg_extension.extversion` shows `0.1`), so the update goes straight to 0.3. Install the new version, restart the server so it loads the new library, then update the extension in each database where it is installed: ```sql ALTER EXTENSION pg_stat_log UPDATE; ``` - **The first start with the new library discards all cumulative statistics**, not only pg_stat_log's. Version 0.3 stores a query identifier in every entry, so PostgreSQL cannot read the statistics file written with 0.1 or 0.2: it logs a `could not read ...` warning and `corrupted statistics file`, and starts over with every cumulative statistic reset. Later clean restarts keep statistics as usual. - **Until the update, the 0.1 view and function keep working** with the new library, without the `query_id` column. - **The update recreates the `pg_stat_log` view and the `pg_stat_log_data()` function.** Drop your own views that depend on them before the update and create them again afterwards. Privileges you granted on them must be granted again; the default grants to `pg_read_all_stats` are restored. ## Configuration | Parameter | Type | Default | Context | Description | |-----------|------|---------|---------|-------------| | `pg_stat_log.enabled` | bool | `true` | SUSET | Enable or disable collection at runtime | | `pg_stat_log.min_error_level` | enum | `warning` | SUSET | Minimum severity to track (`debug5` through `panic`) | | `pg_stat_log.max_entries` | int | `1024` | POSTMASTER | Maximum distinct combinations to track; requires restart | | `pg_stat_log.compute_query_id` | bool | `off` | POSTMASTER | Also group by query identifier; see [Query identifiers](#query-identifiers) | ## Usage ### Viewing statistics Query the `pg_stat_log` view: ```sql SELECT * FROM pg_stat_log; ``` ``` backend_type | database_oid | database_name | user_oid | user_name | query_id | elevel | sqlerrcode | sqlerrcode_name | count ----------------+--------------+---------------+----------+-----------+----------+---------+------------+------------------+------- client backend | 16384 | mydb | 10 | postgres | | WARNING | 01000 | warning | 42 client backend | 16384 | mydb | 10 | postgres | | ERROR | 22012 | division_by_zero | 3 client backend | 16384 | mydb | 10 | postgres | | ERROR | 42P01 | undefined_table | 7 ``` ### View columns | Column | Type | Description | |--------|------|-------------| | `backend_type` | text | Process type (e.g. `client backend`, `autovacuum worker`) | | `database_oid` | oid | Database OID, NULL for shared or background processes | | `database_name` | text | Resolved database name | | `user_oid` | oid | Role OID, NULL for background processes | | `user_name` | text | Resolved role name | | `query_id` | bigint | Query identifier, NULL unless `pg_stat_log.compute_query_id` is on; see [Query identifiers](#query-identifiers) | | `elevel` | text | Error severity (WARNING, ERROR, FATAL, PANIC, ...) | | `sqlerrcode` | text | 5-character SQLSTATE code | | `sqlerrcode_name` | text | Condition name (e.g. `division_by_zero`) | | `count` | bigint | Cumulative message count | ### Example queries Top 10 most frequent errors: ```sql SELECT elevel, sqlerrcode, sqlerrcode_name, sum(count) AS total FROM pg_stat_log WHERE elevel = 'ERROR' GROUP BY elevel, sqlerrcode, sqlerrcode_name ORDER BY total DESC LIMIT 10; ``` Errors by database: ```sql SELECT database_name, elevel, count FROM pg_stat_log WHERE database_name IS NOT NULL ORDER BY count DESC; ``` Errors from background processes: ```sql SELECT backend_type, elevel, sqlerrcode_name, count FROM pg_stat_log WHERE backend_type <> 'client backend' ORDER BY count DESC; ``` ### Connection lifecycle records PostgreSQL writes the `log_connections` and `log_disconnections` records at `LOG` level with SQLSTATE `00000`. `pg_stat_log` counts them under extension-defined SQLSTATEs instead, attributed to the connected database and the login role: | SQLSTATE | `sqlerrcode_name` | Record | Logged when | |----------|-------------------|--------|-------------| | `PSL01` | `connection_authenticated` | `connection authenticated: ...` | `log_connections` includes `authentication` | | `PSL02` | `connection_authorized` | `connection authorized: ...`, `replication connection authorized: ...` | `log_connections` includes `authorization` | | `PSL03` | `disconnection` | `disconnection: session time: ...` | `log_disconnections = on` | A record is counted only if it reaches the server log, and only if `pg_stat_log.min_error_level` is `log` or lower: ``` # postgresql.conf pg_stat_log.min_error_level = 'log' log_connections = 'authentication,authorization' # or 'on', 'all' log_disconnections = on ``` Connections and disconnections per database and user: ```sql SELECT database_name, user_name, backend_type, sqlerrcode_name, count FROM pg_stat_log WHERE sqlerrcode LIKE 'PSL%' ORDER BY database_name, user_name, backend_type, sqlerrcode; ``` - **The user is the login role**, the role that connected, even if the session later runs `SET ROLE` or `SET SESSION AUTHORIZATION`. - **Physical replication connections** are counted with `backend_type = 'walsender'` and a NULL database. - **A startup that fails after authentication** counts the records logged so far with the attribution known at that point. For example, a connection to a database that does not exist counts `PSL01` and `PSL02` with a NULL database, and no `PSL03`, since its disconnection is not logged. - **Counting is not simultaneous with logging.** `PSL01` and `PSL02` are counted when backend startup completes or fails; `PSL03` is counted as the session exits, just before its record is written. - **Per-session settings only apply at connection start.** Superusers and roles granted `SET` on `log_disconnections` can change it for a session with startup options, such as `PGOPTIONS='-c log_disconnections=on'` or the `options` connection parameter. `ALTER ROLE ... SET` and `ALTER DATABASE ... SET` cannot set it. A configuration reload does not change it for running sessions, so PostgreSQL still logs, and pg_stat_log still counts, the disconnection of a session that started with `log_disconnections = on`. Startup options for `log_connections` come too late for `PSL01` and `PSL02`: PostgreSQL logs those records before it applies the options. - **The other `log_connections` records are not mapped.** `receipt` (`connection received: ...`) is logged before the backend has a `PGPROC` and is never counted; `setup_durations` (`connection ready: ...`) is counted as an ordinary `LOG` record with SQLSTATE `00000`. - **Each (backend type, database, user) combination** that connects can use up to three entries; account for them when sizing `pg_stat_log.max_entries`. Before version 0.3, the authenticated and authorized records were counted as `00000` (`successful_completion`) with a NULL database and user, and disconnections were not counted. ### Query identifiers With `pg_stat_log.compute_query_id = on`, messages are also grouped by the query identifier of the statement that logged them, so the same error raised by different statements is counted in separate rows: ``` # postgresql.conf pg_stat_log.compute_query_id = on compute_query_id = auto # the default; 'on' also works ``` `query_id` is the identifier that `%Q` prints in `log_line_prefix`, the same one shown by `pg_stat_activity.query_id` and `pg_stat_statements.queryid`. Errors per statement, with the statement text from `pg_stat_statements`: ```sql SELECT l.sqlerrcode_name, l.count, s.query FROM pg_stat_log l LEFT JOIN pg_stat_statements s ON s.queryid = l.query_id AND s.dbid = l.database_oid AND s.userid = l.user_oid AND s.toplevel WHERE l.query_id IS NOT NULL ORDER BY l.count DESC; ``` `pg_stat_statements` lists a statement only after it has completed once, so `query` is NULL for statements that always fail. - **`compute_query_id` must not be `off`.** Under `auto`, pg_stat_log has PostgreSQL compute query identifiers, as `pg_stat_statements` does; it decides this at server start, so `pg_stat_log.compute_query_id` requires a restart. While `compute_query_id` is `off`, in the server configuration or in a session, messages are counted with a NULL `query_id`. - **Some messages have no query identifier** and are counted with a NULL `query_id`: messages logged outside any statement, for example by the checkpointer or during connection startup; errors raised before parse analysis completes, such as syntax errors and references to tables that do not exist; the connection lifecycle records; and every message while `track_activities` is off. - **Statements that differ only in constants share an identifier**, such as `SELECT 1/0` and `SELECT 2/0`. - **Messages from nested statements**, such as an error raised inside a function, are counted under the identifier of the top-level statement. - **An idle session keeps its last statement's identifier.** A message logged between statements, such as an `idle_session_timeout` FATAL, is counted under the identifier of the session's most recent statement. ### Resetting statistics ```sql SELECT pg_stat_log_reset(); ``` Resetting also reclaims the tracked-combination slots, so new distinct combinations can be tracked after a reset even if `max_entries` had been reached previously. ### Inspecting capacity and drops `pg_stat_log_info()` exposes metadata about the tracking state: ```sql SELECT * FROM pg_stat_log_info(); ``` ``` max_entries | num_entries | n_dropped | stats_reset -------------+-------------+-----------+------------------------------- 1024 | 37 | 0 | 2026-04-22 16:40:39.124+00 ``` | Column | Type | Description | |--------|------|-------------| | `max_entries` | int | Configured `pg_stat_log.max_entries` capacity | | `num_entries` | int | Distinct combinations currently tracked | | `n_dropped` | bigint | Log messages that could not be tracked because `max_entries` was full | | `stats_reset` | timestamptz | Timestamp of the last reset (or shared-memory init) | Monitor `n_dropped` to detect when `max_entries` is too small for your workload: if it grows over time, increase `pg_stat_log.max_entries` (and restart) so every distinct combination fits. `stats_reset` advances whenever `pg_stat_log_reset()` is called. ## How it works `pg_stat_log` uses the fixed-amount Custom Cumulative Stats API, following the same pattern as PostgreSQL's in-tree `test_custom_fixed_stats` test module. 1. **Startup** (`_PG_init`): registers custom stats kind 28, defines GUCs, and installs the `emit_log_hook`. 2. **Hook** (`emit_log_hook`): on each log message at or above the configured minimum level, acquires a shared-memory LWLock, scans the entry array for a matching (backend_type, database, user, query_id, elevel, sqlerrcode) combination, and either increments an existing counter or creates a new entry. `query_id` is 0, shown as NULL, unless `pg_stat_log.compute_query_id` is on. Uses the changecount protocol for atomic reads. 3. **pgstat callbacks**: three callbacks mirror the test module -- `init_shmem_cb` initializes the LWLock and array header, `reset_all_cb` zeroes counters and reclaims slots, `snapshot_cb` copies stats for reporting. 4. **Reporting**: `pg_stat_log_data()` is a set-returning C function that reads from the pgstat snapshot. The `pg_stat_log` view joins its output with `pg_database` and `pg_roles` to resolve OIDs to names. 5. **Connection lifecycle records**: the hook recognizes the `log_connections` and `log_disconnections` records by the source file and function that emit them. Authentication and authorization are logged before the session has a database and login role, so they are held in backend-local counters and counted when the startup transaction commits or aborts. The disconnection record is logged after the backend has released its `PGPROC`, when shared memory can no longer be locked, so a `before_shmem_exit` callback counts it just before, under the same conditions that decide whether the record reaches the server log. `check-upstream-anchors.pl` verifies in CI that the PostgreSQL sources still match these assumptions. ### Design notes: why a custom hash table The entries live in an index-based separate-chaining hash table built directly inside the stats block, rather than in one of PostgreSQL's standard hash tables. The reason is a hard constraint of the fixed-amount custom stats API: the block is snapshotted with a raw `memcpy` and persisted/restored verbatim (`fwrite`/`fread`) with no serialize callbacks, so every structure inside it must be position-independent (no raw pointers) and fully self-contained. None of the standard tables meet that bar. `dynahash` chains its entries, segment directory, and freelists with absolute pointers, which become garbage in a memcpy'd snapshot or after a restart (this is why `pg_stat_statements` serializes its dynahash logically at shutdown instead of dumping it raw -- an option fixed-amount stats kinds do not have). `simplehash` is process-local and grows by reallocation, so it cannot live in shared memory at all. `dshash` would require abandoning the fixed-amount API for variable-amount stats, which brings a 64-bit key limit (pg_stat_log's signature needs ~212 bits), unbounded DSA memory growth, no extension-visible API to enumerate entries for reporting, and memory allocation inside `emit_log_hook` -- which runs in every process type, including the postmaster. The index-chained design sidesteps all of this: chain links are array indices instead of pointers, so the block survives snapshots and restarts unchanged; the footprint is a hard cap of ~1.15x the raw entry data, fixed at startup; and lookups, inserts, and drops stay O(1) expected even at 100% load. ## Caveats A few things to keep in mind when interpreting the counters: - **`emit_log_hook` only fires for messages that actually reach the server log.** Lowering `pg_stat_log.min_error_level` on its own is not enough -- `log_min_messages` acts as a floor on what the hook ever sees. For example, `pg_stat_log.min_error_level = 'notice'` requires `log_min_messages = 'notice'` (or lower, e.g. `info`, `debug1`) to have any effect. With the default `log_min_messages = 'warning'`, NOTICE and INFO messages are filtered out before the hook runs and will not be counted. - **NULL `database_name` / `user_name` rows are expected.** A log message can be emitted before a backend has bound to a database or role, so the database and user OIDs may be unset. Typical cases include authentication failures (e.g. FATAL with SQLSTATE `28P01`), early startup messages, and messages logged in the postmaster context. This is not a bug -- those rows record real events that simply have no database/user to attribute them to. Connection lifecycle records of a failed startup follow the same rule; see [Connection lifecycle records](#connection-lifecycle-records). - **Parallel workers appear as a separate `backend_type`.** If an error is raised during a parallel query, you may see one row with `backend_type = 'client backend'` (the leader) plus one row per parallel worker that hit the error with `backend_type = 'parallel worker'`. Keep this in mind when aggregating so you do not double-count a single logical error. - **`max_entries` is a hard cap.** Once the configured number of distinct (backend_type, database, user, query_id, elevel, sqlerrcode) combinations has been reached, new distinct combinations are dropped until `pg_stat_log_reset()` is called, which reclaims all slots. Monitor `n_dropped` via `pg_stat_log_info()` to detect when `max_entries` is too small. Size it to cover the cardinality you expect -- roughly `N_databases x N_roles x typical_distinct_sqlstates x backend_types` -- and remember it is `POSTMASTER`-context, so changes require a restart. With `pg_stat_log.compute_query_id = on`, multiply that by the number of distinct statements that log messages. - **Changing `max_entries` discards all cumulative statistics.** The persisted block changes length, so at the next start PostgreSQL cannot read the statistics file: it logs a `could not read ...` warning and `corrupted statistics file`, and starts over with every cumulative statistic reset, not only pg_stat_log's. - **`n_dropped` and `stats_reset` do not persist across restarts.** These metadata fields are reinitialized at server startup. The per-combination counters persist across clean restarts, but `n_dropped` resets to zero and `stats_reset` is set to the startup timestamp. This is by design -- drop telemetry measures pressure since the last startup or reset. - **Caught PL/pgSQL exceptions are not counted.** If a `RAISE` inside a `BEGIN ... EXCEPTION WHEN OTHERS` block is caught by the exception handler, the message never reaches the server log and `emit_log_hook` does not fire. Only messages that actually make it to the log are tracked. ## Files | File | Purpose | |------|---------| | `pg_stat_log.c` | C implementation (hook, callbacks, SQL functions) | | `pg_stat_log--0.1.sql` | SQL objects (functions, view) | | `pg_stat_log--0.1--0.3.sql` | Update to 0.3, adding the `query_id` column | | `pg_stat_log.control` | Extension metadata | | `Makefile` | PGXS build | | `generate-errcode-names.pl` | Generates SQLSTATE-to-name lookup header from `errcodes.txt` | | `check-upstream-anchors.pl` | Checks the PostgreSQL source assumptions behind the connection lifecycle counters and query identifiers | | `t/001_pg_stat_log.pl` | TAP regression tests | | `t/002_connection_lifecycle.pl` | TAP tests for the connection lifecycle counters | | `t/003_query_id.pl` | TAP tests for the query identifiers | ## License Released under the [PostgreSQL License](https://opensource.org/licenses/PostgreSQL). fabriziomello-pg_stat_log-20b57e2/check-upstream-anchors.pl000066400000000000000000000152751526422552700241270ustar00rootroot00000000000000#!/usr/bin/perl # # Check the PostgreSQL source assumptions behind the connection lifecycle # counters and the query identifiers # # pg_stat_log recognizes the log_connections and log_disconnections records # by the functions that emit them, attributes them when the startup # transaction ends, and relies on log_disconnections keeping its startup # value for the rest of the session. At exit, it predicts whether the # disconnection record reaches the server log. It records the query # identifier that log_line_prefix's %Q prints. Each check names the # assumption it guards; a failure means pg_stat_log.c must be revisited for # that PostgreSQL version. # # Usage: check-upstream-anchors.pl # use strict; use warnings FATAL => 'all'; die "Usage: $0 \n" unless @ARGV == 1; my $srcdir = $ARGV[0]; my $failures = 0; # The first four checks must match pg_stat_log_lifecycle_origins in # pg_stat_log.c my @function_checks = ( { file => 'src/backend/libpq/auth.c', function => 'set_authn_id', contains => [ 'ereport(LOG', 'connection authenticated: identity=' ], why => 'PSL01 is logged by set_authn_id()', }, { file => 'src/backend/libpq/auth.c', function => 'ClientAuthentication', contains => [ 'ereport(LOG', 'connection authenticated: user=' ], why => 'PSL01 without an authenticated identity is logged by ClientAuthentication()', }, { file => 'src/backend/utils/init/postinit.c', function => 'PerformAuthentication', contains => [ 'ereport(LOG', 'connection authorized: user=' ], why => 'PSL02 is logged by PerformAuthentication()', }, { file => 'src/backend/tcop/postgres.c', function => 'log_disconnections', contains => [ 'ereport(LOG', 'disconnection: session time:' ], why => 'PSL03 is logged by log_disconnections()', }, { file => 'src/backend/tcop/postgres.c', function => 'PostgresMain', contains => [ 'if (IsUnderPostmaster && Log_disconnections)', 'on_proc_exit(log_disconnections, 0);' ], why => 'log_disconnections() is installed under the condition pg_stat_log_startup_xact_cb() uses', }, { file => 'src/backend/tcop/postgres.c', function => 'log_disconnections', lacks => ['Log_disconnections'], why => 'log_disconnections() logs unconditionally once installed', }, { file => 'src/backend/utils/misc/guc.c', function => 'set_config_with_handle', in_order => [ 'case PGC_SU_BACKEND:', 'case PGC_BACKEND:', 'if (context == PGC_SIGHUP)', 'if (IsUnderPostmaster && changeVal && !is_reload)', 'return -1;' ], why => 'a configuration reload does not change PGC_SU_BACKEND settings in running backends', }, { file => 'src/backend/utils/init/postinit.c', function => 'InitPostgres', in_order => [ 'before_shmem_exit(ShutdownPostgres, 0);', 'StartTransactionCommand();', 'PerformAuthentication(', 'InitializeSessionUserId(', 'process_startup_options(', 'CommitTransactionCommand();' ], why => 'authentication runs in the startup transaction, which a FATAL aborts and a walsender commits after its startup options', }, { file => 'src/backend/utils/init/postinit.c', function => 'InitPostgres', in_order => [ 'MyDatabaseId = ', 'process_startup_options(', 'CommitTransactionCommand();' ], why => 'the startup transaction commits after the database and the startup options are set', }, { file => 'src/backend/utils/init/miscinit.c', function => 'SetAuthenticatedUserId', contains => ['MyProc->roleId = userid;'], why => 'MyProc->roleId holds the login role', }, { file => 'src/backend/utils/error/elog.c', function => 'is_log_level_output', contains => ['if (log_min_level == LOG || log_min_level <= ERROR)'], why => 'pg_stat_log_log_reaches_server() mirrors the LOG output rule', }, { file => 'src/backend/utils/error/elog.c', function => 'log_status_format', contains => ['pgstat_get_my_query_id()'], why => 'query_id is the identifier that %Q prints', }, ); sub slurp { my ($file) = @_; my $path = "$srcdir/$file"; open my $fh, '<', $path or die "$0: could not open '$path': $!\n"; local $/; my $contents = <$fh>; close $fh; return $contents; } sub report { my ($ok, $what, $why, $problem) = @_; if ($ok) { print "ok - $what: $why\n"; } else { print "not ok - $what: $why\n"; print " $problem\n"; $failures++; } } # From the definition line, which starts with the function name, to the # first closing brace in column 1 sub function_bodies { my ($file, $function) = @_; return slurp($file) =~ /^(\Q$function\E\(.*?^\})/msg; } my ($major) = slurp('configure.ac') =~ /^AC_INIT\(\[PostgreSQL\], \[(\d+)/m or die "$0: could not find the PostgreSQL version in configure.ac\n"; print "# PostgreSQL $major sources in $srcdir\n"; foreach my $check (@function_checks) { my $what = "$check->{file}: $check->{function}()"; my @bodies = function_bodies($check->{file}, $check->{function}); if (@bodies != 1) { report(0, $what, $check->{why}, "expected one definition, found " . scalar(@bodies)); next; } my $body = $bodies[0]; my @problems; if ($check->{contains}) { my @missing = grep { index($body, $_) < 0 } @{ $check->{contains} }; push @problems, "missing " . join(', ', map { "'$_'" } @missing) if @missing; } elsif ($check->{in_order}) { my $pos = 0; foreach my $text (@{ $check->{in_order} }) { $pos = index($body, $text, $pos); if ($pos < 0) { push @problems, "missing '$text' in order"; last; } $pos += length($text); } } if ($check->{lacks}) { my @present = grep { index($body, $_) >= 0 } @{ $check->{lacks} }; push @problems, "unexpected " . join(', ', map { "'$_'" } @present) if @present; } report(!@problems, $what, $check->{why}, join('; ', @problems)); } my $decl = $major >= 19 ? 'extern PGDLLIMPORT int log_min_messages[];' : 'extern PGDLLIMPORT int log_min_messages;'; report( index(slurp('src/include/utils/guc.h'), $decl) >= 0, "src/include/utils/guc.h: log_min_messages", "pg_stat_log_log_reaches_server() reads log_min_messages as declared for PostgreSQL $major", "missing '$decl'"); my $guc_file = $major >= 19 ? 'src/backend/utils/misc/guc_parameters.dat' : 'src/backend/utils/misc/guc_tables.c'; my $guc_definition = $major >= 19 ? qr/\{ name => 'log_disconnections',[^}]*context => 'PGC_SU_BACKEND'/ : qr/\{"log_disconnections", PGC_SU_BACKEND,/; report( scalar(slurp($guc_file) =~ $guc_definition), "$guc_file: log_disconnections", 'log_disconnections is PGC_SU_BACKEND, so it can only change at connection start', 'log_disconnections is not defined as PGC_SU_BACKEND'); if ($failures) { print "# $failures assumption(s) no longer hold for PostgreSQL $major\n"; exit 1; } print "# all assumptions hold for PostgreSQL $major\n"; fabriziomello-pg_stat_log-20b57e2/demo/000077500000000000000000000000001526422552700201365ustar00rootroot00000000000000fabriziomello-pg_stat_log-20b57e2/demo/Dockerfile000066400000000000000000000004331526422552700221300ustar00rootroot00000000000000FROM postgres:18 RUN apt-get update \ && apt-get install -y --no-install-recommends \ postgresql-18-stat-log \ postgresql-contrib-18 \ && rm -rf /var/lib/apt/lists/* COPY initdb/ /docker-entrypoint-initdb.d/ COPY workload/ /demo/ RUN chmod +x /demo/loop.sh fabriziomello-pg_stat_log-20b57e2/demo/README.md000066400000000000000000000102351526422552700214160ustar00rootroot00000000000000# Hands-on demo: live error rates from `pg_stat_log` Self-contained Docker Compose stack: PostgreSQL 18 with `pg_stat_log` from PGDG (`postgresql-18-stat-log`), two infinite mixed workloads, Prometheus, and Grafana. Counters in `pg_stat_log` are **cumulative**; the dashboard uses `rate()` / `increase()`. Do not graph `sum(count)` as a rate without those functions. ``` app_shop \ postgres_exporter app_analytics --> Postgres 18 + pg_stat_log --> Prometheus --> Grafana :3000 ``` ## Start Needs Docker Compose and a network for the image build (apt). From this directory: ```bash docker compose up --build ``` Open **http://127.0.0.1:3000** (anonymous admin). The provisioned dashboard is `pg_stat_log`. Collection uses `pg_stat_log.min_error_level=log` so LOG and above are counted. `log_min_messages` stays at **warning**: in PostgreSQL, `log_min_messages=log` is a special rank that *drops* WARNING and ERROR from the server log (only LOG/FATAL/PANIC remain), so the hook would never see the workload errors. LOG messages such as checkpoints still reach the hook when `log_min_messages` is warning. The error-rate graphs still filter `ERROR|FATAL|PANIC`; **All log messages** is the unfiltered group-by (`backend_type`, `elevel`, SQLSTATE), matching: ```sql SELECT backend_type, elevel, sqlerrcode, sqlerrcode_name, sum(count) FROM pg_stat_log GROUP BY 1, 2, 3, 4; ``` First start initializes the data volume. Later starts reuse it; to wipe counters and data: ```bash docker compose down -v docker compose up --build ``` If Postgres exits complaining about data in `/var/lib/postgresql/data` ("unused mount/volume"), you have a volume from an older layout: `postgres:18` expects a single mount at `/var/lib/postgresql`. Run `docker compose down -v` and start again. ## Session (~30 minutes) 1. Grafana: both `shop` and `analytics` series should move (error rate by database and by user). 2. Isolate a tenant: `docker compose stop app_analytics` — shop keeps rising, analytics goes quiet. 3. Bring it back: `docker compose start app_analytics`. 4. Same numbers in SQL (no log file): ```bash docker compose exec postgres psql -U postgres -c \ "SELECT elevel, sqlerrcode, sqlerrcode_name, database_name, user_name, count FROM pg_stat_log ORDER BY count DESC;" ``` Optional reset during the talk: ```bash docker compose exec postgres psql -U postgres -c 'SELECT pg_stat_log_reset();' ``` ## What is running | Service | Role | |---------|------| | `postgres` | Official `postgres:18` + PGDG `postgresql-18-stat-log`, `min_error_level=log`, `max_connections=300` | | `app_shop` | pgbench bursts as role `app_shop` / database `shop` (random `-c` 1..50) | | `app_analytics` | Same for `app_analytics` / `analytics` | | `postgres_exporter` | Scrapes `pg_stat_log` (`count` as a COUNTER) and `pg_stat_log_info` | | `prometheus` | 5s scrape | | `grafana` | Dashboard on port 3000 | Each workload is an infinite loop of **pgbench bursts**. Every burst picks `-c` uniformly in 1..50 and runs **one transaction per client** (pgbench aborts a client on SQL ERROR, so a long mixed `-T` run would collapse to healthy-only). Script weights are ~80% healthy and ~15% real errors (`22012`, `23505`, `42P01`, `RAISE WARNING` — no `EXCEPTION` handler). Failed logins are occasional `psql` attempts between bursts (FATAL; names often NULL), because pgbench cannot authenticate as the wrong user mid-session. `max_connections=300` leaves room for two tenants at 50 clients plus exporter and `psql`. Ports on the host: Postgres `5432`, Grafana `3000`, Prometheus `9090`, exporter `9187`. Demo passwords: `postgres` / `shop` / `analytics` / `demo_monitor`=`monitor`. Do not reuse this compose file in production. ## Files | Path | Role | |------|------| | `Dockerfile` | `postgres:18` + PGDG `postgresql-18-stat-log` and `postgresql-contrib-18` | | `initdb/` | `CREATE EXTENSION`, tenants, seed row | | `docker-compose.yml` | Full stack | | `workload/loop.sh` | Infinite pgbench bursts, random `-c` 1..50, then failed logins | | `workload/healthy.sql` | Non-error pgbench script (`@80`) | | `workload/error_*.sql` | Log-producing pgbench scripts | | `grafana/` | Exporter queries, Prometheus, provisioned dashboard | fabriziomello-pg_stat_log-20b57e2/demo/docker-compose.yml000066400000000000000000000057311526422552700236010ustar00rootroot00000000000000services: postgres: build: . environment: POSTGRES_PASSWORD: postgres command: - postgres - -c - shared_preload_libraries=pg_stat_log - -c - log_min_messages=warning - -c - pg_stat_log.min_error_level=log - -c - max_connections=300 ports: - "5432:5432" healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 2s timeout: 5s retries: 30 volumes: - pgdata:/var/lib/postgresql app_shop: build: . depends_on: postgres: condition: service_healthy environment: PGHOST: postgres PGUSER: app_shop PGPASSWORD: shop PGDATABASE: shop entrypoint: ["/demo/loop.sh"] restart: unless-stopped app_analytics: build: . depends_on: postgres: condition: service_healthy environment: PGHOST: postgres PGUSER: app_analytics PGPASSWORD: analytics PGDATABASE: analytics entrypoint: ["/demo/loop.sh"] restart: unless-stopped postgres_exporter: image: prometheuscommunity/postgres-exporter:v0.15.0 environment: DATA_SOURCE_NAME: postgresql://demo_monitor:monitor@postgres:5432/postgres?sslmode=disable command: - "--extend.query-path=/etc/postgres_exporter/queries.yaml" - "--disable-default-metrics" - "--disable-settings-metrics" # Built-in collectors are irrelevant here and some of them fail on PG 18 # as demo_monitor, which would show up as demo noise in pg_stat_log. - "--no-collector.database" - "--no-collector.locks" - "--no-collector.replication" - "--no-collector.replication_slot" - "--no-collector.stat_bgwriter" - "--no-collector.stat_database" - "--no-collector.stat_user_tables" - "--no-collector.statio_user_tables" - "--no-collector.wal" volumes: - ./grafana/queries.yaml:/etc/postgres_exporter/queries.yaml:ro ports: - "9187:9187" depends_on: postgres: condition: service_healthy prometheus: image: prom/prometheus:v2.55.1 volumes: - ./grafana/prometheus.yml:/etc/prometheus/prometheus.yml:ro command: - "--config.file=/etc/prometheus/prometheus.yml" - "--storage.tsdb.path=/prometheus" - "--storage.tsdb.retention.time=6h" ports: - "9090:9090" depends_on: - postgres_exporter grafana: image: grafana/grafana:11.3.0 environment: GF_AUTH_ANONYMOUS_ENABLED: "true" GF_AUTH_ANONYMOUS_ORG_ROLE: Admin GF_AUTH_DISABLE_LOGIN_FORM: "true" GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/pg_stat_log.json volumes: - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro - ./grafana/dashboards:/var/lib/grafana/dashboards:ro ports: - "3000:3000" depends_on: - prometheus volumes: pgdata: fabriziomello-pg_stat_log-20b57e2/demo/grafana/000077500000000000000000000000001526422552700215355ustar00rootroot00000000000000fabriziomello-pg_stat_log-20b57e2/demo/grafana/dashboards/000077500000000000000000000000001526422552700236475ustar00rootroot00000000000000fabriziomello-pg_stat_log-20b57e2/demo/grafana/dashboards/pg_stat_log.json000066400000000000000000000154571526422552700270600ustar00rootroot00000000000000{ "annotations": { "list": [] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], "liveNow": true, "panels": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": false }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "id": 1, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (database_name) (rate(pg_stat_log_count{elevel=~\"ERROR|FATAL|PANIC\"}[1m]))", "legendFormat": "{{database_name}}", "refId": "A" } ], "title": "Error rate by database", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 2, "showPoints": "never" }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "id": 2, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (user_name) (rate(pg_stat_log_count{elevel=~\"ERROR|FATAL|PANIC\"}[1m]))", "legendFormat": "{{user_name}}", "refId": "A" } ], "title": "Error rate by user", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 2, "showPoints": "never", "spanNulls": false }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "id": 7, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (backend_type, elevel, sqlerrcode, sqlerrcode_name) (rate(pg_stat_log_count[1m]))", "legendFormat": "{{backend_type}} {{elevel}} {{sqlerrcode}} {{sqlerrcode_name}}", "refId": "A" } ], "title": "All log messages", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "custom": { "align": "auto" } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [{ "id": "displayName", "value": "count" }] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "id": 8, "options": { "showHeader": true, "sortBy": [{ "displayName": "count", "desc": true }] }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (backend_type, elevel, sqlerrcode, sqlerrcode_name) (pg_stat_log_count)", "format": "table", "instant": true, "refId": "A" } ], "title": "All log messages (snapshot)", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": { "backend_type": 0, "elevel": 1, "sqlerrcode": 2, "sqlerrcode_name": 3, "Value": 4 } } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "bars", "fillOpacity": 40, "lineWidth": 1 }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 16, "x": 0, "y": 16 }, "id": 3, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (sqlerrcode, sqlerrcode_name) (increase(pg_stat_log_count{elevel=~\"WARNING|ERROR|FATAL|PANIC\"}[5m]))", "legendFormat": "{{sqlerrcode}} {{sqlerrcode_name}}", "refId": "A" } ], "title": "Top SQLSTATEs (increase over 5m)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 16, "y": 16 }, "id": 4, "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "horizontal" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "pg_stat_log_info_num_entries", "legendFormat": "num_entries", "refId": "A" } ], "title": "Tracked combinations", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 20, "y": 16 }, "id": 5, "options": { "reduceOptions": { "calcs": ["lastNotNull"] } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "pg_stat_log_info_n_dropped", "legendFormat": "n_dropped", "refId": "A" } ], "title": "n_dropped", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "max": 1, "min": 0, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 4, "w": 8, "x": 16, "y": 20 }, "id": 6, "options": { "reduceOptions": { "calcs": ["lastNotNull"] } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "pg_stat_log_info_num_entries / pg_stat_log_info_max_entries", "legendFormat": "fill", "refId": "A" } ], "title": "Hash table fill", "type": "gauge" } ], "refresh": "5s", "schemaVersion": 39, "tags": ["pg_stat_log"], "templating": { "list": [] }, "time": { "from": "now-15m", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "pg_stat_log", "uid": "pg-stat-log", "version": 1 } fabriziomello-pg_stat_log-20b57e2/demo/grafana/prometheus.yml000066400000000000000000000002511526422552700244510ustar00rootroot00000000000000global: scrape_interval: 5s evaluation_interval: 5s scrape_configs: - job_name: postgres_exporter static_configs: - targets: ["postgres_exporter:9187"] fabriziomello-pg_stat_log-20b57e2/demo/grafana/provisioning/000077500000000000000000000000001526422552700242635ustar00rootroot00000000000000fabriziomello-pg_stat_log-20b57e2/demo/grafana/provisioning/dashboards/000077500000000000000000000000001526422552700263755ustar00rootroot00000000000000fabriziomello-pg_stat_log-20b57e2/demo/grafana/provisioning/dashboards/dashboards.yml000066400000000000000000000003451526422552700312340ustar00rootroot00000000000000apiVersion: 1 providers: - name: pg_stat_log orgId: 1 folder: "" type: file disableDeletion: false editable: true options: path: /var/lib/grafana/dashboards foldersFromFilesStructure: false fabriziomello-pg_stat_log-20b57e2/demo/grafana/provisioning/datasources/000077500000000000000000000000001526422552700266005ustar00rootroot00000000000000fabriziomello-pg_stat_log-20b57e2/demo/grafana/provisioning/datasources/prometheus.yml000066400000000000000000000002641526422552700315200ustar00rootroot00000000000000apiVersion: 1 datasources: - name: Prometheus uid: prometheus type: prometheus access: proxy url: http://prometheus:9090 isDefault: true editable: false fabriziomello-pg_stat_log-20b57e2/demo/grafana/queries.yaml000066400000000000000000000025431526422552700241020ustar00rootroot00000000000000pg_stat_log: query: | SELECT backend_type, coalesce(database_name, 'none') AS database_name, coalesce(user_name, 'none') AS user_name, elevel, sqlerrcode, coalesce(sqlerrcode_name, '') AS sqlerrcode_name, count FROM pg_stat_log master: true cache_seconds: 0 metrics: - backend_type: usage: "LABEL" description: "Backend type" - database_name: usage: "LABEL" description: "Database name (none if unset)" - user_name: usage: "LABEL" description: "Role name (none if unset)" - elevel: usage: "LABEL" description: "Log severity" - sqlerrcode: usage: "LABEL" description: "SQLSTATE" - sqlerrcode_name: usage: "LABEL" description: "SQLSTATE name" - count: usage: "COUNTER" description: "Cumulative log-message count from pg_stat_log" pg_stat_log_info: query: | SELECT max_entries, num_entries, n_dropped FROM pg_stat_log_info() master: true cache_seconds: 0 metrics: - max_entries: usage: "GAUGE" description: "pg_stat_log.max_entries" - num_entries: usage: "GAUGE" description: "Distinct combinations currently tracked" - n_dropped: usage: "COUNTER" description: "Messages dropped because the table was full" fabriziomello-pg_stat_log-20b57e2/demo/initdb/000077500000000000000000000000001526422552700214075ustar00rootroot00000000000000fabriziomello-pg_stat_log-20b57e2/demo/initdb/01-extension.sql000066400000000000000000000000541526422552700243610ustar00rootroot00000000000000CREATE EXTENSION IF NOT EXISTS pg_stat_log; fabriziomello-pg_stat_log-20b57e2/demo/initdb/02-demo.sql000066400000000000000000000016471526422552700233030ustar00rootroot00000000000000CREATE ROLE app_shop LOGIN PASSWORD 'shop'; CREATE ROLE app_analytics LOGIN PASSWORD 'analytics'; CREATE ROLE demo_monitor LOGIN PASSWORD 'monitor'; CREATE DATABASE shop OWNER app_shop; CREATE DATABASE analytics OWNER app_analytics; GRANT pg_read_all_stats TO demo_monitor; GRANT CONNECT ON DATABASE postgres TO demo_monitor; \c shop CREATE TABLE demo_orders ( id serial PRIMARY KEY, sku text UNIQUE, note text ); INSERT INTO demo_orders (sku, note) VALUES ('FIXED', 'seed'); GRANT ALL ON TABLE demo_orders TO app_shop; GRANT USAGE, SELECT ON SEQUENCE demo_orders_id_seq TO app_shop; \c analytics CREATE TABLE demo_orders ( id serial PRIMARY KEY, sku text UNIQUE, note text ); INSERT INTO demo_orders (sku, note) VALUES ('FIXED', 'seed'); GRANT ALL ON TABLE demo_orders TO app_analytics; GRANT USAGE, SELECT ON SEQUENCE demo_orders_id_seq TO app_analytics; \c postgres SELECT pg_stat_log_reset(); fabriziomello-pg_stat_log-20b57e2/demo/workload/000077500000000000000000000000001526422552700217605ustar00rootroot00000000000000fabriziomello-pg_stat_log-20b57e2/demo/workload/error_div.sql000066400000000000000000000000601526422552700244700ustar00rootroot00000000000000SET client_min_messages TO error; SELECT 1 / 0; fabriziomello-pg_stat_log-20b57e2/demo/workload/error_undef.sql000066400000000000000000000000771526422552700250170ustar00rootroot00000000000000SET client_min_messages TO error; SELECT * FROM no_such_table; fabriziomello-pg_stat_log-20b57e2/demo/workload/error_unique.sql000066400000000000000000000001371526422552700252210ustar00rootroot00000000000000SET client_min_messages TO error; INSERT INTO demo_orders (sku, note) VALUES ('FIXED', 'dup'); fabriziomello-pg_stat_log-20b57e2/demo/workload/error_warn.sql000066400000000000000000000001241526422552700246560ustar00rootroot00000000000000SET client_min_messages TO error; DO $$ BEGIN RAISE WARNING 'demo warning'; END $$; fabriziomello-pg_stat_log-20b57e2/demo/workload/healthy.sql000066400000000000000000000002751526422552700241430ustar00rootroot00000000000000-- Cheap work that should not produce WARNING/ERROR log traffic. SET client_min_messages TO error; SELECT count(*) FROM demo_orders; INSERT INTO demo_orders (note) VALUES ('ok'); SELECT 1; fabriziomello-pg_stat_log-20b57e2/demo/workload/loop.sh000077500000000000000000000025311526422552700232710ustar00rootroot00000000000000#!/usr/bin/env bash # Infinite pgbench load: random 1..50 clients per burst, weighted custom # scripts, then a few failed logins (auth cannot run inside a connected # pgbench session). Target DB/user come from libpq env. set -u : "${PGHOST:?}" : "${PGUSER:?}" : "${PGPASSWORD:?}" : "${PGDATABASE:?}" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" export PGCONNECT_TIMEOUT=5 echo "workload loop starting: user=$PGUSER db=$PGDATABASE host=$PGHOST" while true; do clients=$((1 + RANDOM % 50)) if (( clients < 8 )); then jobs=$clients else jobs=8 fi echo "pgbench burst: clients=$clients jobs=$jobs txns=1/client" # One transaction per client, then start over. pgbench aborts a client on # SQL ERROR, so a long -T mixed burst would shed error-producing clients # and the mix would collapse to healthy-only. pgbench -n \ -h "$PGHOST" -U "$PGUSER" -d "$PGDATABASE" \ -c "$clients" -j "$jobs" -t 1 --max-tries=1 \ -f "$SCRIPT_DIR/healthy.sql@80" \ -f "$SCRIPT_DIR/error_div.sql@4" \ -f "$SCRIPT_DIR/error_unique.sql@4" \ -f "$SCRIPT_DIR/error_undef.sql@4" \ -f "$SCRIPT_DIR/error_warn.sql@3" \ >/dev/null 2>&1 \ || true if (( RANDOM % 8 == 0 )); then PGPASSWORD='wrong-password' psql -q -c 'SELECT 1' >/dev/null 2>&1 || true fi done fabriziomello-pg_stat_log-20b57e2/expected/000077500000000000000000000000001526422552700210135ustar00rootroot00000000000000fabriziomello-pg_stat_log-20b57e2/expected/pg_stat_log.out000066400000000000000000000077241526422552700240600ustar00rootroot00000000000000-- -- pg_stat_log regression tests -- CREATE EXTENSION pg_stat_log; -- Start clean SELECT pg_stat_log_reset(); pg_stat_log_reset ------------------- (1 row) SELECT pg_stat_force_next_flush(); pg_stat_force_next_flush -------------------------- (1 row) -- -- Test 1: Warnings are counted -- DO $$ BEGIN RAISE WARNING 'test warning 1'; END $$; WARNING: test warning 1 DO $$ BEGIN RAISE WARNING 'test warning 2'; END $$; WARNING: test warning 2 DO $$ BEGIN RAISE WARNING 'test warning 3'; END $$; WARNING: test warning 3 SELECT pg_stat_force_next_flush(); pg_stat_force_next_flush -------------------------- (1 row) SELECT count >= 3 AS warning_count_ok FROM pg_stat_log_data() WHERE elevel = 'WARNING' AND sqlerrcode = '01000'; warning_count_ok ------------------ t (1 row) -- -- Test 2: Errors are tracked -- SELECT 1/0; ERROR: division by zero SELECT pg_stat_force_next_flush(); pg_stat_force_next_flush -------------------------- (1 row) SELECT count >= 1 AS division_by_zero_ok FROM pg_stat_log_data() WHERE elevel = 'ERROR' AND sqlerrcode = '22012'; division_by_zero_ok --------------------- t (1 row) -- -- Test 3: pg_stat_log view works (returns rows with database/user names) -- SELECT count(*) > 0 AS view_has_rows FROM pg_stat_log WHERE count > 0; view_has_rows --------------- t (1 row) -- -- Test 4: Disable via GUC stops counting -- SET pg_stat_log.enabled = off; DO $$ BEGIN RAISE WARNING 'should not be counted'; END $$; WARNING: should not be counted SELECT pg_stat_force_next_flush(); pg_stat_force_next_flush -------------------------- (1 row) -- The warning count should not have increased; we check by looking for the -- specific message-related sqlerrcode that was already counted before. SELECT count >= 3 AS still_same_warning_count FROM pg_stat_log_data() WHERE elevel = 'WARNING' AND sqlerrcode = '01000'; still_same_warning_count -------------------------- t (1 row) SET pg_stat_log.enabled = on; -- -- Test 5: min_error_level filtering -- SET pg_stat_log.min_error_level = 'error'; -- Record warning count before SELECT count AS cnt_before FROM pg_stat_log_data() WHERE elevel = 'WARNING' AND sqlerrcode = '01000' \gset DO $$ BEGIN RAISE WARNING 'filtered out'; END $$; WARNING: filtered out SELECT pg_stat_force_next_flush(); pg_stat_force_next_flush -------------------------- (1 row) -- Warning count should be unchanged SELECT count = :cnt_before AS warning_filtered_ok FROM pg_stat_log_data() WHERE elevel = 'WARNING' AND sqlerrcode = '01000'; warning_filtered_ok --------------------- t (1 row) SET pg_stat_log.min_error_level = 'warning'; -- -- Test 6: Reset zeroes counters -- SELECT pg_stat_log_reset(); pg_stat_log_reset ------------------- (1 row) SELECT pg_stat_force_next_flush(); pg_stat_force_next_flush -------------------------- (1 row) SELECT COALESCE(sum(count), 0) = 0 AS reset_ok FROM pg_stat_log_data(); reset_ok ---------- t (1 row) -- -- Test 7: pg_stat_log_info() returns one row with expected columns -- SELECT count(*) = 1 AS info_one_row FROM pg_stat_log_info(); info_one_row -------------- t (1 row) -- -- Test 8: max_entries matches GUC -- SELECT max_entries = current_setting('pg_stat_log.max_entries')::int AS max_matches_guc FROM pg_stat_log_info(); max_matches_guc ----------------- t (1 row) -- -- Test 9: After reset, num_entries and n_dropped are zero -- SELECT pg_stat_log_reset(); pg_stat_log_reset ------------------- (1 row) SELECT pg_stat_force_next_flush(); pg_stat_force_next_flush -------------------------- (1 row) SELECT num_entries = 0 AS num_zero, n_dropped = 0 AS dropped_zero FROM pg_stat_log_info(); num_zero | dropped_zero ----------+-------------- t | t (1 row) -- -- Test 10: pg_stat_log_reset() is restricted to superusers -- CREATE ROLE regress_pg_stat_log_user; SET ROLE regress_pg_stat_log_user; SELECT pg_stat_log_reset(); ERROR: permission denied for function pg_stat_log_reset RESET ROLE; DROP ROLE regress_pg_stat_log_user; -- Clean up DROP EXTENSION pg_stat_log; fabriziomello-pg_stat_log-20b57e2/expected/pg_stat_log_upgrade.out000066400000000000000000000112011526422552700255500ustar00rootroot00000000000000-- -- pg_stat_log upgrade from 0.1 to 0.3 -- -- -- Test 1: The 0.1 objects keep working with the 0.3 library -- CREATE EXTENSION pg_stat_log VERSION '0.1'; SELECT extversion FROM pg_extension WHERE extname = 'pg_stat_log'; extversion ------------ 0.1 (1 row) SELECT pg_stat_log_reset(); pg_stat_log_reset ------------------- (1 row) DO $$ BEGIN RAISE WARNING 'counted before the update'; END $$; WARNING: counted before the update SELECT pg_get_function_arguments('pg_stat_log_data()'::regprocedure); pg_get_function_arguments ------------------------------------------------------------------------------------------------------------------------------------------------- OUT backend_type text, OUT database_oid oid, OUT user_oid oid, OUT elevel text, OUT sqlerrcode text, OUT sqlerrcode_name text, OUT count bigint (1 row) SELECT backend_type, elevel, sqlerrcode, sqlerrcode_name, count FROM pg_stat_log_data() WHERE sqlerrcode = '01000'; backend_type | elevel | sqlerrcode | sqlerrcode_name | count ----------------+---------+------------+-----------------+------- client backend | WARNING | 01000 | warning | 1 (1 row) SELECT backend_type, database_name = current_database() AS current_database, elevel, sqlerrcode, count FROM pg_stat_log WHERE sqlerrcode = '01000'; backend_type | current_database | elevel | sqlerrcode | count ----------------+------------------+---------+------------+------- client backend | t | WARNING | 01000 | 1 (1 row) -- -- Test 2: ALTER EXTENSION UPDATE adds query_id -- ALTER EXTENSION pg_stat_log UPDATE; SELECT extversion FROM pg_extension WHERE extname = 'pg_stat_log'; extversion ------------ 0.3 (1 row) SELECT pg_get_function_arguments('pg_stat_log_data()'::regprocedure); pg_get_function_arguments ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- OUT backend_type text, OUT database_oid oid, OUT user_oid oid, OUT query_id bigint, OUT elevel text, OUT sqlerrcode text, OUT sqlerrcode_name text, OUT count bigint (1 row) SELECT attname FROM pg_attribute WHERE attrelid = 'pg_stat_log'::regclass AND attnum > 0 ORDER BY attnum; attname ----------------- backend_type database_oid database_name user_oid user_name query_id elevel sqlerrcode sqlerrcode_name count (10 rows) -- The recreated view and function are members of the extension SELECT pg_describe_object(classid, objid, objsubid) AS member FROM pg_depend WHERE refclassid = 'pg_extension'::regclass AND refobjid = (SELECT oid FROM pg_extension WHERE extname = 'pg_stat_log') AND deptype = 'e' AND classid IN ('pg_proc'::regclass, 'pg_class'::regclass) ORDER BY member; member ------------------------------ function pg_stat_log_data() function pg_stat_log_info() function pg_stat_log_reset() view pg_stat_log (4 rows) -- Counts recorded before the update are kept. query_id is NULL because -- pg_stat_log.compute_query_id is off. SELECT backend_type, query_id, elevel, sqlerrcode, count FROM pg_stat_log WHERE sqlerrcode = '01000'; backend_type | query_id | elevel | sqlerrcode | count ----------------+----------+---------+------------+------- client backend | | WARNING | 01000 | 1 (1 row) -- -- Test 3: The recreated objects keep the 0.1 privileges -- SELECT has_function_privilege('pg_read_all_stats', 'pg_stat_log_data()', 'EXECUTE') AS read_all_stats_execute, has_function_privilege('public', 'pg_stat_log_data()', 'EXECUTE') AS public_execute, has_table_privilege('pg_read_all_stats', 'pg_stat_log', 'SELECT') AS read_all_stats_select, has_table_privilege('public', 'pg_stat_log', 'SELECT') AS public_select; read_all_stats_execute | public_execute | read_all_stats_select | public_select ------------------------+----------------+-----------------------+--------------- t | f | t | f (1 row) -- -- Test 4: A definition whose columns do not match its entry point fails -- CREATE FUNCTION pg_stat_log_data_mismatch(OUT backend_type text, OUT count bigint) RETURNS SETOF record AS '$libdir/pg_stat_log', 'pg_stat_log_data_0_3' LANGUAGE C STRICT; SELECT * FROM pg_stat_log_data_mismatch(); ERROR: incorrect number of output arguments DROP FUNCTION pg_stat_log_data_mismatch(); -- Clean up DROP EXTENSION pg_stat_log; fabriziomello-pg_stat_log-20b57e2/generate-errcode-names.pl000066400000000000000000000026771526422552700240770ustar00rootroot00000000000000#!/usr/bin/perl # # Generate pg_stat_log_errcodes.h from errcodes.txt # # Produces a static lookup table mapping packed sqlerrcode values to their # human-readable condition names (e.g. ERRCODE_DIVISION_BY_ZERO -> "division_by_zero"). # use strict; use warnings FATAL => 'all'; use Getopt::Long; my $outfile = ''; GetOptions('outfile=s' => \$outfile) or die "$0: wrong arguments"; open my $errcodes, '<', $ARGV[0] or die "$0: could not open input file '$ARGV[0]': $!\n"; my $outfh; if ($outfile) { open $outfh, '>', $outfile or die "$0: could not open output file '$outfile': $!\n"; } else { $outfh = *STDOUT; } print $outfh "/* autogenerated from errcodes.txt, do not edit */\n\n"; print $outfh "typedef struct PgStatLogErrCode\n"; print $outfh "{\n"; print $outfh "\tint\t\t\tsqlerrcode;\n"; print $outfh "\tconst char *name;\n"; print $outfh "} PgStatLogErrCode;\n\n"; print $outfh "static const PgStatLogErrCode pg_stat_log_errcodes[] = {\n"; while (<$errcodes>) { chomp; next if /^#/; next if /^\s*$/; next if /^Section:/; # Parse: sqlstate E/W/S ERRCODE_MACRO [spec_name] next unless /^([^\s]{5})\s+[EWS]\s+([^\s]+)(?:\s+([^\s]+))?/; my ($sqlstate, $errcode_macro, $spec_name) = ($1, $2, $3); # Skip entries without a spec_name next unless defined $spec_name && $spec_name ne ''; print $outfh "\t{$errcode_macro, \"$spec_name\"},\n"; } print $outfh "\t{0, NULL}\n"; print $outfh "};\n"; close $errcodes; close $outfh if ($outfile); fabriziomello-pg_stat_log-20b57e2/pg_stat_log--0.1--0.3.sql000066400000000000000000000026251526422552700231660ustar00rootroot00000000000000/* pg_stat_log/pg_stat_log--0.1--0.3.sql */ -- complain if script is sourced in psql, rather than via ALTER EXTENSION \echo Use "ALTER EXTENSION pg_stat_log UPDATE TO '0.3'" to load this file. \quit -- First remove the objects whose columns change from the extension ALTER EXTENSION pg_stat_log DROP VIEW pg_stat_log; ALTER EXTENSION pg_stat_log DROP FUNCTION pg_stat_log_data(); -- Then drop them DROP VIEW pg_stat_log; DROP FUNCTION pg_stat_log_data(); -- Now redefine them with the query_id column CREATE FUNCTION pg_stat_log_data( OUT backend_type text, OUT database_oid oid, OUT user_oid oid, OUT query_id bigint, OUT elevel text, OUT sqlerrcode text, OUT sqlerrcode_name text, OUT count bigint ) RETURNS SETOF record AS 'MODULE_PATHNAME', 'pg_stat_log_data_0_3' LANGUAGE C STRICT PARALLEL UNSAFE; CREATE VIEW pg_stat_log AS SELECT s.backend_type, s.database_oid, d.datname AS database_name, s.user_oid, u.rolname AS user_name, s.query_id, s.elevel, s.sqlerrcode, s.sqlerrcode_name, s.count FROM pg_stat_log_data() s LEFT JOIN pg_database d ON d.oid = s.database_oid LEFT JOIN pg_roles u ON u.oid = s.user_oid; REVOKE ALL ON FUNCTION pg_stat_log_data() FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_stat_log_data() TO pg_read_all_stats; REVOKE ALL ON pg_stat_log FROM PUBLIC; GRANT SELECT ON pg_stat_log TO pg_read_all_stats; fabriziomello-pg_stat_log-20b57e2/pg_stat_log--0.1.sql000066400000000000000000000031241526422552700226060ustar00rootroot00000000000000/* pg_stat_log/pg_stat_log--0.1.sql */ -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION pg_stat_log" to load this file. \quit CREATE FUNCTION pg_stat_log_data( OUT backend_type text, OUT database_oid oid, OUT user_oid oid, OUT elevel text, OUT sqlerrcode text, OUT sqlerrcode_name text, OUT count bigint ) RETURNS SETOF record AS 'MODULE_PATHNAME', 'pg_stat_log_data' LANGUAGE C STRICT PARALLEL UNSAFE; CREATE FUNCTION pg_stat_log_reset() RETURNS void AS 'MODULE_PATHNAME', 'pg_stat_log_reset' LANGUAGE C STRICT PARALLEL UNSAFE; CREATE VIEW pg_stat_log AS SELECT s.backend_type, s.database_oid, d.datname AS database_name, s.user_oid, u.rolname AS user_name, s.elevel, s.sqlerrcode, s.sqlerrcode_name, s.count FROM pg_stat_log_data() s LEFT JOIN pg_database d ON d.oid = s.database_oid LEFT JOIN pg_roles u ON u.oid = s.user_oid; REVOKE ALL ON FUNCTION pg_stat_log_reset() FROM PUBLIC; REVOKE ALL ON FUNCTION pg_stat_log_data() FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_stat_log_data() TO pg_read_all_stats; REVOKE ALL ON pg_stat_log FROM PUBLIC; GRANT SELECT ON pg_stat_log TO pg_read_all_stats; CREATE FUNCTION pg_stat_log_info( OUT max_entries int, OUT num_entries int, OUT n_dropped bigint, OUT stats_reset timestamp with time zone ) RETURNS SETOF record AS 'MODULE_PATHNAME', 'pg_stat_log_info' LANGUAGE C STRICT PARALLEL UNSAFE; REVOKE ALL ON FUNCTION pg_stat_log_info() FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_stat_log_info() TO pg_read_all_stats; fabriziomello-pg_stat_log-20b57e2/pg_stat_log.c000066400000000000000000000753521526422552700216740ustar00rootroot00000000000000/*-------------------------------------------------------------------------- * * pg_stat_log.c * Cumulative statistics about log messages. * * Hooks into emit_log_hook to count log messages grouped by * (elevel, sqlerrcode, database_oid, user_oid, backend_type, query_id). * The query identifier is tracked only with pg_stat_log.compute_query_id. * * Connection lifecycle records written by log_connections and * log_disconnections are counted under extension-defined SQLSTATEs * (PSL01-PSL03), attributed to the connected database and login role. * * Uses the fixed-amount Custom Cumulative Stats API introduced in * PostgreSQL 18, following the same pattern as test_custom_fixed_stats.c. * * Copyright (c) 2026, PlanetScale Inc. * * IDENTIFICATION * pg_stat_log/pg_stat_log.c * *-------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" #include "pgstat.h" #include "access/xact.h" #include "common/hashfn.h" #include "nodes/queryjumble.h" #include "storage/ipc.h" #include "storage/proc.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" #include "utils/errcodes.h" #include "utils/guc.h" #include "utils/pgstat_internal.h" #include "utils/timestamp.h" #include "utils/tuplestore.h" #include "pg_stat_log_errcodes.h" #define PGSTAT_LOG_MODULE_NAME "pg_stat_log" #define PGSTAT_LOG_VERSION "0.3" /* default_version in pg_stat_log.control */ #define PGSTAT_LOG_TRANCHE_NAME PGSTAT_LOG_MODULE_NAME PG_MODULE_MAGIC_EXT(.name = PGSTAT_LOG_MODULE_NAME, .version = PGSTAT_LOG_VERSION); /* * Custom stats kind ID — registered at * https://wiki.postgresql.org/wiki/CustomCumulativeStats */ #define PGSTAT_KIND_LOG 28 /* GUC defaults and bounds */ #define PGSTAT_LOG_MAX_DEFAULT 1024 #define PGSTAT_LOG_MIN_ENTRIES 64 #define PGSTAT_LOG_MAX_ENTRIES (PGSTAT_LOG_MAX_DEFAULT * PGSTAT_LOG_MAX_DEFAULT) /* * Extension-defined SQLSTATEs for connection lifecycle records, which * PostgreSQL logs as 00000. Class "PS" is unused by core errcodes.txt. */ #define PGSTAT_LOG_ERRCODE_CONNECTION_AUTHENTICATED MAKE_SQLSTATE('P', 'S', 'L', '0', '1') #define PGSTAT_LOG_ERRCODE_CONNECTION_AUTHORIZED MAKE_SQLSTATE('P', 'S', 'L', '0', '2') #define PGSTAT_LOG_ERRCODE_DISCONNECTION MAKE_SQLSTATE('P', 'S', 'L', '0', '3') /* Extension versions whose pg_stat_log_data() returns different columns */ typedef enum PgStatLogApiVersion { PGSTAT_LOG_V0_1, PGSTAT_LOG_V0_3, } PgStatLogApiVersion; #define PGSTAT_LOG_DATA_COLS_V0_1 7 #define PGSTAT_LOG_DATA_COLS_V0_3 8 #define PGSTAT_LOG_DATA_COLS 8 /* maximum of above */ /* * Data structures */ typedef struct PgStatLogSlot { int32 next; /* next slot index in bucket chain, or -1 */ BackendType backend_type; Oid dboid; Oid userid; int elevel; int sqlerrcode; int64 queryid; /* 0 if not tracked */ PgStat_Counter count; } PgStatLogSlot; /* * Stats data block. Variable-length: a header followed by a payload laid out * as (entries first to keep 8-byte alignment without padding): * * PgStatLogSlot entries[max_entries] insertion-ordered slots * int32 heads[max_entries] bucket chain heads (-1 = empty) * * The slots form an index-based separate-chaining hash table: heads[bucket] * points to the first slot of a chain and each slot's "next" links the rest. * Slots are allocated sequentially (slot i is live iff i < num_entries) and * never evicted, so lookups, inserts, and drops walk only one bucket chain * (average length = load factor) and stay O(1) expected even at full load. * Links are array indices (not pointers), so the block is snapshot- and * persistence-safe. Use the accessors below to reach each sub-array. * * Why not a standard PostgreSQL hash table? The fixed-amount custom stats * API snapshots this block with a raw memcpy and persists/restores it * verbatim (fwrite/fread) with no serialize callbacks, so everything in the * block must be position-independent and self-contained. dynahash chains * entries with raw pointers, simplehash is process-local and grows by * reallocation, and dshash would require switching to variable-amount stats * (64-bit key limit, unbounded DSA growth, no extension-visible enumeration, * and allocation inside emit_log_hook). See "Design notes" in README.md. */ typedef struct PgStatLog { int max_entries; int num_entries; char data[FLEXIBLE_ARRAY_MEMBER]; } PgStatLog; /* * Shared memory wrapper. LWLock + changecount + metadata + data. Metadata * fields (stat_reset_timestamp, n_dropped) live outside the copied stats * block so they can be read directly under the LWLock without the * changecount protocol. The data area holds one PgStatLog block. */ typedef struct PgStatLogShared { LWLock lock; uint32 changecount; TimestampTz stat_reset_timestamp; uint64 n_dropped; char data[FLEXIBLE_ARRAY_MEMBER]; } PgStatLogShared; /* * Connection lifecycle records. PostgreSQL logs them at LOG without an * errcode, so they are recognized by the function that emits them. */ typedef enum PgStatLogLifecycleEvent { PGSTAT_LOG_EVENT_AUTHENTICATED, PGSTAT_LOG_EVENT_AUTHORIZED, PGSTAT_LOG_EVENT_DISCONNECTION, PGSTAT_LOG_NUM_EVENTS } PgStatLogLifecycleEvent; typedef struct PgStatLogLifecycleOrigin { const char *filename; const char *funcname; PgStatLogLifecycleEvent event; } PgStatLogLifecycleOrigin; /* Keep in sync with check-upstream-anchors.pl */ static const PgStatLogLifecycleOrigin pg_stat_log_lifecycle_origins[] = { {"auth.c", "set_authn_id", PGSTAT_LOG_EVENT_AUTHENTICATED}, {"auth.c", "ClientAuthentication", PGSTAT_LOG_EVENT_AUTHENTICATED}, {"postinit.c", "PerformAuthentication", PGSTAT_LOG_EVENT_AUTHORIZED }, {"postgres.c", "log_disconnections", PGSTAT_LOG_EVENT_DISCONNECTION}, }; /* Indexed by PgStatLogLifecycleEvent */ static const PgStatLogErrCode pg_stat_log_lifecycle_codes[] = { {PGSTAT_LOG_ERRCODE_CONNECTION_AUTHENTICATED, "connection_authenticated"}, {PGSTAT_LOG_ERRCODE_CONNECTION_AUTHORIZED, "connection_authorized" }, {PGSTAT_LOG_ERRCODE_DISCONNECTION, "disconnection" }, }; StaticAssertDecl(lengthof(pg_stat_log_lifecycle_codes) == PGSTAT_LOG_NUM_EVENTS, "pg_stat_log_lifecycle_codes must match PgStatLogLifecycleEvent"); /* * GUC variables */ static bool pg_stat_log_enabled = true; static int pg_stat_log_min_elevel = WARNING; static int pg_stat_log_max = PGSTAT_LOG_MAX_DEFAULT; static bool pg_stat_log_compute_query_id = false; /* * Computed sizes (set in _PG_init based on pg_stat_log.max_entries) */ static Size stats_block_size; /* one PgStatLog block */ /* * Hook state */ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static emit_log_hook_type prev_emit_log_hook = NULL; static bool in_emit_log_hook = false; /* * Connection events logged before the session's database and login role are * known; counted at the end of the startup transaction. */ static int pending_connection_events[PGSTAT_LOG_NUM_EVENTS]; /* * Accessor helpers */ static inline PgStatLog * pg_stat_log_get_stats(PgStatLogShared *shmem) { return (PgStatLog *) shmem->data; } static inline PgStatLogSlot * pg_stat_log_entries(PgStatLog *s) { return (PgStatLogSlot *) s->data; } static inline int32 * pg_stat_log_heads(PgStatLog *s) { return (int32 *) (s->data + sizeof(PgStatLogSlot) * (Size) s->max_entries); } /* * Errcode name lookup */ static const char * pg_stat_log_errcode_name(int sqlerrcode) { for (int i = 0; i < PGSTAT_LOG_NUM_EVENTS; i++) { if (pg_stat_log_lifecycle_codes[i].sqlerrcode == sqlerrcode) return pg_stat_log_lifecycle_codes[i].name; } for (int i = 0; pg_stat_log_errcodes[i].name != NULL; i++) { if (pg_stat_log_errcodes[i].sqlerrcode == sqlerrcode) return pg_stat_log_errcodes[i].name; } return NULL; } /* * Hash function for slot lookup — combines all key fields into a uint32 * used to pick a chain bucket (hash % max_entries). */ static inline uint32 pg_stat_log_hash_key(BackendType backend_type, Oid dboid, Oid userid, int elevel, int sqlerrcode, int64 queryid) { uint32 h; h = murmurhash32((uint32) backend_type); h = hash_combine(h, murmurhash32((uint32) dboid)); h = hash_combine(h, murmurhash32((uint32) userid)); h = hash_combine(h, murmurhash32((uint32) elevel)); h = hash_combine(h, murmurhash32((uint32) sqlerrcode)); h = hash_combine(h, (uint32) murmurhash64((uint64) queryid)); return h; } /* * PgStat_KindInfo — filled dynamically in _PG_init */ static void pg_stat_log_init_backend_cb(void); static void pg_stat_log_init_shmem_cb(void *stats); static void pg_stat_log_reset_all_cb(TimestampTz ts); static void pg_stat_log_snapshot_cb(void); static PgStat_KindInfo log_stats_kind; /* * init_backend_cb — per-backend initialization * * Runs in every backend after the stats file has been loaded by the * startup process. Validates that persisted max_entries matches the * current GUC. A changed pg_stat_log.max_entries changes the length of the * persisted block, so PostgreSQL cannot parse the stats file, discards it * and calls reset_all_cb, which rebuilds the block. This check is a safety * net against a restored block with a stale layout. */ static void pg_stat_log_init_backend_cb(void) { PgStatLogShared *shmem; PgStatLog *s; shmem = (PgStatLogShared *) pgstat_get_custom_shmem_data(PGSTAT_KIND_LOG); s = pg_stat_log_get_stats(shmem); if (s->max_entries != pg_stat_log_max) { elog(LOG, "pg_stat_log: discarding persisted stats " "(max_entries changed from %d to %d)", s->max_entries, pg_stat_log_max); LWLockAcquire(&shmem->lock, LW_EXCLUSIVE); pgstat_begin_changecount_write(&shmem->changecount); s->max_entries = pg_stat_log_max; s->num_entries = 0; memset(pg_stat_log_heads(s), 0xFF, sizeof(int32) * (Size) s->max_entries); pgstat_end_changecount_write(&shmem->changecount); shmem->stat_reset_timestamp = GetCurrentTimestamp(); shmem->n_dropped = 0; LWLockRelease(&shmem->lock); } } /* * init_shmem_cb — initialize shared memory * * Only runs in the postmaster (see StatsShmemInit), after the main LWLock * array has been set up. This is the first place it is safe to call * LWLockNewTrancheId(), which needs shared memory to exist. */ static void pg_stat_log_init_shmem_cb(void *stats) { PgStatLogShared *shmem = (PgStatLogShared *) stats; PgStatLog *s; #if PG_VERSION_NUM < 190000 LWLockInitialize(&shmem->lock, LWLockNewTrancheId()); #else LWLockInitialize(&shmem->lock, LWLockNewTrancheId(PGSTAT_LOG_TRANCHE_NAME)); #endif shmem->stat_reset_timestamp = GetCurrentTimestamp(); shmem->n_dropped = 0; s = pg_stat_log_get_stats(shmem); s->max_entries = pg_stat_log_max; s->num_entries = 0; memset(pg_stat_log_heads(s), 0xFF, sizeof(int32) * (Size) s->max_entries); } /* * shmem_startup_hook — register the tranche name in every process. * * On fork-based platforms backends inherit the tranche ID and the * registration from the postmaster, but under EXEC_BACKEND each child has * to re-register, so do it unconditionally here. The tranche ID is read * from shared memory (initialized by the init_shmem_cb above). */ static void pg_stat_log_shmem_startup(void) { if (prev_shmem_startup_hook) prev_shmem_startup_hook(); #if PG_VERSION_NUM < 190000 { PgStatLogShared *shmem = (PgStatLogShared *) pgstat_get_custom_shmem_data(PGSTAT_KIND_LOG); LWLockRegisterTranche(shmem->lock.tranche, PGSTAT_LOG_TRANCHE_NAME); } #endif } /* * reset_all_cb — reset statistics * * Empty all bucket chains and reset num_entries so slots are reclaimed for * reuse. Otherwise, once max_entries is reached, a reset would not free * capacity for new distinct combinations. */ static void pg_stat_log_reset_all_cb(TimestampTz ts) { PgStatLogShared *shmem; PgStatLog *s; shmem = (PgStatLogShared *) pgstat_get_custom_shmem_data(PGSTAT_KIND_LOG); s = pg_stat_log_get_stats(shmem); LWLockAcquire(&shmem->lock, LW_EXCLUSIVE); /* * After a stats file fails to load, the header holds whatever was read * from the file, so size the block from the GUC instead. */ pgstat_begin_changecount_write(&shmem->changecount); s->max_entries = pg_stat_log_max; s->num_entries = 0; memset(pg_stat_log_heads(s), 0xFF, sizeof(int32) * (Size) s->max_entries); pgstat_end_changecount_write(&shmem->changecount); shmem->stat_reset_timestamp = ts; shmem->n_dropped = 0; LWLockRelease(&shmem->lock); } /* * snapshot_cb — build snapshot for reads */ static void pg_stat_log_snapshot_cb(void) { PgStatLogShared *shmem; PgStatLog *snap; shmem = (PgStatLogShared *) pgstat_get_custom_shmem_data(PGSTAT_KIND_LOG); snap = (PgStatLog *) pgstat_get_custom_snapshot_data(PGSTAT_KIND_LOG); /* Copy current stats via changecount protocol */ pgstat_copy_changecounted_stats(snap, pg_stat_log_get_stats(shmem), stats_block_size, &shmem->changecount); } /* * pg_stat_log_count — record one log message in shared memory * * Acquires the LWLock, walks the hash bucket chain to find or create the * entry, increments the counter, and releases the lock. The chain walk is * O(1) expected (average chain length = load factor) for all of lookup, * insert, and drop, even when the table is full. */ static void pg_stat_log_count(BackendType backend_type, Oid dboid, Oid userid, int elevel, int sqlerrcode, int64 queryid) { PgStatLogShared *shmem; PgStatLog *s; PgStatLogSlot *entries; int32 *heads; uint32 hash; uint32 bucket; int32 idx; bool found; shmem = (PgStatLogShared *) pgstat_get_custom_shmem_data(PGSTAT_KIND_LOG); LWLockAcquire(&shmem->lock, LW_EXCLUSIVE); s = pg_stat_log_get_stats(shmem); entries = pg_stat_log_entries(s); heads = pg_stat_log_heads(s); hash = pg_stat_log_hash_key(backend_type, dboid, userid, elevel, sqlerrcode, queryid); bucket = hash % s->max_entries; found = false; for (idx = heads[bucket]; idx != -1; idx = entries[idx].next) { PgStatLogSlot *slot = &entries[idx]; if (slot->backend_type == backend_type && slot->dboid == dboid && slot->userid == userid && slot->elevel == elevel && slot->sqlerrcode == sqlerrcode && slot->queryid == queryid) { pgstat_begin_changecount_write(&shmem->changecount); slot->count++; pgstat_end_changecount_write(&shmem->changecount); found = true; break; } } if (!found) { if (s->num_entries < s->max_entries) { int32 newidx = s->num_entries; PgStatLogSlot *slot = &entries[newidx]; pgstat_begin_changecount_write(&shmem->changecount); slot->backend_type = backend_type; slot->dboid = dboid; slot->userid = userid; slot->elevel = elevel; slot->sqlerrcode = sqlerrcode; slot->queryid = queryid; slot->count = 1; slot->next = heads[bucket]; heads[bucket] = newidx; s->num_entries++; pgstat_end_changecount_write(&shmem->changecount); } else { shmem->n_dropped++; } } LWLockRelease(&shmem->lock); } /* * pg_stat_log_classify_lifecycle — recognize connection lifecycle records * * elog.c has already reduced edata->filename to its base name. */ static bool pg_stat_log_classify_lifecycle(const ErrorData *edata, PgStatLogLifecycleEvent *event) { if (edata->elevel != LOG || edata->sqlerrcode != ERRCODE_SUCCESSFUL_COMPLETION || edata->filename == NULL || edata->funcname == NULL) return false; for (int i = 0; i < lengthof(pg_stat_log_lifecycle_origins); i++) { const PgStatLogLifecycleOrigin *origin = &pg_stat_log_lifecycle_origins[i]; if (strcmp(edata->funcname, origin->funcname) == 0 && strcmp(edata->filename, origin->filename) == 0) { *event = origin->event; return true; } } return false; } /* * pg_stat_log_record_message — count one message that reached the server log * * The connection authenticated/authorized records are logged before the * session's database and login role are known, so they are held until * pg_stat_log_startup_xact_cb. The disconnection record is logged after * ProcKill, when shared memory can no longer be locked, so * pg_stat_log_disconnect_cb counts it ahead of time instead. * * The query identifier is the one log_line_prefix's %Q prints: the current * top-level statement's, or the session's most recent one while idle. */ static void pg_stat_log_record_message(ErrorData *edata) { PgStatLogLifecycleEvent event; Oid userid; int sec_context; int64 queryid = 0; if (pg_stat_log_classify_lifecycle(edata, &event)) { if (event != PGSTAT_LOG_EVENT_DISCONNECTION) pending_connection_events[event]++; return; } if (pg_stat_log_compute_query_id && IsQueryIdEnabled()) queryid = pgstat_get_my_query_id(); GetUserIdAndSecContext(&userid, &sec_context); pg_stat_log_count(MyBackendType, MyDatabaseId, userid, edata->elevel, edata->sqlerrcode, queryid); } /* * pg_stat_log_log_reaches_server — would a LOG record reach the server log? * * Mirrors is_log_level_output() in elog.c; check-upstream-anchors.pl * verifies in CI that the rule is unchanged. */ static bool pg_stat_log_log_reaches_server(void) { #if PG_VERSION_NUM >= 190000 int min_level = log_min_messages[MyBackendType]; #else int min_level = log_min_messages; #endif return min_level == LOG || min_level <= ERROR; } /* * pg_stat_log_disconnect_cb — before_shmem_exit callback that counts the * disconnection record * * Runs before ShutdownPostgres, pgstat_shutdown_hook and ProcKill, while the * process still has its PGPROC, and applies the same conditions that decide * whether log_disconnections' record reaches the server log. */ static void pg_stat_log_disconnect_cb(int code, Datum arg) { /* PostgresMain installs log_disconnections only after InitPostgres() */ if (!IsNormalProcessingMode()) return; if (!pg_stat_log_enabled || LOG < pg_stat_log_min_elevel || !pg_stat_log_log_reaches_server()) return; /* The record is written after MyBEEntry is cleared, so its %Q is 0 */ pg_stat_log_count(MyBackendType, MyDatabaseId, MyProc->roleId, LOG, PGSTAT_LOG_ERRCODE_DISCONNECTION, 0); } /* * pg_stat_log_startup_xact_cb — attribute connection lifecycle events * * The first transaction a client backend or walsender ends is the startup * transaction of InitPostgres(). It commits once the login role, the database * and the startup options are all set, or aborts when startup fails with a * FATAL error, in which case only part of the attribution may be known. * MyProc->roleId is the login role; it is set before the remaining login * checks can fail. */ static void pg_stat_log_startup_xact_cb(XactEvent event, void *arg) { if (event != XACT_EVENT_PRE_COMMIT && event != XACT_EVENT_ABORT) return; UnregisterXactCallback(pg_stat_log_startup_xact_cb, NULL); if (!IsExternalConnectionBackend(MyBackendType)) return; for (int i = 0; i < PGSTAT_LOG_NUM_EVENTS; i++) { for (; pending_connection_events[i] > 0; pending_connection_events[i]--) pg_stat_log_count(MyBackendType, MyDatabaseId, MyProc->roleId, LOG, pg_stat_log_lifecycle_codes[i].sqlerrcode, 0); } /* * Log_disconnections is final from here on; a configuration reload does * not change it in a running backend. */ if (event == XACT_EVENT_PRE_COMMIT && IsUnderPostmaster && Log_disconnections) before_shmem_exit(pg_stat_log_disconnect_cb, 0); } /* * emit_log_hook — intercept log messages for counting * * Always forwards to the previous hook in the chain. Only counts the * message if pg_stat_log is enabled and the severity meets the threshold. */ static void pg_stat_log_emit_hook(ErrorData *edata) { if (in_emit_log_hook) return; /* * pgstat shared memory might not be set up yet during early startup or * in auxiliary processes before attachment. */ if ((!IsUnderPostmaster && IsPostmasterEnvironment) || !MyProc) return; in_emit_log_hook = true; PG_TRY(); { if (prev_emit_log_hook) prev_emit_log_hook(edata); if (pg_stat_log_enabled && edata->elevel >= pg_stat_log_min_elevel) pg_stat_log_record_message(edata); } PG_FINALLY(); { in_emit_log_hook = false; } PG_END_TRY(); } /* * server_message_level_options was introduced in commit * https://github.com/postgres/postgres/commit/0a20ff54f5e6 * but only exported for use by extensions in * https://github.com/postgres/postgres/commit/38e0190ced71, * which will ship in PostgreSQL 19. */ #if PG_VERSION_NUM < 190000 static const struct config_enum_entry server_message_level_options[] = { {"debug5", DEBUG5, false}, {"debug4", DEBUG4, false}, {"debug3", DEBUG3, false}, {"debug2", DEBUG2, false}, {"debug1", DEBUG1, false}, {"debug", DEBUG2, true }, {"info", INFO, false}, {"notice", NOTICE, false}, {"warning", WARNING, false}, {"error", ERROR, false}, {"log", LOG, false}, {"fatal", FATAL, false}, {"panic", PANIC, false}, {NULL, 0, false} }; #endif /* * Module initialization */ void _PG_init(void) { Size shared_size; if (!process_shared_preload_libraries_in_progress) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("pg_stat_log must be loaded via " "shared_preload_libraries"))); /* Define GUCs before computing sizes */ DefineCustomBoolVariable("pg_stat_log.enabled", "Enable collection of log statistics.", NULL, &pg_stat_log_enabled, true, PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomEnumVariable("pg_stat_log.min_error_level", "Minimum error level to track.", NULL, &pg_stat_log_min_elevel, WARNING, server_message_level_options, PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomIntVariable("pg_stat_log.max_entries", "Maximum number of distinct log entry " "combinations to track.", NULL, &pg_stat_log_max, PGSTAT_LOG_MAX_DEFAULT, PGSTAT_LOG_MIN_ENTRIES, PGSTAT_LOG_MAX_ENTRIES, PGC_POSTMASTER, 0, NULL, NULL, NULL); DefineCustomBoolVariable("pg_stat_log.compute_query_id", "Tracks log messages per query identifier.", "Has no effect when compute_query_id is off. With auto, " "pg_stat_log enables query identifier computation.", &pg_stat_log_compute_query_id, false, PGC_POSTMASTER, 0, NULL, NULL, NULL); MarkGUCPrefixReserved(PGSTAT_LOG_MODULE_NAME); /* Like pg_stat_statements; a no-op when compute_query_id is off */ if (pg_stat_log_compute_query_id) EnableQueryId(); /* Compute sizes based on pg_stat_log.max_entries */ stats_block_size = offsetof(PgStatLog, data) + sizeof(PgStatLogSlot) * (Size) pg_stat_log_max + sizeof(int32) * (Size) pg_stat_log_max; shared_size = offsetof(PgStatLogShared, data) + stats_block_size; /* Fill in the KindInfo struct — use memcpy because .name is const */ { PgStat_KindInfo tmp = { .name = PGSTAT_LOG_MODULE_NAME, .fixed_amount = true, .write_to_file = true, .shared_size = shared_size, .shared_data_off = offsetof(PgStatLogShared, data), .shared_data_len = stats_block_size, .init_backend_cb = pg_stat_log_init_backend_cb, .init_shmem_cb = pg_stat_log_init_shmem_cb, .reset_all_cb = pg_stat_log_reset_all_cb, .snapshot_cb = pg_stat_log_snapshot_cb, }; memcpy(&log_stats_kind, &tmp, sizeof(PgStat_KindInfo)); } pgstat_register_kind(PGSTAT_KIND_LOG, &log_stats_kind); /* * Install shmem_startup_hook so every process registers the LWLock * tranche name (needed for EXEC_BACKEND; harmless on fork). */ prev_shmem_startup_hook = shmem_startup_hook; shmem_startup_hook = pg_stat_log_shmem_startup; /* Install emit_log_hook */ prev_emit_log_hook = emit_log_hook; emit_log_hook = pg_stat_log_emit_hook; /* Attribute connection lifecycle events once backend startup completes */ RegisterXactCallback(pg_stat_log_startup_xact_cb, NULL); } /* * SQL-callable functions */ PG_FUNCTION_INFO_V1(pg_stat_log_data); PG_FUNCTION_INFO_V1(pg_stat_log_data_0_3); /* * pg_stat_log_data_internal * Return all tracked log statistics as a set of rows, with the columns * of the given extension version. */ static void pg_stat_log_data_internal(FunctionCallInfo fcinfo, PgStatLogApiVersion api_version) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; PgStatLog *snap; PgStatLogSlot *entries; int expected_cols; int i; pgstat_snapshot_fixed(PGSTAT_KIND_LOG); snap = (PgStatLog *) pgstat_get_custom_snapshot_data(PGSTAT_KIND_LOG); InitMaterializedSRF(fcinfo, 0); expected_cols = api_version == PGSTAT_LOG_V0_1 ? PGSTAT_LOG_DATA_COLS_V0_1 : PGSTAT_LOG_DATA_COLS_V0_3; if (rsinfo->setDesc->natts != expected_cols) elog(ERROR, "incorrect number of output arguments"); entries = pg_stat_log_entries(snap); for (i = 0; i < snap->num_entries; i++) { Datum values[PGSTAT_LOG_DATA_COLS]; bool nulls[PGSTAT_LOG_DATA_COLS] = {0}; int col = 0; PgStatLogSlot *slot = &entries[i]; const char *errname; if (slot->count <= 0) continue; values[col++] = CStringGetTextDatum(GetBackendTypeDesc(slot->backend_type)); if (OidIsValid(slot->dboid)) values[col] = ObjectIdGetDatum(slot->dboid); else nulls[col] = true; col++; if (OidIsValid(slot->userid)) values[col] = ObjectIdGetDatum(slot->userid); else nulls[col] = true; col++; if (api_version >= PGSTAT_LOG_V0_3) { if (slot->queryid != 0) values[col] = Int64GetDatum(slot->queryid); else nulls[col] = true; col++; } values[col++] = CStringGetTextDatum(error_severity(slot->elevel)); values[col++] = CStringGetTextDatum(unpack_sql_state(slot->sqlerrcode)); errname = pg_stat_log_errcode_name(slot->sqlerrcode); if (errname) values[col] = CStringGetTextDatum(errname); else nulls[col] = true; col++; values[col++] = Int64GetDatum(slot->count); Assert(col == expected_cols); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); } } /* * pg_stat_log_data() * The 0.1 definition, without query_id. Databases where the extension * has not been updated to 0.3 still call it. */ Datum pg_stat_log_data(PG_FUNCTION_ARGS) { pg_stat_log_data_internal(fcinfo, PGSTAT_LOG_V0_1); return (Datum) 0; } /* * pg_stat_log_data_0_3() * The 0.3 definition, with query_id. */ Datum pg_stat_log_data_0_3(PG_FUNCTION_ARGS) { pg_stat_log_data_internal(fcinfo, PGSTAT_LOG_V0_3); return (Datum) 0; } PG_FUNCTION_INFO_V1(pg_stat_log_reset); /* * pg_stat_log_reset() * Reset all tracked log statistics. */ Datum pg_stat_log_reset(PG_FUNCTION_ARGS) { pgstat_reset_of_kind(PGSTAT_KIND_LOG); PG_RETURN_VOID(); } PG_FUNCTION_INFO_V1(pg_stat_log_info); /* * pg_stat_log_info() * Return metadata about the pg_stat_log shared memory area: * max_entries capacity, current num_entries, number of dropped * messages due to capacity, and the last reset timestamp. */ Datum pg_stat_log_info(PG_FUNCTION_ARGS) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; PgStatLogShared *shmem; PgStatLog *s; Datum values[4]; bool nulls[4] = {0}; InitMaterializedSRF(fcinfo, 0); shmem = (PgStatLogShared *) pgstat_get_custom_shmem_data(PGSTAT_KIND_LOG); s = pg_stat_log_get_stats(shmem); LWLockAcquire(&shmem->lock, LW_SHARED); values[0] = Int32GetDatum(s->max_entries); values[1] = Int32GetDatum(s->num_entries); values[2] = Int64GetDatum((int64) shmem->n_dropped); values[3] = TimestampTzGetDatum(shmem->stat_reset_timestamp); LWLockRelease(&shmem->lock); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); return (Datum) 0; } fabriziomello-pg_stat_log-20b57e2/pg_stat_log.conf000066400000000000000000000000511526422552700223570ustar00rootroot00000000000000shared_preload_libraries = 'pg_stat_log' fabriziomello-pg_stat_log-20b57e2/pg_stat_log.control000066400000000000000000000002101526422552700231070ustar00rootroot00000000000000comment = 'cumulative statistics about log messages' default_version = '0.3' module_pathname = '$libdir/pg_stat_log' relocatable = true fabriziomello-pg_stat_log-20b57e2/sql/000077500000000000000000000000001526422552700200115ustar00rootroot00000000000000fabriziomello-pg_stat_log-20b57e2/sql/pg_stat_log.sql000066400000000000000000000053501526422552700230370ustar00rootroot00000000000000-- -- pg_stat_log regression tests -- CREATE EXTENSION pg_stat_log; -- Start clean SELECT pg_stat_log_reset(); SELECT pg_stat_force_next_flush(); -- -- Test 1: Warnings are counted -- DO $$ BEGIN RAISE WARNING 'test warning 1'; END $$; DO $$ BEGIN RAISE WARNING 'test warning 2'; END $$; DO $$ BEGIN RAISE WARNING 'test warning 3'; END $$; SELECT pg_stat_force_next_flush(); SELECT count >= 3 AS warning_count_ok FROM pg_stat_log_data() WHERE elevel = 'WARNING' AND sqlerrcode = '01000'; -- -- Test 2: Errors are tracked -- SELECT 1/0; SELECT pg_stat_force_next_flush(); SELECT count >= 1 AS division_by_zero_ok FROM pg_stat_log_data() WHERE elevel = 'ERROR' AND sqlerrcode = '22012'; -- -- Test 3: pg_stat_log view works (returns rows with database/user names) -- SELECT count(*) > 0 AS view_has_rows FROM pg_stat_log WHERE count > 0; -- -- Test 4: Disable via GUC stops counting -- SET pg_stat_log.enabled = off; DO $$ BEGIN RAISE WARNING 'should not be counted'; END $$; SELECT pg_stat_force_next_flush(); -- The warning count should not have increased; we check by looking for the -- specific message-related sqlerrcode that was already counted before. SELECT count >= 3 AS still_same_warning_count FROM pg_stat_log_data() WHERE elevel = 'WARNING' AND sqlerrcode = '01000'; SET pg_stat_log.enabled = on; -- -- Test 5: min_error_level filtering -- SET pg_stat_log.min_error_level = 'error'; -- Record warning count before SELECT count AS cnt_before FROM pg_stat_log_data() WHERE elevel = 'WARNING' AND sqlerrcode = '01000' \gset DO $$ BEGIN RAISE WARNING 'filtered out'; END $$; SELECT pg_stat_force_next_flush(); -- Warning count should be unchanged SELECT count = :cnt_before AS warning_filtered_ok FROM pg_stat_log_data() WHERE elevel = 'WARNING' AND sqlerrcode = '01000'; SET pg_stat_log.min_error_level = 'warning'; -- -- Test 6: Reset zeroes counters -- SELECT pg_stat_log_reset(); SELECT pg_stat_force_next_flush(); SELECT COALESCE(sum(count), 0) = 0 AS reset_ok FROM pg_stat_log_data(); -- -- Test 7: pg_stat_log_info() returns one row with expected columns -- SELECT count(*) = 1 AS info_one_row FROM pg_stat_log_info(); -- -- Test 8: max_entries matches GUC -- SELECT max_entries = current_setting('pg_stat_log.max_entries')::int AS max_matches_guc FROM pg_stat_log_info(); -- -- Test 9: After reset, num_entries and n_dropped are zero -- SELECT pg_stat_log_reset(); SELECT pg_stat_force_next_flush(); SELECT num_entries = 0 AS num_zero, n_dropped = 0 AS dropped_zero FROM pg_stat_log_info(); -- -- Test 10: pg_stat_log_reset() is restricted to superusers -- CREATE ROLE regress_pg_stat_log_user; SET ROLE regress_pg_stat_log_user; SELECT pg_stat_log_reset(); RESET ROLE; DROP ROLE regress_pg_stat_log_user; -- Clean up DROP EXTENSION pg_stat_log; fabriziomello-pg_stat_log-20b57e2/sql/pg_stat_log_upgrade.sql000066400000000000000000000045361526422552700245530ustar00rootroot00000000000000-- -- pg_stat_log upgrade from 0.1 to 0.3 -- -- -- Test 1: The 0.1 objects keep working with the 0.3 library -- CREATE EXTENSION pg_stat_log VERSION '0.1'; SELECT extversion FROM pg_extension WHERE extname = 'pg_stat_log'; SELECT pg_stat_log_reset(); DO $$ BEGIN RAISE WARNING 'counted before the update'; END $$; SELECT pg_get_function_arguments('pg_stat_log_data()'::regprocedure); SELECT backend_type, elevel, sqlerrcode, sqlerrcode_name, count FROM pg_stat_log_data() WHERE sqlerrcode = '01000'; SELECT backend_type, database_name = current_database() AS current_database, elevel, sqlerrcode, count FROM pg_stat_log WHERE sqlerrcode = '01000'; -- -- Test 2: ALTER EXTENSION UPDATE adds query_id -- ALTER EXTENSION pg_stat_log UPDATE; SELECT extversion FROM pg_extension WHERE extname = 'pg_stat_log'; SELECT pg_get_function_arguments('pg_stat_log_data()'::regprocedure); SELECT attname FROM pg_attribute WHERE attrelid = 'pg_stat_log'::regclass AND attnum > 0 ORDER BY attnum; -- The recreated view and function are members of the extension SELECT pg_describe_object(classid, objid, objsubid) AS member FROM pg_depend WHERE refclassid = 'pg_extension'::regclass AND refobjid = (SELECT oid FROM pg_extension WHERE extname = 'pg_stat_log') AND deptype = 'e' AND classid IN ('pg_proc'::regclass, 'pg_class'::regclass) ORDER BY member; -- Counts recorded before the update are kept. query_id is NULL because -- pg_stat_log.compute_query_id is off. SELECT backend_type, query_id, elevel, sqlerrcode, count FROM pg_stat_log WHERE sqlerrcode = '01000'; -- -- Test 3: The recreated objects keep the 0.1 privileges -- SELECT has_function_privilege('pg_read_all_stats', 'pg_stat_log_data()', 'EXECUTE') AS read_all_stats_execute, has_function_privilege('public', 'pg_stat_log_data()', 'EXECUTE') AS public_execute, has_table_privilege('pg_read_all_stats', 'pg_stat_log', 'SELECT') AS read_all_stats_select, has_table_privilege('public', 'pg_stat_log', 'SELECT') AS public_select; -- -- Test 4: A definition whose columns do not match its entry point fails -- CREATE FUNCTION pg_stat_log_data_mismatch(OUT backend_type text, OUT count bigint) RETURNS SETOF record AS '$libdir/pg_stat_log', 'pg_stat_log_data_0_3' LANGUAGE C STRICT; SELECT * FROM pg_stat_log_data_mismatch(); DROP FUNCTION pg_stat_log_data_mismatch(); -- Clean up DROP EXTENSION pg_stat_log; fabriziomello-pg_stat_log-20b57e2/t/000077500000000000000000000000001526422552700174555ustar00rootroot00000000000000fabriziomello-pg_stat_log-20b57e2/t/001_pg_stat_log.pl000066400000000000000000000236461526422552700227070ustar00rootroot00000000000000# Copyright (c) 2026, Fabrizio de Royes Mello # Test pg_stat_log persistence behavior # # These tests require server restart/crash and cannot be covered by # regular regression tests. # # Verifies: # - Stats persist across clean restart # - Stats are lost after crash recovery use strict; use warnings FATAL => 'all'; use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; my $node = PostgreSQL::Test::Cluster->new('main'); $node->init; $node->append_conf('postgresql.conf', "shared_preload_libraries = 'pg_stat_log'"); $node->append_conf('postgresql.conf', "pg_stat_log.min_error_level = 'warning'"); $node->start; $node->safe_psql('postgres', q(CREATE EXTENSION pg_stat_log)); # Generate some data to persist $node->safe_psql('postgres', q( DO $$ BEGIN RAISE WARNING 'persist test'; END $$; )); $node->psql('postgres', q(SELECT 1/0)); $node->safe_psql('postgres', q(SELECT pg_stat_force_next_flush())); my $result = $node->safe_psql('postgres', q( SELECT count FROM pg_stat_log_data() WHERE elevel = 'ERROR' AND sqlerrcode = '22012' )); my $error_count_pre_restart = $result; # --------------------------------------------------------------- # Test 1: Stats persist across clean restart # --------------------------------------------------------------- $node->stop; $node->start; $result = $node->safe_psql('postgres', q( SELECT count FROM pg_stat_log_data() WHERE elevel = 'ERROR' AND sqlerrcode = '22012' )); is($result, $error_count_pre_restart, "error count persists after clean restart"); # --------------------------------------------------------------- # Test 1b: n_dropped and stats_reset do not persist across restart # --------------------------------------------------------------- my $n_dropped = $node->safe_psql('postgres', q( SELECT n_dropped FROM pg_stat_log_info() )); is($n_dropped, "0", "n_dropped resets to 0 after clean restart"); my $reset_ts = $node->safe_psql('postgres', q( SELECT stats_reset FROM pg_stat_log_info() )); ok(defined $reset_ts && $reset_ts ne '', "stats_reset is set to startup timestamp after clean restart"); # --------------------------------------------------------------- # Test 2: Stats lost after crash recovery # --------------------------------------------------------------- $node->stop('immediate'); $node->start; $result = $node->safe_psql('postgres', q( SELECT COALESCE(sum(count), 0) FROM pg_stat_log_data() )); is($result, "0", "all counts are zero after crash recovery"); # --------------------------------------------------------------- # Test 3: pg_stat_log_info() basics # --------------------------------------------------------------- my $info_rows = $node->safe_psql('postgres', q( SELECT count(*) FROM pg_stat_log_info() )); is($info_rows, "1", "pg_stat_log_info() returns one row"); my $max_entries = $node->safe_psql('postgres', q( SELECT max_entries FROM pg_stat_log_info() )); my $guc_max = $node->safe_psql('postgres', q(SHOW pg_stat_log.max_entries)); is($max_entries, $guc_max, "pg_stat_log_info.max_entries matches GUC pg_stat_log.max_entries"); # --------------------------------------------------------------- # Test 4: stats_reset advances on reset # --------------------------------------------------------------- my $reset_before = $node->safe_psql('postgres', q( SELECT extract(epoch FROM stats_reset)::numeric FROM pg_stat_log_info() )); $node->safe_psql('postgres', q(SELECT pg_sleep(0.1); SELECT pg_stat_log_reset();)); my $reset_after = $node->safe_psql('postgres', q( SELECT extract(epoch FROM stats_reset)::numeric FROM pg_stat_log_info() )); ok($reset_after > $reset_before, "stats_reset timestamp advances after pg_stat_log_reset()"); # --------------------------------------------------------------- # Test 5: n_dropped increments and reset reclaims slots # --------------------------------------------------------------- $node->stop('immediate'); $node->append_conf('postgresql.conf', "pg_stat_log.max_entries = 64"); $node->start; $max_entries = $node->safe_psql('postgres', q( SELECT max_entries FROM pg_stat_log_info() )); is($max_entries, "64", "max_entries reflects restart-scoped GUC"); # Generate 100 distinct SQLSTATE codes to overflow the 64-slot capacity $node->safe_psql('postgres', q{ DO $$ DECLARE i int; code text; BEGIN FOR i IN 1..100 LOOP code := 'Z' || lpad(i::text, 4, '0'); BEGIN RAISE WARNING 'overflow test %', i USING ERRCODE = code; EXCEPTION WHEN OTHERS THEN NULL; END; END LOOP; END $$; }); $node->safe_psql('postgres', q(SELECT pg_stat_force_next_flush())); my $num_entries = $node->safe_psql('postgres', q( SELECT num_entries FROM pg_stat_log_info() )); is($num_entries, "64", "num_entries saturates at max_entries"); $n_dropped = $node->safe_psql('postgres', q( SELECT n_dropped FROM pg_stat_log_info() )); ok($n_dropped > 0, "n_dropped > 0 after overflowing max_entries"); # An already-tracked signature must keep counting even when the table is full my $tracked_code = $node->safe_psql('postgres', q( SELECT sqlerrcode FROM pg_stat_log_data() LIMIT 1 )); my $count_before = $node->safe_psql('postgres', "SELECT count FROM pg_stat_log_data() WHERE sqlerrcode = '$tracked_code'"); $node->safe_psql('postgres', "DO \$\$ BEGIN RAISE WARNING 'still counts' USING ERRCODE = '$tracked_code'; END \$\$;"); $node->safe_psql('postgres', q(SELECT pg_stat_force_next_flush())); my $count_after = $node->safe_psql('postgres', "SELECT count FROM pg_stat_log_data() WHERE sqlerrcode = '$tracked_code'"); cmp_ok($count_after, '>', $count_before, "tracked signature still counts when table is full"); # Reset should reclaim slots $node->safe_psql('postgres', q(SELECT pg_stat_log_reset())); $num_entries = $node->safe_psql('postgres', q( SELECT num_entries FROM pg_stat_log_info() )); is($num_entries, "0", "num_entries is 0 after reset when saturated"); $n_dropped = $node->safe_psql('postgres', q( SELECT n_dropped FROM pg_stat_log_info() )); is($n_dropped, "0", "n_dropped is 0 after reset"); # Generate a NEW distinct error and verify it is tracked (slot reclaimed) $node->safe_psql('postgres', q{ DO $$ BEGIN RAISE WARNING 'post-reset' USING ERRCODE = 'Z9999'; EXCEPTION WHEN OTHERS THEN NULL; END $$; }); $node->safe_psql('postgres', q(SELECT pg_stat_force_next_flush())); my $post_reset = $node->safe_psql('postgres', q( SELECT count FROM pg_stat_log_data() WHERE sqlerrcode = 'Z9999' )); is($post_reset, "1", "new distinct error is tracked after reset (slots reclaimed)"); # --------------------------------------------------------------- # Test 6: Stats discarded when max_entries changes across restart # --------------------------------------------------------------- # Generate some stats with the current max_entries=64 $node->safe_psql('postgres', q(SELECT pg_stat_log_reset())); $node->safe_psql('postgres', q{ DO $$ BEGIN RAISE WARNING 'before resize'; END $$; }); $node->safe_psql('postgres', q(SELECT pg_stat_force_next_flush())); $num_entries = $node->safe_psql('postgres', q( SELECT num_entries FROM pg_stat_log_info() )); ok($num_entries > 0, "have entries before max_entries change"); # Clean restart with a different max_entries $node->stop; $node->append_conf('postgresql.conf', "pg_stat_log.max_entries = 128"); $node->start; # Stats should have been discarded due to capacity mismatch $max_entries = $node->safe_psql('postgres', q( SELECT max_entries FROM pg_stat_log_info() )); is($max_entries, "128", "max_entries reflects new GUC after restart"); $num_entries = $node->safe_psql('postgres', q( SELECT num_entries FROM pg_stat_log_info() )); is($num_entries, "0", "persisted stats discarded after max_entries change"); # Verify new entries can be tracked with the new capacity $node->safe_psql('postgres', q{ DO $$ BEGIN RAISE WARNING 'after resize'; END $$; }); $node->safe_psql('postgres', q(SELECT pg_stat_force_next_flush())); $num_entries = $node->safe_psql('postgres', q( SELECT num_entries FROM pg_stat_log_info() )); ok($num_entries > 0, "new entries tracked after max_entries change"); # --------------------------------------------------------------- # Test 7: Stats discarded when max_entries is lowered across restart # --------------------------------------------------------------- # A different max_entries changes the length of the persisted block, so # PostgreSQL cannot parse the stats file and resets every stats kind. The # reset must size the block from the new max_entries, not from the header # restored from the file, or it writes past the smaller block. $node->safe_psql('postgres', q{ CREATE TABLE core_stats (i int); INSERT INTO core_stats SELECT generate_series(1, 10); }); $result = $node->safe_psql('postgres', q( SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'core_stats' )); is($result, "10", "core statistics recorded before restart"); $node->stop; $node->append_conf('postgresql.conf', "pg_stat_log.max_entries = 64"); my $log_offset = -s $node->logfile; $node->start; $result = $node->safe_psql('postgres', q( SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'core_stats' )); is($result, "0", "core statistics discarded along with pg_stat_log's"); $max_entries = $node->safe_psql('postgres', q( SELECT max_entries FROM pg_stat_log_info() )); is($max_entries, "64", "max_entries reflects the lowered GUC after restart"); $num_entries = $node->safe_psql('postgres', q( SELECT num_entries FROM pg_stat_log_info() )); is($num_entries, "0", "persisted stats discarded after lowering max_entries"); ok($node->log_contains(qr/corrupted statistics file/, $log_offset), "stats file written with the old max_entries is discarded"); ok( !$node->log_contains( qr/pg_stat_log: discarding persisted stats/, $log_offset), "reset sized the block from the new max_entries"); $node->safe_psql('postgres', q{ DO $$ BEGIN RAISE WARNING 'after shrink'; END $$; }); $node->safe_psql('postgres', q(SELECT pg_stat_force_next_flush())); $num_entries = $node->safe_psql('postgres', q( SELECT num_entries FROM pg_stat_log_info() )); ok($num_entries > 0, "new entries tracked after lowering max_entries"); done_testing(); fabriziomello-pg_stat_log-20b57e2/t/002_connection_lifecycle.pl000066400000000000000000000245341526422552700245610ustar00rootroot00000000000000# Copyright (c) 2026, Fabrizio de Royes Mello # Test counting of connection lifecycle records # # log_connections and log_disconnections write their records at LOG with # SQLSTATE 00000. pg_stat_log counts them under extension-defined SQLSTATEs, # attributed to the connected database and the login role: # # PSL01 connection_authenticated # PSL02 connection_authorized # PSL03 disconnection # # Verifies: # - Each record is counted once, for both authentication code paths # - A startup failing after authorization keeps the login role # - Physical walsenders are counted without a database # - Other LOG records keep SQLSTATE 00000 # - A disconnection is counted only if its record reaches the server log # - Nothing is counted while log_connections and log_disconnections are off # - log_disconnections enabled with startup options is honored # - A reload does not change log_disconnections for a running session use strict; use warnings FATAL => 'all'; use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; if (!$use_unix_sockets) { plan skip_all => "authentication tests cannot run without Unix-domain sockets"; } # Streaming is needed for the walsender connection my $node = PostgreSQL::Test::Cluster->new('lifecycle'); $node->init(allows_streaming => 1); $node->append_conf( 'postgresql.conf', q{ shared_preload_libraries = 'pg_stat_log' pg_stat_log.min_error_level = 'log' log_connections = 'authentication,authorization' log_disconnections = on log_statement = none }); unlink($node->data_dir . '/pg_hba.conf'); $node->append_conf( 'pg_hba.conf', q{ local app alice scram-sha-256 local replication all trust local all all trust }); $node->start; $node->safe_psql( 'postgres', q{ CREATE EXTENSION pg_stat_log; SET password_encryption = 'scram-sha-256'; CREATE ROLE alice LOGIN PASSWORD 'alice-pass'; CREATE ROLE bob LOGIN; CREATE DATABASE app; }); my $superuser = $node->safe_psql('postgres', q(SELECT current_user)); my $alice_connstr = $node->connstr('app') . " user=alice password=alice-pass"; my $bob_connstr = $node->connstr('app') . " user=bob"; # Lifecycle counters of the pg_stat_log rows matching $filter sub lifecycle_counts { my ($filter) = @_; return $node->safe_psql( 'postgres', qq{ SELECT string_agg(concat_ws(' ', backend_type, sqlerrcode, sqlerrcode_name, count), E'\\n' ORDER BY backend_type, sqlerrcode) FROM pg_stat_log WHERE sqlerrcode LIKE 'PSL%' AND $filter }); } # A backend counts its disconnection while exiting, before it leaves # pg_stat_activity sub wait_for_backends_to_exit { $node->poll_query_until( 'postgres', q{ SELECT count(*) = 0 FROM pg_stat_activity WHERE backend_type IN ('client backend', 'walsender') AND pid <> pg_backend_pid() }) or die "timed out waiting for backends to exit"; } # --------------------------------------------------------------- # Test 1: Records attributed to the database and login role # --------------------------------------------------------------- $node->connect_ok( $alice_connstr, "alice connects with scram-sha-256", log_like => [qr/connection authenticated: identity="alice" method=scram-sha-256/]); wait_for_backends_to_exit(); is( lifecycle_counts(q(database_name = 'app' AND user_name = 'alice')), "client backend PSL01 connection_authenticated 1 client backend PSL02 connection_authorized 1 client backend PSL03 disconnection 1", "scram-sha-256 session counted once per record"); # --------------------------------------------------------------- # Test 2: Authentication without an authenticated identity # --------------------------------------------------------------- $node->connect_ok( $bob_connstr, "bob connects with trust", log_like => [qr/connection authenticated: user="bob" method=trust/]); wait_for_backends_to_exit(); is( lifecycle_counts(q(database_name = 'app' AND user_name = 'bob')), "client backend PSL01 connection_authenticated 1 client backend PSL02 connection_authorized 1 client backend PSL03 disconnection 1", "trust session counted once per record"); # --------------------------------------------------------------- # Test 3: Startup failing after authorization # --------------------------------------------------------------- $node->connect_fails( $node->connstr('no_such_db') . " user=alice", "alice cannot connect to a missing database", expected_stderr => qr/database "no_such_db" does not exist/); wait_for_backends_to_exit(); is( lifecycle_counts(q(database_oid IS NULL AND user_name = 'alice')), "client backend PSL01 connection_authenticated 1 client backend PSL02 connection_authorized 1", "failed startup counted for the login role, without database or disconnection" ); # --------------------------------------------------------------- # Test 4: Physical walsender # --------------------------------------------------------------- $node->safe_psql('postgres', 'IDENTIFY_SYSTEM', replication => 'true'); wait_for_backends_to_exit(); is( lifecycle_counts( qq(backend_type = 'walsender' AND database_oid IS NULL AND user_name = '$superuser') ), "walsender PSL01 connection_authenticated 1 walsender PSL02 connection_authorized 1 walsender PSL03 disconnection 1", "physical walsender counted without a database"); # --------------------------------------------------------------- # Test 5: Other LOG records keep SQLSTATE 00000 # --------------------------------------------------------------- $node->safe_psql('app', q(DO $$ BEGIN RAISE LOG 'not a connection record'; END $$)); my $result = $node->safe_psql( 'postgres', q{ SELECT string_agg(concat_ws(' ', sqlerrcode, sqlerrcode_name, count), E'\n') FROM pg_stat_log WHERE elevel = 'LOG' AND sqlerrcode NOT LIKE 'PSL%' AND database_name = 'app' }); is($result, "00000 successful_completion 1", "RAISE LOG still counted as successful_completion"); $result = $node->safe_psql( 'postgres', q{ SELECT count(*) FROM pg_stat_log WHERE backend_type IN ('client backend', 'walsender') AND elevel = 'LOG' AND database_oid IS NULL AND user_oid IS NULL }); is($result, "0", "connection records are not counted without attribution"); # --------------------------------------------------------------- # Test 6: Disconnection record not reaching the server log # --------------------------------------------------------------- # Authentication and authorization are logged before startup options apply $node->safe_psql('postgres', q(GRANT SET ON PARAMETER log_min_messages TO bob)); $node->connect_ok( $bob_connstr . " options='-c log_min_messages=fatal'", "bob connects with log_min_messages = fatal"); wait_for_backends_to_exit(); is( lifecycle_counts(q(database_name = 'app' AND user_name = 'bob')), "client backend PSL01 connection_authenticated 2 client backend PSL02 connection_authorized 2 client backend PSL03 disconnection 1", "disconnection not counted when LOG does not reach the server log"); # --------------------------------------------------------------- # Test 7: log_connections and log_disconnections off # --------------------------------------------------------------- $node->append_conf( 'postgresql.conf', q{ log_connections = off log_disconnections = off }); $node->reload; $node->poll_query_until( 'postgres', q{ SELECT current_setting('log_connections') = 'off' AND current_setting('log_disconnections') = 'off' }) or die "timed out waiting for the configuration reload"; $node->connect_ok($alice_connstr, "alice connects with log_connections and log_disconnections off"); wait_for_backends_to_exit(); is( lifecycle_counts(q(database_name = 'app' AND user_name = 'alice')), "client backend PSL01 connection_authenticated 1 client backend PSL02 connection_authorized 1 client backend PSL03 disconnection 1", "nothing counted while log_connections and log_disconnections are off"); # --------------------------------------------------------------- # Test 8: log_disconnections enabled with startup options # --------------------------------------------------------------- $node->safe_psql('postgres', q(GRANT SET ON PARAMETER log_disconnections TO alice)); $node->connect_ok( $alice_connstr . " options='-c log_disconnections=on'", "alice enables log_disconnections at connection start", sql => 'SHOW log_disconnections', expected_stdout => qr/^on$/); wait_for_backends_to_exit(); is( lifecycle_counts(q(database_name = 'app' AND user_name = 'alice')), "client backend PSL01 connection_authenticated 1 client backend PSL02 connection_authorized 1 client backend PSL03 disconnection 2", "disconnection counted when log_disconnections is set at connection start" ); # --------------------------------------------------------------- # Test 9: Reload in a running session # --------------------------------------------------------------- # PostgreSQL ignores reloaded PGC_SU_BACKEND settings in running backends, so # the session still logs its disconnection $node->safe_psql( 'postgres', q{ CREATE DATABASE reload_db; ALTER SYSTEM SET log_disconnections = on; SELECT pg_reload_conf(); }); $node->poll_query_until('postgres', q(SELECT current_setting('log_disconnections') = 'on')) or die "timed out waiting for the configuration reload"; my $log_offset = -s $node->logfile; my $session = $node->background_psql('reload_db'); is($session->query_safe('SHOW log_disconnections'), 'on', "session starts with log_disconnections on"); $session->query_safe(q(ALTER SYSTEM SET log_disconnections = off)); $session->query_safe(q(ALTER SYSTEM SET work_mem = '7MB')); $session->query_safe(q(SELECT pg_reload_conf())); $node->poll_query_until('postgres', q(SELECT current_setting('log_disconnections') = 'off')) or die "timed out waiting for the configuration reload"; is($session->query_safe('SHOW work_mem'), '7MB', "session applied the reloaded configuration"); is($session->query_safe('SHOW log_disconnections'), 'on', "session keeps log_disconnections on after the reload"); $session->quit; $node->wait_for_log(qr/disconnection: session time: .* database=reload_db/, $log_offset); wait_for_backends_to_exit(); is( lifecycle_counts(q(database_name = 'reload_db')), "client backend PSL03 disconnection 1", "disconnection logged after the reload is counted"); $node->connect_ok( $node->connstr('reload_db'), "session started after the reload has log_disconnections off", sql => 'SHOW log_disconnections', expected_stdout => qr/^off$/); wait_for_backends_to_exit(); is( lifecycle_counts(q(database_name = 'reload_db')), "client backend PSL03 disconnection 1", "sessions started after the reload are not counted"); done_testing(); fabriziomello-pg_stat_log-20b57e2/t/003_query_id.pl000066400000000000000000000147571526422552700222330ustar00rootroot00000000000000# Copyright (c) 2026, Fabrizio de Royes Mello # Test counting per query identifier # # With pg_stat_log.compute_query_id, messages are also grouped by the query # identifier that %Q prints in log_line_prefix. The server log, written with # log_line_prefix = 'qid=%Q ', gives the expected query_id values. # # Verifies: # - pg_stat_log.compute_query_id has query identifiers computed under # compute_query_id = auto # - Statements differing only in constants share a query_id # - Messages logged without a query identifier have a NULL query_id # - Connection lifecycle records have a NULL query_id # - query_id persists across a clean restart # - Nothing is tracked with pg_stat_log.compute_query_id or # compute_query_id off use strict; use warnings FATAL => 'all'; use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; my $node = PostgreSQL::Test::Cluster->new('query_id'); $node->init; $node->append_conf( 'postgresql.conf', q{ shared_preload_libraries = 'pg_stat_log' pg_stat_log.compute_query_id = on pg_stat_log.min_error_level = 'log' log_connections = 'authentication,authorization' log_disconnections = on log_line_prefix = 'qid=%Q ' }); $node->start; $node->safe_psql('postgres', 'CREATE EXTENSION pg_stat_log'); # query_id and count of the pg_stat_log rows for $sqlerrcode, most frequent # first sub query_id_counts { my ($sqlerrcode) = @_; return $node->safe_psql( 'postgres', qq{ SELECT string_agg(concat_ws(' ', coalesce(query_id::text, 'NULL'), count), E'\\n' ORDER BY count DESC, query_id) FROM pg_stat_log WHERE sqlerrcode = '$sqlerrcode' }); } # Query identifiers that %Q printed for ERROR $message since $offset sub logged_query_ids { my ($offset, $message) = @_; my $log = slurp_file($node->logfile, $offset); return $log =~ /^qid=(-?\d+) ERROR: \Q$message\E/mg; } # --------------------------------------------------------------- # Test 1: query_id matches %Q # --------------------------------------------------------------- my $result = $node->safe_psql( 'postgres', q{ SELECT current_setting('compute_query_id'), context FROM pg_settings WHERE name = 'pg_stat_log.compute_query_id' }); is($result, "auto|postmaster", "compute_query_id is auto and pg_stat_log.compute_query_id needs a restart" ); my $log_offset = -s $node->logfile; $node->psql('postgres', 'SELECT 1/0'); $node->psql('postgres', 'SELECT 2/0'); $node->psql('postgres', 'SELECT 1 % 0'); my @qids = logged_query_ids($log_offset, 'division by zero'); is(scalar @qids, 3, "three division by zero errors logged"); my ($qid_div, $qid_div2, $qid_mod) = @qids; like($qid_div, qr/^-?[1-9][0-9]*$/, "query identifiers computed under compute_query_id = auto"); is($qid_div2, $qid_div, "statements differing only in constants share a query identifier"); isnt($qid_mod, $qid_div, "a different statement has its own query identifier"); is(query_id_counts('22012'), "$qid_div 2\n$qid_mod 1", "query_id matches %Q"); # --------------------------------------------------------------- # Test 2: Messages logged without a query identifier # --------------------------------------------------------------- $log_offset = -s $node->logfile; $node->psql('postgres', 'SET compute_query_id = off; SELECT 3/0'); is(join(' ', logged_query_ids($log_offset, 'division by zero')), "0", "no query identifier with compute_query_id set off in the session"); is( query_id_counts('22012'), "$qid_div 2\n$qid_mod 1\nNULL 1", "query_id is NULL with compute_query_id set off in the session"); $log_offset = -s $node->logfile; $node->psql('postgres', 'SELECT FROM WHERE'); is(join(' ', logged_query_ids($log_offset, 'syntax error')), "0", "no query identifier for a syntax error"); is(query_id_counts('42601'), "NULL 1", "query_id is NULL for a syntax error"); # --------------------------------------------------------------- # Test 3: Connection lifecycle records # --------------------------------------------------------------- # The backend still has its last statement's query identifier while exiting $log_offset = -s $node->logfile; $node->safe_psql('postgres', 'SELECT 1'); $node->wait_for_log(qr/disconnection: session time:/, $log_offset); my @lifecycle_qids = slurp_file($node->logfile) =~ /^qid=(-?\d+) LOG: (?:connection authenticated|connection authorized|disconnection):/mg; cmp_ok(scalar @lifecycle_qids, '>=', 3, "connection lifecycle records logged"); is(join(' ', grep { $_ ne '0' } @lifecycle_qids), '', "no query identifier logged for connection lifecycle records"); $result = $node->safe_psql( 'postgres', q{ SELECT string_agg(concat_ws(' ', sqlerrcode, coalesce(query_id::text, 'NULL')), E'\n' ORDER BY sqlerrcode) FROM pg_stat_log WHERE sqlerrcode LIKE 'PSL%' }); is( $result, "PSL01 NULL\nPSL02 NULL\nPSL03 NULL", "query_id is NULL for connection lifecycle records"); # --------------------------------------------------------------- # Test 4: query_id persists across a clean restart # --------------------------------------------------------------- $node->restart; is( query_id_counts('22012'), "$qid_div 2\n$qid_mod 1\nNULL 1", "query_id persists across a clean restart"); $node->psql('postgres', 'SELECT 4/0'); is( query_id_counts('22012'), "$qid_div 3\n$qid_mod 1\nNULL 1", "statement after the restart counted under the persisted query_id"); # --------------------------------------------------------------- # Test 5: pg_stat_log.compute_query_id off # --------------------------------------------------------------- $node->append_conf( 'postgresql.conf', q{ pg_stat_log.compute_query_id = off compute_query_id = on }); $node->restart; $node->safe_psql('postgres', 'SELECT pg_stat_log_reset()'); $log_offset = -s $node->logfile; $node->psql('postgres', 'SELECT 1/0'); is(join(' ', logged_query_ids($log_offset, 'division by zero')), $qid_div, "compute_query_id = on computes the query identifier"); is(query_id_counts('22012'), "NULL 1", "query_id is NULL with pg_stat_log.compute_query_id off"); # --------------------------------------------------------------- # Test 6: compute_query_id off # --------------------------------------------------------------- $node->append_conf( 'postgresql.conf', q{ pg_stat_log.compute_query_id = on compute_query_id = off }); $node->restart; $node->safe_psql('postgres', 'SELECT pg_stat_log_reset()'); $log_offset = -s $node->logfile; $node->psql('postgres', 'SELECT 1/0'); is(join(' ', logged_query_ids($log_offset, 'division by zero')), "0", "compute_query_id = off computes no query identifier"); is(query_id_counts('22012'), "NULL 1", "query_id is NULL with compute_query_id off"); done_testing();