From fa55eb333d160bb597c2baa0f7afebfe237e375f Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Mon, 16 Aug 2021 17:28:36 -0400 Subject: [PATCH 01/56] PG-220: Fix pgsm_overflow_target defaults. The GUC variable pgsm_overflow_target was pointing to the wrong index (12, pgsm_track_planning) in the "GucVariable conf[MAX_SETTINGS]" array. Adjusted it to the right index, 11. --- pg_stat_monitor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pg_stat_monitor.h b/pg_stat_monitor.h index 2ddfe01..a1ab4d5 100644 --- a/pg_stat_monitor.h +++ b/pg_stat_monitor.h @@ -411,7 +411,7 @@ void pgss_startup(void); #define PGSM_HISTOGRAM_MAX get_conf(8)->guc_variable #define PGSM_HISTOGRAM_BUCKETS get_conf(9)->guc_variable #define PGSM_QUERY_SHARED_BUFFER get_conf(10)->guc_variable -#define PGSM_OVERFLOW_TARGET get_conf(12)->guc_variable +#define PGSM_OVERFLOW_TARGET get_conf(11)->guc_variable #define PGSM_QUERY_PLAN get_conf(12)->guc_variable #define PGSM_TRACK_PLANNING get_conf(13)->guc_variable From c3d167e4524d37d92fe4dcc73ccdebf8bd9885ae Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Mon, 16 Aug 2021 18:11:17 -0400 Subject: [PATCH 02/56] PG-220: Check possible query buffer overflow. In SaveQueryText() we check for a possible overflow in the query buffer, but if overflow would happen and pgsm_overflow_target value is 1 (the default), then we dump the query buffer to a temporary file and reset the buffer (start saving queries from the start of the buffer). The problem is that after resetting the buffer we don't check if the current query length would exceed the buffer size of MAX_QUERY_BUFFER_BUCKET, if that is the case the buffer would overflow and probably crash the process or in the worst case become an attack vector for exploitation. This commit fix the problem by adding an additional check for overflow after resetting the query buffer. --- pg_stat_monitor.c | 48 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 41c23c8..127d6a3 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -76,7 +76,7 @@ static struct pg_hook_stats_t *pg_hook_stats; static void extract_query_comments(const char *query, char *comments, size_t max_len); static int get_histogram_bucket(double q_time); static bool IsSystemInitialized(void); -static void dump_queries_buffer(int bucket_id, unsigned char *buf, int buf_len); +static bool dump_queries_buffer(int bucket_id, unsigned char *buf, int buf_len); static double time_diff(struct timeval end, struct timeval start); @@ -3092,11 +3092,39 @@ SaveQueryText(uint64 bucketid, return false; case OVERFLOW_TARGET_DISK: { - dump_queries_buffer(bucketid, buf, MAX_QUERY_BUFFER_BUCKET); + bool dump_ok; + + /* + * If the query buffer is empty, there is nothing to dump, this also + * means that the current query length exceeds MAX_QUERY_BUFFER_BUCKET. + */ + if (buf_len <= sizeof (uint64)) + return false; + + dump_ok = dump_queries_buffer(bucketid, buf, buf_len); buf_len = sizeof (uint64); + + /* + * We must check for overflow again, as the query length may + * exceed the size allocated to the buffer (MAX_QUERY_BUFFER_BUCKET). + */ + if (QUERY_BUFFER_OVERFLOW(buf_len, query_len)) + { + /* + * If we successfully dumped the query buffer to disk, then + * reset the buffer, otherwise we could end up dumping the + * same buffer again. + */ + if (dump_ok) + *(uint64 *)buf = 0; + + return false; + } + } break; default: + Assert(false); break; } } @@ -3287,27 +3315,37 @@ IsSystemInitialized(void) return (system_init && IsHashInitialize()); } -static void +static bool dump_queries_buffer(int bucket_id, unsigned char *buf, int buf_len) { - int fd = 0; + int fd = 0; char file_name[1024]; + bool success = true; snprintf(file_name, 1024, "%s.%d", PGSM_TEXT_FILE, bucket_id); fd = OpenTransientFile(file_name, O_RDWR | O_CREAT | O_APPEND | PG_BINARY); - if (fd < 0) + if (fd < 0) + { ereport(LOG, (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", file_name))); + return false; + } if (write(fd, buf, buf_len) != buf_len) + { ereport(LOG, (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", file_name))); + success = false; + } + if (fd > 0) CloseTransientFile(fd); + + return success; } int From e593dbccc33978e3297bb5c91df5a0ac058e8989 Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Tue, 17 Aug 2021 11:19:22 -0400 Subject: [PATCH 03/56] PG-220: Fix GUC regression test (pgsm_overflow_target). Updated expected output for pgsm_overflow_target to match the default value of 1. --- regression/expected/guc.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression/expected/guc.out b/regression/expected/guc.out index 934c719..0da2986 100644 --- a/regression/expected/guc.out +++ b/regression/expected/guc.out @@ -23,7 +23,7 @@ SELECT * FROM pg_stat_monitor_settings ORDER BY name COLLATE "C"; pg_stat_monitor.pgsm_max | 100 | 100 | Sets the maximum size of shared memory in (MB) used for statement's metadata tracked by pg_stat_monitor. | 1 | 1000 | 1 pg_stat_monitor.pgsm_max_buckets | 10 | 10 | Sets the maximum number of buckets. | 1 | 10 | 1 pg_stat_monitor.pgsm_normalized_query | 1 | 1 | Selects whether save query in normalized format. | 0 | 0 | 0 - pg_stat_monitor.pgsm_overflow_target | 0 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | 1 + pg_stat_monitor.pgsm_overflow_target | 1 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | 1 pg_stat_monitor.pgsm_query_max_len | 1024 | 1024 | Sets the maximum length of query. | 1024 | 2147483647 | 1 pg_stat_monitor.pgsm_query_shared_buffer | 20 | 20 | Sets the maximum size of shared memory in (MB) used for query tracked by pg_stat_monitor. | 1 | 10000 | 1 pg_stat_monitor.pgsm_track_planning | 1 | 1 | Selects whether planning statistics are tracked. | 0 | 0 | 0 From d5e25f961fbf4fa7a9baba7184b66443cb5234a5 Mon Sep 17 00:00:00 2001 From: Anastasia Alexadrova Date: Tue, 10 Aug 2021 14:58:50 +0300 Subject: [PATCH 04/56] PG-215 User Guide updates Moved pg_stat_monitor column description and compaison to separate docs new file: REFERENCE.md new file: COMPARISON.md --- docs/COMPARISON.md | 73 ++++ docs/REFERENCE.md | 62 ++++ docs/USER_GUIDE.md | 863 ++++++++++++++++++++++++++------------------- 3 files changed, 634 insertions(+), 364 deletions(-) create mode 100644 docs/COMPARISON.md create mode 100644 docs/REFERENCE.md diff --git a/docs/COMPARISON.md b/docs/COMPARISON.md new file mode 100644 index 0000000..bbeaf11 --- /dev/null +++ b/docs/COMPARISON.md @@ -0,0 +1,73 @@ +# Comparing pg_stat_monitor and pg_stat_statements + +The `pg_stat_monitor` extension is developed on the basis of `pg_stat_statements` as its more advanced replacement. + +Thus, `pg_stat_monitor` inherits the columns available in `pg_stat_statements` plus provides additional ones. + +To see all available columns, run the following command from the `psql` terminal: + +```sql + postgres=# \d pg_stat_monitor; +``` + +The following table compares the `pg_stat_monitor` view with that of `pg_stat_statements`. + +Note that the column names differ depending on the PostgreSQL version you are running. + + +| Column name for PostgreSQL 13 and above | Column name for PostgreSQL 11 and 12 | pg_stat_monitor | pg_stat_statements +|--------------------|--------------------------|-----------------------------|---------------------- + bucket | bucket | :heavy_check_mark: | :x: +bucket_start_time | bucket_start_time | :heavy_check_mark: | :x: +userid | userid | :heavy_check_mark: | :heavy_check_mark: +datname | datname | :heavy_check_mark: | :heavy_check_mark: +client_ip | client_ip | :heavy_check_mark:| :x: +queryid | queryid | :heavy_check_mark: | :heavy_check_mark: +planid | planid | :heavy_check_mark:| :x: +query_plan | query_plan | :heavy_check_mark: | :x: +top_query | top_query | :heavy_check_mark: | :x: +top_queryid | top_queryid | :heavy_check_mark: | :x: +query | query | :heavy_check_mark: | :heavy_check_mark: +application_name | application_name | :heavy_check_mark:| :x: +relations | relations | :heavy_check_mark: | :x: +cmd_type | cmd_type | :heavy_check_mark: | :x: +elevel | elevel | :heavy_check_mark: | :x: +sqlcode | sqlcode | :heavy_check_mark: | :x: +message | message | :heavy_check_mark: | :x: +plans_calls | plans_calls | :heavy_check_mark: | :heavy_check_mark: +plan_total_time | plan_total_time | :heavy_check_mark: | :heavy_check_mark: +plan_min_time | plan_min_time | :heavy_check_mark: | :heavy_check_mark: +plan_max_time | plan_max_time | :heavy_check_mark: | :heavy_check_mark: +plan_mean_time | plan_mean_time | :heavy_check_mark: | :heavy_check_mark: +calls | calls | :heavy_check_mark: | :heavy_check_mark: +total_time | total_time | :heavy_check_mark: | :heavy_check_mark: +min_time | min_time | :heavy_check_mark: | :heavy_check_mark: +max_time | max_time | :heavy_check_mark: | :heavy_check_mark: +mean_time | mean_time | :heavy_check_mark: | :heavy_check_mark: +stddev_time | stddev_time | :heavy_check_mark: | :heavy_check_mark: +rows_retrieved | rows_retrieved | :heavy_check_mark: | :heavy_check_mark: +shared_blks_hit | shared_blks_hit | :heavy_check_mark: | :heavy_check_mark: +shared_blks_read | shared_blks_read | :heavy_check_mark: | :heavy_check_mark: +shared_blks_dirtied | shared_blks_dirtied | :heavy_check_mark: | :heavy_check_mark: +shared_blks_written | shared_blks_written | :heavy_check_mark: | :heavy_check_mark: +local_blks_hit | local_blks_hit | :heavy_check_mark: | :heavy_check_mark: +local_blks_read | local_blks_read | :heavy_check_mark: | :heavy_check_mark: +local_blks_dirtied | local_blks_dirtied | :heavy_check_mark: | :heavy_check_mark: +local_blks_written | local_blks_written | :heavy_check_mark: | :heavy_check_mark: +temp_blks_read | temp_blks_read | :heavy_check_mark: | :heavy_check_mark: +temp_blks_written | temp_blks_written | :heavy_check_mark:  | :heavy_check_mark: +blk_read_time | blk_read_time | :heavy_check_mark: | :heavy_check_mark: +blk_write_time | blk_write_time | :heavy_check_mark: | :heavy_check_mark: +resp_calls | resp_calls | :heavy_check_mark: | :x: +cpu_user_time | cpu_user_time | :heavy_check_mark: | :x: +cpu_sys_time | cpu_sys_time | :heavy_check_mark: | :x: +wal_records | wal_records | :heavy_check_mark:  | :heavy_check_mark: +wal_fpi | wal_fpi | :heavy_check_mark:  | :heavy_check_mark: +wal_bytes | wal_bytes | :heavy_check_mark:  | :heavy_check_mark: +state_code | state_code | :heavy_check_mark: | :x: +state | state | :heavy_check_mark: | :x: + +To learn more about the features in `pg_stat_monitor`, please see the [User guide](https://github.com/percona/pg_stat_monitor/blob/master/docs/USER_GUIDE.md). + + +Additional reading: [pg_stat_statements](https://www.postgresql.org/docs/current/pgstatstatements.html) diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md new file mode 100644 index 0000000..0247747 --- /dev/null +++ b/docs/REFERENCE.md @@ -0,0 +1,62 @@ +# `pg_stat_monitor` view reference + +`pg_stat_monitor` provides a view where the statistics data is displayed. To see all available columns, run the following command from `psql`: + +```sql +postgres=# \d pg_stat_monitor +``` + +Depending on the PostgreSQL version, some column names may differ. The following table describes the `pg_stat_monitor` view for PostgreSQL 13 and higher versions. + + +| Column | Type | Description +|--------------------|--------------------------|------------------ + bucket | integer | Data collection unit. The number shows what bucket in a chain a record belongs to +bucket_start_time | timestamp with time zone | The start time of the bucket| +userid | regrole | An ID of the user who run a query | +datname | name | The name of a database where the query was executed +client_ip | inet | The IP address of a client that run the query +queryid | text | The internal hash code serving to identify every query in a statement +planid | text | An internally generated ID of a query plan +query_plan | text | The sequence of steps used to execute a query. This parameter is available only when the `pgsm_enable_query_plan` is enabled. +top_query | text | Shows the top query used in a statement | +query | text | The actual text of the query | +application_name | text | Shows the name of the application connected to the database +relations | text[] | The list of tables involved in the query +cmd_type | text[] | Type of the query executed +elevel | integer | Shows the error level of a query (WARNING, ERROR, LOG) +sqlcode | integer | +message | text | The error message +plans_calls | bigint | The number of times the statement was planned +plan_total_time | double precision | The total time (in ms) spent on planning the statement +plan_min_time | double precision | Minimum time (in ms) spent on planning the statement +plan_max_time | double precision | Maximum time (in ms) spent on planning the statement +plan_mean_time | double precision | The mean (average) time (in ms) spent on planning the statement +plan_stddev_time | double precision | The standard deviation of time (in ms) spent on planning the statement +calls | bigint | The number of times a particular query was executed +total_time | double precision | The total time (in ms) spent on executing a query +min_time | double precision | The minimum time (in ms) it took to execute a query +max_time | double precision | The maximum time (in ms) it took to execute a query +mean_time | double precision | The mean (average) time (in ms) it took to execute a query +stddev_time | double precision | The standard deviation of time (in ms) spent on executing a query +rows_retrieved | bigint | The number of rows retrieved when executing a query +shared_blks_hit | bigint | Shows the total number of shared memory blocks returned from the cache +shared_blks_read | bigint | Shows the total number of shared blocks returned not from the cache +shared_blks_dirtied | bigint | Shows the number of shared memory blocks "dirtied" by the query execution (i.e. a query modified at least one tuple in a block and this block must be written to a drive) +shared_blks_written | bigint | Shows the number of shared memory blocks written simultaneously to a drive during the query execution +local_blks_hit | bigint | The number of blocks which are considered as local by the backend and thus are used for temporary tables +local_blks_read | bigint | Total number of local blocks read during the query execution +local_blks_dirtied | bigint | Total number of local blocks "dirtied" during the query execution (i.e. a query modified at least one tuple in a block and this block must be written to a drive) +local_blks_written | bigint | Total number of local blocks written simultaneously to a drive during the query execution +temp_blks_read | bigint | Total number of blocks of temporary files read from a drive. Temporary files are used when there's not enough memory to execute a query +temp_blks_written | bigint | Total number of blocks of temporary files written to a drive +blk_read_time | double precision | Total waiting time (in ms) for reading blocks +blk_write_time | double precision | Total waiting time (in ms) for writing blocks to a drive +resp_calls | text[] | Call histogram +cpu_user_time | double precision | The time (in ms) the CPU spent on running the query +cpu_sys_time | double precision | The time (in ms) the CPU spent on executing the kernel code +wal_records | bigint | The total number of WAL (Write Ahead Logs) generated by the query +wal_fpi | bigint | The total number of WAL FPI (Full Page Images) generated by the query +wal_bytes | numeric | Total number of bytes used for the WAL generated by the query +state_code | bigint | Shows the state code of a query +state | text | The state message diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 8cdd750..a671e1d 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -1,173 +1,328 @@ # User Guide -This document describes the configuration, key features and usage of ``pg_stat_monitor`` extension and compares it with ``pg_stat_statements``. +* [Introduction](#introduction) +* [Features](#features) +* [Views](#views) +* [Functions](#functions) +* [Configuration](#configuration) +* [Usage examples](#usage-examples) -For how to install and set up ``pg_stat_monitor``, see [README](https://github.com/percona/pg_stat_monitor/blob/master/README.md). +## Introduction -After you've installed and enabled ``pg_stat_monitor``, create the ``pg_stat_monitor`` extension using the ``CREATE EXTENSION`` command. +This document describes the features, functions and configuration of the ``pg_stat_monitor`` extension and gives some usage examples. For how to install and set up ``pg_stat_monitor``, see [Installation in README](https://github.com/percona/pg_stat_monitor/blob/master/README.md#installation). -```sql -CREATE EXTENSION pg_stat_monitor; -CREATE EXTENSION +## Features + +The following are the key features of pg_stat_monitor: + +* [Time buckets](#time-buckets), +* [Table and index access statistics per statement](#table-and-index-access-statistics-per-statement), +* Query statistics: + * [Query and client information](#query-and-client-information), + * [Query timing information](#query-timing-information), + * [Query execution plan information](#query-execution-plan-information), + * [Use of actual data or parameters placeholders in queries](#use-of-actual-data-or-parameters-placeholders-in-queries), + * [Query type filtering](#query-type-filtering), + * [Query metadata supporting Google’s Sqlcommentor](#query-metadata), +* [Top query tracking](#top-query-tracking), +* [Relations](#relations) - showing tables involved in a query, +* [Monitoring of queries terminated with ERROR, WARNING and LOG error levels](#monitoring-of-queries-terminated-with-error-warning-and-log-error-levels), +* [Integration with Percona Monitoring and Management (PMM) tool](#integration-with-pmm), +* [Histograms](#histogram) - visual representation of query performance. + +### Time buckets + +Instead of supplying one set of ever-increasing counts, `pg_stat_monitor` computes stats for a configured number of time intervals; time buckets. This allows for much better data accuracy, especially in the case of high-resolution or unreliable networks. + +### Table and index access statistics per statement + +`pg_stat_monitor` collects the information about what tables were accessed by a statement. This allows you to identify all queries which access a given table easily. + +### Query and client information + +`pg_stat_monitor` provides additional metrics for detailed analysis of query performance from various perspectives, including client connection details like user name, application name, IP address to name a few relevant columns. +With this information, `pg_stat_monitor` enables users to track a query to the originating application. More details about the application or query may be incorporated in the SQL query in a [Google’s Sqlcommenter](https://google.github.io/sqlcommenter/) format. + +### Query timing information + +Understanding query execution time stats helps you identify what affects query performance and take measures to optimize it. `pg_stat_monitor` collects the total, min, max and average (mean) time it took to execute a particular query and provides this data in separate columns. See the [Query timing information](#usage-examples-query-timing-information) example for the sample output. + +### Query execution plan information + +Every query has a plan that was constructed for its executing. Collecting the query plan information as well as monitoring query plan timing helps you understand how you can modify the query to optimize its execution. It also helps make communication about the query clearer when discussing query performance with other DBAs and application developers. + +See the [Query execution plan](#usage-examples-query-execution-time) example for the sample output. + +### Use of actual data or parameters placeholders in queries + +You can select whether to see queries with parameters placeholders or actual query data. The benefit of having the full query example is in being able to run the [EXPLAIN](https://www.postgresql.org/docs/current/sql-explain.html) command on it to see how its execution was planned. As a result, you can modify the query to make it run better. + +### Query type filtering + +`pg_stat_monitor` monitors queries per type (``SELECT``, `INSERT`, `UPDATE` or `DELETE`) and classifies them accordingly in the `cmd_type` column. This way you can separate the queries you are interested in and focus on identifying the issues and optimizing query performance. + +See the [Query type filtering example](#usage-examples-query-time-filtering) for the sample output. + +### Query metadata + +Google’s Sqlcommenter is a useful tool that in a way bridges that gap between ORM libraries and understanding database performance. And ``pg_stat_monitor`` supports it. So, you can now put any key-value data (like what client executed a query or if it is testing vs production query) in the comments in `/* … */` syntax in your SQL statements, and the information will be parsed by `pg_stat_monitor` and made available in the comments column in the `pg_stat_monitor` view. For details on the comments’ syntax, see [Sqlcommenter documentation](https://google.github.io/sqlcommenter/). + +To see how it works, see the [Query metadata](#isage-examples-query-metadata) example. + +### Top query tracking + +Using functions is common. While running, functions can execute queries internally. `pg_stat_monitor` not only keeps track of all executed queries within a function, but also marks that function as top query. + +Top query indicates the main query. To illustrate, for the SELECT query that is invoked within a function, the top query is calling this function. + +This enables you to backtrack to the originating function and thus simplifies the tracking and analysis. + +Find more details in the [usage example](#usage-examples-function-execution-tracking). + +### Relations + +`pg_stat_monitor` provides the list of tables involved in the query in the relations column. This reduces time on identifying the tables and simplifies the analysis. + +### Monitoring queries terminated with ERROR, WARNING and LOG error levels + +Monitoring queries that terminate with ERROR, WARNING, LOG states can give useful information to debug an issue. Such messages have the error level (`elevel`), error code (`sqlcode`), and error message (`message`). `pg_stat_monitor` collects all this information and aggregates it so that you can measure performance for successful and failed queries separately, as well as understand why a particular query failed to execute successfully. + +### Integration with PMM + +To timely identify and react on issues, performance should be automated and alerts should be sent when an issue occurs. There are many monitoring tools available for PostgreSQL, some of them (like Nagios) supporting custom metrics provided via extensions. Though you can integrate `pg_stat_monitor` with these tools, it natively supports integration with Percona Management and Monitoring (PMM). This integration allows you to enjoy all the features provided by both solutions: advanced statistics data provided by `pg_stat_monitor` and automated monitoring with data visualization on dashboards, security threat checks and alerting, available in PMM out of the box. + +To learn how to integrate pg_stat_monitor with PMM, see [Configure pg_stat_monitor in PMM](https://www.percona.com/doc/percona-monitoring-and-management/2.x/setting-up/client/postgresql.html#pg_stat_monitor) + +### Histogram + +Histogram (the `resp_calls` parameter) provides a visual representation of query performance. With the help of the histogram function, you can view a timing/calling data histogram in response to an SQL query. + +Learn more about using histograms from the [usage example](#usage-examples-histogram). + +## Views + +`pg_stat_monitor` provides the following views: +* `pg_stat_monitor` is the view where statistics data is presented. +* `pg_stat_monitor_settings` view shows available configuration options which you can change. + +### `pg_stat_monitor` view + +The statistics gathered by the module are made available via the view named `pg_stat_monitor`. This view contains one row for each distinct combination of metrics and whether it is a top-level statement or not (up to the maximum number of distinct statements that the module can track). For details about available counters, refer to the [`pg_stat_monitor` view reference](https://github.com/percona/pg_stat_monitor/blob/master/docs/REFERENCE.md). + +The following are the primary keys for pg_stat_monitor: + +* `bucket`, +* `userid`, +* `dbid`, +* `client_ip`, +* `application_name`. + +A new row is created for each key in the `pg_stat_monitor` view. + +`pg_stat_monitor` inherits the metrics available in `pg_stat_statements`, plus provides additional ones. See the [`pg_stat_monitor` vs `pg_stat_statements` comparison](https://github.com/percona/pg_stat_monitor/blob/master/docs/REFERENCE.md) for details. + +For security reasons, only superusers and members of the `pg_read_all_stats` role are allowed to see the SQL text and `queryid` of queries executed by other users. Other users can see the statistics, however, if the view has been installed in their database. + +### pg_stat_monitor_settings view + +The `pg_stat_monitor_settings` view shows one row per `pg_stat_monitor` configuration parameter. It displays configuration parameter name, value, default value, description, minimum and maximum values, and whether a restart is required for a change in value to be effective. + +## Functions + +### pg_stat_monitor_reset() + +This function resets all the statistics and clears the view. Eventually, the function will delete all the previous data. + +### pg_stat_monitor_version() +This function provides the build version of `pg_stat_monitor` version. + +``` +postgres=# select pg_stat_monitor_version(); + pg_stat_monitor_version +------------------------- + devel +(1 row) ``` -### Configuration -Here is the complete list of configuration parameters. +### histogram(bucket id, query id) + +It is used to generate the histogram, you can refer to histogram sections. + +## Configuration + +Use the following command to view available configuration parameters in the `pg_stat_monitor_settings` view: + ```sql SELECT * FROM pg_stat_monitor_settings; - name | value | default_value | description | minimum | maximum | restart -------------------------------------------+--------+---------------+----------------------------------------------------------------------------------------------------------+---------+------------+--------- - pg_stat_monitor.pgsm_max | 100 | 100 | Sets the maximum size of shared memory in (MB) used for statement's metadata tracked by pg_stat_monitor. | 1 | 1000 | 1 - pg_stat_monitor.pgsm_query_max_len | 1024 | 1024 | Sets the maximum length of query. | 1024 | 2147483647 | 1 - pg_stat_monitor.pgsm_enable | 1 | 1 | Enable/Disable statistics collector. | 0 | 0 | 0 - pg_stat_monitor.pgsm_track_utility | 1 | 1 | Selects whether utility commands are tracked. | 0 | 0 | 0 - pg_stat_monitor.pgsm_normalized_query | 1 | 1 | Selects whether save query in normalized format. | 0 | 0 | 0 - pg_stat_monitor.pgsm_max_buckets | 10 | 10 | Sets the maximum number of buckets. | 1 | 10 | 1 - pg_stat_monitor.pgsm_bucket_time | 300 | 300 | Sets the time in seconds per bucket. | 1 | 2147483647 | 1 - pg_stat_monitor.pgsm_histogram_min | 0 | 0 | Sets the time in millisecond. | 0 | 2147483647 | 1 - pg_stat_monitor.pgsm_histogram_max | 100000 | 100000 | Sets the time in millisecond. | 10 | 2147483647 | 1 - pg_stat_monitor.pgsm_histogram_buckets | 10 | 10 | Sets the maximum number of histogram buckets | 2 | 2147483647 | 1 - pg_stat_monitor.pgsm_query_shared_buffer | 20 | 20 | Sets the maximum size of shared memory in (MB) used for query tracked by pg_stat_monitor. | 1 | 10000 | 1 - pg_stat_monitor.pgsm_overflow_target | 1 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | 1 - pg_stat_monitor.pgsm_track_planning | 0 | 1 | Selects whether planning statistics are tracked. | 0 | 0 | 0 -(13 rows) - - ``` -Some configuration parameters require the server restart and should be set before the server startup. These must be set in the ``postgresql.conf`` file. Other parameters do not require server restart and can be set permanently either in the ``postgresql.conf`` or from the client (``psql``). -The table below shows set up options for each configuration parameter and whether the server restart is required to apply the parameter's value: +To amend the `pg_stat_monitor` configuration, use the General Configuration Unit (GCU) system. Some configuration parameters require the server restart and should be set before the server startup. These must be set in the `postgresql.conf` file. Other parameters do not require server restart and can be set permanently either in the `postgresql.conf` or from the client (`psql`) using the SET or ALTER SYSTEM SET commands. +The following table shows setup options for each configuration parameter and whether the server restart is required to apply the parameter's value: -| Parameter Name                                |  postgresql.conf   | SET | ALTER SYSTEM SET  |  server restart   | configuration reload +| Parameter Name | postgresql.conf | SET | ALTER SYSTEM SET | server restart | configuration reload | ----------------------------------------------|--------------------|-----|-------------------|-------------------|--------------------- -| pg_stat_monitor.pgsm_max                      | :heavy_check_mark: | :x:                |:x:                |:heavy_check_mark: | :x: -| pg_stat_monitor.pgsm_query_max_len            | :heavy_check_mark: | :x:                |:x:                |:heavy_check_mark: | :x: -| pg_stat_monitor.pgsm_enable                   | :heavy_check_mark: | :x:                |:heavy_check_mark: |:x: | :x: -| pg_stat_monitor.pgsm_track_utility            | :heavy_check_mark: | :heavy_check_mark: |:heavy_check_mark: |:x: | :heavy_check_mark: -| pg_stat_monitor.pgsm_normalized_query         | :heavy_check_mark: | :heavy_check_mark: |:heavy_check_mark: |:x: | :heavy_check_mark: -| pg_stat_monitor.pgsm_max_buckets              | :heavy_check_mark: | :x:                |:x:                |:heavy_check_mark: | :heavy_check_mark: -| pg_stat_monitor.pgsm_bucket_time              | :heavy_check_mark: | :x:                |:x:                |:heavy_check_mark: | :x: -| pg_stat_monitor.pgsm_object_cache             | :heavy_check_mark: | :x:                |:x:                |:heavy_check_mark: | :x: -| pg_stat_monitor.pgsm_respose_time_lower_bound | :heavy_check_mark: | :x:                |:x:                |:heavy_check_mark: | :x: -| pg_stat_monitor.pgsm_respose_time_step        | :heavy_check_mark: | :x:                |:x:                |:heavy_check_mark: | :x: -| pg_stat_monitor.pgsm_query_shared_buffer      | :heavy_check_mark: | :x:                |:x:                |:heavy_check_mark: | :x: -  +| [pg_stat_monitor.pgsm_max](#pg-stat-monitorpgsm-max) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: +| [pg_stat_monitor.pgsm_query_max_len](#pg-stat-monitorpgsm-query-max-len) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: +| [pg_stat_monitor.pgsm_enable](#pg-stat-monitorpgsm-enable) | :heavy_check_mark: | :x: |:heavy_check_mark: |:x: | :x: +| [pg_stat_monitor.pgsm_track_utility](#pg-stat-monitorpgsm-track-utility) | :heavy_check_mark: | :heavy_check_mark: |:heavy_check_mark: |:x: | :heavy_check_mark: +| [pg_stat_monitor.pgsm_normalized_query](#pg-stat-monitorpgsm-normalized-query) | :heavy_check_mark: | :heavy_check_mark: |:heavy_check_mark: |:x: | :heavy_check_mark: +| [pg_stat_monitor.pgsm_max_buckets](#pg-stat-monitorpgsm-max-buckets) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :heavy_check_mark: +| [pg_stat_monitor.pgsm_bucket_time](#pg-stat-monitorpgsm-bucket-time) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: +| [pg_stat_monitor.pgsm_object_cache](#pg-stat-monitorpgsm-object-cache) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: +| [pg_stat_monitor.pgsm_histogram_min](#pg-stat-monitorpgsm-histogram-min) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: +| [pg_stat_monitor.pgsm_histogram_max](#pg-stat-monitorpgsm-histogram-max) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: +| [pg_stat_monitor.pgsm_histogram_buckets](#pg-stat-monitorpgsm-histogram-buckets) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: +| [pg_stat_monitor.pgsm_query_shared_buffer](#pg-stat-monitorpgsm-query-shared-buffer) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: +| [pg_stat_monitor.pgsm_overflow_target](#pg-stat-monitorpgsm-overflow-target) | :heavy_check_mark: | :x: | :x: | :heavy_check_mark: | :x: | +| [pg_stat_monitor.pgsm_enable_query_plan](pg-stat-monitorpgsm-enable-query-plan) | :heavy_check_mark: | :x: | :x: | :heavy_check_mark: | :x: | +| [pg_stat_monitor.pgsm_track_planning](#pg-stat-monitorpgsm-track-planning) | :heavy_check_mark: | :x: | :x: | :heavy_check_mark: | :x: | + #### Parameters description: -- **pg_stat_monitor.pgsm_max**: This parameter defines the limit of shared memory for ``pg_stat_monitor``. This memory is used by buckets in a circular manner. The memory is divided between the buckets equally, at the start of the PostgreSQL. -- **pg_stat_monitor.pgsm_query_max_len**: Sets the maximum size of the query. This parameter can only be set at the start of PostgreSQL. For long queries, the query is truncated to that particular length. This is to avoid unnecessary usage of shared memory. -- **pg_stat_monitor.pgsm_enable**: This parameter enables or disables the monitoring. "Disable" means that ``pg_stat_monitor`` will not collect the statistics for the whole cluster. -- **pg_stat_monitor.pgsm_track_utility**: This parameter controls whether utility commands are tracked by the module. Utility commands are all those other than ``SELECT``, ``INSERT``, ``UPDATE``, and ``DELETE``. -- **pg_stat_monitor.pgsm_normalized_query**: By default, the query shows the actual parameter instead of the placeholder. It is quite useful when users want to use that query and try to run that query to check the abnormalities. But in most cases users like the queries with a placeholder. This parameter is used to toggle between the two said options. -- **pg_stat_monitor.pgsm_max_buckets**: ``pg_stat_monitor`` accumulates the information in the form of buckets. All the aggregated information is bucket based. This parameter is used to set the number of buckets the system can have. For example, if this parameter is set to 2, then the system will create two buckets. First, the system will add all the information into the first bucket. After its lifetime (defined in the  pg_stat_monitor.pgsm_bucket_time parameter) expires, it will switch to the second bucket,  reset all the counters and repeat the process. -- **pg_stat_monitor.pgsm_bucket_time**: This parameter is used to set the lifetime of the bucket. System switches between buckets on the basis of ``pg_stat_monitor.pgsm_bucket_time``. -- **pg_stat_monitor.pgsm_respose_time_lower_bound**: ``pg_stat_monitor`` also stores the execution time histogram. This parameter is used to set the lower bound of the histogram. -- **pg_stat_monitor.pgsm_respose_time_step:** This parameter is used to set the steps for the histogram. +##### pg_stat_monitor.pgsm_max + +Values: +- Min: 1 +- Max: 1000 +- Default: 100 + +This parameter defines the limit of shared memory (in MB) for ``pg_stat_monitor``. This memory is used by buckets in a circular manner. The memory is divided between the buckets equally, at the start of the PostgreSQL. Requires the server restart. + +##### pg_stat_monitor.pgsm_query_max_len + +Values: +- Min: 1024 +- Max: 2147483647 +- Default: 1024 + +Sets the maximum size of the query. This parameter can only be set at the start of PostgreSQL. For long queries, the query is truncated to that particular length. This is to avoid unnecessary usage of shared memory. Requires the server restart. + +##### pg_stat_monitor.pgsm_enable + +Type: boolean. Default: 1 + +Enables or disables the monitoring. "Disable" (0) means that ``pg_stat_monitor`` will not collect the statistics for the whole cluster. + +##### pg_stat_monitor.pgsm_track_utility + +Type: boolean. Default: 1 + +This parameter controls whether utility commands are tracked by the module. Utility commands are all those other than ``SELECT``, ``INSERT``, ``UPDATE``, and ``DELETE``. + +##### pg_stat_monitor.pgsm_normalized_query + +Type: boolean. Default: 1 + +By default, the query shows the actual parameter instead of the placeholder. It is quite useful when users want to use that query and try to run that query to check the abnormalities. But in most cases users like the queries with a placeholder. This parameter is used to toggle between the two said options. + +##### pg_stat_monitor.pgsm_max_buckets + +Values: +- Min: 1 +- Max: 10 +- Default: 10 + +``pg_stat_monitor`` accumulates the information in the form of buckets. All the aggregated information is bucket based. This parameter is used to set the number of buckets the system can have. For example, if this parameter is set to 2, then the system will create two buckets. First, the system will add all the information into the first bucket. After its lifetime (defined in the [pg_stat_monitor.pgsm_bucket_time](#pg-stat-monitorpgsm-bucket-time) parameter) expires, it will switch to the second bucket, reset all the counters and repeat the process. + +Requires the server restart. + +#### pg_stat_monitor.pgsm_bucket_time + +Values: +- Min: 1 +- Max: 2147483647 +- Default: 300 + +This parameter is used to set the lifetime of the bucket. System switches between buckets on the basis of [pg_stat_monitor.pgsm_bucket_time](#pg-stat-monitorpgsm-bucket-time). + +Requires the server restart. + +##### pg_stat_monitor.pgsm_histogram_min + +Values: +- Min: 0 +- Max: 2147483647 +- Default: 0 + +``pg_stat_monitor`` also stores the execution time histogram. This parameter is used to set the lower bound of the histogram (in ms). + +Requires the server restart. + +##### pg_stat_monitor.pgsm_histogram_max + +Values: +- Min: 10 +- Max: 2147483647 +- Default: 100000 + +This parameter sets the upper bound of the execution time histogram (in ms). Requires the server restart. + +##### pg_stat_monitor.pgsm_histogram_buckets + +Values: +- Min: 2 +- Max: 2147483647 +- Default: 10 + +This parameter sets the maximum number of histogram buckets. Requires the server restart. + +##### pg_stat_monitor.pgsm_query_shared_buffer + +Values: +- Min: 1 +- Max: 10000 +- Default: 20 + +This parameter defines the shared memory limit (in MB) allocated for a query tracked by ``pg_stat_monitor``. Requires the server restart. + +##### pg_stat_monitor.pgsm_overflow_target + +Type: boolean. Default: 1 + +Sets the overflow target for the `pg_stat_monitor`. Requires the server restart. + +#### pg_stat_monitor.pgsm_enable_query_plan + +Type: boolean. Default: 1 + +Enables or disables query plan monitoring. When the `pgsm_enable_query_plan` is disabled (0), the query plan will not be captured by `pg_stat_monitor`. Enabling it may adversely affect the database performance. Requires the server restart. -### Usage +##### pg_stat_monitor.pgsm_track_planning -pg_stat_monitor extension contains a view called pg_stat_monitor, which contains all the monitoring information. Find the list of columns in pg_stat_monitor view in the following table. The table also shows whether a particular column is available in pg_stat_statements. +Type: boolean. Default: 0 +This parameter instructs ``pg_stat_monitor`` to monitor query planning statistics. Requires the server restart. -|      Column        |           Type           | pg_stat_monitor      | pg_stat_statements -|--------------------|--------------------------|----------------------|------------------ - bucket              | integer                  | :heavy_check_mark:  | :x: - bucket_start_time   | timestamp with time zone | :heavy_check_mark:  | :x: - userid              | oid                      | :heavy_check_mark:  | :heavy_check_mark: - dbid                | oid                      | :heavy_check_mark:  | :heavy_check_mark: - client_ip           | inet                     | :heavy_check_mark:  | :x: - queryid             | text                     | :heavy_check_mark:  | :heavy_check_mark: - planid             | text                     | :heavy_check_mark:  | :x: - query_plan         | text                     | :heavy_check_mark:  | :x: - top_query           | text                     | :heavy_check_mark:  | :x: - query               | text                     | :heavy_check_mark:  | :heavy_check_mark: - application_name    | text                     | :heavy_check_mark:  | :x: - relations           | text[]                   | :heavy_check_mark:  | :x: - cmd_type            | text[]                   | :heavy_check_mark:  | :x: - elevel              | integer                  | :heavy_check_mark:  | :x: - sqlcode             | integer                  | :heavy_check_mark:  | :x: - message             | text                     | :heavy_check_mark:  | :x: - plans               | bigint                   | :heavy_check_mark:  | :heavy_check_mark: - plan_total_time     | double precision         | :heavy_check_mark:  | :heavy_check_mark: - plan_min_timei      | double precision         | :heavy_check_mark:  | :heavy_check_mark: - plan_max_time       | double precision         | :heavy_check_mark:  | :heavy_check_mark: - plan_mean_time      | double precision         | :heavy_check_mark:  | :heavy_check_mark: - plan_stddev_time    | double precision         | :heavy_check_mark:  | :heavy_check_mark: - calls               | bigint                   | :heavy_check_mark:  | :heavy_check_mark: - total_time          | double precision         | :heavy_check_mark:  | :heavy_check_mark: - min_time            | double precision         | :heavy_check_mark:  | :heavy_check_mark: - max_time            | double precision         | :heavy_check_mark:  | :heavy_check_mark: - mean_time           | double precision         | :heavy_check_mark:  | :heavy_check_mark: - stddev_time         | double precision         | :heavy_check_mark:  | :heavy_check_mark: - rows_retrieved      | bigint                   | :heavy_check_mark:  | :heavy_check_mark: - shared_blks_hit     | bigint                   | :heavy_check_mark:  | :heavy_check_mark: - shared_blks_read    | bigint                   | :heavy_check_mark:  | :heavy_check_mark: - shared_blks_dirtied | bigint                   | :heavy_check_mark:  | :heavy_check_mark: - shared_blks_written | bigint                   | :heavy_check_mark:  | :heavy_check_mark: - local_blks_hit      | bigint                   | :heavy_check_mark:  | :heavy_check_mark: - local_blks_read     | bigint                   | :heavy_check_mark:  | :heavy_check_mark: - local_blks_dirtied  | bigint                   | :heavy_check_mark:  | :heavy_check_mark: - local_blks_written  | bigint                   | :heavy_check_mark:  | :heavy_check_mark: - temp_blks_read      | bigint                   | :heavy_check_mark:  | :heavy_check_mark: - temp_blks_written   | bigint                   | :heavy_check_mark:  | :heavy_check_mark: - blk_read_time       | double precision         | :heavy_check_mark:  | :heavy_check_mark: - blk_write_time      | double precision         | :heavy_check_mark:  | :heavy_check_mark: - resp_calls          | text[]                   | :heavy_check_mark:  | :x: - cpu_user_time       | double precision         | :heavy_check_mark:  | :x: - cpu_sys_time        | double precision         | :heavy_check_mark:  | :x: - wal_records | bigint | :heavy_check_mark:  | :heavy_check_mark: - wal_fpi | bigint | :heavy_check_mark:  | :heavy_check_mark: - wal_bytes | numeric | :heavy_check_mark:  | :heavy_check_mark: - state_code | bigint | :heavy_check_mark:  | :x: - state           | text                   | :heavy_check_mark:  | :x: +## Usage examples +Note that the column names differ depending on the PostgreSQL version you are using. The following usage examples are provided for PostgreSQL version 13. +For versions 11 and 12, please consult the [pg_stat_monitor reference](https://github.com/percona/pg_stat_monitor/blob/master/docs/REFERENCE.md). - -The following are some key features of pg_stat_monitor and usage examples. - -#### Buckets - -**`bucket`**: Accumulates the statistics per bucket. All the information and aggregate reset for each bucket. The bucket will be a number showing the number of buckets for which this record belongs. - -**`bucket_start_time`**: shows the start time of the bucket. +### Querying buckets ```sql postgres=# select bucket, bucket_start_time, query,calls from pg_stat_monitor order by bucket; - -bucket | bucket_start_time | query | calls ---------+---------------------+---------------------------------------------------------------------------------------------------------------+------- - 3 | 11-01-2021 17:30:45 | copy pgbench_accounts from stdin | 1 - 3 | 11-01-2021 17:30:45 | alter table pgbench_accounts add primary key (aid) | 1 - 3 | 11-01-2021 17:30:45 | vacuum analyze pgbench_accounts | 1 - 3 | 11-01-2021 17:30:45 | vacuum analyze pgbench_tellers | 1 - 3 | 11-01-2021 17:30:45 | insert into pgbench_branches(bid,bbalance) values($1,$2) | 100 - 5 | 11-01-2021 17:31:15 | vacuum analyze pgbench_branches | 1 - 5 | 11-01-2021 17:31:15 | copy pgbench_accounts from stdin | 1 - 5 | 11-01-2021 17:31:15 | vacuum analyze pgbench_tellers | 1 - 5 | 11-01-2021 17:31:15 | commit | 1 - 6 | 11-01-2021 17:31:30 | alter table pgbench_branches add primary key (bid) | 1 - 6 | 11-01-2021 17:31:30 | vacuum analyze pgbench_accounts | 1 +-[ RECORD 1 ]-----+------------------------------------------------------------------------------------ +bucket | 0 +bucket_start_time | 2021-10-22 11:10:00 +query | select bucket, bucket_start_time, query,calls from pg_stat_monitor order by bucket; +calls | 1 ``` -#### Query Information +The `bucket` parameter shows the number of a bucket for which a given record belongs. +The `bucket_start_time` shows the start time of the bucket. +`query` shows the actual query text. +`calls` shows how many times a given query was called. -**`userid`**: An ID of the user to whom that query belongs. pg_stat_monitor collects queries from all the users and uses the `userid` to segregate the queries based on different users. +### Query information -**`dbid`**: The database ID of the query. pg_stat_monitor accumulates queries from all the databases; therefore, this column is used to identify the database. +**Example 1: Shows the usename, database name, unique queryid hash, query, and the total number of calls of that query.** -**`queryid`**:  pg_stat_monitor generates a unique ID for each query (queryid). - -**`query`**: The query column contains the actual text of the query. This parameter depends on the **`pg_stat_monitor.pgsm_normalized_query`** configuration parameters, in which format to show the query. - -**`calls`**: Number of calls of that particular query. - - -##### Example 1: Shows the usename, database name, unique queryid hash, query, and the total number of calls of that query. ```sql postgres=# SELECT userid, datname, queryid, substr(query,0, 50) AS query, calls FROM pg_stat_monitor; - userid | datname | queryid | query | calls + userid | datname | queryid | query | calls ---------+----------+------------------+---------------------------------------------------+------- vagrant | postgres | 939C2F56E1F6A174 | END | 561 vagrant | postgres | 2A4437C4905E0E23 | SELECT abalance FROM pgbench_accounts WHERE aid = | 561 @@ -186,219 +341,43 @@ postgres=# SELECT userid, datname, queryid, substr(query,0, 50) AS query, calls ``` -##### Example 4: Shows the connected application_name. +**Example 2: Shows the connected application details.** ```sql -SELECT application_name, query FROM pg_stat_monitor; - application_name |                                                query                                                 -------------------+------------------------------------------------------------------------------------------------------ - pgbench          | UPDATE pgbench_branches SET bbalance = bbalance + $1 WHERE bid = $2 - pgbench          | UPDATE pgbench_accounts SET abalance = abalance + $1 WHERE aid = $2 - pgbench          | vacuum pgbench_tellers - pgbench          | SELECT abalance FROM pgbench_accounts WHERE aid = $1 - pgbench          | END - pgbench          | select count(*) from pgbench_branches - pgbench          | BEGIN - pgbench          | INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES ($1, $2, $3, $4, CURRENT_TIMESTAMP) - psql             | select application_name, query from pg_stat_monitor - pgbench          | vacuum pgbench_branches - psql             | select application_name query from pg_stat_monitor - pgbench          | truncate pgbench_history - pgbench          | UPDATE pgbench_tellers SET tbalance = tbalance + $1 WHERE tid = $2 +postgres=# SELECT application_name, client_ip, substr(query,0,100) as query FROM pg_stat_monitor; + application_name | client_ip | query +------------------+-----------+----------------------------------------------------------------------------------------------------- + pgbench | 127.0.0.1 | truncate pgbench_history + pgbench | 127.0.0.1 | SELECT abalance FROM pgbench_accounts WHERE aid = $1 + pgbench | 127.0.0.1 | UPDATE pgbench_accounts SET abalance = abalance + $1 WHERE aid = $2 + pgbench | 127.0.0.1 | BEGIN; + pgbench | 127.0.0.1 | INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES ($1, $2, $3, $4, CURRENT_TIMESTAMP + pgbench | 127.0.0.1 | END; + pgbench | 127.0.0.1 | vacuum pgbench_branches + pgbench | 127.0.0.1 | UPDATE pgbench_tellers SET tbalance = tbalance + $1 WHERE tid = $2 + pgbench | 127.0.0.1 | vacuum pgbench_tellers + pgbench | 127.0.0.1 | UPDATE pgbench_branches SET bbalance = bbalance + $1 WHERE bid = $2 + pgbench | 127.0.0.1 | select o.n, p.partstrat, pg_catalog.count(i.inhparent) from pg_catalog.pg_class as c join pg_catalo + psql | 127.0.0.1 | SELECT application_name, client_ip, substr(query,$1,$2) as query FROM pg_stat_monitor + pgbench | 127.0.0.1 | select count(*) from pgbench_branches (13 rows) + ``` -#### Error Messages / Error Codes and Error Level - -**`elevel`**, **`sqlcode`**,**`message`**,: error level / sql code and  log/warning/ error message +### Query timing information ```sql -SELECT substr(query,0,50) AS query, decode_error_level(elevel) AS elevel,sqlcode, calls, substr(message,0,50) message -FROM pg_stat_monitor; -                       query                       | elevel | sqlcode | calls |                      message                       ----------------------------------------------------+--------+---------+-------+--------------------------------------------------- - select substr(query,$1,$2) as query, decode_error |        |       0 |     1 | - select bucket,substr(query,$1,$2),decode_error_le |        |       0 |     3 | -                                                   | LOG    |       0 |     1 | database system is ready to accept connections - select 1/0;                                       | ERROR  |     130 |     1 | division by zero -                                                   | LOG    |       0 |     1 | database system was shut down at 2020-11-11 11:37 - select $1/$2                                      |        |       0 |     1 | -(6 rows) -``` - -#### Query Timing Information - -**`total_time`**,  **`min_time`**, **`max_time`**, **`mean_time`**: The total / minimum / maximum and mean time spent for the same query. - - -``` -SELECT userid,  total_time, min_time, max_time, mean_time, query FROM pg_stat_monitor; - userid |     total_time     |      min_time      |      max_time      |     mean_time      |                              query                               +SELECT userid, total_time, min_time, max_time, mean_time, query FROM pg_stat_monitor; + userid | total_time | min_time | max_time | mean_time | query --------+--------------------+--------------------+--------------------+--------------------+------------------------------------------------------------------ -     10 |               0.14 |               0.14 |               0.14 |               0.14 | select * from pg_stat_monitor_reset() -     10 |               0.19 |               0.19 |               0.19 |               0.19 | select userid,  dbid, queryid, query from pg_stat_monitor -     10 |               0.30 |               0.13 |               0.16 |               0.15 | select bucket, bucket_start_time, query from pg_stat_monitor -     10 |               0.29 |               0.29 |               0.29 |               0.29 | select userid,  dbid, queryid, query, calls from pg_stat_monitor -     10 |           11277.79 |           11277.79 |           11277.79 |           11277.79 | SELECT * FROM foo + 10 | 0.14 | 0.14 | 0.14 | 0.14 | select * from pg_stat_monitor_reset() + 10 | 0.19 | 0.19 | 0.19 | 0.19 | select userid, dbid, queryid, query from pg_stat_monitor + 10 | 0.30 | 0.13 | 0.16 | 0.15 | select bucket, bucket_start_time, query from pg_stat_monitor + 10 | 0.29 | 0.29 | 0.29 | 0.29 | select userid, dbid, queryid, query, calls from pg_stat_monitor + 10 | 11277.79 | 11277.79 | 11277.79 | ``` -#### Client IP address - -**`client_ip`**: The IP address of the client that originated the query. - -```sql -SELECT userid::regrole, datname, substr(query,0, 50) AS query, calls, client_ip -FROM pg_stat_monitor, pg_database -WHERE dbid = oid; -userid  | datname  |                       query                       | calls | client_ip ----------+----------+---------------------------------------------------+-------+----------- - vagrant | postgres | UPDATE pgbench_branches SET bbalance = bbalance + |  1599 | 10.0.2.15 - vagrant | postgres | select userid::regrole, datname, substr(query,$1, |     5 | 10.0.2.15 - vagrant | postgres | UPDATE pgbench_accounts SET abalance = abalance + |  1599 | 10.0.2.15 - vagrant | postgres | select userid::regrole, datname, substr(query,$1, |     1 | 127.0.0.1 - vagrant | postgres | vacuum pgbench_tellers                            |     1 | 10.0.2.15 - vagrant | postgres | SELECT abalance FROM pgbench_accounts WHERE aid = |  1599 | 10.0.2.15 - vagrant | postgres | END                                               |  1599 | 10.0.2.15 - vagrant | postgres | select count(*) from pgbench_branches             |     1 | 10.0.2.15 - vagrant | postgres | BEGIN                                             |  1599 | 10.0.2.15 - vagrant | postgres | INSERT INTO pgbench_history (tid, bid, aid, delta |  1599 | 10.0.2.15 - vagrant | postgres | vacuum pgbench_branches                           |     1 | 10.0.2.15 - vagrant | postgres | truncate pgbench_history                          |     1 | 10.0.2.15 - vagrant | postgres | UPDATE pgbench_tellers SET tbalance = tbalance +  |  1599 | 10.0.2.15 -``` - -#### Call Timings Histogram - -**`resp_calls`**: Call histogram - -```sql -SELECT resp_calls, query FROM pg_stat_monitor; -                    resp_calls                    |                 query                                         ---------------------------------------------------+---------------------------------------------- -{1," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 0"} | select client_ip, query from pg_stat_monitor -{3," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 1"} | select * from pg_stat_monitor_reset() -{3," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 1"} | SELECT * FROM foo - -postgres=# SELECT * FROM histogram(0, 'F44CD1B4B33A47AF') AS a(range TEXT, freq INT, bar TEXT); - range | freq | bar ---------------------+------+-------------------------------- - (0 - 3)} | 2 | ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■ - (3 - 10)} | 0 | - (10 - 31)} | 1 | ■■■■■■■■■■■■■■■ - (31 - 100)} | 0 | - (100 - 316)} | 0 | - (316 - 1000)} | 0 | - (1000 - 3162)} | 0 | - (3162 - 10000)} | 0 | - (10000 - 31622)} | 0 | - (31622 - 100000)} | 0 | -(10 rows) -``` - -There are 10 timebase buckets of the time **`pg_stat_monitor.pgsm_respose_time_step`** in the field ``resp_calls``. The value in the field shows how many queries run in that period of time. - - -#### Object Information. - -**`relations`**: The list of tables involved in the query - -##### Example 1: List all the table names involved in the query. -```sql -postgres=# SELECT relations,query FROM pg_stat_monitor; - relations | query --------------------------------+------------------------------------------------------------------------------------------------------ - | END - {pgbench_accounts} | SELECT abalance FROM pgbench_accounts WHERE aid = $1 - | vacuum pgbench_branches - {pgbench_branches} | select count(*) from pgbench_branches - {pgbench_accounts} | UPDATE pgbench_accounts SET abalance = abalance + $1 WHERE aid = $2 - | truncate pgbench_history - {pgbench_history} | INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES ($1, $2, $3, $4, CURRENT_TIMESTAMP) - {pg_stat_monitor,pg_database} | SELECT relations query FROM pg_stat_monitor - | vacuum pgbench_tellers - | BEGIN - {pgbench_tellers} | UPDATE pgbench_tellers SET tbalance = tbalance + $1 WHERE tid = $2 - {pgbench_branches} | UPDATE pgbench_branches SET bbalance = bbalance + $1 WHERE bid = $2 -(12 rows) -``` - -##### Example 2: List all the views and the name of the table in the view. Here we have a view "test_view" -```sql -\d+ test_view -                          View "public.test_view" - Column |  Type   | Collation | Nullable | Default | Storage | Description ---------+---------+-----------+----------+---------+---------+------------- - foo_a  | integer |           |          |         | plain   | - bar_a  | integer |           |          |         | plain   | -View definition: - SELECT f.a AS foo_a, -    b.a AS bar_a -   FROM foo f, -    bar b; -``` - -Now when we query the pg_stat_monitor, it will show the view name and also all the table names in the view. -```sql -SELECT relations, query FROM pg_stat_monitor; -      relations      |                                                query                                                 ----------------------+------------------------------------------------------------------------------------------------------ - {test_view,foo,bar} | select * from test_view - {foo,bar}           | select * from foo,bar -(2 rows) -``` - -#### Query command Type (SELECT, INSERT, UPDATE OR DELETE) - -**`cmd_type`**: List the command type of the query. - -```sql -postgres=# SELECT bucket, substr(query,0, 50) AS query, cmd_type FROM pg_stat_monitor WHERE elevel = 0; - bucket | query | cmd_type ---------+---------------------------------------------------+---------- - 4 | END | - 4 | SELECT abalance FROM pgbench_accounts WHERE aid = | SELECT - 4 | vacuum pgbench_branches | - 4 | select count(*) from pgbench_branches | SELECT - 4 | UPDATE pgbench_accounts SET abalance = abalance + | UPDATE - 4 | truncate pgbench_history | - 4 | INSERT INTO pgbench_history (tid, bid, aid, delta | INSERT - 5 | SELECT relations query FROM pg_stat_monitor | SELECT - 9 | SELECT bucket, substr(query,$1, $2) AS query, cmd | - 4 | vacuum pgbench_tellers | - 4 | BEGIN | - 5 | SELECT relations,query FROM pg_stat_monitor | SELECT - 4 | UPDATE pgbench_tellers SET tbalance = tbalance + | UPDATE - 4 | UPDATE pgbench_branches SET bbalance = bbalance + | UPDATE -(14 rows) -``` - -#### Function Execution Tracking - -**`top_queryid`**: Outer layer caller's query id. - -```sql -CREATE OR REPLACE function add2(int, int) RETURNS int as -$$ -BEGIN - return (select $1 + $2); -END; -$$ language plpgsql; - -SELECT add2(1,2); - add2 ------ - 3 -(1 row) - -postgres=# SELECT queryid, top_queryid, query, top_query FROM pg_stat_monitor; - queryid | top_queryid | query. | top_query -------------------+------------------+-------------------------------------------------------------------------+------------------- - 3408CA84B2353094 | | select add2($1,$2) | - 762B99349F6C7F31 | 3408CA84B2353094 | SELECT (select $1 + $2) | select add2($1,$2) -(2 rows) -``` - -#### Monitor Query Execution Plan. +### Query execution plan ```sql postgres=# SELECT substr(query,0,50), query_plan from pg_stat_monitor limit 10; @@ -445,9 +424,13 @@ postgres=# SELECT substr(query,0,50), query_plan from pg_stat_monitor limit 10; vacuum pgbench_tellers | UPDATE pgbench_accounts SET abalance = abalance + | (10 rows) - ``` -#### SQL Commenter / tags. + +The `plan` column does not contain costing, width and other values. This is an expected behavior as each row is an accumulation of statistics based on `plan` and amongst other key columns. Plan is only available when the `pgsm_enable_query_plan` configuration parameter is enabled. + +### Query metadata + +The `comments` column contains any text wrapped in `“/*”` and `“*/”` comment tags. The `pg_stat_monitor` extension picks up these comments and makes them available in the comments column. Please note that only the latest comment value is preserved per row. The comments may be put in any format that can be parsed by a tool. ```sql CREATE EXTENSION hstore; @@ -459,68 +442,220 @@ EXCEPTION WHEN OTHERS THEN END; $$ LANGUAGE plpgsql STRICT; postgres=# SELECT 1 AS num /* { "application", java_app, "real_ip", 192.168.1.1} */; - num + num ----- 1 (1 row) postgres=# SELECT 1 AS num1,2 AS num2 /* { "application", java_app, "real_ip", 192.168.1.2} */; - num1 | num2 + num1 | num2 ------+------ 1 | 2 (1 row) postgres=# SELECT 1 AS num1,2 AS num2, 3 AS num3 /* { "application", java_app, "real_ip", 192.168.1.3} */; - num1 | num2 | num3 + num1 | num2 | num3 ------+------+------ 1 | 2 | 3 (1 row) postgres=# SELECT 1 AS num1,2 AS num2, 3 AS num3, 4 AS num4 /* { "application", psql_app, "real_ip", 192.168.1.3} */; - num1 | num2 | num3 | num4 + num1 | num2 | num3 | num4 ------+------+------+------ 1 | 2 | 3 | 4 (1 row) postgres=# select query, text_to_hstore(comments) as comments_tags from pg_stat_monitor; - query | comments_tags + query | comments_tags ---------------------------------------------------------------------------------------------------------------+----------------------------------------------------- SELECT $1 AS num /* { "application", psql_app, "real_ip", 192.168.1.3) */ | "real_ip"=>"192.168.1.1", "application"=>"java_app" - SELECT pg_stat_monitor_reset(); | - select query, comments, text_to_hstore(comments) from pg_stat_monitor; | + SELECT pg_stat_monitor_reset(); | + select query, comments, text_to_hstore(comments) from pg_stat_monitor; | SELECT $1 AS num1,$2 AS num2, $3 AS num3 /* { "application", java_app, "real_ip", 192.168.1.3} */ | "real_ip"=>"192.168.1.3", "application"=>"java_app" - select query, text_to_hstore(comments) as comments_tags from pg_stat_monitor; | + select query, text_to_hstore(comments) as comments_tags from pg_stat_monitor; | SELECT $1 AS num1,$2 AS num2 /* { "application", java_app, "real_ip", 192.168.1.2} */ | "real_ip"=>"192.168.1.2", "application"=>"java_app" SELECT $1 AS num1,$2 AS num2, $3 AS num3, $4 AS num4 /* { "application", psql_app, "real_ip", 192.168.1.3} */ | "real_ip"=>"192.168.1.3", "application"=>"psql_app" (7 rows) postgres=# select query, text_to_hstore(comments)->'application' as application_name from pg_stat_monitor; - query | application_name + query | application_name ---------------------------------------------------------------------------------------------------------------+---------- SELECT $1 AS num /* { "application", psql_app, "real_ip", 192.168.1.3) */ | java_app - SELECT pg_stat_monitor_reset(); | - select query, text_to_hstore(comments)->"real_ip" as comments_tags from pg_stat_monitor; | - select query, text_to_hstore(comments)->$1 from pg_stat_monitor | - select query, text_to_hstore(comments) as comments_tags from pg_stat_monitor; | - select query, text_to_hstore(comments)->"application" as comments_tags from pg_stat_monitor; | + SELECT pg_stat_monitor_reset(); | + select query, text_to_hstore(comments)->"real_ip" as comments_tags from pg_stat_monitor; | + select query, text_to_hstore(comments)->$1 from pg_stat_monitor | + select query, text_to_hstore(comments) as comments_tags from pg_stat_monitor; | + select query, text_to_hstore(comments)->"application" as comments_tags from pg_stat_monitor; | SELECT $1 AS num1,$2 AS num2 /* { "application", java_app, "real_ip", 192.168.1.2} */ | java_app SELECT $1 AS num1,$2 AS num2, $3 AS num3 /* { "application", java_app, "real_ip", 192.168.1.3} */ | java_app - select query, comments, text_to_hstore(comments) from pg_stat_monitor; | + select query, comments, text_to_hstore(comments) from pg_stat_monitor; | SELECT $1 AS num1,$2 AS num2, $3 AS num3, $4 AS num4 /* { "application", psql_app, "real_ip", 192.168.1.3} */ | psql_app (10 rows) postgres=# select query, text_to_hstore(comments)->'real_ip' as real_ip from pg_stat_monitor; - query | real_ip + query | real_ip ---------------------------------------------------------------------------------------------------------------+------------- SELECT $1 AS num /* { "application", psql_app, "real_ip", 192.168.1.3) */ | 192.168.1.1 - SELECT pg_stat_monitor_reset(); | - select query, text_to_hstore(comments)->"real_ip" as comments_tags from pg_stat_monitor; | - select query, text_to_hstore(comments)->$1 from pg_stat_monitor | - select query, text_to_hstore(comments) as comments_tags from pg_stat_monitor; | - select query, text_to_hstore(comments)->"application" as comments_tags from pg_stat_monitor; | + SELECT pg_stat_monitor_reset(); | + select query, text_to_hstore(comments)->"real_ip" as comments_tags from pg_stat_monitor; | + select query, text_to_hstore(comments)->$1 from pg_stat_monitor | + select query, text_to_hstore(comments) as comments_tags from pg_stat_monitor; | + select query, text_to_hstore(comments)->"application" as comments_tags from pg_stat_monitor; | SELECT $1 AS num1,$2 AS num2 /* { "application", java_app, "real_ip", 192.168.1.2} */ | 192.168.1.2 SELECT $1 AS num1,$2 AS num2, $3 AS num3 /* { "application", java_app, "real_ip", 192.168.1.3} */ | 192.168.1.3 - select query, comments, text_to_hstore(comments) from pg_stat_monitor; | + select query, comments, text_to_hstore(comments) from pg_stat_monitor; | SELECT $1 AS num1,$2 AS num2, $3 AS num3, $4 AS num4 /* { "application", psql_app, "real_ip", 192.168.1.3} */ | 192.168.1.3 (10 rows) ``` + +### Query type filtering + +``pg_stat_monitor`` monitors queries per type (SELECT, INSERT, UPDATE OR DELETE) and classifies them accordingly in the ``cmd_type`` column thus reducing your efforts. + +```sql +postgres=# SELECT bucket, substr(query,0, 50) AS query, cmd_type FROM pg_stat_monitor WHERE elevel = 0; + bucket | query | cmd_type +--------+---------------------------------------------------+---------- + 4 | END | + 4 | SELECT abalance FROM pgbench_accounts WHERE aid = | SELECT + 4 | vacuum pgbench_branches | + 4 | select count(*) from pgbench_branches | SELECT + 4 | UPDATE pgbench_accounts SET abalance = abalance + | UPDATE + 4 | truncate pgbench_history | + 4 | INSERT INTO pgbench_history (tid, bid, aid, delta | INSERT + 5 | SELECT relations query FROM pg_stat_monitor | SELECT + 9 | SELECT bucket, substr(query,$1, $2) AS query, cmd | + 4 | vacuum pgbench_tellers | + 4 | BEGIN | + 5 | SELECT relations,query FROM pg_stat_monitor | SELECT + 4 | UPDATE pgbench_tellers SET tbalance = tbalance + | UPDATE + 4 | UPDATE pgbench_branches SET bbalance = bbalance + | UPDATE +(14 rows) +``` + +### Queries terminated with errors + +```sql +SELECT substr(query,0,50) AS query, decode_error_level(elevel) AS elevel,sqlcode, calls, substr(message,0,50) message +FROM pg_stat_monitor; + query | elevel | sqlcode | calls | message +---------------------------------------------------+--------+---------+-------+--------------------------------------------------- + select substr(query,$1,$2) as query, decode_error | | 0 | 1 | + select bucket,substr(query,$1,$2),decode_error_le | | 0 | 3 | + | LOG | 0 | 1 | database system is ready to accept connections + select 1/0; | ERROR | 130 | 1 | division by zero + | LOG | 0 | 1 | database system was shut down at 2020-11-11 11:37 + select $1/$2 | | 0 | 1 | +(6 rows) + 11277.79 | SELECT * FROM foo +``` + +### Histogram + +Histogram (the `resp_calls` parameter) provides a visual representation of query performance. With the help of the histogram function, you can view a timing/calling data histogram in response to a SQL query. + + +```sql +SELECT resp_calls, query FROM pg_stat_monitor; +                    resp_calls                    |                 query                                         +--------------------------------------------------+---------------------------------------------- +{1," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 0"} | select client_ip, query from pg_stat_monitor +{3," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 1"} | select * from pg_stat_monitor_reset() +{3," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 1"} | SELECT * FROM foo + +postgres=# SELECT * FROM histogram(0, 'F44CD1B4B33A47AF') AS a(range TEXT, freq INT, bar TEXT); + range | freq | bar +--------------------+------+-------------------------------- + (0 - 3)} | 2 | ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■ + (3 - 10)} | 0 | + (10 - 31)} | 1 | ■■■■■■■■■■■■■■■ + (31 - 100)} | 0 | + (100 - 316)} | 0 | + (316 - 1000)} | 0 | + (1000 - 3162)} | 0 | + (3162 - 10000)} | 0 | + (10000 - 31622)} | 0 | + (31622 - 100000)} | 0 | +(10 rows) +``` + +There are 10 time based buckets of the time **`pg_stat_monitor.pgsm_respose_time_step`** in the field ``resp_calls``. The value in the field shows how many queries run in that period of time. + +### Top query tracking + +In the following example we create a function `add2` that adds one parameter value to another one and call this function to calculate 1+2. + + +```sql +CREATE OR REPLACE function add2(int, int) RETURNS int as +$$ +BEGIN + return (select $1 + $2); +END; +$$ language plpgsql; + +SELECT add2(1,2); + add2 +----- + 3 +(1 row) + +The ``pg_stat_monitor`` view shows all executed queries and shows the very first query in a row - calling the `add2` function. + +postgres=# SELECT queryid, top_queryid, query, top_query FROM pg_stat_monitor; + queryid | top_queryid | query. | top_query +------------------+------------------+-------------------------------------------------------------------------+------------------- + 3408CA84B2353094 | | select add2($1,$2) | + 762B99349F6C7F31 | 3408CA84B2353094 | SELECT (select $1 + $2) | select add2($1,$2) +(2 rows) +``` + +### Relations + +**Example 1: List all the table names involved in the query.** + +```sql +postgres=# SELECT relations,query FROM pg_stat_monitor; + relations | query +-------------------------------+------------------------------------------------------------------------------------------------------ + | END + {public.pgbench_accounts} | SELECT abalance FROM pgbench_accounts WHERE aid = $1 + | vacuum pgbench_branches + {public.pgbench_branches} | select count(*) from pgbench_branches + {public.pgbench_accounts} | UPDATE pgbench_accounts SET abalance = abalance + $1 WHERE aid = $2 + | truncate pgbench_history + {public.pgbench_history} | INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES ($1, $2, $3, $4, CURRENT_TIMESTAMP) + {public.pg_stat_monitor,pg_catalog.pg_database} | SELECT relations query FROM pg_stat_monitor + | vacuum pgbench_tellers + | BEGIN + {public.pgbench_tellers} | UPDATE pgbench_tellers SET tbalance = tbalance + $1 WHERE tid = $2 + {public.pgbench_branches} | UPDATE pgbench_branches SET bbalance = bbalance + $1 WHERE bid = $2 +(12 rows) +``` + +**Example 2: List all the views and the name of the table in the view. Here we have a view "test_view"** + +```sql +\d+ test_view + View "public.test_view" + Column | Type | Collation | Nullable | Default | Storage | Description +--------+---------+-----------+----------+---------+---------+------------- + foo_a | integer | | | | plain | + bar_a | integer | | | | plain | +View definition: + SELECT f.a AS foo_a, + b.a AS bar_a + FROM foo f, + bar b; +``` + +Now when we query the ``pg_stat_monitor``, it will show the view name and also all the table names in the view. Note that the view name is followed by an asterisk (*). + +```sql +SELECT relations, query FROM pg_stat_monitor; + relations | query +---------------------+---------------------------------------------------- + {test_view*,foo,bar} | select * from test_view + {foo,bar} | select * from foo,bar +(2 rows) +``` From f66b45afd60756c5c6604664984b754fcf0f3e4f Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Mon, 25 Oct 2021 16:55:30 -0300 Subject: [PATCH 05/56] PG-220: Fix read/write of dumped query buffer to files. This commit fix some issues when the query buffer overflows and pg_stat_monitor attempts to dump its contents to a file. The dump process is now as follows: 1. The dump will always be a full dump of the current query buffer, meaning pg_stat_monitor will dump MAX_QUERY_BUFFER_BUCKET bytes to the dump file. 2. When scanning the dump file, read chunks of size MAX_QUERY_BUFFER_BUCKET, then look for the query ID using that chunk and the query position metadata, this allows pg_stat_monitor to avoid scanning the whole chunk when looking for a query ID. The code in charge to read from/write to the dump file now takes into account that read() and write() may return less bytes than what it was asked for, the code now ensures that we actually read or write the amount of bytes required (MAX_QUERY_BUFFER_BUCKET), also it handles rare but posssible interrupts when doing those operations. --- pg_stat_monitor.c | 125 ++++++++++++++++++++++++++++++++-------------- pg_stat_monitor.h | 2 +- 2 files changed, 88 insertions(+), 39 deletions(-) diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 127d6a3..93789c3 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -1701,9 +1701,9 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, if (read_query(buf, queryid, query_txt, entry->query_pos) == 0) { - int len; - len = read_query_buffer(bucketid, queryid, query_txt); - if (len != MAX_QUERY_BUFFER_BUCKET) + int rc; + rc = read_query_buffer(bucketid, queryid, query_txt, entry->query_pos); + if (rc != 1) snprintf(query_txt, 32, "%s", ""); } @@ -1726,11 +1726,11 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, if (tmp.info.parentid != UINT64CONST(0)) { - int len = 0; + int rc = 0; if (read_query(buf, tmp.info.parentid, parent_query_txt, 0) == 0) { - len = read_query_buffer(bucketid, tmp.info.parentid, parent_query_txt); - if (len != MAX_QUERY_BUFFER_BUCKET) + rc = read_query_buffer(bucketid, tmp.info.parentid, parent_query_txt, 0); + if (rc != 1) snprintf(parent_query_txt, 32, "%s", ""); } } @@ -3101,7 +3101,7 @@ SaveQueryText(uint64 bucketid, if (buf_len <= sizeof (uint64)) return false; - dump_ok = dump_queries_buffer(bucketid, buf, buf_len); + dump_ok = dump_queries_buffer(bucketid, buf, MAX_QUERY_BUFFER_BUCKET); buf_len = sizeof (uint64); /* @@ -3321,6 +3321,8 @@ dump_queries_buffer(int bucket_id, unsigned char *buf, int buf_len) int fd = 0; char file_name[1024]; bool success = true; + int off = 0; + int tries = 0; snprintf(file_name, 1024, "%s.%d", PGSM_TEXT_FILE, bucket_id); fd = OpenTransientFile(file_name, O_RDWR | O_CREAT | O_APPEND | PG_BINARY); @@ -3333,14 +3335,24 @@ dump_queries_buffer(int bucket_id, unsigned char *buf, int buf_len) return false; } - if (write(fd, buf, buf_len) != buf_len) - { + /* Loop until write buf_len bytes to the file. */ + do { + ssize_t nwrite = write(fd, buf + off, buf_len - off); + if (nwrite == -1) + { + if (errno == EINTR && tries++ < 3) + continue; + + success = false; + break; + } + off += nwrite; + } while (off < buf_len); + + if (!success) ereport(LOG, - (errcode_for_file_access(), - errmsg("could not write file \"%s\": %m", - file_name))); - success = false; - } + (errcode_for_file_access(), + errmsg("could not write file \"%s\": %m", file_name))); if (fd > 0) CloseTransientFile(fd); @@ -3348,14 +3360,25 @@ dump_queries_buffer(int bucket_id, unsigned char *buf, int buf_len) return success; } +/* + * Try to locate query text in a dumped file for bucket_id. + * + * Returns: + * 1 Query sucessfully read, query_text will contain the query text. + * 0 Query not found. + * -1 I/O Error. + */ int -read_query_buffer(int bucket_id, uint64 queryid, char *query_txt) +read_query_buffer(int bucket_id, uint64 queryid, char *query_txt, size_t pos) { int fd = 0; - int buf_len = 0; char file_name[1024]; unsigned char *buf = NULL; + ssize_t nread = 0; int off = 0; + int tries = 0; + bool done = false; + bool found = false; snprintf(file_name, 1024, "%s.%d", PGSM_TEXT_FILE, bucket_id); fd = OpenTransientFile(file_name, O_RDONLY | PG_BINARY); @@ -3363,40 +3386,66 @@ read_query_buffer(int bucket_id, uint64 queryid, char *query_txt) goto exit; buf = (unsigned char*) palloc(MAX_QUERY_BUFFER_BUCKET); - for(;;) + while (!done) { - if (lseek(fd, off, SEEK_SET) != off) - goto exit; + off = 0; + /* read a chunck of MAX_QUERY_BUFFER_BUCKET size. */ + do { + nread = read(fd, buf + off, MAX_QUERY_BUFFER_BUCKET - off); + if (nread == -1) + { + if (errno == EINTR && tries++ < 3) /* read() was interrupted, attempt to read again (max attempts=3) */ + continue; - buf_len = read(fd, buf, MAX_QUERY_BUFFER_BUCKET); - if (buf_len != MAX_QUERY_BUFFER_BUCKET) - { - if (errno != ENOENT) goto exit; - - if (buf_len == 0) + } + else if (nread == 0) /* EOF */ + { + done = true; break; + } + + off += nread; + } while (off < MAX_QUERY_BUFFER_BUCKET); + + if (off == MAX_QUERY_BUFFER_BUCKET) + { + /* we have a chunck, scan it looking for queryid. */ + if (read_query(buf, queryid, query_txt, pos) != 0) + { + + found = true; + /* query was found, don't need to read another chunck. */ + break; + } } - off += buf_len; - if (read_query(buf, queryid, query_txt, 0)) + else + /* + * Either done=true or file has a size not multiple of MAX_QUERY_BUFFER_BUCKET. + * It is safe to assume that the file was truncated or corrupted. + */ break; } - if (fd > 0) - CloseTransientFile(fd); - if (buf) - pfree(buf); - return buf_len; exit: - ereport(LOG, - (errcode_for_file_access(), - errmsg("could not read file \"%s\": %m", - file_name))); - if (fd > 0) + if (fd < 0 || nread == -1) + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not read file \"%s\": %m", + file_name))); + + if (fd >= 0) CloseTransientFile(fd); + if (buf) pfree(buf); - return buf_len; + + if (found) + return 1; + else if (fd == -1 || nread == -1) + return -1; /* I/O error. */ + else + return 0; /* Not found. */ } static double diff --git a/pg_stat_monitor.h b/pg_stat_monitor.h index a1ab4d5..be32103 100644 --- a/pg_stat_monitor.h +++ b/pg_stat_monitor.h @@ -390,7 +390,7 @@ void hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *que pgssEntry* hash_entry_alloc(pgssSharedState *pgss, pgssHashKey *key, int encoding); Size hash_memsize(void); -int read_query_buffer(int bucket_id, uint64 queryid, char *query_txt); +int read_query_buffer(int bucket_id, uint64 queryid, char *query_txt, size_t pos); uint64 read_query(unsigned char *buf, uint64 queryid, char * query, size_t pos); pgssQueryEntry* hash_find_query_entry(uint64 bucket_id, uint64 queryid, uint64 dbid, uint64 userid, uint64 ip, uint64 appid); pgssQueryEntry* hash_create_query_entry(uint64 bucket_id, uint64 queryid, uint64 dbid, uint64 userid, uint64 ip, uint64 appid); From 5b5979c0aa6a4ecd161a54fbb439a2d95bda40ba Mon Sep 17 00:00:00 2001 From: Denys Kondratenko Date: Fri, 29 Oct 2021 23:20:54 +0200 Subject: [PATCH 06/56] add dependencies for build --- CONTRIBUTING.md | 2 ++ README.md | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 78ec34d..f8ee2e8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,6 +64,8 @@ To build `pg_stat_monitor` from source code, you require the following: * git * make +* gcc +* pg_config Refer to the [Building from source code](https://github.com/percona/pg_stat_monitor#installing-from-source-code) section for guidelines. diff --git a/README.md b/README.md index 8beb28d..c5b2206 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,9 @@ * [Setup](#setup) * [Building from source code](#building-from-source) * [How to contribute](#how-to-contribute) +* [Support, discussions and forums](#support-discussions-and-forums) * [License](#license) -* [Copyright](#copyright) +* [Copyright notice](#copyright-notice) ## Overview @@ -209,6 +210,13 @@ To learn more about `pg_stat_monitor` features and usage, see [User Guide](https ### Building from source +To build `pg_stat_monitor` from source code, you require the following: + +* git +* make +* gcc +* pg_config + You can download the source code of the latest release of `pg_stat_monitor` from [the releases page on GitHub](https://github.com/Percona/pg_stat_monitor/releases) or using git: From ac7aa57995ea6c13e0a77f23f842f49593bced32 Mon Sep 17 00:00:00 2001 From: Evgeniy Patlan Date: Mon, 1 Nov 2021 14:49:35 +0200 Subject: [PATCH 07/56] PG-264 fix version --- pg_stat_monitor.c | 2 +- regression/expected/version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 41c23c8..5fb38eb 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -26,7 +26,7 @@ PG_MODULE_MAGIC; -#define BUILD_VERSION "1.0.0 - Beta 2" +#define BUILD_VERSION "1.0.0-beta-2" #define PG_STAT_STATEMENTS_COLS 53 /* maximum of above */ #define PGSM_TEXT_FILE "/tmp/pg_stat_monitor_query" diff --git a/regression/expected/version.out b/regression/expected/version.out index bdda4d2..77ff028 100644 --- a/regression/expected/version.out +++ b/regression/expected/version.out @@ -2,7 +2,7 @@ CREATE EXTENSION pg_stat_monitor; SELECT pg_stat_monitor_version(); pg_stat_monitor_version ------------------------- - 1.0.0 - Beta 2 + 1.0.0-beta-2 (1 row) DROP EXTENSION pg_stat_monitor; From d9c281a40897fb0b4c58acc87ce1aeecb71bae9e Mon Sep 17 00:00:00 2001 From: Anastasia Alexadrova Date: Fri, 5 Nov 2021 14:42:26 +0200 Subject: [PATCH 08/56] Fixed cmd_type* columns in the Reference doc --- docs/REFERENCE.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 0247747..5002cd6 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -23,9 +23,10 @@ top_query | text | Shows the top query used in a query | text | The actual text of the query | application_name | text | Shows the name of the application connected to the database relations | text[] | The list of tables involved in the query -cmd_type | text[] | Type of the query executed +cmd_type | integer | Type of the query executed +cmd_type_text | text[] | The description of the query executed elevel | integer | Shows the error level of a query (WARNING, ERROR, LOG) -sqlcode | integer | +sqlcode | integer | SQL error code message | text | The error message plans_calls | bigint | The number of times the statement was planned plan_total_time | double precision | The total time (in ms) spent on planning the statement From 09f298b8d695a02560a40bb70faacf06ae382aab Mon Sep 17 00:00:00 2001 From: Roma Novikov Date: Wed, 10 Nov 2021 15:47:29 +0200 Subject: [PATCH 09/56] PG-256: DOC about Views presentation --- docs/USER_GUIDE.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 666ec33..31e586f 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -301,6 +301,7 @@ There are 10 timebase buckets of the time **`pg_stat_monitor.pgsm_respose_time_s #### Object Information. **`relations`**: The list of tables involved in the query +Views will be added with * like VIEW_NAME* ##### Example 1: List all the table names involved in the query. ```sql @@ -342,7 +343,7 @@ Now when we query ``pg_stat_monitor``, it will show the view name and also all t SELECT relations, query FROM pg_stat_monitor;       relations      |                                                query                                                 ---------------------+------------------------------------------------------------------------------------------------------ - {test_view,foo,bar} | select * from test_view + {test_view*,foo,bar} | select * from test_view  {foo,bar}           | select * from foo,bar (2 rows) ``` From 20f3d8c04722cf29a7ecc68243b8475459e9eb27 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Wed, 10 Nov 2021 18:33:02 +0000 Subject: [PATCH 10/56] PG-210: Columns names should match upstream pg_stat_statements column names. --- Makefile | 23 ++ pg_stat_monitor--1.0.13.dat | 262 ++++++++++++++++++ ...nitor--1.0.sql => pg_stat_monitor--1.0.dat | 0 3 files changed, 285 insertions(+) create mode 100644 pg_stat_monitor--1.0.13.dat rename pg_stat_monitor--1.0.sql => pg_stat_monitor--1.0.dat (100%) diff --git a/Makefile b/Makefile index ef1a66c..c7b98b1 100644 --- a/Makefile +++ b/Makefile @@ -17,8 +17,31 @@ REGRESS = basic version guc counters relations database top_query application_na # which typical installcheck users do not have (e.g. buildfarm clients). # NO_INSTALLCHECK = 1 + ifdef USE_PGXS PG_CONFIG = pg_config +PG_VERSION := $(shell pg_config --version) +MAJOR := $(shell echo $(PG_VERSION) | sed -e 's/\.[^./]*$$//') + +all: + echo $(MAJOR) + +ifneq (,$(findstring PostgreSQL 14,$(MAJOR))) + CP := $(shell cp pg_stat_monitor--1.0.13.dat pg_stat_monitor--1.0.sql) +endif + +ifneq (,$(findstring PostgreSQL 13,$(MAJOR))) + CP := $(shell cp pg_stat_monitor--1.0.13.dat pg_stat_monitor--1.0.sql) +endif + +ifneq (,$(findstring PostgreSQL 12,$(MAJOR))) + CP := $(shell cp pg_stat_monitor--1.0.dat pg_stat_monitor--1.0.sql) +endif + +ifneq (,$(findstring PostgreSQL 11,$(MAJOR))) + CP := $(shell cp pg_stat_monitor--1.0.dat pg_stat_monitor--1.0.sql) +endif + PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) else diff --git a/pg_stat_monitor--1.0.13.dat b/pg_stat_monitor--1.0.13.dat new file mode 100644 index 0000000..768a5c5 --- /dev/null +++ b/pg_stat_monitor--1.0.13.dat @@ -0,0 +1,262 @@ +/* contrib/pg_stat_monitor/pg_stat_monitor--1.1.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION pg_stat_monitor" to load this file. \quit + +-- Register functions. +CREATE FUNCTION pg_stat_monitor_reset() +RETURNS void +AS 'MODULE_PATHNAME' +LANGUAGE C PARALLEL SAFE; + +CREATE FUNCTION pg_stat_monitor_version() +RETURNS text +AS 'MODULE_PATHNAME' +LANGUAGE C PARALLEL SAFE; + +CREATE FUNCTION get_histogram_timings() +RETURNS text +AS 'MODULE_PATHNAME' +LANGUAGE C PARALLEL SAFE; + +CREATE FUNCTION range() +RETURNS text[] AS $$ +SELECT string_to_array(get_histogram_timings(), ','); +$$ LANGUAGE SQL; + +CREATE FUNCTION pg_stat_monitor_internal(IN showtext boolean, + OUT bucket int8, -- 0 + OUT userid oid, + OUT dbid oid, + OUT client_ip int8, + + OUT queryid text, -- 4 + OUT planid text, + OUT query text, + OUT query_plan text, + OUT state_code int8, + OUT top_queryid text, + OUT top_query text, + OUT application_name text, + + OUT relations text, -- 11 + OUT cmd_type int, + OUT elevel int, + OUT sqlcode TEXT, + OUT message text, + OUT bucket_start_time text, + + OUT calls int8, -- 16 + + OUT total_exec_time float8, + OUT min_exec_time float8, + OUT max_exec_time float8, + OUT mean_exec_time float8, + OUT stddev_exec_time float8, + + OUT rows_retrieved int8, + + OUT plans_calls int8, -- 23 + + OUT total_plan_time float8, + OUT min_plan_time float8, + OUT max_plan_time float8, + OUT mean_plan_time float8, + OUT stddev_plan_time float8, + + OUT shared_blks_hit int8, -- 29 + OUT shared_blks_read int8, + OUT shared_blks_dirtied int8, + OUT shared_blks_written int8, + OUT local_blks_hit int8, + OUT local_blks_read int8, + OUT local_blks_dirtied int8, + OUT local_blks_written int8, + OUT temp_blks_read int8, + OUT temp_blks_written int8, + OUT blk_read_time float8, + OUT blk_write_time float8, + OUT resp_calls text, -- 41 + OUT cpu_user_time float8, + OUT cpu_sys_time float8, + OUT wal_records int8, + OUT wal_fpi int8, + OUT wal_bytes numeric, + OUT comments TEXT +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stat_monitor' +LANGUAGE C STRICT VOLATILE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION get_state(state_code int8) RETURNS TEXT AS +$$ +SELECT + CASE + WHEN state_code = 0 THEN 'PARSING' + WHEN state_code = 1 THEN 'PLANNING' + WHEN state_code = 2 THEN 'ACTIVE' + WHEN state_code = 3 THEN 'FINISHED' + WHEN state_code = 4 THEN 'FINISHED WITH ERROR' + END +$$ +LANGUAGE SQL PARALLEL SAFE; + +CREATE or REPLACE FUNCTION get_cmd_type (cmd_type INTEGER) RETURNS TEXT AS +$$ +SELECT + CASE + WHEN cmd_type = 0 THEN '' + WHEN cmd_type = 1 THEN 'SELECT' + WHEN cmd_type = 2 THEN 'UPDATE' + WHEN cmd_type = 3 THEN 'INSERT' + WHEN cmd_type = 4 THEN 'DELETE' + WHEN cmd_type = 5 THEN 'UTILITY' + WHEN cmd_type = 6 THEN 'NOTHING' + END +$$ +LANGUAGE SQL PARALLEL SAFE; + +CREATE FUNCTION pg_stat_monitor_settings( + OUT name text, + OUT value INTEGER, + OUT default_value INTEGER, + OUT description text, + OUT minimum INTEGER, + OUT maximum INTEGER, + OUT restart INTEGER +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stat_monitor_settings' +LANGUAGE C STRICT VOLATILE PARALLEL SAFE; + +CREATE VIEW pg_stat_monitor_settings AS SELECT + name, + value, + default_value, + description, + minimum, + maximum, + restart +FROM pg_stat_monitor_settings(); + +-- Register a view on the function for ease of use. +CREATE VIEW pg_stat_monitor AS SELECT + bucket, + bucket_start_time AS bucket_start_time, + userid::regrole, + datname, + '0.0.0.0'::inet + client_ip AS client_ip, + queryid, + top_queryid, + query, + comments, + planid, + query_plan, + top_query, + application_name, + string_to_array(relations, ',') AS relations, + cmd_type, + get_cmd_type(cmd_type) AS cmd_type_text, + elevel, + sqlcode, + message, + calls, + total_exec_time, + min_exec_time, + max_exec_time, + mean_exec_time, + stddev_exec_time, + + rows_retrieved, + + plans_calls, + + total_plan_time, + min_plan_time, + max_plan_time, + mean_plan_time, + stddev_plan_time, + + shared_blks_hit, + shared_blks_read, + shared_blks_dirtied, + shared_blks_written, + local_blks_hit, + local_blks_read, + local_blks_dirtied, + local_blks_written, + temp_blks_read, + temp_blks_written, + blk_read_time, + blk_write_time, + (string_to_array(resp_calls, ',')) resp_calls, + cpu_user_time, + cpu_sys_time, + wal_records, + wal_fpi, + wal_bytes, + state_code, + get_state(state_code) as state +FROM pg_stat_monitor_internal(TRUE) p, pg_database d WHERE dbid = oid +ORDER BY bucket_start_time; + +CREATE FUNCTION decode_error_level(elevel int) +RETURNS text +AS +$$ +SELECT + CASE + WHEN elevel = 0 THEN '' + WHEN elevel = 10 THEN 'DEBUG5' + WHEN elevel = 11 THEN 'DEBUG4' + WHEN elevel = 12 THEN 'DEBUG3' + WHEN elevel = 13 THEN 'DEBUG2' + WHEN elevel = 14 THEN 'DEBUG1' + WHEN elevel = 15 THEN 'LOG' + WHEN elevel = 16 THEN 'LOG_SERVER_ONLY' + WHEN elevel = 17 THEN 'INFO' + WHEN elevel = 18 THEN 'NOTICE' + WHEN elevel = 19 THEN 'WARNING' + WHEN elevel = 20 THEN 'ERROR' + END +$$ +LANGUAGE SQL PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION histogram(_bucket int, _quryid text) +RETURNS SETOF RECORD AS $$ +DECLARE + rec record; +BEGIN +for rec in + with stat as (select queryid, bucket, unnest(range()) as range, unnest(resp_calls)::int freq from pg_stat_monitor) select range, freq, repeat('■', (freq::float / max(freq) over() * 30)::int) as bar from stat where queryid = _quryid and bucket = _bucket +loop +return next rec; +end loop; +END +$$ language plpgsql; + +CREATE FUNCTION pg_stat_monitor_hook_stats( + OUT hook text, + OUT min_time float8, + OUT max_time float8, + OUT total_time float8, + OUT ncalls int8 +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stat_monitor_hook_stats' +LANGUAGE C STRICT VOLATILE PARALLEL SAFE; + +CREATE VIEW pg_stat_monitor_hook_stats AS SELECT + hook, + min_time, + max_time, + total_time, + total_time / greatest(ncalls, 1) as avg_time, + ncalls, + ROUND(CAST(total_time / greatest(sum(total_time) OVER(), 0.00000001) * 100 as numeric), 2)::text || '%' as load_comparison +FROM pg_stat_monitor_hook_stats(); + +GRANT SELECT ON pg_stat_monitor TO PUBLIC; +GRANT SELECT ON pg_stat_monitor_settings TO PUBLIC; +-- Don't want this to be available to non-superusers. +REVOKE ALL ON FUNCTION pg_stat_monitor_reset() FROM PUBLIC; diff --git a/pg_stat_monitor--1.0.sql b/pg_stat_monitor--1.0.dat similarity index 100% rename from pg_stat_monitor--1.0.sql rename to pg_stat_monitor--1.0.dat From 06b5e4c5fea9088bbabe011025bf0078d696caa7 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Wed, 10 Nov 2021 19:30:32 +0000 Subject: [PATCH 11/56] PG-210: Columns names should match upstream pg_stat_statements column names. --- Makefile | 15 ++++++--------- pg_stat_monitor.c | 2 +- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index c7b98b1..023e13a 100644 --- a/Makefile +++ b/Makefile @@ -18,30 +18,27 @@ REGRESS = basic version guc counters relations database top_query application_na # NO_INSTALLCHECK = 1 -ifdef USE_PGXS PG_CONFIG = pg_config -PG_VERSION := $(shell pg_config --version) +PG_VERSION := $(shell pg_config --version | awk {'print $$1 $$2'}) MAJOR := $(shell echo $(PG_VERSION) | sed -e 's/\.[^./]*$$//') -all: - echo $(MAJOR) - -ifneq (,$(findstring PostgreSQL 14,$(MAJOR))) +ifneq (,$(findstring PostgreSQL14,$(MAJOR))) CP := $(shell cp pg_stat_monitor--1.0.13.dat pg_stat_monitor--1.0.sql) endif -ifneq (,$(findstring PostgreSQL 13,$(MAJOR))) +ifneq (,$(findstring PostgreSQL13,$(MAJOR))) CP := $(shell cp pg_stat_monitor--1.0.13.dat pg_stat_monitor--1.0.sql) endif -ifneq (,$(findstring PostgreSQL 12,$(MAJOR))) +ifneq (,$(findstring PostgreSQL12,$(MAJOR))) CP := $(shell cp pg_stat_monitor--1.0.dat pg_stat_monitor--1.0.sql) endif -ifneq (,$(findstring PostgreSQL 11,$(MAJOR))) +ifneq (,$(findstring PostgreSQL11,$(MAJOR))) CP := $(shell cp pg_stat_monitor--1.0.dat pg_stat_monitor--1.0.sql) endif +ifdef USE_PGXS PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) else diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 5fb38eb..f08e716 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -1779,7 +1779,7 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, if (enc != query_txt) pfree(enc); /* plan at column number 7 */ - if (planid && tmp.planinfo.plan_text) + if (planid && strlen(tmp.planinfo.plan_text) > 0) values[i++] = CStringGetTextDatum(tmp.planinfo.plan_text); else nulls[i++] = true; From 18f357284a3d540c26d2dd9716e00e48821105e5 Mon Sep 17 00:00:00 2001 From: Anastasia Alexadrova Date: Fri, 12 Nov 2021 12:57:08 +0200 Subject: [PATCH 12/56] PG-215 Doc Fixed links to usage examples, reaorded usage examples to align with feature order modified: docs/USER_GUIDE.md --- docs/USER_GUIDE.md | 206 +++++++++++++++++++++++---------------------- 1 file changed, 105 insertions(+), 101 deletions(-) diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 02f9fc7..88ed7bb 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -33,7 +33,7 @@ The following are the key features of pg_stat_monitor: ### Time buckets -Instead of supplying one set of ever-increasing counts, `pg_stat_monitor` computes stats for a configured number of time intervals; time buckets. This allows for much better data accuracy, especially in the case of high-resolution or unreliable networks. +Instead of supplying one set of ever-increasing counts, `pg_stat_monitor` computes stats for a configured number of time intervals; time buckets. This allows for much better data accuracy, especially in the case of high-resolution or unreliable networks. ### Table and index access statistics per statement @@ -42,20 +42,21 @@ Instead of supplying one set of ever-increasing counts, `pg_stat_monitor` comput ### Query and client information - `pg_stat_monitor` provides additional metrics for detailed analysis of query performance from various perspectives, including client connection details like user name, application name, IP address to name a few relevant columns. With this information, `pg_stat_monitor` enables users to track a query to the originating application. More details about the application or query may be incorporated in the SQL query in a [Google’s Sqlcommenter](https://google.github.io/sqlcommenter/) format. +To see how it works, refer to the [usage example](#query-information) + ### Query timing information -Understanding query execution time stats helps you identify what affects query performance and take measures to optimize it. `pg_stat_monitor` collects the total, min, max and average (mean) time it took to execute a particular query and provides this data in separate columns. See the [Query timing information](#usage-examples-query-timing-information) example for the sample output. +Understanding query execution time stats helps you identify what affects query performance and take measures to optimize it. `pg_stat_monitor` collects the total, min, max and average (mean) time it took to execute a particular query and provides this data in separate columns. See the [Query timing information](#query-timing-information-1) example for the sample output. ### Query execution plan information Every query has a plan that was constructed for its executing. Collecting the query plan information as well as monitoring query plan timing helps you understand how you can modify the query to optimize its execution. It also helps make communication about the query clearer when discussing query performance with other DBAs and application developers. -See the [Query execution plan](#usage-examples-query-execution-time) example for the sample output. +See the [Query execution plan](##query-execution-plan) example for the sample output. ### Use of actual data or parameters placeholders in queries @@ -65,13 +66,13 @@ You can select whether to see queries with parameters placeholders or actual que `pg_stat_monitor` monitors queries per type (``SELECT``, `INSERT`, `UPDATE` or `DELETE`) and classifies them accordingly in the `cmd_type` column. This way you can separate the queries you are interested in and focus on identifying the issues and optimizing query performance. -See the [Query type filtering example](#usage-examples-query-time-filtering) for the sample output. +See the [Query type filtering example](#query-type-filtering-1) for the sample output. ### Query metadata Google’s Sqlcommenter is a useful tool that in a way bridges that gap between ORM libraries and understanding database performance. And ``pg_stat_monitor`` supports it. So, you can now put any key-value data (like what client executed a query or if it is testing vs production query) in the comments in `/* … */` syntax in your SQL statements, and the information will be parsed by `pg_stat_monitor` and made available in the comments column in the `pg_stat_monitor` view. For details on the comments’ syntax, see [Sqlcommenter documentation](https://google.github.io/sqlcommenter/). -To see how it works, see the [Query metadata](#isage-examples-query-metadata) example. +To see how it works, see the [Query metadata](#query-metadata-1) example. ### Top query tracking @@ -81,16 +82,18 @@ Top query indicates the main query. To illustrate, for the SELECT query that is This enables you to backtrack to the originating function and thus simplifies the tracking and analysis. -Find more details in the [usage example](#usage-examples-function-execution-tracking). +Find more details in the [usage example](#top-query-tracking-1). ### Relations -`pg_stat_monitor` provides the list of tables involved in the query in the relations column. This reduces time on identifying the tables and simplifies the analysis. +`pg_stat_monitor` provides the list of tables involved in the query in the relations column. This reduces time on identifying the tables and simplifies the analysis. To learn more, see the [usage examples](#relations-1) ### Monitoring queries terminated with ERROR, WARNING and LOG error levels Monitoring queries that terminate with ERROR, WARNING, LOG states can give useful information to debug an issue. Such messages have the error level (`elevel`), error code (`sqlcode`), and error message (`message`). `pg_stat_monitor` collects all this information and aggregates it so that you can measure performance for successful and failed queries separately, as well as understand why a particular query failed to execute successfully. +Find details in the [usage example](#queries-terminated-with-errors) + ### Integration with PMM To timely identify and react on issues, performance should be automated and alerts should be sent when an issue occurs. There are many monitoring tools available for PostgreSQL, some of them (like Nagios) supporting custom metrics provided via extensions. Though you can integrate `pg_stat_monitor` with these tools, it natively supports integration with Percona Management and Monitoring (PMM). This integration allows you to enjoy all the features provided by both solutions: advanced statistics data provided by `pg_stat_monitor` and automated monitoring with data visualization on dashboards, security threat checks and alerting, available in PMM out of the box. @@ -101,7 +104,7 @@ To learn how to integrate pg_stat_monitor with PMM, see [Configure pg_stat_monit Histogram (the `resp_calls` parameter) provides a visual representation of query performance. With the help of the histogram function, you can view a timing/calling data histogram in response to an SQL query. -Learn more about using histograms from the [usage example](#usage-examples-histogram). +Learn more about using histograms from the [usage example](#histogram-1). ## Views @@ -166,21 +169,21 @@ The following table shows setup options for each configuration parameter and whe | Parameter Name | postgresql.conf | SET | ALTER SYSTEM SET | server restart | configuration reload | ----------------------------------------------|--------------------|-----|-------------------|-------------------|--------------------- -| [pg_stat_monitor.pgsm_max](#pg-stat-monitorpgsm-max) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: -| [pg_stat_monitor.pgsm_query_max_len](#pg-stat-monitorpgsm-query-max-len) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: -| [pg_stat_monitor.pgsm_enable](#pg-stat-monitorpgsm-enable) | :heavy_check_mark: | :x: |:heavy_check_mark: |:x: | :x: -| [pg_stat_monitor.pgsm_track_utility](#pg-stat-monitorpgsm-track-utility) | :heavy_check_mark: | :heavy_check_mark: |:heavy_check_mark: |:x: | :heavy_check_mark: -| [pg_stat_monitor.pgsm_normalized_query](#pg-stat-monitorpgsm-normalized-query) | :heavy_check_mark: | :heavy_check_mark: |:heavy_check_mark: |:x: | :heavy_check_mark: -| [pg_stat_monitor.pgsm_max_buckets](#pg-stat-monitorpgsm-max-buckets) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :heavy_check_mark: -| [pg_stat_monitor.pgsm_bucket_time](#pg-stat-monitorpgsm-bucket-time) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: +| [pg_stat_monitor.pgsm_max](#pg_stat_monitorpgsm_max) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: +| [pg_stat_monitor.pgsm_query_max_len](#pg_stat_monitorpgsm_query_max_len) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: +| [pg_stat_monitor.pgsm_enable](#pg_stat_monitorpgsm_enable) | :heavy_check_mark: | :x: |:heavy_check_mark: |:x: | :x: +| [pg_stat_monitor.pgsm_track_utility](#pg_stat_monitorpgsm_track_utility) | :heavy_check_mark: | :heavy_check_mark: |:heavy_check_mark: |:x: | :heavy_check_mark: +| [pg_stat_monitor.pgsm_normalized_query](#pg_stat_monitorpgsm_normalized_query) | :heavy_check_mark: | :heavy_check_mark: |:heavy_check_mark: |:x: | :heavy_check_mark: +| [pg_stat_monitor.pgsm_max_buckets](#pg_stat_monitorpgsm_max_buckets) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :heavy_check_mark: +| [pg_stat_monitor.pgsm_bucket_time](#pg_stat_monitorpgsm_bucket_time) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: | [pg_stat_monitor.pgsm_object_cache](#pg-stat-monitorpgsm-object-cache) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: -| [pg_stat_monitor.pgsm_histogram_min](#pg-stat-monitorpgsm-histogram-min) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: -| [pg_stat_monitor.pgsm_histogram_max](#pg-stat-monitorpgsm-histogram-max) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: -| [pg_stat_monitor.pgsm_histogram_buckets](#pg-stat-monitorpgsm-histogram-buckets) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: -| [pg_stat_monitor.pgsm_query_shared_buffer](#pg-stat-monitorpgsm-query-shared-buffer) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: -| [pg_stat_monitor.pgsm_overflow_target](#pg-stat-monitorpgsm-overflow-target) | :heavy_check_mark: | :x: | :x: | :heavy_check_mark: | :x: | -| [pg_stat_monitor.pgsm_enable_query_plan](pg-stat-monitorpgsm-enable-query-plan) | :heavy_check_mark: | :x: | :x: | :heavy_check_mark: | :x: | -| [pg_stat_monitor.pgsm_track_planning](#pg-stat-monitorpgsm-track-planning) | :heavy_check_mark: | :x: | :x: | :heavy_check_mark: | :x: | +| [pg_stat_monitor.pgsm_histogram_min](#pg_stat_monitorpgsm_histogram_min) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: +| [pg_stat_monitor.pgsm_histogram_max](#pg_stat_monitorpgsm_histogram_max) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: +| [pg_stat_monitor.pgsm_histogram_buckets](#pg_stat_monitorpgsm_histogram_buckets) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: +| [pg_stat_monitor.pgsm_query_shared_buffer](#pg_stat_monitorpgsm_query_shared_buffer) | :heavy_check_mark: | :x: |:x: |:heavy_check_mark: | :x: +| [pg_stat_monitor.pgsm_overflow_target](#pg_stat_monitorpgsm_overflow_target) | :heavy_check_mark: | :x: | :x: | :heavy_check_mark: | :x: | +| [pg_stat_monitor.pgsm_enable_query_plan](#pg_stat_monitorpgsm_enable_query_plan) | :heavy_check_mark: | :x: | :x: | :heavy_check_mark: | :x: | +| [pg_stat_monitor.pgsm_track_planning](#pg_stat_monitorpgsm_track_planning) | :heavy_check_mark: | :x: | :x: | :heavy_check_mark: | :x: | #### Parameters description: @@ -233,12 +236,12 @@ Values: Requires the server restart. -#### pg_stat_monitor.pgsm_bucket_time +##### pg_stat_monitor.pgsm_bucket_time Values: - Min: 1 - Max: 2147483647 -- Default: 300 +- Default: 60 This parameter is used to set the lifetime of the bucket. System switches between buckets on the basis of [pg_stat_monitor.pgsm_bucket_time](#pg-stat-monitorpgsm-bucket-time). @@ -288,7 +291,7 @@ Type: boolean. Default: 1 Sets the overflow target for the `pg_stat_monitor`. Requires the server restart. -#### pg_stat_monitor.pgsm_enable_query_plan +##### pg_stat_monitor.pgsm_enable_query_plan Type: boolean. Default: 1 @@ -308,7 +311,7 @@ For versions 11 and 12, please consult the [pg_stat_monitor reference](https://g ### Querying buckets - +```sql postgres=# select bucket, bucket_start_time, query,calls from pg_stat_monitor order by bucket; -[ RECORD 1 ]-----+------------------------------------------------------------------------------------ bucket | 0 @@ -434,6 +437,31 @@ postgres=# SELECT substr(query,0,50), query_plan from pg_stat_monitor limit 10; The `plan` column does not contain costing, width and other values. This is an expected behavior as each row is an accumulation of statistics based on `plan` and amongst other key columns. Plan is only available when the `pgsm_enable_query_plan` configuration parameter is enabled. +### Query type filtering + +``pg_stat_monitor`` monitors queries per type (SELECT, INSERT, UPDATE OR DELETE) and classifies them accordingly in the ``cmd_type`` column thus reducing your efforts. + +```sql +postgres=# SELECT bucket, substr(query,0, 50) AS query, cmd_type FROM pg_stat_monitor WHERE elevel = 0; + bucket | query | cmd_type +--------+---------------------------------------------------+---------- + 4 | END | + 4 | SELECT abalance FROM pgbench_accounts WHERE aid = | SELECT + 4 | vacuum pgbench_branches | + 4 | select count(*) from pgbench_branches | SELECT + 4 | UPDATE pgbench_accounts SET abalance = abalance + | UPDATE + 4 | truncate pgbench_history | + 4 | INSERT INTO pgbench_history (tid, bid, aid, delta | INSERT + 5 | SELECT relations query FROM pg_stat_monitor | SELECT + 9 | SELECT bucket, substr(query,$1, $2) AS query, cmd | + 4 | vacuum pgbench_tellers | + 4 | BEGIN | + 5 | SELECT relations,query FROM pg_stat_monitor | SELECT + 4 | UPDATE pgbench_tellers SET tbalance = tbalance + | UPDATE + 4 | UPDATE pgbench_branches SET bbalance = bbalance + | UPDATE +(14 rows) +``` + ### Query metadata The `comments` column contains any text wrapped in `“/*”` and `“*/”` comment tags. The `pg_stat_monitor` extension picks up these comments and makes them available in the comments column. Please note that only the latest comment value is preserved per row. The comments may be put in any format that can be parsed by a tool. @@ -514,79 +542,6 @@ postgres=# select query, text_to_hstore(comments)->'real_ip' as real_ip from pg_ (10 rows) ``` -### Query type filtering - -``pg_stat_monitor`` monitors queries per type (SELECT, INSERT, UPDATE OR DELETE) and classifies them accordingly in the ``cmd_type`` column thus reducing your efforts. - -```sql -postgres=# SELECT bucket, substr(query,0, 50) AS query, cmd_type FROM pg_stat_monitor WHERE elevel = 0; - bucket | query | cmd_type ---------+---------------------------------------------------+---------- - 4 | END | - 4 | SELECT abalance FROM pgbench_accounts WHERE aid = | SELECT - 4 | vacuum pgbench_branches | - 4 | select count(*) from pgbench_branches | SELECT - 4 | UPDATE pgbench_accounts SET abalance = abalance + | UPDATE - 4 | truncate pgbench_history | - 4 | INSERT INTO pgbench_history (tid, bid, aid, delta | INSERT - 5 | SELECT relations query FROM pg_stat_monitor | SELECT - 9 | SELECT bucket, substr(query,$1, $2) AS query, cmd | - 4 | vacuum pgbench_tellers | - 4 | BEGIN | - 5 | SELECT relations,query FROM pg_stat_monitor | SELECT - 4 | UPDATE pgbench_tellers SET tbalance = tbalance + | UPDATE - 4 | UPDATE pgbench_branches SET bbalance = bbalance + | UPDATE -(14 rows) -``` - -### Queries terminated with errors - -```sql -SELECT substr(query,0,50) AS query, decode_error_level(elevel) AS elevel,sqlcode, calls, substr(message,0,50) message -FROM pg_stat_monitor; - query | elevel | sqlcode | calls | message ----------------------------------------------------+--------+---------+-------+--------------------------------------------------- - select substr(query,$1,$2) as query, decode_error | | 0 | 1 | - select bucket,substr(query,$1,$2),decode_error_le | | 0 | 3 | - | LOG | 0 | 1 | database system is ready to accept connections - select 1/0; | ERROR | 130 | 1 | division by zero - | LOG | 0 | 1 | database system was shut down at 2020-11-11 11:37 - select $1/$2 | | 0 | 1 | -(6 rows) - 11277.79 | SELECT * FROM foo -``` - -### Histogram - -Histogram (the `resp_calls` parameter) provides a visual representation of query performance. With the help of the histogram function, you can view a timing/calling data histogram in response to a SQL query. - - -```sql -SELECT resp_calls, query FROM pg_stat_monitor; -                    resp_calls                    |                 query                                         ---------------------------------------------------+---------------------------------------------- -{1," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 0"} | select client_ip, query from pg_stat_monitor -{3," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 1"} | select * from pg_stat_monitor_reset() -{3," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 1"} | SELECT * FROM foo - -postgres=# SELECT * FROM histogram(0, 'F44CD1B4B33A47AF') AS a(range TEXT, freq INT, bar TEXT); - range | freq | bar ---------------------+------+-------------------------------- - (0 - 3)} | 2 | ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■ - (3 - 10)} | 0 | - (10 - 31)} | 1 | ■■■■■■■■■■■■■■■ - (31 - 100)} | 0 | - (100 - 316)} | 0 | - (316 - 1000)} | 0 | - (1000 - 3162)} | 0 | - (3162 - 10000)} | 0 | - (10000 - 31622)} | 0 | - (31622 - 100000)} | 0 | -(10 rows) -``` - -There are 10 time based buckets of the time **`pg_stat_monitor.pgsm_respose_time_step`** in the field ``resp_calls``. The value in the field shows how many queries run in that period of time. - ### Top query tracking In the following example we create a function `add2` that adds one parameter value to another one and call this function to calculate 1+2. @@ -596,7 +551,7 @@ In the following example we create a function `add2` that adds one parameter val CREATE OR REPLACE function add2(int, int) RETURNS int as $$ BEGIN - return (select $1 + $2); + return (select $1 + $2); END; $$ language plpgsql; @@ -665,3 +620,52 @@ SELECT relations, query FROM pg_stat_monitor; {foo,bar} | select * from foo,bar (2 rows) ``` + +### Queries terminated with errors + +```sql +SELECT substr(query,0,50) AS query, decode_error_level(elevel) AS elevel,sqlcode, calls, substr(message,0,50) message +FROM pg_stat_monitor; + query | elevel | sqlcode | calls | message +---------------------------------------------------+--------+---------+-------+--------------------------------------------------- + select substr(query,$1,$2) as query, decode_error | | 0 | 1 | + select bucket,substr(query,$1,$2),decode_error_le | | 0 | 3 | + | LOG | 0 | 1 | database system is ready to accept connections + select 1/0; | ERROR | 130 | 1 | division by zero + | LOG | 0 | 1 | database system was shut down at 2020-11-11 11:37 + select $1/$2 | | 0 | 1 | +(6 rows) + 11277.79 | SELECT * FROM foo +``` + +### Histogram + +Histogram (the `resp_calls` parameter) provides a visual representation of query performance. With the help of the histogram function, you can view a timing/calling data histogram in response to a SQL query. + + +```sql +SELECT resp_calls, query FROM pg_stat_monitor; +                    resp_calls                    |                 query                                         +--------------------------------------------------+---------------------------------------------- +{1," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 0"} | select client_ip, query from pg_stat_monitor +{3," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 1"} | select * from pg_stat_monitor_reset() +{3," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 0"," 1"} | SELECT * FROM foo + +postgres=# SELECT * FROM histogram(0, 'F44CD1B4B33A47AF') AS a(range TEXT, freq INT, bar TEXT); + range | freq | bar +--------------------+------+-------------------------------- + (0 - 3)} | 2 | ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■ + (3 - 10)} | 0 | + (10 - 31)} | 1 | ■■■■■■■■■■■■■■■ + (31 - 100)} | 0 | + (100 - 316)} | 0 | + (316 - 1000)} | 0 | + (1000 - 3162)} | 0 | + (3162 - 10000)} | 0 | + (10000 - 31622)} | 0 | + (31622 - 100000)} | 0 | +(10 rows) +``` + +There are 10 time based buckets of the time generated automatically based on total buckets in the field ``resp_calls``. The value in the field shows how many queries run in that period of time. + From 997639c067a8af1ae11c58d7ef74ef42dd7cab32 Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Fri, 12 Nov 2021 10:58:56 -0300 Subject: [PATCH 13/56] PG-272: Fix server crash when calling pg_stat_monitor_reset(). The loop that resets the query buffers was incorrecly using MAX_BUCKETS to indicate the number of buckets to clear, which defaults to 10. If a user lowers this value the loop would access a pointer beyond the number of query buffers allocated. Fix the problem by using the correct PGSM_MAX_BUCKETS GUC as the limit to the loop. --- pg_stat_monitor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 41c23c8..4afee9d 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -1579,7 +1579,7 @@ pg_stat_monitor_reset(PG_FUNCTION_ARGS) LWLockAcquire(pgss->lock, LW_EXCLUSIVE); hash_entry_dealloc(-1, -1, NULL); /* Reset query buffers. */ - for (size_t i = 0; i < MAX_BUCKETS; ++i) + for (size_t i = 0; i < PGSM_MAX_BUCKETS; ++i) { *(uint64 *)pgss_qbuf[i] = 0; } From 3be31d67e925251f997ae3cec8b91289a79bac7b Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Fri, 12 Nov 2021 14:42:17 -0300 Subject: [PATCH 14/56] PG-275: Fix regression tests. Removal of redundant file guc_1.out. Adjusted guc.out to match query planning disabled by default. --- regression/expected/guc.out | 3 +-- regression/expected/guc_1.out | 38 ----------------------------------- 2 files changed, 1 insertion(+), 40 deletions(-) delete mode 100644 regression/expected/guc_1.out diff --git a/regression/expected/guc.out b/regression/expected/guc.out index 0da2986..ef3da0b 100644 --- a/regression/expected/guc.out +++ b/regression/expected/guc.out @@ -26,9 +26,8 @@ SELECT * FROM pg_stat_monitor_settings ORDER BY name COLLATE "C"; pg_stat_monitor.pgsm_overflow_target | 1 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | 1 pg_stat_monitor.pgsm_query_max_len | 1024 | 1024 | Sets the maximum length of query. | 1024 | 2147483647 | 1 pg_stat_monitor.pgsm_query_shared_buffer | 20 | 20 | Sets the maximum size of shared memory in (MB) used for query tracked by pg_stat_monitor. | 1 | 10000 | 1 - pg_stat_monitor.pgsm_track_planning | 1 | 1 | Selects whether planning statistics are tracked. | 0 | 0 | 0 pg_stat_monitor.pgsm_track_utility | 1 | 1 | Selects whether utility commands are tracked. | 0 | 0 | 0 -(14 rows) +(13 rows) SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset diff --git a/regression/expected/guc_1.out b/regression/expected/guc_1.out deleted file mode 100644 index 1425109..0000000 --- a/regression/expected/guc_1.out +++ /dev/null @@ -1,38 +0,0 @@ -CREATE EXTENSION pg_stat_monitor; -SELECT pg_stat_monitor_reset(); - pg_stat_monitor_reset ------------------------ - -(1 row) - -select pg_sleep(.5); - pg_sleep ----------- - -(1 row) - -SELECT * FROM pg_stat_monitor_settings ORDER BY name COLLATE "C"; - name | value | default_value | description | minimum | maximum | restart -------------------------------------------+--------+---------------+----------------------------------------------------------------------------------------------------------+---------+------------+--------- - pg_stat_monitor.pgsm_bucket_time | 60 | 60 | Sets the time in seconds per bucket. | 1 | 2147483647 | 1 - pg_stat_monitor.pgsm_enable | 1 | 1 | Enable/Disable statistics collector. | 0 | 0 | 0 - pg_stat_monitor.pgsm_enable_query_plan | 0 | 0 | Enable/Disable query plan monitoring | 0 | 0 | 0 - pg_stat_monitor.pgsm_histogram_buckets | 10 | 10 | Sets the maximum number of histogram buckets | 2 | 2147483647 | 1 - pg_stat_monitor.pgsm_histogram_max | 100000 | 100000 | Sets the time in millisecond. | 10 | 2147483647 | 1 - pg_stat_monitor.pgsm_histogram_min | 0 | 0 | Sets the time in millisecond. | 0 | 2147483647 | 1 - pg_stat_monitor.pgsm_max | 100 | 100 | Sets the maximum size of shared memory in (MB) used for statement's metadata tracked by pg_stat_monitor. | 1 | 1000 | 1 - pg_stat_monitor.pgsm_max_buckets | 10 | 10 | Sets the maximum number of buckets. | 1 | 10 | 1 - pg_stat_monitor.pgsm_normalized_query | 1 | 1 | Selects whether save query in normalized format. | 0 | 0 | 0 - pg_stat_monitor.pgsm_overflow_target | 0 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | 1 - pg_stat_monitor.pgsm_query_max_len | 1024 | 1024 | Sets the maximum length of query. | 1024 | 2147483647 | 1 - pg_stat_monitor.pgsm_query_shared_buffer | 20 | 20 | Sets the maximum size of shared memory in (MB) used for query tracked by pg_stat_monitor. | 1 | 10000 | 1 - pg_stat_monitor.pgsm_track_utility | 1 | 1 | Selects whether utility commands are tracked. | 0 | 0 | 0 -(13 rows) - -SELECT pg_stat_monitor_reset(); - pg_stat_monitor_reset ------------------------ - -(1 row) - -DROP EXTENSION pg_stat_monitor; From 0a45fc740f16d0a33141b3c1474fb8b1cd2b2d82 Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Fri, 12 Nov 2021 16:55:51 -0300 Subject: [PATCH 15/56] PG-276: Fix regression tests. The guc_1.out is used for PG >= 13, where query track planning is available, so it has been restored. --- regression/expected/guc_1.out | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 regression/expected/guc_1.out diff --git a/regression/expected/guc_1.out b/regression/expected/guc_1.out new file mode 100644 index 0000000..0da2986 --- /dev/null +++ b/regression/expected/guc_1.out @@ -0,0 +1,39 @@ +CREATE EXTENSION pg_stat_monitor; +SELECT pg_stat_monitor_reset(); + pg_stat_monitor_reset +----------------------- + +(1 row) + +select pg_sleep(.5); + pg_sleep +---------- + +(1 row) + +SELECT * FROM pg_stat_monitor_settings ORDER BY name COLLATE "C"; + name | value | default_value | description | minimum | maximum | restart +------------------------------------------+--------+---------------+----------------------------------------------------------------------------------------------------------+---------+------------+--------- + pg_stat_monitor.pgsm_bucket_time | 60 | 60 | Sets the time in seconds per bucket. | 1 | 2147483647 | 1 + pg_stat_monitor.pgsm_enable | 1 | 1 | Enable/Disable statistics collector. | 0 | 0 | 0 + pg_stat_monitor.pgsm_enable_query_plan | 0 | 0 | Enable/Disable query plan monitoring | 0 | 0 | 0 + pg_stat_monitor.pgsm_histogram_buckets | 10 | 10 | Sets the maximum number of histogram buckets | 2 | 2147483647 | 1 + pg_stat_monitor.pgsm_histogram_max | 100000 | 100000 | Sets the time in millisecond. | 10 | 2147483647 | 1 + pg_stat_monitor.pgsm_histogram_min | 0 | 0 | Sets the time in millisecond. | 0 | 2147483647 | 1 + pg_stat_monitor.pgsm_max | 100 | 100 | Sets the maximum size of shared memory in (MB) used for statement's metadata tracked by pg_stat_monitor. | 1 | 1000 | 1 + pg_stat_monitor.pgsm_max_buckets | 10 | 10 | Sets the maximum number of buckets. | 1 | 10 | 1 + pg_stat_monitor.pgsm_normalized_query | 1 | 1 | Selects whether save query in normalized format. | 0 | 0 | 0 + pg_stat_monitor.pgsm_overflow_target | 1 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | 1 + pg_stat_monitor.pgsm_query_max_len | 1024 | 1024 | Sets the maximum length of query. | 1024 | 2147483647 | 1 + pg_stat_monitor.pgsm_query_shared_buffer | 20 | 20 | Sets the maximum size of shared memory in (MB) used for query tracked by pg_stat_monitor. | 1 | 10000 | 1 + pg_stat_monitor.pgsm_track_planning | 1 | 1 | Selects whether planning statistics are tracked. | 0 | 0 | 0 + pg_stat_monitor.pgsm_track_utility | 1 | 1 | Selects whether utility commands are tracked. | 0 | 0 | 0 +(14 rows) + +SELECT pg_stat_monitor_reset(); + pg_stat_monitor_reset +----------------------- + +(1 row) + +DROP EXTENSION pg_stat_monitor; From 5f6177daa30a6d60a1c39e3e0bd3fcd73df8cf37 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Tue, 16 Nov 2021 10:48:11 +0000 Subject: [PATCH 16/56] PG-210: Add new column toplevel. --- Makefile | 8 +- ...0.13.dat => pg_stat_monitor--1.0.13.sql.in | 3 +- pg_stat_monitor--1.0.14.sql.in | 264 ++++++++++++++++++ ...or--1.0.dat => pg_stat_monitor--1.0.sql.in | 11 +- pg_stat_monitor.c | 6 +- pg_stat_monitor.h | 1 + regression/expected/guc.out | 5 +- 7 files changed, 282 insertions(+), 16 deletions(-) rename pg_stat_monitor--1.0.13.dat => pg_stat_monitor--1.0.13.sql.in (99%) create mode 100644 pg_stat_monitor--1.0.14.sql.in rename pg_stat_monitor--1.0.dat => pg_stat_monitor--1.0.sql.in (98%) diff --git a/Makefile b/Makefile index 023e13a..ac1225d 100644 --- a/Makefile +++ b/Makefile @@ -23,19 +23,19 @@ PG_VERSION := $(shell pg_config --version | awk {'print $$1 $$2'}) MAJOR := $(shell echo $(PG_VERSION) | sed -e 's/\.[^./]*$$//') ifneq (,$(findstring PostgreSQL14,$(MAJOR))) - CP := $(shell cp pg_stat_monitor--1.0.13.dat pg_stat_monitor--1.0.sql) + CP := $(shell cp pg_stat_monitor--1.0.14.sql.in pg_stat_monitor--1.0.sql) endif ifneq (,$(findstring PostgreSQL13,$(MAJOR))) - CP := $(shell cp pg_stat_monitor--1.0.13.dat pg_stat_monitor--1.0.sql) + CP := $(shell cp pg_stat_monitor--1.0.13.sql.in pg_stat_monitor--1.0.sql) endif ifneq (,$(findstring PostgreSQL12,$(MAJOR))) - CP := $(shell cp pg_stat_monitor--1.0.dat pg_stat_monitor--1.0.sql) + CP := $(shell cp pg_stat_monitor--1.0.sql.in pg_stat_monitor--1.0.sql) endif ifneq (,$(findstring PostgreSQL11,$(MAJOR))) - CP := $(shell cp pg_stat_monitor--1.0.dat pg_stat_monitor--1.0.sql) + CP := $(shell cp pg_stat_monitor--1.0.sql.in pg_stat_monitor--1.0.sql) endif ifdef USE_PGXS diff --git a/pg_stat_monitor--1.0.13.dat b/pg_stat_monitor--1.0.13.sql.in similarity index 99% rename from pg_stat_monitor--1.0.13.dat rename to pg_stat_monitor--1.0.13.sql.in index 768a5c5..e9e855e 100644 --- a/pg_stat_monitor--1.0.13.dat +++ b/pg_stat_monitor--1.0.13.sql.in @@ -82,7 +82,8 @@ CREATE FUNCTION pg_stat_monitor_internal(IN showtext boolean, OUT wal_records int8, OUT wal_fpi int8, OUT wal_bytes numeric, - OUT comments TEXT + OUT comments TEXT, + OUT toplevel BOOLEAN ) RETURNS SETOF record AS 'MODULE_PATHNAME', 'pg_stat_monitor' diff --git a/pg_stat_monitor--1.0.14.sql.in b/pg_stat_monitor--1.0.14.sql.in new file mode 100644 index 0000000..76ec2d5 --- /dev/null +++ b/pg_stat_monitor--1.0.14.sql.in @@ -0,0 +1,264 @@ +/* contrib/pg_stat_monitor/pg_stat_monitor--1.1.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION pg_stat_monitor" to load this file. \quit + +-- Register functions. +CREATE FUNCTION pg_stat_monitor_reset() +RETURNS void +AS 'MODULE_PATHNAME' +LANGUAGE C PARALLEL SAFE; + +CREATE FUNCTION pg_stat_monitor_version() +RETURNS text +AS 'MODULE_PATHNAME' +LANGUAGE C PARALLEL SAFE; + +CREATE FUNCTION get_histogram_timings() +RETURNS text +AS 'MODULE_PATHNAME' +LANGUAGE C PARALLEL SAFE; + +CREATE FUNCTION range() +RETURNS text[] AS $$ +SELECT string_to_array(get_histogram_timings(), ','); +$$ LANGUAGE SQL; + +CREATE FUNCTION pg_stat_monitor_internal(IN showtext boolean, + OUT bucket int8, -- 0 + OUT userid oid, + OUT dbid oid, + OUT client_ip int8, + + OUT queryid text, -- 4 + OUT planid text, + OUT query text, + OUT query_plan text, + OUT state_code int8, + OUT top_queryid text, + OUT top_query text, + OUT application_name text, + + OUT relations text, -- 11 + OUT cmd_type int, + OUT elevel int, + OUT sqlcode TEXT, + OUT message text, + OUT bucket_start_time text, + + OUT calls int8, -- 16 + + OUT total_exec_time float8, + OUT min_exec_time float8, + OUT max_exec_time float8, + OUT mean_exec_time float8, + OUT stddev_exec_time float8, + + OUT rows_retrieved int8, + + OUT plans_calls int8, -- 23 + + OUT total_plan_time float8, + OUT min_plan_time float8, + OUT max_plan_time float8, + OUT mean_plan_time float8, + OUT stddev_plan_time float8, + + OUT shared_blks_hit int8, -- 29 + OUT shared_blks_read int8, + OUT shared_blks_dirtied int8, + OUT shared_blks_written int8, + OUT local_blks_hit int8, + OUT local_blks_read int8, + OUT local_blks_dirtied int8, + OUT local_blks_written int8, + OUT temp_blks_read int8, + OUT temp_blks_written int8, + OUT blk_read_time float8, + OUT blk_write_time float8, + OUT resp_calls text, -- 41 + OUT cpu_user_time float8, + OUT cpu_sys_time float8, + OUT wal_records int8, + OUT wal_fpi int8, + OUT wal_bytes numeric, + OUT comments TEXT, + OUT toplevel BOOLEAN +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stat_monitor' +LANGUAGE C STRICT VOLATILE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION get_state(state_code int8) RETURNS TEXT AS +$$ +SELECT + CASE + WHEN state_code = 0 THEN 'PARSING' + WHEN state_code = 1 THEN 'PLANNING' + WHEN state_code = 2 THEN 'ACTIVE' + WHEN state_code = 3 THEN 'FINISHED' + WHEN state_code = 4 THEN 'FINISHED WITH ERROR' + END +$$ +LANGUAGE SQL PARALLEL SAFE; + +CREATE or REPLACE FUNCTION get_cmd_type (cmd_type INTEGER) RETURNS TEXT AS +$$ +SELECT + CASE + WHEN cmd_type = 0 THEN '' + WHEN cmd_type = 1 THEN 'SELECT' + WHEN cmd_type = 2 THEN 'UPDATE' + WHEN cmd_type = 3 THEN 'INSERT' + WHEN cmd_type = 4 THEN 'DELETE' + WHEN cmd_type = 5 THEN 'UTILITY' + WHEN cmd_type = 6 THEN 'NOTHING' + END +$$ +LANGUAGE SQL PARALLEL SAFE; + +CREATE FUNCTION pg_stat_monitor_settings( + OUT name text, + OUT value INTEGER, + OUT default_value INTEGER, + OUT description text, + OUT minimum INTEGER, + OUT maximum INTEGER, + OUT restart INTEGER +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stat_monitor_settings' +LANGUAGE C STRICT VOLATILE PARALLEL SAFE; + +CREATE VIEW pg_stat_monitor_settings AS SELECT + name, + value, + default_value, + description, + minimum, + maximum, + restart +FROM pg_stat_monitor_settings(); + +-- Register a view on the function for ease of use. +CREATE VIEW pg_stat_monitor AS SELECT + bucket, + bucket_start_time AS bucket_start_time, + userid::regrole, + datname, + '0.0.0.0'::inet + client_ip AS client_ip, + queryid, + toplevel, + top_queryid, + query, + comments, + planid, + query_plan, + top_query, + application_name, + string_to_array(relations, ',') AS relations, + cmd_type, + get_cmd_type(cmd_type) AS cmd_type_text, + elevel, + sqlcode, + message, + calls, + total_exec_time, + min_exec_time, + max_exec_time, + mean_exec_time, + stddev_exec_time, + + rows_retrieved, + + plans_calls, + + total_plan_time, + min_plan_time, + max_plan_time, + mean_plan_time, + stddev_plan_time, + + shared_blks_hit, + shared_blks_read, + shared_blks_dirtied, + shared_blks_written, + local_blks_hit, + local_blks_read, + local_blks_dirtied, + local_blks_written, + temp_blks_read, + temp_blks_written, + blk_read_time, + blk_write_time, + (string_to_array(resp_calls, ',')) resp_calls, + cpu_user_time, + cpu_sys_time, + wal_records, + wal_fpi, + wal_bytes, + state_code, + get_state(state_code) as state +FROM pg_stat_monitor_internal(TRUE) p, pg_database d WHERE dbid = oid +ORDER BY bucket_start_time; + +CREATE FUNCTION decode_error_level(elevel int) +RETURNS text +AS +$$ +SELECT + CASE + WHEN elevel = 0 THEN '' + WHEN elevel = 10 THEN 'DEBUG5' + WHEN elevel = 11 THEN 'DEBUG4' + WHEN elevel = 12 THEN 'DEBUG3' + WHEN elevel = 13 THEN 'DEBUG2' + WHEN elevel = 14 THEN 'DEBUG1' + WHEN elevel = 15 THEN 'LOG' + WHEN elevel = 16 THEN 'LOG_SERVER_ONLY' + WHEN elevel = 17 THEN 'INFO' + WHEN elevel = 18 THEN 'NOTICE' + WHEN elevel = 19 THEN 'WARNING' + WHEN elevel = 20 THEN 'ERROR' + END +$$ +LANGUAGE SQL PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION histogram(_bucket int, _quryid text) +RETURNS SETOF RECORD AS $$ +DECLARE + rec record; +BEGIN +for rec in + with stat as (select queryid, bucket, unnest(range()) as range, unnest(resp_calls)::int freq from pg_stat_monitor) select range, freq, repeat('■', (freq::float / max(freq) over() * 30)::int) as bar from stat where queryid = _quryid and bucket = _bucket +loop +return next rec; +end loop; +END +$$ language plpgsql; + +CREATE FUNCTION pg_stat_monitor_hook_stats( + OUT hook text, + OUT min_time float8, + OUT max_time float8, + OUT total_time float8, + OUT ncalls int8 +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stat_monitor_hook_stats' +LANGUAGE C STRICT VOLATILE PARALLEL SAFE; + +CREATE VIEW pg_stat_monitor_hook_stats AS SELECT + hook, + min_time, + max_time, + total_time, + total_time / greatest(ncalls, 1) as avg_time, + ncalls, + ROUND(CAST(total_time / greatest(sum(total_time) OVER(), 0.00000001) * 100 as numeric), 2)::text || '%' as load_comparison +FROM pg_stat_monitor_hook_stats(); + +GRANT SELECT ON pg_stat_monitor TO PUBLIC; +GRANT SELECT ON pg_stat_monitor_settings TO PUBLIC; +-- Don't want this to be available to non-superusers. +REVOKE ALL ON FUNCTION pg_stat_monitor_reset() FROM PUBLIC; diff --git a/pg_stat_monitor--1.0.dat b/pg_stat_monitor--1.0.sql.in similarity index 98% rename from pg_stat_monitor--1.0.dat rename to pg_stat_monitor--1.0.sql.in index 67cb3c2..544f668 100644 --- a/pg_stat_monitor--1.0.dat +++ b/pg_stat_monitor--1.0.sql.in @@ -79,8 +79,9 @@ CREATE FUNCTION pg_stat_monitor_internal(IN showtext boolean, OUT wal_records int8, OUT wal_fpi int8, OUT wal_bytes numeric, - OUT comments TEXT -) + OUT comments TEXT, + OUT toplevel BOOLEAN + ) RETURNS SETOF record AS 'MODULE_PATHNAME', 'pg_stat_monitor' LANGUAGE C STRICT VOLATILE PARALLEL SAFE; @@ -164,12 +165,6 @@ CREATE VIEW pg_stat_monitor AS SELECT mean_time, stddev_time, rows_retrieved, - plans_calls, - plan_total_time, - plan_min_time, - plan_max_time, - plan_mean_time, - shared_blks_hit, shared_blks_read, shared_blks_dirtied, diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 1b68146..d7d49a1 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -1499,6 +1499,7 @@ pgss_store(uint64 queryid, key.ip = pg_get_client_addr(); key.planid = planid; key.appid = appid; + key.toplevel = (nested_level == 0); pgss_hash = pgsm_get_hash(); @@ -1664,7 +1665,7 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "pg_stat_monitor: return type must be a row type"); - if (tupdesc->natts != 50) + if (tupdesc->natts != 51) elog(ERROR, "pg_stat_monitor: incorrect number of output arguments, required %d", tupdesc->natts); tupstore = tuplestore_begin_heap(true, false, work_mem); @@ -1694,9 +1695,11 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, uint64 planid = entry->key.planid; unsigned char *buf = pgss_qbuf[bucketid]; #if PG_VERSION_NUM < 140000 + bool toplevel = true; bool is_allowed_role = is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_ALL_STATS); #else bool is_allowed_role = is_member_of_role(GetUserId(), ROLE_PG_READ_ALL_STATS); + bool toplevel = entry->key.toplevel; #endif if (read_query(buf, queryid, query_txt, entry->query_pos) == 0) @@ -1980,6 +1983,7 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, else nulls[i++] = true; } + values[i++] = BoolGetDatum(toplevel); tuplestore_putvalues(tupstore, tupdesc, values, nulls); } pfree(query_txt); diff --git a/pg_stat_monitor.h b/pg_stat_monitor.h index be32103..04b5c52 100644 --- a/pg_stat_monitor.h +++ b/pg_stat_monitor.h @@ -203,6 +203,7 @@ typedef struct pgssHashKey uint64 ip; /* client ip address */ uint64 planid; /* plan identifier */ uint64 appid; /* hash of application name */ + bool toplevel; /* query executed at top level */ } pgssHashKey; typedef struct QueryInfo diff --git a/regression/expected/guc.out b/regression/expected/guc.out index ef3da0b..934c719 100644 --- a/regression/expected/guc.out +++ b/regression/expected/guc.out @@ -23,11 +23,12 @@ SELECT * FROM pg_stat_monitor_settings ORDER BY name COLLATE "C"; pg_stat_monitor.pgsm_max | 100 | 100 | Sets the maximum size of shared memory in (MB) used for statement's metadata tracked by pg_stat_monitor. | 1 | 1000 | 1 pg_stat_monitor.pgsm_max_buckets | 10 | 10 | Sets the maximum number of buckets. | 1 | 10 | 1 pg_stat_monitor.pgsm_normalized_query | 1 | 1 | Selects whether save query in normalized format. | 0 | 0 | 0 - pg_stat_monitor.pgsm_overflow_target | 1 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | 1 + pg_stat_monitor.pgsm_overflow_target | 0 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | 1 pg_stat_monitor.pgsm_query_max_len | 1024 | 1024 | Sets the maximum length of query. | 1024 | 2147483647 | 1 pg_stat_monitor.pgsm_query_shared_buffer | 20 | 20 | Sets the maximum size of shared memory in (MB) used for query tracked by pg_stat_monitor. | 1 | 10000 | 1 + pg_stat_monitor.pgsm_track_planning | 1 | 1 | Selects whether planning statistics are tracked. | 0 | 0 | 0 pg_stat_monitor.pgsm_track_utility | 1 | 1 | Selects whether utility commands are tracked. | 0 | 0 | 0 -(13 rows) +(14 rows) SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset From 680c7fda428a5a618c4d4c325ded97f1ea3ba2a8 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Tue, 16 Nov 2021 11:23:59 +0000 Subject: [PATCH 17/56] PG-210: Add new column toplevel. --- pg_stat_monitor.c | 7 +++++-- pg_stat_monitor.h | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index d7d49a1..768fa8c 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -1499,8 +1499,11 @@ pgss_store(uint64 queryid, key.ip = pg_get_client_addr(); key.planid = planid; key.appid = appid; +#if PG_VERSION_NUM < 140000 + key.toplevel = 1; +#else key.toplevel = (nested_level == 0); - +#endif pgss_hash = pgsm_get_hash(); LWLockAcquire(pgss->lock, LW_SHARED); @@ -1695,7 +1698,7 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, uint64 planid = entry->key.planid; unsigned char *buf = pgss_qbuf[bucketid]; #if PG_VERSION_NUM < 140000 - bool toplevel = true; + bool toplevel = 1; bool is_allowed_role = is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_ALL_STATS); #else bool is_allowed_role = is_member_of_role(GetUserId(), ROLE_PG_READ_ALL_STATS); diff --git a/pg_stat_monitor.h b/pg_stat_monitor.h index 04b5c52..84ed936 100644 --- a/pg_stat_monitor.h +++ b/pg_stat_monitor.h @@ -203,7 +203,7 @@ typedef struct pgssHashKey uint64 ip; /* client ip address */ uint64 planid; /* plan identifier */ uint64 appid; /* hash of application name */ - bool toplevel; /* query executed at top level */ + uint64 toplevel; /* query executed at top level */ } pgssHashKey; typedef struct QueryInfo From c8d72091492e2eb37588b86f6e2005c5486ee92f Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Tue, 16 Nov 2021 11:38:21 +0000 Subject: [PATCH 18/56] PG-273: pg_stat_monitor_hook_stats disable to end user. --- pg_stat_monitor--1.0.13.sql.in | 38 +++++++++++++++---------------- pg_stat_monitor--1.0.14.sql.in | 40 ++++++++++++++++----------------- pg_stat_monitor--1.0.sql.in | 41 ++++++++++++++++++---------------- 3 files changed, 61 insertions(+), 58 deletions(-) diff --git a/pg_stat_monitor--1.0.13.sql.in b/pg_stat_monitor--1.0.13.sql.in index e9e855e..990e62b 100644 --- a/pg_stat_monitor--1.0.13.sql.in +++ b/pg_stat_monitor--1.0.13.sql.in @@ -236,26 +236,26 @@ end loop; END $$ language plpgsql; -CREATE FUNCTION pg_stat_monitor_hook_stats( - OUT hook text, - OUT min_time float8, - OUT max_time float8, - OUT total_time float8, - OUT ncalls int8 -) -RETURNS SETOF record -AS 'MODULE_PATHNAME', 'pg_stat_monitor_hook_stats' -LANGUAGE C STRICT VOLATILE PARALLEL SAFE; +--CREATE FUNCTION pg_stat_monitor_hook_stats( +-- OUT hook text, +-- OUT min_time float8, +-- OUT max_time float8, +-- OUT total_time float8, +-- OUT ncalls int8 +--) +--RETURNS SETOF record +--AS 'MODULE_PATHNAME', 'pg_stat_monitor_hook_stats' +--LANGUAGE C STRICT VOLATILE PARALLEL SAFE; -CREATE VIEW pg_stat_monitor_hook_stats AS SELECT - hook, - min_time, - max_time, - total_time, - total_time / greatest(ncalls, 1) as avg_time, - ncalls, - ROUND(CAST(total_time / greatest(sum(total_time) OVER(), 0.00000001) * 100 as numeric), 2)::text || '%' as load_comparison -FROM pg_stat_monitor_hook_stats(); +--CREATE VIEW pg_stat_monitor_hook_stats AS SELECT +-- hook, +-- min_time, +-- max_time, +-- total_time, +-- total_time / greatest(ncalls, 1) as avg_time, +-- ncalls, +-- ROUND(CAST(total_time / greatest(sum(total_time) OVER(), 0.00000001) * 100 as numeric), 2)::text || '%' as load_comparison +--FROM pg_stat_monitor_hook_stats(); GRANT SELECT ON pg_stat_monitor TO PUBLIC; GRANT SELECT ON pg_stat_monitor_settings TO PUBLIC; diff --git a/pg_stat_monitor--1.0.14.sql.in b/pg_stat_monitor--1.0.14.sql.in index 76ec2d5..dae64b8 100644 --- a/pg_stat_monitor--1.0.14.sql.in +++ b/pg_stat_monitor--1.0.14.sql.in @@ -237,26 +237,26 @@ end loop; END $$ language plpgsql; -CREATE FUNCTION pg_stat_monitor_hook_stats( - OUT hook text, - OUT min_time float8, - OUT max_time float8, - OUT total_time float8, - OUT ncalls int8 -) -RETURNS SETOF record -AS 'MODULE_PATHNAME', 'pg_stat_monitor_hook_stats' -LANGUAGE C STRICT VOLATILE PARALLEL SAFE; - -CREATE VIEW pg_stat_monitor_hook_stats AS SELECT - hook, - min_time, - max_time, - total_time, - total_time / greatest(ncalls, 1) as avg_time, - ncalls, - ROUND(CAST(total_time / greatest(sum(total_time) OVER(), 0.00000001) * 100 as numeric), 2)::text || '%' as load_comparison -FROM pg_stat_monitor_hook_stats(); +--CREATE FUNCTION pg_stat_monitor_hook_stats( +-- OUT hook text, +-- OUT min_time float8, +-- OUT max_time float8, +-- OUT total_time float8, +-- OUT ncalls int8 +--) +--RETURNS SETOF record +--AS 'MODULE_PATHNAME', 'pg_stat_monitor_hook_stats' +--LANGUAGE C STRICT VOLATILE PARALLEL SAFE; +-- +--CREATE VIEW pg_stat_monitor_hook_stats AS SELECT +-- hook, +-- min_time, +-- max_time, +-- total_time, +-- total_time / greatest(ncalls, 1) as avg_time, +-- ncalls, +-- ROUND(CAST(total_time / greatest(sum(total_time) OVER(), 0.00000001) * 100 as numeric), 2)::text || '%' as load_comparison +--FROM pg_stat_monitor_hook_stats(); GRANT SELECT ON pg_stat_monitor TO PUBLIC; GRANT SELECT ON pg_stat_monitor_settings TO PUBLIC; diff --git a/pg_stat_monitor--1.0.sql.in b/pg_stat_monitor--1.0.sql.in index 544f668..fd2495d 100644 --- a/pg_stat_monitor--1.0.sql.in +++ b/pg_stat_monitor--1.0.sql.in @@ -223,28 +223,31 @@ end loop; END $$ language plpgsql; -CREATE FUNCTION pg_stat_monitor_hook_stats( - OUT hook text, - OUT min_time float8, - OUT max_time float8, - OUT total_time float8, - OUT ncalls int8 -) -RETURNS SETOF record -AS 'MODULE_PATHNAME', 'pg_stat_monitor_hook_stats' -LANGUAGE C STRICT VOLATILE PARALLEL SAFE; +-- CREATE FUNCTION pg_stat_monitor_hook_stats( +-- OUT hook text, +-- OUT min_time float8, +-- OUT max_time float8, +-- OUT total_time float8, +-- OUT ncalls int8 +--) +--RETURNS SETOF record +--AS 'MODULE_PATHNAME', 'pg_stat_monitor_hook_stats' +--LANGUAGE C STRICT VOLATILE PARALLEL SAFE; -CREATE VIEW pg_stat_monitor_hook_stats AS SELECT - hook, - min_time, - max_time, - total_time, - total_time / greatest(ncalls, 1) as avg_time, - ncalls, - ROUND(CAST(total_time / greatest(sum(total_time) OVER(), 0.00000001) * 100 as numeric), 2)::text || '%' as load_comparison -FROM pg_stat_monitor_hook_stats(); +--CREATE VIEW pg_stat_monitor_hook_stats AS SELECT +-- hook, +-- min_time, +-- max_time, +-- total_time, +-- total_time / greatest(ncalls, 1) as avg_time, +-- ncalls, +-- ROUND(CAST(total_time / greatest(sum(total_time) OVER(), 0.00000001) * 100 as numeric), 2)::text || '%' as load_comparison +-- FROM pg_stat_monitor_hook_stats(); GRANT SELECT ON pg_stat_monitor TO PUBLIC; GRANT SELECT ON pg_stat_monitor_settings TO PUBLIC; + -- Don't want this to be available to non-superusers. REVOKE ALL ON FUNCTION pg_stat_monitor_reset() FROM PUBLIC; + + From 74dd7c80d8be05e4e2c6a49c67fb3ae654c6cff0 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Tue, 16 Nov 2021 11:46:15 +0000 Subject: [PATCH 19/56] Regression Fix. --- regression/expected/guc_1.out | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/regression/expected/guc_1.out b/regression/expected/guc_1.out index 0da2986..1425109 100644 --- a/regression/expected/guc_1.out +++ b/regression/expected/guc_1.out @@ -23,12 +23,11 @@ SELECT * FROM pg_stat_monitor_settings ORDER BY name COLLATE "C"; pg_stat_monitor.pgsm_max | 100 | 100 | Sets the maximum size of shared memory in (MB) used for statement's metadata tracked by pg_stat_monitor. | 1 | 1000 | 1 pg_stat_monitor.pgsm_max_buckets | 10 | 10 | Sets the maximum number of buckets. | 1 | 10 | 1 pg_stat_monitor.pgsm_normalized_query | 1 | 1 | Selects whether save query in normalized format. | 0 | 0 | 0 - pg_stat_monitor.pgsm_overflow_target | 1 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | 1 + pg_stat_monitor.pgsm_overflow_target | 0 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | 1 pg_stat_monitor.pgsm_query_max_len | 1024 | 1024 | Sets the maximum length of query. | 1024 | 2147483647 | 1 pg_stat_monitor.pgsm_query_shared_buffer | 20 | 20 | Sets the maximum size of shared memory in (MB) used for query tracked by pg_stat_monitor. | 1 | 10000 | 1 - pg_stat_monitor.pgsm_track_planning | 1 | 1 | Selects whether planning statistics are tracked. | 0 | 0 | 0 pg_stat_monitor.pgsm_track_utility | 1 | 1 | Selects whether utility commands are tracked. | 0 | 0 | 0 -(14 rows) +(13 rows) SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset From d3725790d4cdac7654c099d52bb2f1d41cfd3a74 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Tue, 16 Nov 2021 12:01:46 +0000 Subject: [PATCH 20/56] Regression Fix. --- regression/expected/guc.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression/expected/guc.out b/regression/expected/guc.out index 934c719..0da2986 100644 --- a/regression/expected/guc.out +++ b/regression/expected/guc.out @@ -23,7 +23,7 @@ SELECT * FROM pg_stat_monitor_settings ORDER BY name COLLATE "C"; pg_stat_monitor.pgsm_max | 100 | 100 | Sets the maximum size of shared memory in (MB) used for statement's metadata tracked by pg_stat_monitor. | 1 | 1000 | 1 pg_stat_monitor.pgsm_max_buckets | 10 | 10 | Sets the maximum number of buckets. | 1 | 10 | 1 pg_stat_monitor.pgsm_normalized_query | 1 | 1 | Selects whether save query in normalized format. | 0 | 0 | 0 - pg_stat_monitor.pgsm_overflow_target | 0 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | 1 + pg_stat_monitor.pgsm_overflow_target | 1 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | 1 pg_stat_monitor.pgsm_query_max_len | 1024 | 1024 | Sets the maximum length of query. | 1024 | 2147483647 | 1 pg_stat_monitor.pgsm_query_shared_buffer | 20 | 20 | Sets the maximum size of shared memory in (MB) used for query tracked by pg_stat_monitor. | 1 | 10000 | 1 pg_stat_monitor.pgsm_track_planning | 1 | 1 | Selects whether planning statistics are tracked. | 0 | 0 | 0 From c577750538d60dce59e398f6588ae8e2d2968478 Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Tue, 16 Nov 2021 16:31:44 -0300 Subject: [PATCH 21/56] PG-277: Fix regression tests (pgsm_overflow_target defaults). Adjust guc_1.out to match guc.out defaults for pgsm_overflow_target. --- regression/expected/guc_1.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression/expected/guc_1.out b/regression/expected/guc_1.out index 1425109..ef3da0b 100644 --- a/regression/expected/guc_1.out +++ b/regression/expected/guc_1.out @@ -23,7 +23,7 @@ SELECT * FROM pg_stat_monitor_settings ORDER BY name COLLATE "C"; pg_stat_monitor.pgsm_max | 100 | 100 | Sets the maximum size of shared memory in (MB) used for statement's metadata tracked by pg_stat_monitor. | 1 | 1000 | 1 pg_stat_monitor.pgsm_max_buckets | 10 | 10 | Sets the maximum number of buckets. | 1 | 10 | 1 pg_stat_monitor.pgsm_normalized_query | 1 | 1 | Selects whether save query in normalized format. | 0 | 0 | 0 - pg_stat_monitor.pgsm_overflow_target | 0 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | 1 + pg_stat_monitor.pgsm_overflow_target | 1 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | 1 pg_stat_monitor.pgsm_query_max_len | 1024 | 1024 | Sets the maximum length of query. | 1024 | 2147483647 | 1 pg_stat_monitor.pgsm_query_shared_buffer | 20 | 20 | Sets the maximum size of shared memory in (MB) used for query tracked by pg_stat_monitor. | 1 | 10000 | 1 pg_stat_monitor.pgsm_track_utility | 1 | 1 | Selects whether utility commands are tracked. | 0 | 0 | 0 From b5cbd98ae80f200d1bac8ecabc355372ea444944 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Wed, 17 Nov 2021 01:19:32 +0500 Subject: [PATCH 22/56] Update and rename pg11test.yml to postgresql-11-build.yml --- .github/workflows/{pg11test.yml => postgresql-11-build.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{pg11test.yml => postgresql-11-build.yml} (99%) diff --git a/.github/workflows/pg11test.yml b/.github/workflows/postgresql-11-build.yml similarity index 99% rename from .github/workflows/pg11test.yml rename to .github/workflows/postgresql-11-build.yml index a1f7e53..1853c94 100644 --- a/.github/workflows/pg11test.yml +++ b/.github/workflows/postgresql-11-build.yml @@ -1,4 +1,4 @@ -name: Test-with-pg11-build +name: postgresql-11-build on: [push] jobs: From 5e2b1074710f5eccdedca6eec42455856373962d Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Wed, 17 Nov 2021 01:20:08 +0500 Subject: [PATCH 23/56] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c5b2206..82507f0 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![PostgreSQL-11](https://github.com/percona/pg_stat_monitor/workflows/pg11-test/badge.svg) +![PostgreSQL-11](https://github.com/percona/pg_stat_monitor/workflows/postgresql-11-build/badge.svg) ![PostgreSQL-12](https://github.com/percona/pg_stat_monitor/workflows/pg12-test/badge.svg) ![PostgreSQL-13](https://github.com/percona/pg_stat_monitor/workflows/pg13-test/badge.svg) ![PostgreSQL-14](https://github.com/percona/pg_stat_monitor/workflows/pg14-test/badge.svg) From 88523792f571e3208e5feb9da8542dca8f8ba72b Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Tue, 16 Nov 2021 20:30:34 +0000 Subject: [PATCH 24/56] Rename workflows. --- ...pg11packagetest.yml => postgresql-11-package.yml} | 4 ++-- ...g-packages.yml => postgresql-11-pgdg-package.yml} | 4 ++-- .../{pg12test.yml => postgresql-12-build.yml} | 4 ++-- ...pg12packagetest.yml => postgresql-12-package.yml} | 4 ++-- ...g-packages.yml => postgresql-12-pgdg-package.yml} | 4 ++-- .../{pg13test.yml => postgresql-13-build.yml} | 4 ++-- ...pg13packagetest.yml => postgresql-13-package.yml} | 4 ++-- ...g-packages.yml => postgresql-13-pgdg-package.yml} | 4 ++-- .../{pg14test.yml => postgresql-14-build.yml} | 4 ++-- README.md | 12 ++++++------ 10 files changed, 24 insertions(+), 24 deletions(-) rename .github/workflows/{pg11packagetest.yml => postgresql-11-package.yml} (97%) rename .github/workflows/{pg11test-pgdg-packages.yml => postgresql-11-pgdg-package.yml} (96%) rename .github/workflows/{pg12test.yml => postgresql-12-build.yml} (98%) rename .github/workflows/{pg12packagetest.yml => postgresql-12-package.yml} (97%) rename .github/workflows/{pg12test-pgdg-packages.yml => postgresql-12-pgdg-package.yml} (96%) rename .github/workflows/{pg13test.yml => postgresql-13-build.yml} (98%) rename .github/workflows/{pg13packagetest.yml => postgresql-13-package.yml} (97%) rename .github/workflows/{pg13test-pgdg-packages.yml => postgresql-13-pgdg-package.yml} (96%) rename .github/workflows/{pg14test.yml => postgresql-14-build.yml} (98%) diff --git a/.github/workflows/pg11packagetest.yml b/.github/workflows/postgresql-11-package.yml similarity index 97% rename from .github/workflows/pg11packagetest.yml rename to .github/workflows/postgresql-11-package.yml index 71913bc..e9ef1d3 100644 --- a/.github/workflows/pg11packagetest.yml +++ b/.github/workflows/postgresql-11-package.yml @@ -1,9 +1,9 @@ -name: pg11package-test +name: postgresql-11-package on: [push] jobs: build: - name: pg11package-test + name: postgresql-11-package runs-on: ubuntu-20.04 steps: - name: Clone pg_stat_monitor repository diff --git a/.github/workflows/pg11test-pgdg-packages.yml b/.github/workflows/postgresql-11-pgdg-package.yml similarity index 96% rename from .github/workflows/pg11test-pgdg-packages.yml rename to .github/workflows/postgresql-11-pgdg-package.yml index ad8ef4d..f456798 100644 --- a/.github/workflows/pg11test-pgdg-packages.yml +++ b/.github/workflows/postgresql-11-pgdg-package.yml @@ -1,9 +1,9 @@ -name: Test-with-pg11-pgdg-packages +name: postgresql-11-pgdg-package on: [push] jobs: build: - name: pg11-test-with-pgdg-packages + name: postgresql-11-pgdg-package runs-on: ubuntu-18.04 steps: - name: Clone pg_stat_monitor repository diff --git a/.github/workflows/pg12test.yml b/.github/workflows/postgresql-12-build.yml similarity index 98% rename from .github/workflows/pg12test.yml rename to .github/workflows/postgresql-12-build.yml index 919614b..b4d6e19 100644 --- a/.github/workflows/pg12test.yml +++ b/.github/workflows/postgresql-12-build.yml @@ -1,9 +1,9 @@ -name: Test-with-pg12-build +name: postgresql-12-build on: [push] jobs: build: - name: pg12-test + name: postgresql-12-build runs-on: ubuntu-latest steps: - name: Clone postgres repository diff --git a/.github/workflows/pg12packagetest.yml b/.github/workflows/postgresql-12-package.yml similarity index 97% rename from .github/workflows/pg12packagetest.yml rename to .github/workflows/postgresql-12-package.yml index d8207a3..2daed64 100644 --- a/.github/workflows/pg12packagetest.yml +++ b/.github/workflows/postgresql-12-package.yml @@ -1,9 +1,9 @@ -name: pg12package-test +name: postgresql-12-package on: [push] jobs: build: - name: pg12package-test + name: postgresql-12-package runs-on: ubuntu-20.04 steps: - name: Clone pg_stat_monitor repository diff --git a/.github/workflows/pg12test-pgdg-packages.yml b/.github/workflows/postgresql-12-pgdg-package.yml similarity index 96% rename from .github/workflows/pg12test-pgdg-packages.yml rename to .github/workflows/postgresql-12-pgdg-package.yml index 274e082..7d7e635 100644 --- a/.github/workflows/pg12test-pgdg-packages.yml +++ b/.github/workflows/postgresql-12-pgdg-package.yml @@ -1,9 +1,9 @@ -name: Test-with-pg12-pgdg-packages +name: postgresql-12-pgdg-package on: [push] jobs: build: - name: pg12-test-with-pgdg-packages + name: postgresql-12-pgdg-package runs-on: ubuntu-latest steps: - name: Clone pg_stat_monitor repository diff --git a/.github/workflows/pg13test.yml b/.github/workflows/postgresql-13-build.yml similarity index 98% rename from .github/workflows/pg13test.yml rename to .github/workflows/postgresql-13-build.yml index 6a85cd5..67d0eff 100644 --- a/.github/workflows/pg13test.yml +++ b/.github/workflows/postgresql-13-build.yml @@ -1,9 +1,9 @@ -name: Test-with-pg13-build +name: postgresql-13-build on: [push] jobs: build: - name: pg13-test + name: postgresql-13-build runs-on: ubuntu-latest steps: - name: Clone postgres repository diff --git a/.github/workflows/pg13packagetest.yml b/.github/workflows/postgresql-13-package.yml similarity index 97% rename from .github/workflows/pg13packagetest.yml rename to .github/workflows/postgresql-13-package.yml index 7bc1176..9a15a0e 100644 --- a/.github/workflows/pg13packagetest.yml +++ b/.github/workflows/postgresql-13-package.yml @@ -1,9 +1,9 @@ -name: pg13package-test +name: postgresql-13-package on: [push] jobs: build: - name: pg13package-test + name: postgresql-13-package runs-on: ubuntu-20.04 steps: - name: Clone pg_stat_monitor repository diff --git a/.github/workflows/pg13test-pgdg-packages.yml b/.github/workflows/postgresql-13-pgdg-package.yml similarity index 96% rename from .github/workflows/pg13test-pgdg-packages.yml rename to .github/workflows/postgresql-13-pgdg-package.yml index 1a718d2..47e4ee1 100644 --- a/.github/workflows/pg13test-pgdg-packages.yml +++ b/.github/workflows/postgresql-13-pgdg-package.yml @@ -1,9 +1,9 @@ -name: Test-with-pg13-pgdg-packages +name: postgresql-13-pgdg-package on: [push] jobs: build: - name: pg13-test-with-pgdg-packages + name: postgresql-13-pgdg-package runs-on: ubuntu-latest steps: - name: Clone pg_stat_monitor repository diff --git a/.github/workflows/pg14test.yml b/.github/workflows/postgresql-14-build.yml similarity index 98% rename from .github/workflows/pg14test.yml rename to .github/workflows/postgresql-14-build.yml index 0824bd4..df6109d 100644 --- a/.github/workflows/pg14test.yml +++ b/.github/workflows/postgresql-14-build.yml @@ -1,9 +1,9 @@ -name: Test-with-pg14-build +name: postgresql-14-build on: [push] jobs: build: - name: pg14-test + name: postgresql-14-build runs-on: ubuntu-latest steps: - name: Clone postgres repository diff --git a/README.md b/README.md index 82507f0..392bd25 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ ![PostgreSQL-11](https://github.com/percona/pg_stat_monitor/workflows/postgresql-11-build/badge.svg) -![PostgreSQL-12](https://github.com/percona/pg_stat_monitor/workflows/pg12-test/badge.svg) -![PostgreSQL-13](https://github.com/percona/pg_stat_monitor/workflows/pg13-test/badge.svg) -![PostgreSQL-14](https://github.com/percona/pg_stat_monitor/workflows/pg14-test/badge.svg) -![PostgreSQL-11-Package](https://github.com/percona/pg_stat_monitor/workflows/pg11package-test/badge.svg) -![PostgreSQL-12-Packages](https://github.com/percona/pg_stat_monitor/workflows/pg12package-test/badge.svg) -![PostgreSQL-13-Packages](https://github.com/percona/pg_stat_monitor/workflows/pg13package-test/badge.svg) +![PostgreSQL-12](https://github.com/percona/pg_stat_monitor/workflows/postgresql-12-build/badge.svg) +![PostgreSQL-13](https://github.com/percona/pg_stat_monitor/workflows/postgresql-13-build/badge.svg) +![PostgreSQL-14](https://github.com/percona/pg_stat_monitor/workflows/postgresql-14-build/badge.svg) +![PostgreSQL-11-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-11-package/badge.svg) +![PostgreSQL-12-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-12-package/badge.svg) +![PostgreSQL-13-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-13-package/badge.svg) [![Coverage Status](https://coveralls.io/repos/github/percona/pg_stat_monitor/badge.svg)](https://coveralls.io/github/percona/pg_stat_monitor) From a0c848749cc1b2da0cfd3c5fde256ddd88ec6147 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Wed, 17 Nov 2021 01:36:38 +0500 Subject: [PATCH 25/56] Update README.md --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 392bd25..68ac46a 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ -![PostgreSQL-11](https://github.com/percona/pg_stat_monitor/workflows/postgresql-11-build/badge.svg) -![PostgreSQL-12](https://github.com/percona/pg_stat_monitor/workflows/postgresql-12-build/badge.svg) -![PostgreSQL-13](https://github.com/percona/pg_stat_monitor/workflows/postgresql-13-build/badge.svg) -![PostgreSQL-14](https://github.com/percona/pg_stat_monitor/workflows/postgresql-14-build/badge.svg) -![PostgreSQL-11-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-11-package/badge.svg) -![PostgreSQL-12-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-12-package/badge.svg) -![PostgreSQL-13-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-13-package/badge.svg) +* ![PostgreSQL-11](https://github.com/percona/pg_stat_monitor/workflows/postgresql-11-build/badge.svg) ![PostgreSQL-11-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-11-package/badge.svg) +* ![PostgreSQL-12](https://github.com/percona/pg_stat_monitor/workflows/postgresql-12-build/badge.svg) ![PostgreSQL-12-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-12-package/badge.svg) + +* ![PostgreSQL-13](https://github.com/percona/pg_stat_monitor/workflows/postgresql-13-build/badge.svg) ![PostgreSQL-13-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-13-package/badge.svg) + +* ![PostgreSQL-14](https://github.com/percona/pg_stat_monitor/workflows/postgresql-14-build/badge.svg) + + [![Coverage Status](https://coveralls.io/repos/github/percona/pg_stat_monitor/badge.svg)](https://coveralls.io/github/percona/pg_stat_monitor) From ff7d82be6e198601719d20cb3b970a3d67a32dcc Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Wed, 17 Nov 2021 01:38:24 +0500 Subject: [PATCH 26/56] Update README.md --- README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 68ac46a..d5d6883 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,6 @@ -* ![PostgreSQL-11](https://github.com/percona/pg_stat_monitor/workflows/postgresql-11-build/badge.svg) ![PostgreSQL-11-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-11-package/badge.svg) -* ![PostgreSQL-12](https://github.com/percona/pg_stat_monitor/workflows/postgresql-12-build/badge.svg) ![PostgreSQL-12-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-12-package/badge.svg) +![PostgreSQL-11](https://github.com/percona/pg_stat_monitor/workflows/postgresql-11-build/badge.svg) ![PostgreSQL-11-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-11-package/badge.svg) ![PostgreSQL-12](https://github.com/percona/pg_stat_monitor/workflows/postgresql-12-build/badge.svg) ![PostgreSQL-12-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-12-package/badge.svg) -* ![PostgreSQL-13](https://github.com/percona/pg_stat_monitor/workflows/postgresql-13-build/badge.svg) ![PostgreSQL-13-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-13-package/badge.svg) - -* ![PostgreSQL-14](https://github.com/percona/pg_stat_monitor/workflows/postgresql-14-build/badge.svg) +![PostgreSQL-13](https://github.com/percona/pg_stat_monitor/workflows/postgresql-13-build/badge.svg) ![PostgreSQL-13-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-13-package/badge.svg) ![PostgreSQL-14](https://github.com/percona/pg_stat_monitor/workflows/postgresql-14-build/badge.svg) From 0248c1fe5e0b85aac3ccac52bbd5e754285c7e8b Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Tue, 16 Nov 2021 20:41:04 +0000 Subject: [PATCH 27/56] Rename workflows. --- .github/workflows/pg14test-pgdg-packages.yml | 63 -------------------- 1 file changed, 63 deletions(-) delete mode 100644 .github/workflows/pg14test-pgdg-packages.yml diff --git a/.github/workflows/pg14test-pgdg-packages.yml b/.github/workflows/pg14test-pgdg-packages.yml deleted file mode 100644 index ad6bc91..0000000 --- a/.github/workflows/pg14test-pgdg-packages.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Test-with-pg14-pgdg-packages -on: [push] - -jobs: - build: - name: pg14-test-with-pgdg-packages - runs-on: ubuntu-latest - steps: - - name: Clone pg_stat_monitor repository - uses: actions/checkout@v2 - with: - path: 'src/pg_stat_monitor' - - - name: Delete old postgresql files - run: | - sudo apt-get update - sudo apt purge postgresql-client-common postgresql-common postgresql postgresql* - sudo rm -rf /var/lib/postgresql/ - sudo rm -rf /var/log/postgresql/ - sudo rm -rf /etc/postgresql/ - sudo rm -rf /usr/lib/postgresql - sudo rm -rf /usr/include/postgresql - sudo rm -rf /usr/share/postgresql - sudo rm -rf /etc/postgresql - sudo rm -f /usr/bin/pg_config - - - name: Install PG Distribution Postgresql 14 - run: | - sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main 14" > /etc/apt/sources.list.d/pgdg.list' - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - - sudo apt-get update - sudo apt-get -y install postgresql-14 - sudo apt-get update - sudo apt-get -y install postgresql-client-14 - sudo apt-get update - sudo apt-get -y install postgresql-server-dev-14 - sudo chown -R postgres:postgres src/ - - - name: Build pg_stat_monitor - run: | - export PATH="/usr/lib/postgresql/14/bin:$PATH" - sudo cp /usr/lib/postgresql/14/bin/pg_config /usr/bin - sudo make USE_PGXS=1 - sudo make USE_PGXS=1 install - working-directory: src/pg_stat_monitor/ - - - name: Start pg_stat_monitor_tests - run: | - sudo service postgresql stop - echo "shared_preload_libraries = 'pg_stat_monitor'" | sudo tee -a /etc/postgresql/14/main/postgresql.conf - sudo service postgresql start - sudo -u postgres bash -c 'make installcheck USE_PGXS=1' - working-directory: src/pg_stat_monitor/ - - - name: Report on test fail - uses: actions/upload-artifact@v2 - if: ${{ failure() }} - with: - name: Regressions diff and postgresql log - path: | - src/pg_stat_monitor/regression.diffs - src/pg_stat_monitor/logfile - retention-days: 1 From 973cb16d345dcfd3a63170637b05a84e53fab286 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Wed, 17 Nov 2021 01:42:10 +0500 Subject: [PATCH 28/56] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d5d6883..981e8ff 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ![PostgreSQL-11](https://github.com/percona/pg_stat_monitor/workflows/postgresql-11-build/badge.svg) ![PostgreSQL-11-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-11-package/badge.svg) ![PostgreSQL-12](https://github.com/percona/pg_stat_monitor/workflows/postgresql-12-build/badge.svg) ![PostgreSQL-12-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-12-package/badge.svg) -![PostgreSQL-13](https://github.com/percona/pg_stat_monitor/workflows/postgresql-13-build/badge.svg) ![PostgreSQL-13-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-13-package/badge.svg) ![PostgreSQL-14](https://github.com/percona/pg_stat_monitor/workflows/postgresql-14-build/badge.svg) +![PostgreSQL-13](https://github.com/percona/pg_stat_monitor/workflows/postgresql-13-build/badge.svg) ![PostgreSQL-13-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-13-package/badge.svg) ![PostgreSQL-14](https://github.com/percona/pg_stat_monitor/workflows/postgresql-14-build/badge.svg) ![PostgreSQL-14-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-14-package/badge.svg) From 6cf798416e10cb0220a74ff572962566300524e6 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Wed, 17 Nov 2021 01:42:55 +0500 Subject: [PATCH 29/56] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 981e8ff..5097660 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ![PostgreSQL-11](https://github.com/percona/pg_stat_monitor/workflows/postgresql-11-build/badge.svg) ![PostgreSQL-11-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-11-package/badge.svg) ![PostgreSQL-12](https://github.com/percona/pg_stat_monitor/workflows/postgresql-12-build/badge.svg) ![PostgreSQL-12-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-12-package/badge.svg) -![PostgreSQL-13](https://github.com/percona/pg_stat_monitor/workflows/postgresql-13-build/badge.svg) ![PostgreSQL-13-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-13-package/badge.svg) ![PostgreSQL-14](https://github.com/percona/pg_stat_monitor/workflows/postgresql-14-build/badge.svg) ![PostgreSQL-14-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-14-package/badge.svg) +![PostgreSQL-13](https://github.com/percona/pg_stat_monitor/workflows/postgresql-13-build/badge.svg) ![PostgreSQL-13-Package](https://github.com/percona/pg_stat_monitor/workflows/postgresql-13-package/badge.svg) ![PostgreSQL-14](https://github.com/percona/pg_stat_monitor/workflows/postgresql-14-build/badge.svg) From 892a117487ebad76809f76f00f9214e8d363a7b5 Mon Sep 17 00:00:00 2001 From: Evgeniy Patlan Date: Wed, 17 Nov 2021 07:19:18 +0200 Subject: [PATCH 30/56] DISTPG-307 update pg_stat_stat_monitor --- percona-packaging/rpm/pg-stat-monitor.spec | 12 +++++++----- percona-packaging/scripts/pg_stat_monitor_builder.sh | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/percona-packaging/rpm/pg-stat-monitor.spec b/percona-packaging/rpm/pg-stat-monitor.spec index 243dddc..e59ab61 100644 --- a/percona-packaging/rpm/pg-stat-monitor.spec +++ b/percona-packaging/rpm/pg-stat-monitor.spec @@ -1,4 +1,4 @@ -%global sname percona-pg-stat-monitor +%global sname percona-pg_stat_monitor %global pgrel @@PG_REL@@ %global rpm_release @@RPM_RELEASE@@ %global pginstdir /usr/pgsql-@@PG_REL@@/ @@ -8,12 +8,14 @@ Name: %{sname}%{pgrel} Version: @@VERSION@@ Release: %{rpm_release}%{?dist} License: PostgreSQL -Source0: %{sname}%{pgrel}-%{version}.tar.gz +Source0: percona-pg-stat-monitor%{pgrel}-%{version}.tar.gz URL: https://github.com/Percona-Lab/pg_stat_monitor BuildRequires: percona-postgresql%{pgrel}-devel Requires: postgresql-server +Provides: percona-pg-stat-monitor%{pgrel} Epoch: 1 - +Packager: Percona Development Team +Vendor: Percona, Inc %description The pg_stat_monitor is statistics collector tool @@ -25,7 +27,7 @@ It provides all the features of pg_stat_statment plus its own feature set. %prep -%setup -q -n %{sname}%{pgrel}-%{version} +%setup -q -n percona-pg-stat-monitor%{pgrel}-%{version} %build @@ -61,5 +63,5 @@ sed -i 's:PG_CONFIG = pg_config:PG_CONFIG = /usr/pgsql-%{pgrel}/bin/pg_config:' %changelog -* Thu Dec 19 2019 Oleksandr Miroshnychenko - 1.0.0-1 +* Wed Nov 17 2021 Evgeniy Patlan - 1.0.0-1 - Initial build diff --git a/percona-packaging/scripts/pg_stat_monitor_builder.sh b/percona-packaging/scripts/pg_stat_monitor_builder.sh index 27a27b9..52b2ccb 100644 --- a/percona-packaging/scripts/pg_stat_monitor_builder.sh +++ b/percona-packaging/scripts/pg_stat_monitor_builder.sh @@ -333,10 +333,10 @@ build_rpm(){ echo "It is not possible to build rpm here" exit 1 fi - SRC_RPM=$(basename $(find $WORKDIR/srpm -name 'percona-pg-stat-monitor*.src.rpm' | sort | tail -n1)) + SRC_RPM=$(basename $(find $WORKDIR/srpm -name 'percona-pg_stat_monitor*.src.rpm' | sort | tail -n1)) if [ -z $SRC_RPM ] then - SRC_RPM=$(basename $(find $CURDIR/srpm -name 'percona-pg-stat-monitor*.src.rpm' | sort | tail -n1)) + SRC_RPM=$(basename $(find $CURDIR/srpm -name 'percona-pg_stat_monitor*.src.rpm' | sort | tail -n1)) if [ -z $SRC_RPM ] then echo "There is no src rpm for build" From bb7fd54b743db73299a7570f485df320a1c19e97 Mon Sep 17 00:00:00 2001 From: Anastasia Alexadrova Date: Tue, 16 Nov 2021 18:15:45 +0200 Subject: [PATCH 31/56] PG-210 Doc: Updated column names depending on PG version added a footenote about toplevel being available starting from PG14 only modified: docs/COMPARISON.md modified: docs/REFERENCE.md --- docs/COMPARISON.md | 25 ++++++++++++++++--------- docs/REFERENCE.md | 23 +++++++++++++---------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/docs/COMPARISON.md b/docs/COMPARISON.md index bbeaf11..92aa8a4 100644 --- a/docs/COMPARISON.md +++ b/docs/COMPARISON.md @@ -21,6 +21,7 @@ Note that the column names differ depending on the PostgreSQL version you are ru bucket_start_time | bucket_start_time | :heavy_check_mark: | :x: userid | userid | :heavy_check_mark: | :heavy_check_mark: datname | datname | :heavy_check_mark: | :heavy_check_mark: +toplevel[^1] | | :heavy_check_mark: | :heavy_check_mark: client_ip | client_ip | :heavy_check_mark:| :x: queryid | queryid | :heavy_check_mark: | :heavy_check_mark: planid | planid | :heavy_check_mark:| :x: @@ -35,16 +36,17 @@ elevel | elevel | :heavy_check_mark: | :x: sqlcode | sqlcode | :heavy_check_mark: | :x: message | message | :heavy_check_mark: | :x: plans_calls | plans_calls | :heavy_check_mark: | :heavy_check_mark: -plan_total_time | plan_total_time | :heavy_check_mark: | :heavy_check_mark: -plan_min_time | plan_min_time | :heavy_check_mark: | :heavy_check_mark: -plan_max_time | plan_max_time | :heavy_check_mark: | :heavy_check_mark: -plan_mean_time | plan_mean_time | :heavy_check_mark: | :heavy_check_mark: +total_plan_time | | :heavy_check_mark: | :heavy_check_mark: +min_plan_time | | :heavy_check_mark: | :heavy_check_mark: +max_plan_time | | :heavy_check_mark: | :heavy_check_mark: +mean_plan_time | | :heavy_check_mark: | :heavy_check_mark: +stddev_plan_time | | :heavy_check_mark: | :heavy_check_mark: calls | calls | :heavy_check_mark: | :heavy_check_mark: -total_time | total_time | :heavy_check_mark: | :heavy_check_mark: -min_time | min_time | :heavy_check_mark: | :heavy_check_mark: -max_time | max_time | :heavy_check_mark: | :heavy_check_mark: -mean_time | mean_time | :heavy_check_mark: | :heavy_check_mark: -stddev_time | stddev_time | :heavy_check_mark: | :heavy_check_mark: +total_exec_time | total_time | :heavy_check_mark: | :heavy_check_mark: +min_exec_time | min_time | :heavy_check_mark: | :heavy_check_mark: +max_exec_time | max_time | :heavy_check_mark: | :heavy_check_mark: +mean_exec_time | mean_time | :heavy_check_mark: | :heavy_check_mark: +stddev_exec_time | stddev_time | :heavy_check_mark: | :heavy_check_mark: rows_retrieved | rows_retrieved | :heavy_check_mark: | :heavy_check_mark: shared_blks_hit | shared_blks_hit | :heavy_check_mark: | :heavy_check_mark: shared_blks_read | shared_blks_read | :heavy_check_mark: | :heavy_check_mark: @@ -71,3 +73,8 @@ To learn more about the features in `pg_stat_monitor`, please see the [User guid Additional reading: [pg_stat_statements](https://www.postgresql.org/docs/current/pgstatstatements.html) + + + + +[^1]: Available starting from PostgreSQL 14 and above diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 5002cd6..9831871 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -6,7 +6,7 @@ postgres=# \d pg_stat_monitor ``` -Depending on the PostgreSQL version, some column names may differ. The following table describes the `pg_stat_monitor` view for PostgreSQL 13 and higher versions. +Depending on the PostgreSQL version, some column names may differ. The following table describes the `pg_stat_monitor` view for PostgreSQL 14 and higher versions. | Column | Type | Description @@ -15,6 +15,7 @@ Depending on the PostgreSQL version, some column names may differ. The following bucket_start_time | timestamp with time zone | The start time of the bucket| userid | regrole | An ID of the user who run a query | datname | name | The name of a database where the query was executed +toplevel | bool | True means that a query was executed as a top-level statement client_ip | inet | The IP address of a client that run the query queryid | text | The internal hash code serving to identify every query in a statement planid | text | An internally generated ID of a query plan @@ -29,17 +30,17 @@ elevel | integer | Shows the error level of a que sqlcode | integer | SQL error code message | text | The error message plans_calls | bigint | The number of times the statement was planned -plan_total_time | double precision | The total time (in ms) spent on planning the statement -plan_min_time | double precision | Minimum time (in ms) spent on planning the statement -plan_max_time | double precision | Maximum time (in ms) spent on planning the statement -plan_mean_time | double precision | The mean (average) time (in ms) spent on planning the statement -plan_stddev_time | double precision | The standard deviation of time (in ms) spent on planning the statement +total_plan_time | double precision | The total time (in ms) spent on planning the statement +min_plan_time | double precision | Minimum time (in ms) spent on planning the statement +max_plan_time | double precision | Maximum time (in ms) spent on planning the statement +mean_plan_time | double precision | The mean (average) time (in ms) spent on planning the statement +stddev_plan_time | double precision | The standard deviation of time (in ms) spent on planning the statement calls | bigint | The number of times a particular query was executed -total_time | double precision | The total time (in ms) spent on executing a query -min_time | double precision | The minimum time (in ms) it took to execute a query -max_time | double precision | The maximum time (in ms) it took to execute a query +total_exec_time | double precision | The total time (in ms) spent on executing a query +min_exec_time | double precision | The minimum time (in ms) it took to execute a query +max_exec_time | double precision | The maximum time (in ms) it took to execute a query mean_time | double precision | The mean (average) time (in ms) it took to execute a query -stddev_time | double precision | The standard deviation of time (in ms) spent on executing a query +stddev_exec_time | double precision | The standard deviation of time (in ms) spent on executing a query rows_retrieved | bigint | The number of rows retrieved when executing a query shared_blks_hit | bigint | Shows the total number of shared memory blocks returned from the cache shared_blks_read | bigint | Shows the total number of shared blocks returned not from the cache @@ -61,3 +62,5 @@ wal_fpi | bigint | The total number of WAL FPI (Full Pag wal_bytes | numeric | Total number of bytes used for the WAL generated by the query state_code | bigint | Shows the state code of a query state | text | The state message + + From 47e84f96c362b3d09feaa575c09b59dd84daadce Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Mon, 22 Nov 2021 15:13:30 -0300 Subject: [PATCH 32/56] PG-234: Fix loading both pg_stat_monitor and pg_stat_statements. If both modules are loaded then pg_stat_monitor detects that and avoid calling standard_ProcessUtility() in ProcessUtility_hook hook, as calling it twice is an error and triggers an assertion on PostgreSQL. On PostgreSQL 13, pg_stat_monitor must be loaded after pg_stat_statements, as pg_stat_statements doesn't do such verifications, it end calling standard_ProcessUtility() and other functions even if another module is registered, that is an error. They fixed this problem with pg_stat_statements in PostgreSQL 14 and onward. --- pg_stat_monitor.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 768fa8c..8ae1ac5 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -1046,7 +1046,8 @@ static void pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, context, params, queryEnv, dest, qc); - standard_ProcessUtility(pstmt, queryString, + else + standard_ProcessUtility(pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, @@ -1057,7 +1058,8 @@ static void pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, context, params, queryEnv, dest, qc); - standard_ProcessUtility(pstmt, queryString, + else + standard_ProcessUtility(pstmt, queryString, context, params, queryEnv, dest, qc); @@ -1067,7 +1069,8 @@ static void pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, context, params, queryEnv, dest, completionTag); - standard_ProcessUtility(pstmt, queryString, + else + standard_ProcessUtility(pstmt, queryString, context, params, queryEnv, dest, completionTag); From f7275071cd76fd84dddf8a64be0514f3a368c433 Mon Sep 17 00:00:00 2001 From: Anastasia Alexadrova Date: Wed, 24 Nov 2021 12:43:24 +0200 Subject: [PATCH 33/56] PG-285 Doc: Order of modules Added a note about strict order of modules for PG 13 and earlier versions modified: README.md --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5097660..4c516d8 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,11 @@ ALTER SYSTEM SET shared_preload_libraries = 'pg_stat_monitor'; ALTER SYSTEM ``` -**NOTE**: If you’ve added other values to the shared_preload_libraries parameter, list all of them separated by commas for the `ALTER SYSTEM` command. For example, `ALTER SYSTEM SET shared_preload_libraries = 'foo, bar, pg_stat_monitor'` +> **NOTE**: If you’ve added other modules to the `shared_preload_libraries` parameter (for example, `pg_stat_statements`), list all of them separated by commas for the `ALTER SYSTEM` command. +> +>:warning: For PostgreSQL 13 and earlier versions,`pg_stat_monitor` **must** follow `pg_stat_statements`. For example, `ALTER SYSTEM SET shared_preload_libraries = 'foo, pg_stat_statements, pg_stat_monitor'`. +> +>In PostgreSQL 14, modules can be specified in any order. Start or restart the `postgresql` instance to apply the changes. @@ -187,6 +191,7 @@ CREATE EXTENSION This allows you to see the stats collected by `pg_stat_monitor`. +By default, `pg_stat_monitor` is created for the `postgres` database. To access the statistics from other databases, you need to create the extension for every database. ``` -- Select some of the query information, like client_ip, username and application_name etc. From 8fe76769232ebf33cfdf8cd891807a1eb6ecbbf3 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Wed, 24 Nov 2021 18:55:13 +0000 Subject: [PATCH 34/56] PG-284: Bump version to 1.0.0-rc.1. --- pg_stat_monitor.c | 2 +- regression/expected/version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 8ae1ac5..4042de1 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -26,7 +26,7 @@ PG_MODULE_MAGIC; -#define BUILD_VERSION "1.0.0-beta-2" +#define BUILD_VERSION "1.0.0-rc.1" #define PG_STAT_STATEMENTS_COLS 53 /* maximum of above */ #define PGSM_TEXT_FILE "/tmp/pg_stat_monitor_query" diff --git a/regression/expected/version.out b/regression/expected/version.out index 77ff028..27f6dd1 100644 --- a/regression/expected/version.out +++ b/regression/expected/version.out @@ -2,7 +2,7 @@ CREATE EXTENSION pg_stat_monitor; SELECT pg_stat_monitor_version(); pg_stat_monitor_version ------------------------- - 1.0.0-beta-2 + 1.0.0-rc.1 (1 row) DROP EXTENSION pg_stat_monitor; From 3b155bd643181b8cc41b34c24e5646a75f9179fe Mon Sep 17 00:00:00 2001 From: Oleksandr Miroshnychenko Date: Fri, 26 Nov 2021 16:47:00 +0200 Subject: [PATCH 35/56] DISTPG-7 fir debian rules for 1.0.0-rc.1 version --- percona-packaging/debian/rules | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/percona-packaging/debian/rules b/percona-packaging/debian/rules index 2667c7f..f4f90c0 100755 --- a/percona-packaging/debian/rules +++ b/percona-packaging/debian/rules @@ -12,18 +12,17 @@ export USE_PGXS=1 dh $@ override_dh_auto_build: - +pg_buildext build build-%v + +pg_buildext clean . build-%v + +pg_buildext build . build-%v override_dh_auto_test: # nothing to do here override_dh_auto_install: - +pg_buildext install build-%v percona-pg-stat-monitor@@PG_REL@@ + +pg_buildext install . build-%v percona-pg-stat-monitor@@PG_REL@@ override_dh_installdocs: dh_installdocs --all README.* override_dh_auto_clean: - +pg_buildext clean build-%v - rm -rf results regression.* - mkdir results + rm -rf results/* regression.* From 5d526fbbd4daf42ccd996acdf66feee69ed6b6fd Mon Sep 17 00:00:00 2001 From: Evgeniy Patlan Date: Mon, 6 Dec 2021 14:43:51 +0200 Subject: [PATCH 36/56] Fix rpm packaging --- percona-packaging/rpm/pg-stat-monitor.spec | 2 ++ 1 file changed, 2 insertions(+) diff --git a/percona-packaging/rpm/pg-stat-monitor.spec b/percona-packaging/rpm/pg-stat-monitor.spec index e59ab61..e4c852c 100644 --- a/percona-packaging/rpm/pg-stat-monitor.spec +++ b/percona-packaging/rpm/pg-stat-monitor.spec @@ -13,6 +13,8 @@ URL: https://github.com/Percona-Lab/pg_stat_monitor BuildRequires: percona-postgresql%{pgrel}-devel Requires: postgresql-server Provides: percona-pg-stat-monitor%{pgrel} +Conflicts: percona-pg-stat-monitor%{pgrel} +Obsoletes: percona-pg-stat-monitor%{pgrel} Epoch: 1 Packager: Percona Development Team Vendor: Percona, Inc From 3433c77d9d3d298061fc93ba4acd8f534e7db29c Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Tue, 30 Nov 2021 16:15:53 -0300 Subject: [PATCH 37/56] PG-290: Fix crash when enabling debugging log level on PostgreSQL. There were couple issues to handle, the main one was that our log hook (pgsm_emit_log_hook) was being called after the shared memory hook completed (pgss_shmem_startup) but before PostgreSQL boostraping code finished, thus triggering the following assertion during a call to LWLockAcquire(): Assert(!(proc == NULL && IsUnderPostmaster)); proc is a pointer to MyProc, a PostgreSQL's shared global variable that was not yet initalized by PostgreSQL. We must also check for a NULL pointer return in pg_get_backend_status() the pgstat_fetch_stat_local_beentry() function may return a NULL pointer during initialization, in which case we use "127.0.0.1" for the client address, and "postmaster" for application name. --- pg_stat_monitor.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 4042de1..ec43578 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -1123,6 +1123,9 @@ pg_get_backend_status(void) PgBackendStatus *beentry; local_beentry = pgstat_fetch_stat_local_beentry(i); + if (!local_beentry) + continue; + beentry = &local_beentry->backendStatus; if (beentry->st_procpid == MyProcPid) @@ -1135,6 +1138,8 @@ static int pg_get_application_name(char *application_name) { PgBackendStatus *beentry = pg_get_backend_status(); + if (!beentry) + return snprintf(application_name, APPLICATIONNAME_LEN, "%s", "postmaster"); snprintf(application_name, APPLICATIONNAME_LEN, "%s", beentry->st_appname); return strlen(application_name); @@ -1157,6 +1162,9 @@ pg_get_client_addr(void) char remote_host[NI_MAXHOST]; int ret; + if (!beentry) + return ntohl(inet_addr("127.0.0.1")); + memset(remote_host, 0x0, NI_MAXHOST); ret = pg_getnameinfo_all(&beentry->st_clientaddr.addr, beentry->st_clientaddr.salen, @@ -1481,7 +1489,13 @@ pgss_store(uint64 queryid, return; Assert(query != NULL); - userid = GetUserId(); + if (kind == PGSS_ERROR) + { + int sec_ctx; + GetUserIdAndSecContext((Oid *)&userid, &sec_ctx); + } + else + userid = GetUserId(); application_name_len = pg_get_application_name(application_name); planid = plan_info ? plan_info->planid: 0; @@ -3303,6 +3317,10 @@ pgsm_emit_log_hook(ErrorData *edata) if (IsParallelWorker()) return; + /* Check if PostgreSQL has finished its own bootstraping code. */ + if (MyProc == NULL) + return; + if ((edata->elevel == ERROR || edata->elevel == WARNING || edata->elevel == INFO || edata->elevel == DEBUG1)) { uint64 queryid = 0; From 150df586eee5b132f82ba7c8e79d4eaf682f6c41 Mon Sep 17 00:00:00 2001 From: Umair Shahid Date: Tue, 28 Dec 2021 17:58:36 +0400 Subject: [PATCH 38/56] Update README.md Proposed changes for https://jira.percona.com/browse/PG-155 --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 4c516d8..24d1ad5 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ * [Setup](#setup) * [Building from source code](#building-from-source) * [How to contribute](#how-to-contribute) +* [Report a Bug](#report-a-bug) * [Support, discussions and forums](#support-discussions-and-forums) * [License](#license) * [Copyright notice](#copyright-notice) @@ -241,6 +242,10 @@ We welcome and strongly encourage community participation and contributions, and The [Contributing Guide](https://github.com/percona/pg_stat_monitor/blob/master/CONTRIBUTING.md) contains the guidelines on how you can contribute. +### Report a Bug + +Please report all bugs to Percona's Jira: https://jira.percona.com/projects/PG/issues + ### Support, discussions and forums From c37713b9d50190a6070e8ac5d5dc23721bf7ddbc Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Thu, 30 Sep 2021 11:07:40 -0300 Subject: [PATCH 39/56] PG-228: Add new view, pg_stat_monitor_errors. The pg_stat_monitor_errors view was created in order to help inspecting internal errors that may occur in pg_stat_monitor module itself, it contains the error message, its severity, the last time the error occurred and the number of times that any given error ocurred. Implementation details: - A new lwlock was added to the pgssSharedState structure in order to control access to the errors hash table. - Increased RequestNamedLWLockTranche() no. of locks requested to 2. - The function GetNamedLWLockTranche() returns an array of locks, we use the first lock for controlling access to the usual buckets hash table, and the second one to control access to the errors hash table. - During module initialization (_PG_init) we increased the amount of shared memory space requested to include space to the new errors hash table: RequestAddinShmemSpace(... + pgsm_errors_size()) - The implementation in pgsm_errors.c simple uses a hash table to track error messages, the message is also used as the key. --- Makefile | 2 +- hash_query.c | 23 ++-- pg_stat_monitor--1.0.sql.in | 22 +++- pg_stat_monitor.c | 85 +++++++++----- pg_stat_monitor.h | 2 + pgsm_errors.c | 215 ++++++++++++++++++++++++++++++++++++ pgsm_errors.h | 53 +++++++++ 7 files changed, 365 insertions(+), 37 deletions(-) create mode 100644 pgsm_errors.c create mode 100644 pgsm_errors.h diff --git a/Makefile b/Makefile index ac1225d..5a3278c 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # contrib/pg_stat_monitor/Makefile MODULE_big = pg_stat_monitor -OBJS = hash_query.o guc.o pg_stat_monitor.o $(WIN32RES) +OBJS = hash_query.o guc.o pgsm_errors.o pg_stat_monitor.o $(WIN32RES) EXTENSION = pg_stat_monitor DATA = pg_stat_monitor--1.0.sql diff --git a/hash_query.c b/hash_query.c index 29d0cc8..bf488e6 100644 --- a/hash_query.c +++ b/hash_query.c @@ -17,13 +17,13 @@ #include "postgres.h" #include "nodes/pg_list.h" +#include "pgsm_errors.h" #include "pg_stat_monitor.h" static pgssSharedState *pgss; static HTAB *pgss_hash; -static HTAB* hash_init(const char *hash_name, int key_size, int entry_size, int hash_size); /* * Copy query from src_buffer to dst_buff. * Use query_id and query_pos to fast locate query in source buffer. @@ -66,7 +66,9 @@ pgss_startup(void) if (!found) { /* First time through ... */ - pgss->lock = &(GetNamedLWLockTranche("pg_stat_monitor"))->lock; + LWLockPadded *pgsm_locks = GetNamedLWLockTranche("pg_stat_monitor"); + pgss->lock = &(pgsm_locks[0].lock); + pgss->errors_lock = &(pgsm_locks[1].lock); SpinLockInit(&pgss->mutex); ResetSharedState(pgss); } @@ -86,6 +88,8 @@ pgss_startup(void) pgss_hash = hash_init("pg_stat_monitor: bucket hashtable", sizeof(pgssHashKey), sizeof(pgssEntry), MAX_BUCKET_ENTRIES); + psgm_errors_init(); + LWLockRelease(AddinShmemInitLock); /* @@ -144,10 +148,14 @@ hash_entry_alloc(pgssSharedState *pgss, pgssHashKey *key, int encoding) { pgssEntry *entry = NULL; bool found = false; + long bucket_entries_cnt; - if (hash_get_num_entries(pgss_hash) >= MAX_BUCKET_ENTRIES) + bucket_entries_cnt = hash_get_num_entries(pgss_hash); + + if (bucket_entries_cnt >= MAX_BUCKET_ENTRIES) { - elog(DEBUG1, "%s", "pg_stat_monitor: out of memory"); + pgsm_log_error("hash_entry_alloc: BUCKET OVERFLOW. entries(%d) >= max_entries(%d)", + bucket_entries_cnt, MAX_BUCKET_ENTRIES); return NULL; } /* Find or create an entry with desired hash code */ @@ -165,7 +173,8 @@ hash_entry_alloc(pgssSharedState *pgss, pgssHashKey *key, int encoding) entry->encoding = encoding; } if (entry == NULL) - elog(DEBUG1, "%s", "pg_stat_monitor: out of memory"); + pgsm_log_error("hash_entry_alloc: OUT OF MEMORY"); + return entry; } @@ -230,7 +239,7 @@ hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_bu if (!bkp_entry) { /* No memory, remove pending query entry from the previous bucket. */ - elog(ERROR, "hash_entry_dealloc: out of memory"); + pgsm_log_error("hash_entry_dealloc: out of memory"); entry = hash_search(pgss_hash, &entry->key, HASH_REMOVE, NULL); continue; } @@ -261,7 +270,7 @@ hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_bu new_entry = (pgssEntry *) hash_search(pgss_hash, &old_entry->key, HASH_ENTER_NULL, &found); if (new_entry == NULL) - elog(DEBUG1, "%s", "pg_stat_monitor: out of memory"); + pgsm_log_error("hash_entry_dealloc: out of memory"); else if (!found) { /* Restore counters and other data. */ diff --git a/pg_stat_monitor--1.0.sql.in b/pg_stat_monitor--1.0.sql.in index fd2495d..06d0c9a 100644 --- a/pg_stat_monitor--1.0.sql.in +++ b/pg_stat_monitor--1.0.sql.in @@ -244,10 +244,28 @@ $$ language plpgsql; -- ROUND(CAST(total_time / greatest(sum(total_time) OVER(), 0.00000001) * 100 as numeric), 2)::text || '%' as load_comparison -- FROM pg_stat_monitor_hook_stats(); +CREATE FUNCTION pg_stat_monitor_errors( + OUT message text, + OUT msgtime text, + OUT calls int8 +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stat_monitor_errors' +LANGUAGE C STRICT VOLATILE PARALLEL SAFE; + +CREATE VIEW pg_stat_monitor_errors AS SELECT + message, msgtime, calls +FROM pg_stat_monitor_errors(); + +CREATE FUNCTION pg_stat_monitor_reset_errors() +RETURNS void +AS 'MODULE_PATHNAME' +LANGUAGE C PARALLEL SAFE; + GRANT SELECT ON pg_stat_monitor TO PUBLIC; GRANT SELECT ON pg_stat_monitor_settings TO PUBLIC; +GRANT SELECT ON pg_stat_monitor_errors TO PUBLIC; -- Don't want this to be available to non-superusers. REVOKE ALL ON FUNCTION pg_stat_monitor_reset() FROM PUBLIC; - - +REVOKE ALL ON FUNCTION pg_stat_monitor_reset_errors() FROM PUBLIC; diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index ec43578..1584117 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -22,6 +22,7 @@ #include /* clock() */ #endif #include "commands/explain.h" +#include "pgsm_errors.h" #include "pg_stat_monitor.h" PG_MODULE_MAGIC; @@ -75,7 +76,6 @@ static struct pg_hook_stats_t *pg_hook_stats; static void extract_query_comments(const char *query, char *comments, size_t max_len); static int get_histogram_bucket(double q_time); -static bool IsSystemInitialized(void); static bool dump_queries_buffer(int bucket_id, unsigned char *buf, int buf_len); static double time_diff(struct timeval end, struct timeval start); @@ -262,8 +262,8 @@ _PG_init(void) * the postmaster process.) We'll allocate or attach to the shared * resources in pgss_shmem_startup(). */ - RequestAddinShmemSpace(hash_memsize() + HOOK_STATS_SIZE); - RequestNamedLWLockTranche("pg_stat_monitor", 1); + RequestAddinShmemSpace(hash_memsize() + pgsm_errors_size() + HOOK_STATS_SIZE); + RequestNamedLWLockTranche("pg_stat_monitor", 2); /* * Install hooks. @@ -486,7 +486,7 @@ pgss_ExecutorStart(QueryDesc *queryDesc, int eflags) uint64 queryId = queryDesc->plannedstmt->queryId; if(getrusage(RUSAGE_SELF, &rusage_start) != 0) - elog(DEBUG1, "pg_stat_monitor: failed to execute getrusage"); + pgsm_log_error("pgss_ExecutorStart: failed to execute getrusage"); if (prev_ExecutorStart) prev_ExecutorStart(queryDesc, eflags); @@ -674,7 +674,7 @@ pgss_ExecutorEnd(QueryDesc *queryDesc) */ InstrEndLoop(queryDesc->totaltime); if(getrusage(RUSAGE_SELF, &rusage_end) != 0) - elog(DEBUG1, "pg_stat_monitor: failed to execute getrusage"); + pgsm_log_error("pgss_ExecutorEnd: failed to execute getrusage"); sys_info.utime = time_diff(rusage_end.ru_utime, rusage_start.ru_utime); sys_info.stime = time_diff(rusage_end.ru_stime, rusage_start.ru_stime); @@ -1549,7 +1549,7 @@ pgss_store(uint64 queryid, if (!SaveQueryText(bucketid, queryid, pgss_qbuf[bucketid], query, query_len, &qpos)) { LWLockRelease(pgss->lock); - elog(DEBUG1, "pg_stat_monitor: insufficient shared space for query."); + pgsm_log_error("pgss_store: insufficient shared space for query."); return; } @@ -1560,7 +1560,6 @@ pgss_store(uint64 queryid, /* Restore previous query buffer length. */ memcpy(pgss_qbuf[bucketid], &prev_qbuf_len, sizeof(prev_qbuf_len)); LWLockRelease(pgss->lock); - elog(DEBUG1, "pg_stat_monitor: out of memory"); return; } entry->query_pos = qpos; @@ -1683,7 +1682,10 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, /* Build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "pg_stat_monitor: return type must be a row type"); + { + pgsm_log_error("pg_stat_monitor_internal: call_result_type must return a row type"); + return; + } if (tupdesc->natts != 51) elog(ERROR, "pg_stat_monitor: incorrect number of output arguments, required %d", tupdesc->natts); @@ -1724,10 +1726,24 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, if (read_query(buf, queryid, query_txt, entry->query_pos) == 0) { - int rc; - rc = read_query_buffer(bucketid, queryid, query_txt, entry->query_pos); - if (rc != 1) - snprintf(query_txt, 32, "%s", ""); + /* + * Failed to locate the query in memory, + * If overflow is enabled we try to find the query in a dump file in disk. + */ + if (PGSM_OVERFLOW_TARGET == OVERFLOW_TARGET_DISK) + { + int rc; + rc = read_query_buffer(bucketid, queryid, query_txt, entry->query_pos); + if (rc != 1) + { + pgsm_log_error("pg_stat_monitor_internal: can't find query in dump file."); + continue; + } + } else + { + pgsm_log_error("pg_stat_monitor_internal: insufficient shared space."); + continue; + } } /* copy counters to a local variable to keep locking time short */ @@ -1749,12 +1765,20 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, if (tmp.info.parentid != UINT64CONST(0)) { - int rc = 0; if (read_query(buf, tmp.info.parentid, parent_query_txt, 0) == 0) { - rc = read_query_buffer(bucketid, tmp.info.parentid, parent_query_txt, 0); - if (rc != 1) - snprintf(parent_query_txt, 32, "%s", ""); + if (PGSM_OVERFLOW_TARGET == OVERFLOW_TARGET_DISK) + { + int rc; + rc = read_query_buffer(bucketid, tmp.info.parentid, parent_query_txt, 0); + if (rc != 1) + { + pgsm_log_error("pg_stat_monitor_internal: can't find parent query in dump file."); + } + } else + { + pgsm_log_error("pg_stat_monitor_internal: insufficient shared space for parent query."); + } } } /* bucketid at column number 0 */ @@ -2216,7 +2240,7 @@ JumbleRangeTable(JumbleState *jstate, List *rtable) APP_JUMB_STRING(rte->enrname); break; default: - elog(ERROR, "unrecognized RTE kind: %d", (int) rte->rtekind); + pgsm_log_error("JumbleRangeTable: unrecognized RTE kind: %d", (int) rte->rtekind); break; } } @@ -2717,7 +2741,7 @@ JumbleExpr(JumbleState *jstate, Node *node) break; default: /* Only a warning, since we can stumble along anyway */ - elog(INFO, "unrecognized node type: %d", + pgsm_log_error("JumbleExpr: unrecognized node type: %d", (int) nodeTag(node)); break; } @@ -3086,11 +3110,6 @@ read_query(unsigned char *buf, uint64 queryid, char * query, size_t pos) rlen += query_len; } exit: - if (PGSM_OVERFLOW_TARGET == OVERFLOW_TARGET_NONE) - { - sprintf(query, "%s", ""); - return -1; - } return 0; } @@ -3196,10 +3215,16 @@ pg_stat_monitor_settings(PG_FUNCTION_ARGS) /* Build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "pg_stat_monitor: return type must be a row type"); + { + pgsm_log_error("pg_stat_monitor_settings: return type must be a row type"); + return (Datum) 0; + } if (tupdesc->natts != 7) - elog(ERROR, "pg_stat_monitor: incorrect number of output arguments, required %d", tupdesc->natts); + { + pgsm_log_error("pg_stat_monitor_settings: incorrect number of output arguments, required: 7, found %d", tupdesc->natts); + return (Datum) 0; + } tupstore = tuplestore_begin_heap(true, false, work_mem); rsinfo->returnMode = SFRM_Materialize; @@ -3259,10 +3284,16 @@ pg_stat_monitor_hook_stats(PG_FUNCTION_ARGS) /* Build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "pg_stat_monitor: return type must be a row type"); + { + pgsm_log_error("pg_stat_monitor_hook_stats: return type must be a row type"); + return (Datum) 0; + } if (tupdesc->natts != 5) - elog(ERROR, "pg_stat_monitor: incorrect number of output arguments, required %d", tupdesc->natts); + { + pgsm_log_error("pg_stat_monitor_hook_stats: incorrect number of output arguments, required: 5, found %d", tupdesc->natts); + return (Datum) 0; + } tupstore = tuplestore_begin_heap(true, false, work_mem); rsinfo->returnMode = SFRM_Materialize; diff --git a/pg_stat_monitor.h b/pg_stat_monitor.h index 84ed936..508ed43 100644 --- a/pg_stat_monitor.h +++ b/pg_stat_monitor.h @@ -313,6 +313,7 @@ typedef struct pgssSharedState uint64 bucket_entry[MAX_BUCKETS]; int64 query_buf_size_bucket; char bucket_start_time[MAX_BUCKETS][60]; /* start time of the bucket */ + LWLock *errors_lock; /* protects errors hashtable search/modification */ } pgssSharedState; #define ResetSharedState(x) \ @@ -374,6 +375,7 @@ bool SaveQueryText(uint64 bucketid, void init_guc(void); GucVariable *get_conf(int i); +bool IsSystemInitialized(void); /* hash_create.c */ bool IsHashInitialize(void); void pgss_shmem_startup(void); diff --git a/pgsm_errors.c b/pgsm_errors.c new file mode 100644 index 0000000..5706748 --- /dev/null +++ b/pgsm_errors.c @@ -0,0 +1,215 @@ +/*------------------------------------------------------------------------- + * + * pgsm_errors.c + * Track pg_stat_monitor internal error messages. + * + * Copyright © 2021, Percona LLC and/or its affiliates + * + * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group + * + * Portions Copyright (c) 1994, The Regents of the University of California + * + * IDENTIFICATION + * contrib/pg_stat_monitor/pgsm_errors.c + * + *------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "pg_stat_monitor.h" +#include "pgsm_errors.h" + + +PG_FUNCTION_INFO_V1(pg_stat_monitor_errors); +PG_FUNCTION_INFO_V1(pg_stat_monitor_reset_errors); + + +/* + * Maximum number of error messages tracked. + * This should be set to a sensible value in order to track + * the different type of errors that pg_stat_monitor may + * report, e.g. out of memory. + */ +#define PSGM_ERRORS_MAX 128 + +static HTAB *pgsm_errors_ht = NULL; + +void psgm_errors_init(void) +{ + HASHCTL info; + + memset(&info, 0, sizeof(info)); + info.keysize = ERROR_MSG_MAX_LEN; + info.entrysize = sizeof(ErrorEntry); + pgsm_errors_ht = ShmemInitHash("pg_stat_monitor: errors hashtable", + PSGM_ERRORS_MAX, /* initial size */ + PSGM_ERRORS_MAX, /* maximum size */ + &info, + HASH_ELEM | HASH_STRINGS); +} + +size_t pgsm_errors_size(void) +{ + return hash_estimate_size(PSGM_ERRORS_MAX, sizeof(ErrorEntry)); +} + +void pgsm_log_error(const char *format, ...) +{ + char key[ERROR_MSG_MAX_LEN]; + ErrorEntry *entry; + bool found = false; + va_list ap; + int n; + struct timeval tv; + struct tm *lt; + pgssSharedState *pgss; + + va_start(ap, format); + n = vsnprintf(key, ERROR_MSG_MAX_LEN, format, ap); + va_end(ap); + + if (n < 0) + return; + + pgss = pgsm_get_ss(); + LWLockAcquire(pgss->errors_lock, LW_EXCLUSIVE); + + entry = (ErrorEntry *) hash_search(pgsm_errors_ht, key, HASH_ENTER_NULL, &found); + if (!entry) + { + LWLockRelease(pgss->errors_lock); + /* + * We're out of memory, can't track this error message. + * In this case we must fallback to PostgreSQL log facility. + */ + elog(WARNING, "pgsm_log_error: "); + return; + } + + if (!found) + entry->calls = 0; + + /* Update message timestamp. */ + gettimeofday(&tv, NULL); + lt = localtime(&tv.tv_sec); + snprintf(entry->time, sizeof(entry->time), + "%04d-%02d-%02d %02d:%02d:%02d", + lt->tm_year + 1900, + lt->tm_mon + 1, + lt->tm_mday, + lt->tm_hour, + lt->tm_min, + lt->tm_sec); + + entry->calls++; + + LWLockRelease(pgss->errors_lock); +} + +/* + * Clear all entries from the hash table. + */ +Datum +pg_stat_monitor_reset_errors(PG_FUNCTION_ARGS) +{ + HASH_SEQ_STATUS hash_seq; + ErrorEntry *entry; + pgssSharedState *pgss = pgsm_get_ss(); + + /* Safety check... */ + if (!IsSystemInitialized()) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("pg_stat_monitor: must be loaded via shared_preload_libraries"))); + + LWLockAcquire(pgss->errors_lock, LW_EXCLUSIVE); + + hash_seq_init(&hash_seq, pgsm_errors_ht); + while ((entry = hash_seq_search(&hash_seq)) != NULL) + entry = hash_search(pgsm_errors_ht, &entry->message, HASH_REMOVE, NULL); + + LWLockRelease(pgss->errors_lock); + PG_RETURN_VOID(); +} + +/* + * Invoked when users query the view pg_stat_monitor_errors. + * This function creates tuples with error messages from data present in + * the hash table, then return the dataset to the caller. + */ +Datum +pg_stat_monitor_errors(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + TupleDesc tupdesc; + Tuplestorestate *tupstore; + MemoryContext per_query_ctx; + MemoryContext oldcontext; + HASH_SEQ_STATUS hash_seq; + ErrorEntry *error_entry; + pgssSharedState *pgss = pgsm_get_ss(); + + /* Safety check... */ + if (!IsSystemInitialized()) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("pg_stat_monitor: must be loaded via shared_preload_libraries"))); + + /* check to see if caller supports us returning a tuplestore */ + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("pg_stat_monitor: set-valued function called in context that cannot accept a set"))); + + /* Switch into long-lived context to construct returned data structures */ + per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; + oldcontext = MemoryContextSwitchTo(per_query_ctx); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "pg_stat_monitor: return type must be a row type"); + + if (tupdesc->natts != 3) + elog(ERROR, "pg_stat_monitor: incorrect number of output arguments, required 3, found %d", tupdesc->natts); + + tupstore = tuplestore_begin_heap(true, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + + MemoryContextSwitchTo(oldcontext); + + LWLockAcquire(pgss->errors_lock, LW_SHARED); + + hash_seq_init(&hash_seq, pgsm_errors_ht); + while ((error_entry = hash_seq_search(&hash_seq)) != NULL) + { + Datum values[3]; + bool nulls[3]; + int i = 0; + memset(values, 0, sizeof(values)); + memset(nulls, 0, sizeof(nulls)); + + values[i++] = CStringGetTextDatum(error_entry->message); + values[i++] = CStringGetTextDatum(error_entry->time); + values[i++] = Int64GetDatumFast(error_entry->calls); + + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + + LWLockRelease(pgss->errors_lock); + /* clean up and return the tuplestore */ + tuplestore_donestoring(tupstore); + + return (Datum)0; +} \ No newline at end of file diff --git a/pgsm_errors.h b/pgsm_errors.h new file mode 100644 index 0000000..9956fdf --- /dev/null +++ b/pgsm_errors.h @@ -0,0 +1,53 @@ +/*------------------------------------------------------------------------- + * + * pgsm_errors.h + * Track pg_stat_monitor internal error messages. + * + * Copyright © 2021, Percona LLC and/or its affiliates + * + * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group + * + * Portions Copyright (c) 1994, The Regents of the University of California + * + * IDENTIFICATION + * contrib/pg_stat_monitor/pgsm_errors.h + * + *------------------------------------------------------------------------- + */ + +#ifndef PGSM_ERRORS_H +#define PGSM_ERRORS_H + +#include + +#include + + +/* Maximum allowed error message length. */ +#define ERROR_MSG_MAX_LEN 128 + +typedef struct { + char message[ERROR_MSG_MAX_LEN]; /* message is also the hash key (MUST BE FIRST). */ + char time[60]; /* last timestamp in which this error was reported. */ + int64 calls; /* how many times this error was reported. */ +} ErrorEntry; + +/* + * Must be called during module startup, creates the hash table + * used to store pg_stat_monitor error messages. + */ +void psgm_errors_init(void); + +/* + * Returns an estimate of the hash table size. + * Used to reserve space on Postmaster's shared memory. + */ +size_t pgsm_errors_size(void); + +/* + * Add an error message to the hash table. + * Increment no. of calls if message already exists. + */ +void pgsm_log_error(const char *format, ...); + +#endif /* PGSM_ERRORS_H */ \ No newline at end of file From 6f353a5596e62b1afff58f783281cd3d2d6ab49e Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Thu, 30 Sep 2021 10:39:49 -0300 Subject: [PATCH 40/56] PG-228: Add severity to the internal message logging API. Add support to include the severity of messages added to the pg_stat_monitor_errors view. --- pg_stat_monitor--1.0.sql.in | 14 +++++++++++++- pgsm_errors.c | 12 ++++++++---- pgsm_errors.h | 16 ++++++++++++++-- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/pg_stat_monitor--1.0.sql.in b/pg_stat_monitor--1.0.sql.in index 06d0c9a..106106d 100644 --- a/pg_stat_monitor--1.0.sql.in +++ b/pg_stat_monitor--1.0.sql.in @@ -245,6 +245,7 @@ $$ language plpgsql; -- FROM pg_stat_monitor_hook_stats(); CREATE FUNCTION pg_stat_monitor_errors( + OUT severity int, OUT message text, OUT msgtime text, OUT calls int8 @@ -253,8 +254,19 @@ RETURNS SETOF record AS 'MODULE_PATHNAME', 'pg_stat_monitor_errors' LANGUAGE C STRICT VOLATILE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION pgsm_log_severity_as_text(severity int) RETURNS TEXT AS +$$ +SELECT + CASE + WHEN severity = 0 THEN 'INFO' + WHEN severity = 1 THEN 'WARNING' + WHEN severity = 2 THEN 'ERROR' + END +$$ +LANGUAGE SQL PARALLEL SAFE; + CREATE VIEW pg_stat_monitor_errors AS SELECT - message, msgtime, calls + pgsm_log_severity_as_text(severity) as severity, message, msgtime, calls FROM pg_stat_monitor_errors(); CREATE FUNCTION pg_stat_monitor_reset_errors() diff --git a/pgsm_errors.c b/pgsm_errors.c index 5706748..0a4bcde 100644 --- a/pgsm_errors.c +++ b/pgsm_errors.c @@ -63,7 +63,7 @@ size_t pgsm_errors_size(void) return hash_estimate_size(PSGM_ERRORS_MAX, sizeof(ErrorEntry)); } -void pgsm_log_error(const char *format, ...) +void pgsm_log(PgsmLogSeverity severity, const char *format, ...) { char key[ERROR_MSG_MAX_LEN]; ErrorEntry *entry; @@ -97,7 +97,10 @@ void pgsm_log_error(const char *format, ...) } if (!found) + { + entry->severity = severity; entry->calls = 0; + } /* Update message timestamp. */ gettimeofday(&tv, NULL); @@ -179,7 +182,7 @@ pg_stat_monitor_errors(PG_FUNCTION_ARGS) if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "pg_stat_monitor: return type must be a row type"); - if (tupdesc->natts != 3) + if (tupdesc->natts != 4) elog(ERROR, "pg_stat_monitor: incorrect number of output arguments, required 3, found %d", tupdesc->natts); tupstore = tuplestore_begin_heap(true, false, work_mem); @@ -194,12 +197,13 @@ pg_stat_monitor_errors(PG_FUNCTION_ARGS) hash_seq_init(&hash_seq, pgsm_errors_ht); while ((error_entry = hash_seq_search(&hash_seq)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; int i = 0; memset(values, 0, sizeof(values)); memset(nulls, 0, sizeof(nulls)); + values[i++] = Int64GetDatumFast(error_entry->severity); values[i++] = CStringGetTextDatum(error_entry->message); values[i++] = CStringGetTextDatum(error_entry->time); values[i++] = Int64GetDatumFast(error_entry->calls); diff --git a/pgsm_errors.h b/pgsm_errors.h index 9956fdf..8f2865d 100644 --- a/pgsm_errors.h +++ b/pgsm_errors.h @@ -26,8 +26,16 @@ /* Maximum allowed error message length. */ #define ERROR_MSG_MAX_LEN 128 +/* Log message severity. */ +typedef enum { + PSGM_LOG_INFO, + PGSM_LOG_WARNING, + PGSM_LOG_ERROR +} PgsmLogSeverity; + typedef struct { char message[ERROR_MSG_MAX_LEN]; /* message is also the hash key (MUST BE FIRST). */ + PgsmLogSeverity severity; char time[60]; /* last timestamp in which this error was reported. */ int64 calls; /* how many times this error was reported. */ } ErrorEntry; @@ -45,9 +53,13 @@ void psgm_errors_init(void); size_t pgsm_errors_size(void); /* - * Add an error message to the hash table. + * Add a message to the hash table. * Increment no. of calls if message already exists. */ -void pgsm_log_error(const char *format, ...); +void pgsm_log(PgsmLogSeverity severity, const char *format, ...); + +#define pgsm_log_info(msg, ...) pgsm_log(PGSM_LOG_INFO, msg, ##__VA_ARGS__) +#define pgsm_log_warning(msg, ...) pgsm_log(PGSM_LOG_WARNING, msg, ##__VA_ARGS__) +#define pgsm_log_error(msg, ...) pgsm_log(PGSM_LOG_ERROR, msg, ##__VA_ARGS__) #endif /* PGSM_ERRORS_H */ \ No newline at end of file From 8ea02b0f2a183cab1f7c471550d5b662ad9f1fad Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Tue, 16 Nov 2021 13:10:56 -0300 Subject: [PATCH 41/56] PG-228: Fix hash table creation flags on PG <= 13. Before PostgreSQL 14, HASH_STRINGS flag was not available when creating a hash table with ShmemInitHash(). Use HASH_BLOBS for previous PostgreSQL versions. --- pgsm_errors.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pgsm_errors.c b/pgsm_errors.c index 0a4bcde..95f0197 100644 --- a/pgsm_errors.c +++ b/pgsm_errors.c @@ -47,6 +47,12 @@ static HTAB *pgsm_errors_ht = NULL; void psgm_errors_init(void) { HASHCTL info; +#if PG_VERSION_NUM >= 140000 + int flags = HASH_ELEM | HASH_STRINGS; +#else + int flags = HASH_ELEM | HASH_BLOBS; +#endif + memset(&info, 0, sizeof(info)); info.keysize = ERROR_MSG_MAX_LEN; @@ -55,7 +61,7 @@ void psgm_errors_init(void) PSGM_ERRORS_MAX, /* initial size */ PSGM_ERRORS_MAX, /* maximum size */ &info, - HASH_ELEM | HASH_STRINGS); + flags); } size_t pgsm_errors_size(void) From ad1187b9daafe04b1776c7c905259e59bd129bfc Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Mon, 29 Nov 2021 11:09:47 -0300 Subject: [PATCH 42/56] PG-286: Avoid duplicate queries in text buffer. The memory area reserved for query text (pgsm_query_shared_buffer) was divided evenly for each bucket, this allowed to have the same query, e.g. "SELECT 1", duplicated in different buckets, thus wasting space. This commit fix the query text duplication by adding a new hash table whose only purpose is to verify if a given query is already added to the buffer (by using the queryID). This allows different buckets that share the same query to point to a unique entry in the query buffer (pgss_qbuf). When pg_stat_monitor moves to a new bucket id, by avoiding adding a query that already exists in the buffer it can also save some CPU time. --- hash_query.c | 86 +++++++------------------------------ pg_stat_monitor.c | 105 +++++++++++++++++++++++++++------------------- pg_stat_monitor.h | 41 +++++++----------- 3 files changed, 93 insertions(+), 139 deletions(-) diff --git a/hash_query.c b/hash_query.c index bf488e6..d6e4051 100644 --- a/hash_query.c +++ b/hash_query.c @@ -23,18 +23,8 @@ static pgssSharedState *pgss; static HTAB *pgss_hash; +static HTAB *pgss_query_hash; -/* - * Copy query from src_buffer to dst_buff. - * Use query_id and query_pos to fast locate query in source buffer. - * Store updated query position in the destination buffer into param query_pos. - */ -static bool copy_query(uint64 bucket_id, - uint64 query_id, - uint64 query_pos, - unsigned char *dst_buf, - unsigned char *src_buf, - size_t *new_query_pos); static HTAB* hash_init(const char *hash_name, int key_size, int entry_size, int hash_size) @@ -50,7 +40,6 @@ void pgss_startup(void) { bool found = false; - int32 i; /* reset in case this is a restart within the postmaster */ @@ -77,16 +66,10 @@ pgss_startup(void) init_hook_stats(); #endif - pgss->query_buf_size_bucket = MAX_QUERY_BUF / PGSM_MAX_BUCKETS; - - for (i = 0; i < PGSM_MAX_BUCKETS; i++) - { - unsigned char *buf = (unsigned char *)ShmemAlloc(pgss->query_buf_size_bucket); - set_qbuf(i, buf); - memset(buf, 0, sizeof (uint64)); - } + set_qbuf((unsigned char *)ShmemAlloc(MAX_QUERY_BUF)); pgss_hash = hash_init("pg_stat_monitor: bucket hashtable", sizeof(pgssHashKey), sizeof(pgssEntry), MAX_BUCKET_ENTRIES); + pgss_query_hash = hash_init("pg_stat_monitor: queryID hashtable", sizeof(uint64), sizeof(pgssQueryEntry), MAX_BUCKET_ENTRIES); psgm_errors_init(); @@ -111,6 +94,12 @@ pgsm_get_hash(void) return pgss_hash; } +HTAB* +pgsm_get_query_hash(void) +{ + return pgss_query_hash; +} + /* * shmem_shutdown hook: Dump statistics into file. * @@ -191,22 +180,15 @@ hash_entry_alloc(pgssSharedState *pgss, pgssHashKey *key, int encoding) * Caller must hold an exclusive lock on pgss->lock. */ void -hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_buffer[]) +hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_buffer) { HASH_SEQ_STATUS hash_seq; pgssEntry *entry = NULL; - pgssSharedState *pgss = pgsm_get_ss(); /* Store pending query ids from the previous bucket. */ List *pending_entries = NIL; ListCell *pending_entry; - if (new_bucket_id != -1) - { - /* Clear all queries in the query buffer for the new bucket. */ - memset(query_buffer[new_bucket_id], 0, pgss->query_buf_size_bucket); - } - /* Iterate over the hash table. */ hash_seq_init(&hash_seq, pgss_hash); while ((entry = hash_seq_search(&hash_seq)) != NULL) @@ -219,6 +201,11 @@ hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_bu (entry->key.bucket_id == new_bucket_id && (entry->counters.state == PGSS_FINISHED || entry->counters.state == PGSS_ERROR))) { + if (new_bucket_id == -1) { + /* pg_stat_monitor_reset(), remove entry from query hash table too. */ + hash_search(pgss_query_hash, &(entry->key.queryid), HASH_REMOVE, NULL); + } + entry = hash_search(pgss_hash, &entry->key, HASH_REMOVE, NULL); } @@ -277,13 +264,6 @@ hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_bu new_entry->counters = old_entry->counters; SpinLockInit(&new_entry->mutex); new_entry->encoding = old_entry->encoding; - /* copy query's text from previous bucket to the new one. */ - copy_query(new_bucket_id, - new_entry->key.queryid, /* query id */ - old_entry->query_pos, /* query position in buffer */ - query_buffer[new_bucket_id], /* destination query buffer */ - query_buffer[old_bucket_id], /* source query buffer */ - &new_entry->query_pos); /* position in which query was inserted into destination buffer */ } free(old_entry); @@ -319,39 +299,3 @@ IsHashInitialize(void) return (pgss != NULL && pgss_hash != NULL); } - -static bool copy_query(uint64 bucket_id, - uint64 query_id, - uint64 query_pos, - unsigned char *dst_buf, - unsigned char *src_buf, - size_t *new_query_pos) -{ - uint64 query_len = 0; - uint64 buf_len = 0; - - memcpy(&buf_len, src_buf, sizeof (uint64)); - if (buf_len <= 0) - return false; - - /* Try to locate the query directly. */ - if (query_pos != 0 && (query_pos + sizeof(uint64) + sizeof(uint64)) < buf_len) - { - if (*(uint64 *)&src_buf[query_pos] != query_id) - return false; - - query_pos += sizeof(uint64); - - memcpy(&query_len, &src_buf[query_pos], sizeof(uint64)); /* query len */ - query_pos += sizeof(uint64); - - if (query_pos + query_len > buf_len) /* avoid reading past buffer's length. */ - return false; - - return SaveQueryText(bucket_id, query_id, dst_buf, - (const char *)&src_buf[query_pos], - query_len, new_query_pos); - } - - return false; -} diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 1584117..3bde8c4 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -68,7 +68,8 @@ static int num_relations; /* Number of relation in the query */ static bool system_init = false; static struct rusage rusage_start; static struct rusage rusage_end; -static unsigned char *pgss_qbuf[MAX_BUCKETS]; +/* Query buffer, store queries' text. */ +static unsigned char *pgss_qbuf = NULL; static char *pgss_explain(QueryDesc *queryDesc); #ifdef BENCHMARK static struct pg_hook_stats_t *pg_hook_stats; @@ -1311,7 +1312,6 @@ pgss_update_entry(pgssEntry *entry, e->counters.blocks.blk_write_time += INSTR_TIME_GET_MILLISEC(bufusage->blk_write_time); } e->counters.calls.usage += USAGE_EXEC(total_time); - e->counters.info.host = pg_get_client_addr(); if (sys_info) { e->counters.sysinfo.utime = sys_info->utime; @@ -1478,14 +1478,13 @@ pgss_store(uint64 queryid, uint64 planid; uint64 appid; char comments[512] = ""; - size_t query_len; /* Monitoring is disabled */ if (!PGSM_ENABLED) return; /* Safety check... */ - if (!IsSystemInitialized() || !pgss_qbuf[pg_atomic_read_u64(&pgss->current_wbucket)]) + if (!IsSystemInitialized()) return; Assert(query != NULL); @@ -1528,41 +1527,62 @@ pgss_store(uint64 queryid, entry = (pgssEntry *) hash_search(pgss_hash, &key, HASH_FIND, NULL); if (!entry) { - uint64 prev_qbuf_len; - /* position in which the query's text was inserted into the query buffer. */ - size_t qpos = 0; + pgssQueryEntry *query_entry; + size_t query_len = 0; + bool query_found = false; + uint64 prev_qbuf_len = 0; + HTAB *pgss_query_hash; - query_len = strlen(query); - if (query_len > PGSM_QUERY_MAX_LEN) - query_len = PGSM_QUERY_MAX_LEN; + pgss_query_hash = pgsm_get_query_hash(); + + query_entry = hash_search(pgss_query_hash, &queryid, HASH_ENTER_NULL, &query_found); + if (query_entry == NULL) + { + LWLockRelease(pgss->lock); + pgsm_log_error("pgss_store: out of memory (pgss_query_hash)."); + return; + } + else if (!query_found) + { + /* New query, must add it to the buffer, calculate its length. */ + query_len = strlen(query); + if (query_len > PGSM_QUERY_MAX_LEN) + query_len = PGSM_QUERY_MAX_LEN; + } /* Need exclusive lock to make a new hashtable entry - promote */ LWLockRelease(pgss->lock); LWLockAcquire(pgss->lock, LW_EXCLUSIVE); - /* - * Save current query buffer length, if we fail to add a new - * new entry to the hash table then we must restore the - * original length. - */ - memcpy(&prev_qbuf_len, pgss_qbuf[bucketid], sizeof(prev_qbuf_len)); - if (!SaveQueryText(bucketid, queryid, pgss_qbuf[bucketid], query, query_len, &qpos)) + if (!query_found) { - LWLockRelease(pgss->lock); - pgsm_log_error("pgss_store: insufficient shared space for query."); - return; + if (!SaveQueryText(bucketid, queryid, pgss_qbuf, query, query_len, &query_entry->query_pos)) + { + LWLockRelease(pgss->lock); + pgsm_log_error("pgss_store: insufficient shared space for query."); + return; + } + /* + * Save current query buffer length, if we fail to add a new + * new entry to the hash table then we must restore the + * original length. + */ + memcpy(&prev_qbuf_len, pgss_qbuf, sizeof(prev_qbuf_len)); } /* OK to create a new hashtable entry */ entry = hash_entry_alloc(pgss, &key, GetDatabaseEncoding()); if (entry == NULL) { - /* Restore previous query buffer length. */ - memcpy(pgss_qbuf[bucketid], &prev_qbuf_len, sizeof(prev_qbuf_len)); + if (!query_found) + { + /* Restore previous query buffer length. */ + memcpy(pgss_qbuf, &prev_qbuf_len, sizeof(prev_qbuf_len)); + } LWLockRelease(pgss->lock); return; } - entry->query_pos = qpos; + entry->query_pos = query_entry->query_pos; } if (jstate == NULL) @@ -1598,11 +1618,10 @@ pg_stat_monitor_reset(PG_FUNCTION_ARGS) errmsg("pg_stat_monitor: must be loaded via shared_preload_libraries"))); LWLockAcquire(pgss->lock, LW_EXCLUSIVE); hash_entry_dealloc(-1, -1, NULL); - /* Reset query buffers. */ - for (size_t i = 0; i < PGSM_MAX_BUCKETS; ++i) - { - *(uint64 *)pgss_qbuf[i] = 0; - } + + /* Reset query buffer. */ + *(uint64 *)pgss_qbuf = 0; + #ifdef BENCHMARK for (int i = STATS_START; i < STATS_END; ++i) { pg_hook_stats[i].min_time = 0; @@ -1715,7 +1734,6 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, uint64 userid = entry->key.userid; uint64 ip = entry->key.ip; uint64 planid = entry->key.planid; - unsigned char *buf = pgss_qbuf[bucketid]; #if PG_VERSION_NUM < 140000 bool toplevel = 1; bool is_allowed_role = is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_ALL_STATS); @@ -1724,7 +1742,7 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, bool toplevel = entry->key.toplevel; #endif - if (read_query(buf, queryid, query_txt, entry->query_pos) == 0) + if (read_query(pgss_qbuf, queryid, query_txt, entry->query_pos) == 0) { /* * Failed to locate the query in memory, @@ -1765,7 +1783,7 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, if (tmp.info.parentid != UINT64CONST(0)) { - if (read_query(buf, tmp.info.parentid, parent_query_txt, 0) == 0) + if (read_query(pgss_qbuf, tmp.info.parentid, parent_query_txt, 0) == 0) { if (PGSM_OVERFLOW_TARGET == OVERFLOW_TARGET_DISK) { @@ -1826,7 +1844,7 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, if (enc != query_txt) pfree(enc); /* plan at column number 7 */ - if (planid && strlen(tmp.planinfo.plan_text) > 0) + if (planid && tmp.planinfo.plan_text[0]) values[i++] = CStringGetTextDatum(tmp.planinfo.plan_text); else nulls[i++] = true; @@ -3139,17 +3157,17 @@ SaveQueryText(uint64 bucketid, /* * If the query buffer is empty, there is nothing to dump, this also - * means that the current query length exceeds MAX_QUERY_BUFFER_BUCKET. + * means that the current query length exceeds MAX_QUERY_BUF. */ if (buf_len <= sizeof (uint64)) return false; - dump_ok = dump_queries_buffer(bucketid, buf, MAX_QUERY_BUFFER_BUCKET); + dump_ok = dump_queries_buffer(bucketid, buf, MAX_QUERY_BUF); buf_len = sizeof (uint64); /* * We must check for overflow again, as the query length may - * exceed the size allocated to the buffer (MAX_QUERY_BUFFER_BUCKET). + * exceed the total size allocated to the buffer (MAX_QUERY_BUF). */ if (QUERY_BUFFER_OVERFLOW(buf_len, query_len)) { @@ -3324,9 +3342,10 @@ pg_stat_monitor_hook_stats(PG_FUNCTION_ARGS) } void -set_qbuf(int i, unsigned char *buf) +set_qbuf(unsigned char *buf) { - pgss_qbuf[i] = buf; + pgss_qbuf = buf; + *(uint64 *)pgss_qbuf = 0; } #ifdef BENCHMARK @@ -3444,13 +3463,13 @@ read_query_buffer(int bucket_id, uint64 queryid, char *query_txt, size_t pos) if (fd < 0) goto exit; - buf = (unsigned char*) palloc(MAX_QUERY_BUFFER_BUCKET); + buf = (unsigned char*) palloc(MAX_QUERY_BUF); while (!done) { off = 0; - /* read a chunck of MAX_QUERY_BUFFER_BUCKET size. */ + /* read a chunck of MAX_QUERY_BUF size. */ do { - nread = read(fd, buf + off, MAX_QUERY_BUFFER_BUCKET - off); + nread = read(fd, buf + off, MAX_QUERY_BUF - off); if (nread == -1) { if (errno == EINTR && tries++ < 3) /* read() was interrupted, attempt to read again (max attempts=3) */ @@ -3465,9 +3484,9 @@ read_query_buffer(int bucket_id, uint64 queryid, char *query_txt, size_t pos) } off += nread; - } while (off < MAX_QUERY_BUFFER_BUCKET); + } while (off < MAX_QUERY_BUF); - if (off == MAX_QUERY_BUFFER_BUCKET) + if (off == MAX_QUERY_BUF) { /* we have a chunck, scan it looking for queryid. */ if (read_query(buf, queryid, query_txt, pos) != 0) @@ -3480,7 +3499,7 @@ read_query_buffer(int bucket_id, uint64 queryid, char *query_txt, size_t pos) } else /* - * Either done=true or file has a size not multiple of MAX_QUERY_BUFFER_BUCKET. + * Either done=true or file has a size not multiple of MAX_QUERY_BUF. * It is safe to assume that the file was truncated or corrupted. */ break; diff --git a/pg_stat_monitor.h b/pg_stat_monitor.h index 508ed43..c42649a 100644 --- a/pg_stat_monitor.h +++ b/pg_stat_monitor.h @@ -87,9 +87,8 @@ #define MAX_QUERY_BUF (PGSM_QUERY_SHARED_BUFFER * 1024 * 1024) #define MAX_BUCKETS_MEM (PGSM_MAX * 1024 * 1024) #define BUCKETS_MEM_OVERFLOW() ((hash_get_num_entries(pgss_hash) * sizeof(pgssEntry)) >= MAX_BUCKETS_MEM) -#define MAX_QUERY_BUFFER_BUCKET MAX_QUERY_BUF / PGSM_MAX_BUCKETS #define MAX_BUCKET_ENTRIES (MAX_BUCKETS_MEM / sizeof(pgssEntry)) -#define QUERY_BUFFER_OVERFLOW(x,y) ((x + y + sizeof(uint64) + sizeof(uint64)) > MAX_QUERY_BUFFER_BUCKET) +#define QUERY_BUFFER_OVERFLOW(x,y) ((x + y + sizeof(uint64) + sizeof(uint64)) > MAX_QUERY_BUF) #define QUERY_MARGIN 100 #define MIN_QUERY_LEN 10 #define SQLCODE_LEN 20 @@ -161,7 +160,7 @@ typedef enum AGG_KEY #define MAX_QUERY_LEN 1024 -/* shared nenory storage for the query */ +/* shared memory storage for the query */ typedef struct CallTime { double total_time; /* total execution time, in msec */ @@ -171,21 +170,19 @@ typedef struct CallTime double sum_var_time; /* sum of variances in execution time in msec */ } CallTime; -typedef struct pgssQueryHashKey -{ - uint64 bucket_id; /* bucket number */ - uint64 queryid; /* query identifier */ - uint64 userid; /* user OID */ - uint64 dbid; /* database OID */ - uint64 ip; /* client ip address */ - uint64 appid; /* hash of application name */ -} pgssQueryHashKey; - +/* + * Entry type for queries hash table (query ID). + * + * We use a hash table to keep track of query IDs that have their + * corresponding query text added to the query buffer (pgsm_query_shared_buffer). + * + * This allow us to avoid adding duplicated queries to the buffer, therefore + * leaving more space for other queries and saving some CPU. + */ typedef struct pgssQueryEntry { - pgssQueryHashKey key; /* hash key of entry - MUST BE FIRST */ - uint64 pos; /* bucket number */ - uint64 state; + uint64 queryid; /* query identifier, also the key. */ + size_t query_pos; /* query location within query buffer */ } pgssQueryEntry; typedef struct PlanInfo @@ -208,10 +205,6 @@ typedef struct pgssHashKey typedef struct QueryInfo { - uint64 queryid; /* query identifier */ - Oid userid; /* user OID */ - Oid dbid; /* database OID */ - uint host; /* client IP */ uint64 parentid; /* parent queryid of current query*/ int64 type; /* type of query, options are query, info, warning, error, fatal */ char application_name[APPLICATIONNAME_LEN]; @@ -311,7 +304,6 @@ typedef struct pgssSharedState pg_atomic_uint64 current_wbucket; pg_atomic_uint64 prev_bucket_usec; uint64 bucket_entry[MAX_BUCKETS]; - int64 query_buf_size_bucket; char bucket_start_time[MAX_BUCKETS][60]; /* start time of the bucket */ LWLock *errors_lock; /* protects errors hashtable search/modification */ } pgssSharedState; @@ -384,21 +376,20 @@ int pgsm_get_bucket_size(void); pgssSharedState* pgsm_get_ss(void); HTAB *pgsm_get_plan_hash(void); HTAB *pgsm_get_hash(void); +HTAB *pgsm_get_query_hash(void); HTAB *pgsm_get_plan_hash(void); void hash_entry_reset(void); void hash_query_entryies_reset(void); void hash_query_entries(); void hash_query_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_buffer[]); -void hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_buffer[]); +void hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_buffer); pgssEntry* hash_entry_alloc(pgssSharedState *pgss, pgssHashKey *key, int encoding); Size hash_memsize(void); int read_query_buffer(int bucket_id, uint64 queryid, char *query_txt, size_t pos); uint64 read_query(unsigned char *buf, uint64 queryid, char * query, size_t pos); -pgssQueryEntry* hash_find_query_entry(uint64 bucket_id, uint64 queryid, uint64 dbid, uint64 userid, uint64 ip, uint64 appid); -pgssQueryEntry* hash_create_query_entry(uint64 bucket_id, uint64 queryid, uint64 dbid, uint64 userid, uint64 ip, uint64 appid); void pgss_startup(void); -void set_qbuf(int i, unsigned char *); +void set_qbuf(unsigned char *); /* hash_query.c */ void pgss_startup(void); From 1b51defc68ee66bf48463299223787984df2cd2c Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Mon, 29 Nov 2021 14:39:56 -0300 Subject: [PATCH 43/56] PG-286: Small performance improvements. pgss_ExecutorEnd: Avoid unnecessary memset(plan_info, 0, ...). We only use this object if the condition below is true, in which case we already initialize all the fields in the object, also we now store the plan string length (plan_info.plan_len) to avoid calling strlen on it again later: if (queryDesc->operation == CMD_SELECT && PGSM_QUERY_PLAN) { ... here we initialize plan_info If the condition is false, then we pass a NULL PlanInfo* to the pgss_store to avoid more unnecessary processing. pgss_planner_hook: Similar, avoid memset(plan_info, 0, ...) this object is not used here, so we pass NULL to pgss_store. pg_get_application_name: Remove call to strlen, snprintf already give us the calculated string length, so we just return it. pg_get_client_addr: Cache localhost, avoid calling ntohl(inet_addr("127.0.0.1")) all the time. pgss_update_entry: Make use of PlanInfo->plan_len, avoiding a call to strlen again. intarray_get_datum: Init the string by setting the first byte to '\0'. --- pg_stat_monitor.c | 57 ++++++++++++++++++++++------------------------- pg_stat_monitor.h | 1 + 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 3bde8c4..6202106 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -656,14 +656,15 @@ pgss_ExecutorEnd(QueryDesc *queryDesc) uint64 queryId = queryDesc->plannedstmt->queryId; SysInfo sys_info; PlanInfo plan_info; + PlanInfo *plan_ptr = NULL; - /* Extract the plan information in case of SELECT statment */ - memset(&plan_info, 0, sizeof(PlanInfo)); + /* Extract the plan information in case of SELECT statement */ if (queryDesc->operation == CMD_SELECT && PGSM_QUERY_PLAN) - { + { MemoryContext mct = MemoryContextSwitchTo(TopMemoryContext); - snprintf(plan_info.plan_text, PLAN_TEXT_LEN, "%s", pgss_explain(queryDesc)); - plan_info.planid = DatumGetUInt64(hash_any_extended((const unsigned char*)plan_info.plan_text, strlen(plan_info.plan_text), 0)); + plan_info.plan_len = snprintf(plan_info.plan_text, PLAN_TEXT_LEN, "%s", pgss_explain(queryDesc)); + plan_info.planid = DatumGetUInt64(hash_any_extended((const unsigned char*)plan_info.plan_text, plan_info.plan_len, 0)); + plan_ptr = &plan_info; MemoryContextSwitchTo(mct); } @@ -682,7 +683,7 @@ pgss_ExecutorEnd(QueryDesc *queryDesc) pgss_store(queryId, /* query id */ queryDesc->sourceText, /* query text */ - &plan_info, /* PlanInfo */ + plan_ptr, /* PlanInfo */ queryDesc->operation, /* CmdType */ &sys_info, /* SysInfo */ NULL, /* ErrorInfo */ @@ -784,7 +785,6 @@ pgss_planner_hook(Query *parse, const char *query_string, int cursorOptions, Par if (PGSM_TRACK_PLANNING && query_string && parse->queryId != UINT64CONST(0) && !IsParallelWorker()) { - PlanInfo plan_info; instr_time start; instr_time duration; BufferUsage bufusage_start; @@ -792,7 +792,6 @@ pgss_planner_hook(Query *parse, const char *query_string, int cursorOptions, Par WalUsage walusage_start; WalUsage walusage; - memset(&plan_info, 0, sizeof(PlanInfo)); /* We need to track buffer usage as the planner can access them. */ bufusage_start = pgBufferUsage; @@ -836,7 +835,7 @@ pgss_planner_hook(Query *parse, const char *query_string, int cursorOptions, Par WalUsageAccumDiff(&walusage, &pgWalUsage, &walusage_start); pgss_store(parse->queryId, /* query id */ query_string, /* query */ - &plan_info, /* PlanInfo */ + NULL, /* PlanInfo */ parse->commandType, /* CmdType */ NULL, /* SysInfo */ NULL, /* ErrorInfo */ @@ -1142,41 +1141,41 @@ pg_get_application_name(char *application_name) if (!beentry) return snprintf(application_name, APPLICATIONNAME_LEN, "%s", "postmaster"); - snprintf(application_name, APPLICATIONNAME_LEN, "%s", beentry->st_appname); - return strlen(application_name); + return snprintf(application_name, APPLICATIONNAME_LEN, "%s", beentry->st_appname); } -/* - * Store some statistics for a statement. - * - * If queryId is 0 then this is a utility statement and we should compute - * a suitable queryId internally. - * - * If jstate is not NULL then we're trying to create an entry for which - * we have no statistics as yet; we just want to record the normalized - */ - static uint pg_get_client_addr(void) { PgBackendStatus *beentry = pg_get_backend_status(); char remote_host[NI_MAXHOST]; int ret; + static uint localhost = 0; + + remote_host[0] = '\0'; if (!beentry) return ntohl(inet_addr("127.0.0.1")); - - memset(remote_host, 0x0, NI_MAXHOST); + ret = pg_getnameinfo_all(&beentry->st_clientaddr.addr, beentry->st_clientaddr.salen, remote_host, sizeof(remote_host), NULL, 0, NI_NUMERICHOST | NI_NUMERICSERV); if (ret != 0) - return ntohl(inet_addr("127.0.0.1")); + { + if (localhost == 0) + localhost = ntohl(inet_addr("127.0.0.1")); + return localhost; + } if (strcmp(remote_host, "[local]") == 0) - return ntohl(inet_addr("127.0.0.1")); + { + if (localhost == 0) + localhost = ntohl(inet_addr("127.0.0.1")); + return localhost; + } + return ntohl(inet_addr(remote_host)); } @@ -1204,7 +1203,7 @@ pgss_update_entry(pgssEntry *entry, int message_len = error_info ? strlen (error_info->message) : 0; int comments_len = comments ? strlen (comments) : 0; int sqlcode_len = error_info ? strlen (error_info->sqlcode) : 0; - int plan_text_len = plan_info ? strlen (plan_info->plan_text) : 0; + int plan_text_len = plan_info ? plan_info->plan_len : 0; /* volatile block */ @@ -3043,18 +3042,16 @@ intarray_get_datum(int32 arr[], int len) int j; char str[1024]; char tmp[10]; - bool first = true; - memset(str, 0, sizeof(str)); + str[0] = '\0'; /* Need to calculate the actual size, and avoid unnessary memory usage */ for (j = 0; j < len; j++) { - if (first) + if (!str[0]) { snprintf(tmp, 10, "%d", arr[j]); strcat(str,tmp); - first = false; continue; } snprintf(tmp, 10, ",%d", arr[j]); diff --git a/pg_stat_monitor.h b/pg_stat_monitor.h index c42649a..92f7a0a 100644 --- a/pg_stat_monitor.h +++ b/pg_stat_monitor.h @@ -189,6 +189,7 @@ typedef struct PlanInfo { uint64 planid; /* plan identifier */ char plan_text[PLAN_TEXT_LEN]; /* plan text */ + size_t plan_len; /* strlen(plan_text) */ } PlanInfo; typedef struct pgssHashKey From d32dea0daa6282f98ef8ed230e6da63054a25af9 Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Mon, 29 Nov 2021 15:25:56 -0300 Subject: [PATCH 44/56] PG-286: Fix query buffer overflow management. If pgsm_overflow_target is ON (default, 1) and the query buffer overflows, we now dump the buffer and keep track of how many times pg_stat_monitor changed bucket since that. If an overflow happen again before pg_stat_monitor cycle through pgsm_max_buckets buckets (default 10), then we don't dump the buffer again, but instead report an error, this ensures that only one dump file of size pgsm_query_shared_buffer will be in disk at any time, avoiding slowing down queries to the pg_stat_monitor view. As soon as pg_stat_monitor cycles through all buckets, we remove the dump file and reset the counter (pgss->n_bucket_cycles). --- pg_stat_monitor.c | 46 +++++++++++++++++++++++++++++++++++----------- pg_stat_monitor.h | 14 ++++++++++++++ 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 6202106..33deb2d 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -215,7 +215,8 @@ static uint64 djb2_hash(unsigned char *str, size_t len); void _PG_init(void) { - int i, rc; + int rc; + char file_name[1024]; elog(DEBUG2, "pg_stat_monitor: %s()", __FUNCTION__); /* @@ -240,12 +241,8 @@ _PG_init(void) EnableQueryId(); #endif - for (i = 0; i < PGSM_MAX_BUCKETS; i++) - { - char file_name[1024]; - snprintf(file_name, 1024, "%s.%d", PGSM_TEXT_FILE, i); - unlink(file_name); - } + snprintf(file_name, 1024, "%s", PGSM_TEXT_FILE); + unlink(file_name); EmitWarningsOnPlaceholders("pg_stat_monitor"); @@ -2110,8 +2107,22 @@ get_next_wbucket(pgssSharedState *pgss) LWLockAcquire(pgss->lock, LW_EXCLUSIVE); hash_entry_dealloc(new_bucket_id, prev_bucket_id, pgss_qbuf); - snprintf(file_name, 1024, "%s.%d", PGSM_TEXT_FILE, (int)new_bucket_id); - unlink(file_name); + if (pgss->overflow) + { + pgss->n_bucket_cycles += 1; + if (pgss->n_bucket_cycles >= PGSM_MAX_BUCKETS) + { + /* + * A full rotation of PGSM_MAX_BUCKETS buckets happened since + * we detected a query buffer overflow. + * Reset overflow state and remove the dump file. + */ + pgss->overflow = false; + pgss->n_bucket_cycles = 0; + snprintf(file_name, 1024, "%s", PGSM_TEXT_FILE); + unlink(file_name); + } + } LWLockRelease(pgss->lock); @@ -3151,6 +3162,13 @@ SaveQueryText(uint64 bucketid, case OVERFLOW_TARGET_DISK: { bool dump_ok; + pgssSharedState *pgss = pgsm_get_ss(); + + if (pgss->overflow) + { + pgsm_log_error("query buffer overflowed twice"); + return false; + } /* * If the query buffer is empty, there is nothing to dump, this also @@ -3162,6 +3180,12 @@ SaveQueryText(uint64 bucketid, dump_ok = dump_queries_buffer(bucketid, buf, MAX_QUERY_BUF); buf_len = sizeof (uint64); + if (dump_ok) + { + pgss->overflow = true; + pgss->n_bucket_cycles = 0; + } + /* * We must check for overflow again, as the query length may * exceed the total size allocated to the buffer (MAX_QUERY_BUF). @@ -3399,7 +3423,7 @@ dump_queries_buffer(int bucket_id, unsigned char *buf, int buf_len) int off = 0; int tries = 0; - snprintf(file_name, 1024, "%s.%d", PGSM_TEXT_FILE, bucket_id); + snprintf(file_name, 1024, "%s", PGSM_TEXT_FILE); fd = OpenTransientFile(file_name, O_RDWR | O_CREAT | O_APPEND | PG_BINARY); if (fd < 0) { @@ -3455,7 +3479,7 @@ read_query_buffer(int bucket_id, uint64 queryid, char *query_txt, size_t pos) bool done = false; bool found = false; - snprintf(file_name, 1024, "%s.%d", PGSM_TEXT_FILE, bucket_id); + snprintf(file_name, 1024, "%s", PGSM_TEXT_FILE); fd = OpenTransientFile(file_name, O_RDONLY | PG_BINARY); if (fd < 0) goto exit; diff --git a/pg_stat_monitor.h b/pg_stat_monitor.h index 92f7a0a..648aa88 100644 --- a/pg_stat_monitor.h +++ b/pg_stat_monitor.h @@ -307,6 +307,20 @@ typedef struct pgssSharedState uint64 bucket_entry[MAX_BUCKETS]; char bucket_start_time[MAX_BUCKETS][60]; /* start time of the bucket */ LWLock *errors_lock; /* protects errors hashtable search/modification */ + /* + * These variables are used when pgsm_overflow_target is ON. + * + * overflow is set to true when the query buffer overflows. + * + * n_bucket_cycles counts the number of times we changed bucket + * since the query buffer overflowed. When it reaches pgsm_max_buckets + * we remove the dump file, also reset the counter. + * + * This allows us to avoid having a large file on disk that would also + * slowdown queries to the pg_stat_monitor view. + */ + bool overflow; + size_t n_bucket_cycles; } pgssSharedState; #define ResetSharedState(x) \ From 59c321ebc55d9fc21b1994dbfd94c9c103066efb Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Fri, 3 Dec 2021 14:14:31 -0300 Subject: [PATCH 45/56] PG-286: Reduce calls to pgstat_fetch_stat_numbackends(). After couple CPU profiling sessions with perf, it was detected that the function pgstat_fetch_stat_numbackends() is very expensive, reading the implementation on PostgreSQL's backend_status.c just confirmed that. We use that function on pg_stat_monitor to retrieve the application name and IP address of the client, we now cache the results in order to avoid calling it for every query being processed. --- pg_stat_monitor.c | 75 ++++++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 33deb2d..e77a2cc 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -104,12 +104,11 @@ PG_FUNCTION_INFO_V1(pg_stat_monitor_settings); PG_FUNCTION_INFO_V1(get_histogram_timings); PG_FUNCTION_INFO_V1(pg_stat_monitor_hook_stats); -static uint pg_get_client_addr(void); -static int pg_get_application_name(char* application_name); +static uint pg_get_client_addr(bool *ok); +static int pg_get_application_name(char *application_name, bool *ok); static PgBackendStatus *pg_get_backend_status(void); static Datum intarray_get_datum(int32 arr[], int len); - #if PG_VERSION_NUM < 140000 DECLARE_HOOK(void pgss_post_parse_analyze, ParseState *pstate, Query *query); #else @@ -1112,8 +1111,8 @@ static PgBackendStatus* pg_get_backend_status(void) { LocalPgBackendStatus *local_beentry; - int num_backends = pgstat_fetch_stat_numbackends(); - int i; + int num_backends = pgstat_fetch_stat_numbackends(); + int i; for (i = 1; i <= num_backends; i++) { @@ -1132,46 +1131,44 @@ pg_get_backend_status(void) } static int -pg_get_application_name(char *application_name) +pg_get_application_name(char *application_name, bool *ok) { PgBackendStatus *beentry = pg_get_backend_status(); if (!beentry) return snprintf(application_name, APPLICATIONNAME_LEN, "%s", "postmaster"); - return snprintf(application_name, APPLICATIONNAME_LEN, "%s", beentry->st_appname); + if (!beentry) + return snprintf(application_name, APPLICATIONNAME_LEN, "%s", "unknown"); + + *ok = true; + + return snprintf(application_name, APPLICATIONNAME_LEN, "%s", beentry->st_appname); } static uint -pg_get_client_addr(void) +pg_get_client_addr(bool *ok) { PgBackendStatus *beentry = pg_get_backend_status(); - char remote_host[NI_MAXHOST]; - int ret; - static uint localhost = 0; + char remote_host[NI_MAXHOST]; + int ret; remote_host[0] = '\0'; if (!beentry) return ntohl(inet_addr("127.0.0.1")); - + + *ok = true; + ret = pg_getnameinfo_all(&beentry->st_clientaddr.addr, beentry->st_clientaddr.salen, remote_host, sizeof(remote_host), NULL, 0, NI_NUMERICHOST | NI_NUMERICSERV); if (ret != 0) - { - if (localhost == 0) - localhost = ntohl(inet_addr("127.0.0.1")); - return localhost; - } + return ntohl(inet_addr("127.0.0.1")); if (strcmp(remote_host, "[local]") == 0) - { - if (localhost == 0) - localhost = ntohl(inet_addr("127.0.0.1")); - return localhost; - } + return ntohl(inet_addr("127.0.0.1")); return ntohl(inet_addr(remote_host)); } @@ -1191,11 +1188,11 @@ pgss_update_entry(pgssEntry *entry, BufferUsage *bufusage, WalUsage *walusage, bool reset, - pgssStoreKind kind) + pgssStoreKind kind, + const char *app_name, + size_t app_name_len) { int index; - char application_name[APPLICATIONNAME_LEN]; - int application_name_len = pg_get_application_name(application_name); double old_mean; int message_len = error_info ? strlen (error_info->message) : 0; int comments_len = comments ? strlen (comments) : 0; @@ -1264,11 +1261,11 @@ pgss_update_entry(pgssEntry *entry, e->counters.resp_calls[index]++; } - if (plan_text_len > 0) + if (plan_text_len > 0 && !e->counters.planinfo.plan_text[0]) _snprintf(e->counters.planinfo.plan_text, plan_info->plan_text, plan_text_len + 1, PLAN_TEXT_LEN); - if (application_name_len > 0) - _snprintf(e->counters.info.application_name, application_name, application_name_len + 1, APPLICATIONNAME_LEN); + if (app_name_len > 0 && !e->counters.info.application_name[0]) + _snprintf(e->counters.info.application_name, app_name, app_name_len + 1, APPLICATIONNAME_LEN); e->counters.info.num_relations = num_relations; _snprintf2(e->counters.info.relations, relations, num_relations, REL_LEN); @@ -1465,8 +1462,8 @@ pgss_store(uint64 queryid, pgssHashKey key; pgssEntry *entry; pgssSharedState *pgss = pgsm_get_ss(); - char application_name[APPLICATIONNAME_LEN]; - int application_name_len; + static char application_name[APPLICATIONNAME_LEN] = ""; + static int application_name_len = 0; bool reset = false; uint64 bucketid; uint64 prev_bucket_id; @@ -1474,6 +1471,9 @@ pgss_store(uint64 queryid, uint64 planid; uint64 appid; char comments[512] = ""; + static bool found_app_name = false; + static bool found_client_addr = false; + static uint client_addr = 0; /* Monitoring is disabled */ if (!PGSM_ENABLED) @@ -1492,8 +1492,13 @@ pgss_store(uint64 queryid, else userid = GetUserId(); - application_name_len = pg_get_application_name(application_name); - planid = plan_info ? plan_info->planid: 0; + if (!found_app_name) + application_name_len = pg_get_application_name(application_name, &found_app_name); + + if (!found_client_addr) + client_addr = pg_get_client_addr(&found_client_addr); + + planid = plan_info ? plan_info->planid : 0; appid = djb2_hash((unsigned char *)application_name, application_name_len); extract_query_comments(query, comments, sizeof(comments)); @@ -1508,7 +1513,7 @@ pgss_store(uint64 queryid, key.userid = userid; key.dbid = MyDatabaseId; key.queryid = queryid; - key.ip = pg_get_client_addr(); + key.ip = client_addr; key.planid = planid; key.appid = appid; #if PG_VERSION_NUM < 140000 @@ -1596,7 +1601,9 @@ pgss_store(uint64 queryid, bufusage, /* bufusage */ walusage, /* walusage */ reset, /* reset */ - kind); /* kind */ + kind, /* kind */ + application_name, + application_name_len); LWLockRelease(pgss->lock); } From a071516a0f350da88235bb0964431c3839b910f1 Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Sat, 4 Dec 2021 11:01:39 -0300 Subject: [PATCH 46/56] PG-286: Check for NULL return on hash_search before using object. Check if hash_search() function returns NULL before attempting to use the object in hash_entry_alloc(). --- hash_query.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hash_query.c b/hash_query.c index d6e4051..5074d5f 100644 --- a/hash_query.c +++ b/hash_query.c @@ -149,7 +149,9 @@ hash_entry_alloc(pgssSharedState *pgss, pgssHashKey *key, int encoding) } /* Find or create an entry with desired hash code */ entry = (pgssEntry *) hash_search(pgss_hash, key, HASH_ENTER_NULL, &found); - if (!found) + if (entry == NULL) + pgsm_log_error("hash_entry_alloc: OUT OF MEMORY"); + else if (!found) { pgss->bucket_entry[pg_atomic_read_u64(&pgss->current_wbucket)]++; /* New entry, initialize it */ @@ -161,8 +163,6 @@ hash_entry_alloc(pgssSharedState *pgss, pgssHashKey *key, int encoding) /* ... and don't forget the query text metadata */ entry->encoding = encoding; } - if (entry == NULL) - pgsm_log_error("hash_entry_alloc: OUT OF MEMORY"); return entry; } From 007445a0d5308ab63e421821193f1cb56a33b3c4 Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Sat, 4 Dec 2021 11:05:24 -0300 Subject: [PATCH 47/56] PG-286: Several improvements. This commit introduces serveral improvements: 1. Removal of pgss_store_query and pgss_store_utility functions: To store a query, we just use pgss_store(), this makes the code more uniform. 2. Always pass the query length to the pgss_store function using parse state from PostgreSQL to avoid calculating query length again. 3. Always clean the query (extra spaces, update query location) in pgss_store. 4. Normalize queries right before adding them to the query buffer, but only if user asked for query normalization. 5. Correctly handle utility queries among different PostgreSQL versions: - A word about how utility functions are handled on PG 13 and later versions: - On PostgreSQL <= 13, we have to compute a query ID, on later versions we can call EnableQueryId() to inform Postmaster we want to enable query ID computation. - On PostgreSQL <= 13, post_parse hook is called after process utility hook, on PostgreSQL >= 14, post_parse hook is called before process utility functions. - Based on that information, on PostgreSQL <= 13 / process utility, we pass 0 as queryid to the pgss_store function, then we calculate a queryid after cleaning the query (CleanQueryText) using pgss_hash_string. - On PostgreSQL 14 onward, post_parse() is called before pgss_ProcessUtility, we Clear queryId for prepared statements related utility, on process utility hook, we save the query ID for passing it to the pgss_store function, but mark the query ID with zero to avoid instrumenting it again on executor hooks. --- pg_stat_monitor.c | 549 +++++++++++++++++++++++++--------------------- 1 file changed, 299 insertions(+), 250 deletions(-) diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index e77a2cc..0b93662 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -136,6 +136,7 @@ DECLARE_HOOK(void pgss_ProcessUtility, PlannedStmt *pstmt, const char *queryStri ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *qc); +static uint64 pgss_hash_string(const char *str, int len); #else static void BufferUsageAccumDiff(BufferUsage* bufusage, BufferUsage* pgBufferUsage, BufferUsage* bufusage_start); DECLARE_HOOK(void pgss_ProcessUtility, PlannedStmt *pstmt, const char *queryString, @@ -144,30 +145,28 @@ DECLARE_HOOK(void pgss_ProcessUtility, PlannedStmt *pstmt, const char *queryStri DestReceiver *dest, char *completionTag); #endif - -static uint64 pgss_hash_string(const char *str, int len); char *unpack_sql_state(int sql_state); -static void pgss_store_error(uint64 queryid, const char * query, ErrorData *edata); +#define PGSM_HANDLED_UTILITY(n) (!IsA(n, ExecuteStmt) && \ + !IsA(n, PrepareStmt) && \ + !IsA(n, DeallocateStmt)) -static void pgss_store_utility(const char *query, - double total_time, - uint64 rows, - BufferUsage *bufusage, - WalUsage *walusage); +static void pgss_store_error(uint64 queryid, const char *query, ErrorData *edata); static void pgss_store(uint64 queryid, - const char *query, - PlanInfo *plan_info, - CmdType cmd_type, - SysInfo *sys_info, - ErrorInfo *error_info, - double total_time, - uint64 rows, - BufferUsage *bufusage, - WalUsage *walusage, - JumbleState *jstate, - pgssStoreKind kind); + const char *query, + int query_location, + int query_len, + PlanInfo *plan_info, + CmdType cmd_type, + SysInfo *sys_info, + ErrorInfo *error_info, + double total_time, + uint64 rows, + BufferUsage *bufusage, + WalUsage *walusage, + JumbleState *jstate, + pgssStoreKind kind); static void pg_stat_monitor_internal(FunctionCallInfo fcinfo, bool showtext); @@ -179,6 +178,12 @@ static void JumbleQuery(JumbleState *jstate, Query *query); static void JumbleRangeTable(JumbleState *jstate, List *rtable); static void JumbleExpr(JumbleState *jstate, Node *node); static void RecordConstLocation(JumbleState *jstate, int location); +/* + * Given a possibly multi-statement source string, confine our attention to the + * relevant part of the string. + */ +static const char * +CleanQuerytext(const char *query, int *location, int *len); #endif static char *generate_normalized_query(JumbleState *jstate, const char *query, @@ -188,19 +193,6 @@ static int comp_location(const void *a, const void *b); static uint64 get_next_wbucket(pgssSharedState *pgss); -static void -pgss_store_query(uint64 queryid, - const char * query, - CmdType cmd_type, - int query_location, - int query_len, -#if PG_VERSION_NUM > 130000 - JumbleState *jstate, -#else - JumbleState *jstate, -#endif - pgssStoreKind kind); - #if PG_VERSION_NUM < 140000 static uint64 get_query_id(JumbleState *jstate, Query *query); #endif @@ -357,8 +349,6 @@ pgss_post_parse_analyze_benchmark(ParseState *pstate, Query *query, JumbleState static void pgss_post_parse_analyze(ParseState *pstate, Query *query, JumbleState *jstate) { - pgssStoreKind kind = PGSS_PARSE; - if (prev_post_parse_analyze_hook) prev_post_parse_analyze_hook(pstate, query, jstate); @@ -376,7 +366,8 @@ pgss_post_parse_analyze(ParseState *pstate, Query *query, JumbleState *jstate) */ if (query->utilityStmt) { - query->queryId = UINT64CONST(0); + if (PGSM_TRACK_UTILITY && !PGSM_HANDLED_UTILITY(query->utilityStmt)) + query->queryId = UINT64CONST(0); return; } @@ -387,15 +378,21 @@ pgss_post_parse_analyze(ParseState *pstate, Query *query, JumbleState *jstate) * constants, the normalized string would be the same as the query text * anyway, so there's no need for an early entry. */ - if (jstate == NULL || jstate->clocations_count <= 0) - return; - pgss_store_query(query->queryId, /* queryid */ - pstate->p_sourcetext, /* query */ - query->commandType, /* CmdType */ - query->stmt_location, /* Query Location */ - query->stmt_len, /* Query Len */ - jstate, /* JumbleState */ - kind); /*pgssStoreKind */ + if (jstate && jstate->clocations_count > 0) + pgss_store(query->queryId, /* query id */ + pstate->p_sourcetext, /* query */ + query->stmt_location, /* query location */ + query->stmt_len, /* query length */ + NULL, /* PlanInfo */ + query->commandType, /* CmdType */ + NULL, /* SysInfo */ + NULL, /* ErrorInfo */ + 0, /* totaltime */ + 0, /* rows */ + NULL, /* bufusage */ + NULL, /* walusage */ + jstate, /* JumbleState */ + PGSS_PARSE); /* pgssStoreKind */ } #else @@ -416,7 +413,6 @@ static void pgss_post_parse_analyze(ParseState *pstate, Query *query) { JumbleState jstate; - pgssStoreKind kind = PGSS_PARSE; if (prev_post_parse_analyze_hook) prev_post_parse_analyze_hook(pstate, query); @@ -451,16 +447,21 @@ pgss_post_parse_analyze(ParseState *pstate, Query *query) if (query->queryId == UINT64CONST(0)) query->queryId = UINT64CONST(1); - if (jstate.clocations_count <= 0) - return; - - pgss_store_query(query->queryId, /* queryid */ - pstate->p_sourcetext, /* query */ - query->commandType, /* CmdType */ - query->stmt_location, /* Query Location */ - query->stmt_len, /* Query Len */ - &jstate, /* JumbleState */ - kind); /*pgssStoreKind */ + if (jstate.clocations_count > 0) + pgss_store(query->queryId, /* query id */ + pstate->p_sourcetext, /* query */ + query->stmt_location, /* query location */ + query->stmt_len, /* query length */ + NULL, /* PlanInfo */ + query->commandType, /* CmdType */ + NULL, /* SysInfo */ + NULL, /* ErrorInfo */ + 0, /* totaltime */ + 0, /* rows */ + NULL, /* bufusage */ + NULL, /* walusage */ + &jstate, /* JumbleState */ + PGSS_PARSE); /* pgssStoreKind */ } #endif @@ -480,9 +481,7 @@ pgss_ExecutorStart_benchmark(QueryDesc *queryDesc, int eflags) static void pgss_ExecutorStart(QueryDesc *queryDesc, int eflags) { - uint64 queryId = queryDesc->plannedstmt->queryId; - - if(getrusage(RUSAGE_SELF, &rusage_start) != 0) + if (getrusage(RUSAGE_SELF, &rusage_start) != 0) pgsm_log_error("pgss_ExecutorStart: failed to execute getrusage"); if (prev_ExecutorStart) @@ -517,22 +516,20 @@ pgss_ExecutorStart(QueryDesc *queryDesc, int eflags) #endif MemoryContextSwitchTo(oldcxt); } - pgss_store(queryId, /* query id */ + pgss_store(queryDesc->plannedstmt->queryId, /* query id */ queryDesc->sourceText, /* query text */ + queryDesc->plannedstmt->stmt_location, /* query location */ + queryDesc->plannedstmt->stmt_len, /* query length */ NULL, /* PlanInfo */ queryDesc->operation, /* CmdType */ NULL, /* SysInfo */ NULL, /* ErrorInfo */ 0, /* totaltime */ 0, /* rows */ - NULL, /* bufusage */ -#if PG_VERSION_NUM >= 130000 + NULL, /* bufusage */ NULL, /* walusage */ -#else - NULL, -#endif - NULL, - PGSS_EXEC); /* pgssStoreKind */ + NULL, /* JumbleState */ + PGSS_EXEC); /* pgssStoreKind */ } } @@ -649,17 +646,17 @@ pgss_ExecutorEnd_benchmark(QueryDesc *queryDesc) static void pgss_ExecutorEnd(QueryDesc *queryDesc) { - uint64 queryId = queryDesc->plannedstmt->queryId; - SysInfo sys_info; - PlanInfo plan_info; - PlanInfo *plan_ptr = NULL; + uint64 queryId = queryDesc->plannedstmt->queryId; + SysInfo sys_info; + PlanInfo plan_info; + PlanInfo *plan_ptr = NULL; - /* Extract the plan information in case of SELECT statement */ + /* Extract the plan information in case of SELECT statement */ if (queryDesc->operation == CMD_SELECT && PGSM_QUERY_PLAN) - { + { MemoryContext mct = MemoryContextSwitchTo(TopMemoryContext); plan_info.plan_len = snprintf(plan_info.plan_text, PLAN_TEXT_LEN, "%s", pgss_explain(queryDesc)); - plan_info.planid = DatumGetUInt64(hash_any_extended((const unsigned char*)plan_info.plan_text, plan_info.plan_len, 0)); + plan_info.planid = DatumGetUInt64(hash_any_extended((const unsigned char *)plan_info.plan_text, plan_info.plan_len, 0)); plan_ptr = &plan_info; MemoryContextSwitchTo(mct); } @@ -677,15 +674,17 @@ pgss_ExecutorEnd(QueryDesc *queryDesc) sys_info.utime = time_diff(rusage_end.ru_utime, rusage_start.ru_utime); sys_info.stime = time_diff(rusage_end.ru_stime, rusage_start.ru_stime); - pgss_store(queryId, /* query id */ - queryDesc->sourceText, /* query text */ - plan_ptr, /* PlanInfo */ - queryDesc->operation, /* CmdType */ - &sys_info, /* SysInfo */ - NULL, /* ErrorInfo */ - queryDesc->totaltime->total * 1000.0, /* totaltime */ - queryDesc->estate->es_processed, /* rows */ - &queryDesc->totaltime->bufusage, /* bufusage */ + pgss_store(queryId, /* query id */ + queryDesc->sourceText, /* query text */ + queryDesc->plannedstmt->stmt_location, /* query location */ + queryDesc->plannedstmt->stmt_len, /* query length */ + plan_ptr, /* PlanInfo */ + queryDesc->operation, /* CmdType */ + &sys_info, /* SysInfo */ + NULL, /* ErrorInfo */ + queryDesc->totaltime->total * 1000.0, /* totaltime */ + queryDesc->estate->es_processed, /* rows */ + &queryDesc->totaltime->bufusage, /* bufusage */ #if PG_VERSION_NUM >= 130000 &queryDesc->totaltime->walusage, /* walusage */ #else @@ -829,18 +828,20 @@ pgss_planner_hook(Query *parse, const char *query_string, int cursorOptions, Par /* calc differences of WAL counters. */ memset(&walusage, 0, sizeof(WalUsage)); WalUsageAccumDiff(&walusage, &pgWalUsage, &walusage_start); - pgss_store(parse->queryId, /* query id */ - query_string, /* query */ - NULL, /* PlanInfo */ - parse->commandType, /* CmdType */ - NULL, /* SysInfo */ - NULL, /* ErrorInfo */ - INSTR_TIME_GET_MILLISEC(duration), /* totaltime */ - 0, /* rows */ - &bufusage, /* bufusage */ - &walusage, /* walusage */ - NULL, /* JumbleState */ - PGSS_PLAN); /* pgssStoreKind */ + pgss_store(parse->queryId, /* query id */ + query_string, /* query */ + parse->stmt_location, /* query location */ + parse->stmt_len, /* query length */ + NULL, /* PlanInfo */ + parse->commandType, /* CmdType */ + NULL, /* SysInfo */ + NULL, /* ErrorInfo */ + INSTR_TIME_GET_MILLISEC(duration), /* totaltime */ + 0, /* rows */ + &bufusage, /* bufusage */ + &walusage, /* walusage */ + NULL, /* JumbleState */ + PGSS_PLAN); /* pgssStoreKind */ } else { @@ -930,7 +931,24 @@ static void pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, char *completionTag) #endif { - Node *parsetree = pstmt->utilityStmt; + Node *parsetree = pstmt->utilityStmt; + uint64 queryId = 0; + +#if PG_VERSION_NUM >= 140000 + queryId = pstmt->queryId; + + /* + * Force utility statements to get queryId zero. We do this even in cases + * where the statement contains an optimizable statement for which a + * queryId could be derived (such as EXPLAIN or DECLARE CURSOR). For such + * cases, runtime control will first go through ProcessUtility and then + * the executor, and we don't want the executor hooks to do anything, + * since we are already measuring the statement's costs at the utility + * level. + */ + if (PGSM_TRACK_UTILITY && !IsParallelWorker()) + pstmt->queryId = UINT64CONST(0); +#endif /* * If it's an EXECUTE statement, we don't track it and don't increment the @@ -946,10 +964,8 @@ static void pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, * * Likewise, we don't track execution of DEALLOCATE. */ - if (PGSM_TRACK_UTILITY && - !IsA(parsetree, ExecuteStmt) && - !IsA(parsetree, PrepareStmt) && - !IsA(parsetree, DeallocateStmt) && !IsParallelWorker()) + if (PGSM_TRACK_UTILITY && PGSM_HANDLED_UTILITY(parsetree) && + !IsParallelWorker()) { instr_time start; instr_time duration; @@ -1012,7 +1028,16 @@ static void pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, INSTR_TIME_SUBTRACT(duration, start); #if PG_VERSION_NUM >= 130000 +#if PG_VERSION_NUM >= 140000 + rows = (qc && (qc->commandTag == CMDTAG_COPY || + qc->commandTag == CMDTAG_FETCH || + qc->commandTag == CMDTAG_SELECT || + qc->commandTag == CMDTAG_REFRESH_MATERIALIZED_VIEW)) + ? qc->nprocessed + : 0; +#else rows = (qc && qc->commandTag == CMDTAG_COPY) ? qc->nprocessed : 0; +#endif /* calc differences of WAL counters. */ memset(&walusage, 0, sizeof(WalUsage)); WalUsageAccumDiff(&walusage, &pgWalUsage, &walusage_start); @@ -1027,49 +1052,59 @@ static void pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, /* calc differences of buffer counters. */ memset(&bufusage, 0, sizeof(BufferUsage)); BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &bufusage_start); - pgss_store_utility(queryString, /* query text */ - INSTR_TIME_GET_MILLISEC(duration), /* totaltime */ - rows, /* rows */ - &bufusage, /* bufusage */ - &walusage); /* walusage */ + pgss_store( + queryId, /* query ID */ + queryString, /* query text */ + pstmt->stmt_location, /* query location */ + pstmt->stmt_len, /* query length */ + NULL, /* PlanInfo */ + 0, /* CmdType */ + NULL, /* SysInfo */ + NULL, /* ErrorInfo */ + INSTR_TIME_GET_MILLISEC(duration), /* total_time */ + rows, /* rows */ + &bufusage, /* bufusage */ + &walusage, /* walusage */ + NULL, /* JumbleState */ + PGSS_FINISHED); /* pgssStoreKind */ } else { #if PG_VERSION_NUM >= 140000 - if (prev_ProcessUtility) - prev_ProcessUtility(pstmt, queryString, - readOnlyTree, - context, params, queryEnv, + if (prev_ProcessUtility) + prev_ProcessUtility(pstmt, queryString, + readOnlyTree, + context, params, queryEnv, + dest, + qc); + else + standard_ProcessUtility(pstmt, queryString, + readOnlyTree, + context, params, queryEnv, dest, qc); - else - standard_ProcessUtility(pstmt, queryString, - readOnlyTree, - context, params, queryEnv, - dest, - qc); #elif PG_VERSION_NUM >= 130000 - if (prev_ProcessUtility) - prev_ProcessUtility(pstmt, queryString, - context, params, queryEnv, + if (prev_ProcessUtility) + prev_ProcessUtility(pstmt, queryString, + context, params, queryEnv, + dest, + qc); + else + standard_ProcessUtility(pstmt, queryString, + context, params, queryEnv, dest, qc); - else - standard_ProcessUtility(pstmt, queryString, - context, params, queryEnv, - dest, - qc); #else - if (prev_ProcessUtility) - prev_ProcessUtility(pstmt, queryString, + if (prev_ProcessUtility) + prev_ProcessUtility(pstmt, queryString, + context, params, queryEnv, + dest, + completionTag); + else + standard_ProcessUtility(pstmt, queryString, context, params, queryEnv, dest, completionTag); - else - standard_ProcessUtility(pstmt, queryString, - context, params, queryEnv, - dest, - completionTag); #endif } } @@ -1095,6 +1130,8 @@ BufferUsageAccumDiff(BufferUsage* bufusage, BufferUsage* pgBufferUsage, BufferUs INSTR_TIME_SUBTRACT(bufusage->blk_write_time, bufusage_start->blk_write_time); } #endif + +#if PG_VERSION_NUM < 140000 /* * Given an arbitrarily long query string, produce a hash for the purposes of * identifying the query, without normalizing constants. Used when hashing @@ -1106,6 +1143,7 @@ pgss_hash_string(const char *str, int len) return DatumGetUInt64(hash_any_extended((const unsigned char *) str, len, 0)); } +#endif static PgBackendStatus* pg_get_backend_status(void) @@ -1320,72 +1358,6 @@ pgss_update_entry(pgssEntry *entry, } } -static void -pgss_store_query(uint64 queryid, - const char * query, - CmdType cmd_type, - int query_location, - int query_len, -#if PG_VERSION_NUM > 130000 - JumbleState *jstate, -#else - JumbleState *jstate, -#endif - pgssStoreKind kind) -{ - char *norm_query = NULL; - - if (query_location >= 0) - { - Assert(query_location <= strlen(query)); - query += query_location; - /* Length of 0 (or -1) means "rest of string" */ - if (query_len <= 0) - query_len = strlen(query); - else - Assert(query_len <= strlen(query)); - } - else - { - /* If query location is unknown, distrust query_len as well */ - query_location = 0; - query_len = strlen(query); - } - - /* - * Discard leading and trailing whitespace, too. Use scanner_isspace() - * not libc's isspace(), because we want to match the lexer's behavior. - */ - while (query_len > 0 && scanner_isspace(query[0])) - query++, query_location++, query_len--; - while (query_len > 0 && scanner_isspace(query[query_len - 1])) - query_len--; - - if (jstate) - norm_query = generate_normalized_query(jstate, query, - query_location, - &query_len, - GetDatabaseEncoding()); - /* - * For utility statements, we just hash the query string to get an ID. - */ - if (queryid == UINT64CONST(0)) - queryid = pgss_hash_string(query, query_len); - - pgss_store(queryid, /* query id */ - PGSM_NORMALIZED_QUERY ? (norm_query ? norm_query : query) : query, /* query */ - NULL, /* PlanInfo */ - cmd_type, /* CmdType */ - NULL, /* SysInfo */ - NULL, /* ErrorInfo */ - 0, /* totaltime */ - 0, /* rows */ - NULL, /* bufusage */ - NULL, /* walusage */ - jstate, /* JumbleState */ - kind); /* pgssStoreKind */ -} - static void pgss_store_error(uint64 queryid, const char * query, @@ -1397,41 +1369,20 @@ pgss_store_error(uint64 queryid, snprintf(error_info.message, ERROR_MESSAGE_LEN, "%s", edata->message); snprintf(error_info.sqlcode, SQLCODE_LEN, "%s", unpack_sql_state(edata->sqlerrcode)); - pgss_store(queryid, /* query id */ - query, /* query text */ - NULL, /* PlanInfo */ - 0, /* CmdType */ - NULL, /* SysInfo */ - &error_info, /* ErrorInfo */ - 0, /* total_time */ - 0, /* rows */ - NULL, /* bufusage */ - NULL, /* walusage */ - NULL, /* JumbleState */ - PGSS_ERROR); /* pgssStoreKind */ -} - -static void -pgss_store_utility(const char *query, - double total_time, - uint64 rows, - BufferUsage *bufusage, - WalUsage *walusage) -{ - uint64 queryid = pgss_hash_string(query, strlen(query)); - - pgss_store(queryid, /* query id */ - query, /* query text */ - NULL, /* PlanInfo */ - 0, /* CmdType */ - NULL, /* SysInfo */ - NULL, /* ErrorInfo */ - total_time, /* total_time */ - rows, /* rows */ - bufusage, /* bufusage */ - walusage, /* walusage */ - NULL, /* JumbleState */ - PGSS_FINISHED); /* pgssStoreKind */ + pgss_store(queryid, /* query id */ + query, /* query text */ + 0, /* query location */ + strlen(query), /* query length */ + NULL, /* PlanInfo */ + 0, /* CmdType */ + NULL, /* SysInfo */ + &error_info, /* ErrorInfo */ + 0, /* total_time */ + 0, /* rows */ + NULL, /* bufusage */ + NULL, /* walusage */ + NULL, /* JumbleState */ + PGSS_ERROR); /* pgssStoreKind */ } /* @@ -1446,17 +1397,19 @@ pgss_store_utility(const char *query, */ static void pgss_store(uint64 queryid, - const char *query, - PlanInfo *plan_info, - CmdType cmd_type, - SysInfo *sys_info, - ErrorInfo *error_info, - double total_time, - uint64 rows, - BufferUsage *bufusage, - WalUsage *walusage, - JumbleState *jstate, - pgssStoreKind kind) + const char *query, + int query_location, + int query_len, + PlanInfo *plan_info, + CmdType cmd_type, + SysInfo *sys_info, + ErrorInfo *error_info, + double total_time, + uint64 rows, + BufferUsage *bufusage, + WalUsage *walusage, + JumbleState *jstate, + pgssStoreKind kind) { HTAB *pgss_hash; pgssHashKey key; @@ -1471,6 +1424,7 @@ pgss_store(uint64 queryid, uint64 planid; uint64 appid; char comments[512] = ""; + char *norm_query = NULL; static bool found_app_name = false; static bool found_client_addr = false; static uint client_addr = 0; @@ -1483,6 +1437,34 @@ pgss_store(uint64 queryid, if (!IsSystemInitialized()) return; +#if PG_VERSION_NUM >= 140000 + /* + * Nothing to do if compute_query_id isn't enabled and no other module + * computed a query identifier. + */ + if (queryid == UINT64CONST(0)) + return; +#endif + + query = CleanQuerytext(query, &query_location, &query_len); + +#if PG_VERSION_NUM < 140000 + /* + * For utility statements, we just hash the query string to get an ID. + */ + if (queryid == UINT64CONST(0)) + { + queryid = pgss_hash_string(query, query_len); + /* + * If we are unlucky enough to get a hash of zero(invalid), use + * queryID as 2 instead, queryID 1 is already in use for normal + * statements. + */ + if (queryid == UINT64CONST(0)) + queryid = UINT64CONST(2); + } +#endif + Assert(query != NULL); if (kind == PGSS_ERROR) { @@ -1529,24 +1511,41 @@ pgss_store(uint64 queryid, if (!entry) { pgssQueryEntry *query_entry; - size_t query_len = 0; bool query_found = false; uint64 prev_qbuf_len = 0; HTAB *pgss_query_hash; pgss_query_hash = pgsm_get_query_hash(); + /* + * Create a new, normalized query string if caller asked. We don't + * need to hold the lock while doing this work. (Note: in any case, + * it's possible that someone else creates a duplicate hashtable entry + * in the interval where we don't hold the lock below. That case is + * handled by entry_alloc. + */ + if (jstate && PGSM_NORMALIZED_QUERY) + { + LWLockRelease(pgss->lock); + norm_query = generate_normalized_query(jstate, query, + query_location, + &query_len, + GetDatabaseEncoding()); + LWLockAcquire(pgss->lock, LW_SHARED); + } + query_entry = hash_search(pgss_query_hash, &queryid, HASH_ENTER_NULL, &query_found); if (query_entry == NULL) { LWLockRelease(pgss->lock); + if (norm_query) + pfree(norm_query); pgsm_log_error("pgss_store: out of memory (pgss_query_hash)."); return; } else if (!query_found) { - /* New query, must add it to the buffer, calculate its length. */ - query_len = strlen(query); + /* New query, truncate length if necessary. */ if (query_len > PGSM_QUERY_MAX_LEN) query_len = PGSM_QUERY_MAX_LEN; } @@ -1557,21 +1556,28 @@ pgss_store(uint64 queryid, if (!query_found) { - if (!SaveQueryText(bucketid, queryid, pgss_qbuf, query, query_len, &query_entry->query_pos)) + if (!SaveQueryText(bucketid, + queryid, + pgss_qbuf, + norm_query ? norm_query : query, + query_len, + &query_entry->query_pos)) { LWLockRelease(pgss->lock); + if (norm_query) + pfree(norm_query); pgsm_log_error("pgss_store: insufficient shared space for query."); return; } /* - * Save current query buffer length, if we fail to add a new - * new entry to the hash table then we must restore the - * original length. - */ + * Save current query buffer length, if we fail to add a new + * new entry to the hash table then we must restore the + * original length. + */ memcpy(&prev_qbuf_len, pgss_qbuf, sizeof(prev_qbuf_len)); } - /* OK to create a new hashtable entry */ + /* OK to create a new hashtable entry */ entry = hash_entry_alloc(pgss, &key, GetDatabaseEncoding()); if (entry == NULL) { @@ -1581,6 +1587,8 @@ pgss_store(uint64 queryid, memcpy(pgss_qbuf, &prev_qbuf_len, sizeof(prev_qbuf_len)); } LWLockRelease(pgss->lock); + if (norm_query) + pfree(norm_query); return; } entry->query_pos = query_entry->query_pos; @@ -1606,6 +1614,8 @@ pgss_store(uint64 queryid, application_name_len); LWLockRelease(pgss->lock); + if (norm_query) + pfree(norm_query); } /* * Reset all statement statistics. @@ -2806,6 +2816,45 @@ RecordConstLocation(JumbleState *jstate, int location) jstate->clocations_count++; } } + +static const char * +CleanQuerytext(const char *query, int *location, int *len) +{ + int query_location = *location; + int query_len = *len; + + /* First apply starting offset, unless it's -1 (unknown). */ + if (query_location >= 0) + { + Assert(query_location <= strlen(query)); + query += query_location; + /* Length of 0 (or -1) means "rest of string" */ + if (query_len <= 0) + query_len = strlen(query); + else + Assert(query_len <= strlen(query)); + } + else + { + /* If query location is unknown, distrust query_len as well */ + query_location = 0; + query_len = strlen(query); + } + + /* + * Discard leading and trailing whitespace, too. Use scanner_isspace() + * not libc's isspace(), because we want to match the lexer's behavior. + */ + while (query_len > 0 && scanner_isspace(query[0])) + query++, query_location++, query_len--; + while (query_len > 0 && scanner_isspace(query[query_len - 1])) + query_len--; + + *location = query_location; + *len = query_len; + + return query; +} #endif /* From b702145ac34c1f3f0f26496efd573211603f78c9 Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Mon, 6 Dec 2021 10:13:31 -0300 Subject: [PATCH 48/56] PG-286: Update regression tests. As the query normalization and query cleaning is always done in the right place (pgss_store), no more parsed queries have a trailling comma ';' at the end. Also, on error regression test, after fixing some problems with utility related queries, we now have two entries for the RAISE WARNING case, the first entry is the utility query itself, the second entry is the error message logged by emit_log_hook. Some queries have the order adjusted due to the fix introduced by the previous commits. --- regression/expected/application_name.out | 10 ++-- regression/expected/basic.out | 8 ++-- regression/expected/cmd_type.out | 26 +++++----- regression/expected/counters.out | 16 +++---- regression/expected/database.out | 12 ++--- regression/expected/error.out | 28 ++++++----- regression/expected/error_1.out | 28 ++++++----- regression/expected/relations.out | 60 ++++++++++++------------ regression/expected/rows.out | 16 +++---- regression/expected/rows_1.out | 16 +++---- regression/expected/state.out | 12 ++--- regression/expected/tags.out | 4 +- regression/expected/top_query.out | 34 +++++++------- regression/expected/top_query_1.out | 34 +++++++------- 14 files changed, 156 insertions(+), 148 deletions(-) diff --git a/regression/expected/application_name.out b/regression/expected/application_name.out index 9d40e02..e200c18 100644 --- a/regression/expected/application_name.out +++ b/regression/expected/application_name.out @@ -12,11 +12,11 @@ SELECT 1 AS num; (1 row) SELECT query,application_name FROM pg_stat_monitor ORDER BY query COLLATE "C"; - query | application_name ---------------------------------------------------------------------------------+----------------------------- - SELECT $1 AS num | pg_regress/application_name - SELECT pg_stat_monitor_reset(); | pg_regress/application_name - SELECT query,application_name FROM pg_stat_monitor ORDER BY query COLLATE "C"; | pg_regress/application_name + query | application_name +-------------------------------------------------------------------------------+----------------------------- + SELECT $1 AS num | pg_regress/application_name + SELECT pg_stat_monitor_reset() | pg_regress/application_name + SELECT query,application_name FROM pg_stat_monitor ORDER BY query COLLATE "C" | pg_regress/application_name (3 rows) SELECT pg_stat_monitor_reset(); diff --git a/regression/expected/basic.out b/regression/expected/basic.out index 04b5eb8..adea370 100644 --- a/regression/expected/basic.out +++ b/regression/expected/basic.out @@ -12,11 +12,11 @@ SELECT 1 AS num; (1 row) SELECT query FROM pg_stat_monitor ORDER BY query COLLATE "C"; - query ---------------------------------------------------------------- + query +-------------------------------------------------------------- SELECT $1 AS num - SELECT pg_stat_monitor_reset(); - SELECT query FROM pg_stat_monitor ORDER BY query COLLATE "C"; + SELECT pg_stat_monitor_reset() + SELECT query FROM pg_stat_monitor ORDER BY query COLLATE "C" (3 rows) SELECT pg_stat_monitor_reset(); diff --git a/regression/expected/cmd_type.out b/regression/expected/cmd_type.out index aba4be0..086d230 100644 --- a/regression/expected/cmd_type.out +++ b/regression/expected/cmd_type.out @@ -24,19 +24,19 @@ SELECT b FROM t2 FOR UPDATE; TRUNCATE t1; DROP TABLE t1; SELECT query, cmd_type, cmd_type_text FROM pg_stat_monitor ORDER BY query COLLATE "C"; - query | cmd_type | cmd_type_text ------------------------------------------------------------------------------------------+----------+--------------- - CREATE TABLE t1 (a INTEGER); | 0 | - CREATE TABLE t2 (b INTEGER); | 0 | - DELETE FROM t1; | 4 | DELETE - DROP TABLE t1; | 0 | - INSERT INTO t1 VALUES($1) | 3 | INSERT - SELECT a FROM t1; | 1 | SELECT - SELECT b FROM t2 FOR UPDATE; | 1 | SELECT - SELECT pg_stat_monitor_reset(); | 1 | SELECT - SELECT query, cmd_type, cmd_type_text FROM pg_stat_monitor ORDER BY query COLLATE "C"; | 1 | SELECT - TRUNCATE t1; | 0 | - UPDATE t1 SET a = $1 | 2 | UPDATE + query | cmd_type | cmd_type_text +----------------------------------------------------------------------------------------+----------+--------------- + CREATE TABLE t1 (a INTEGER) | 0 | + CREATE TABLE t2 (b INTEGER) | 0 | + DELETE FROM t1 | 4 | DELETE + DROP TABLE t1 | 0 | + INSERT INTO t1 VALUES($1) | 3 | INSERT + SELECT a FROM t1 | 1 | SELECT + SELECT b FROM t2 FOR UPDATE | 1 | SELECT + SELECT pg_stat_monitor_reset() | 1 | SELECT + SELECT query, cmd_type, cmd_type_text FROM pg_stat_monitor ORDER BY query COLLATE "C" | 1 | SELECT + TRUNCATE t1 | 0 | + UPDATE t1 SET a = $1 | 2 | UPDATE (11 rows) SELECT pg_stat_monitor_reset(); diff --git a/regression/expected/counters.out b/regression/expected/counters.out index ebf26f7..40fed48 100644 --- a/regression/expected/counters.out +++ b/regression/expected/counters.out @@ -36,11 +36,11 @@ SELECT a,b,c,d FROM t1, t2, t3, t4 WHERE t1.a = t2.b AND t3.c = t4.d ORDER BY a; (0 rows) SELECT query,calls FROM pg_stat_monitor ORDER BY query COLLATE "C"; - query | calls -----------------------------------------------------------------------------------+------- - SELECT a,b,c,d FROM t1, t2, t3, t4 WHERE t1.a = t2.b AND t3.c = t4.d ORDER BY a; | 4 - SELECT pg_stat_monitor_reset(); | 1 - SELECT query,calls FROM pg_stat_monitor ORDER BY query COLLATE "C"; | 1 + query | calls +---------------------------------------------------------------------------------+------- + SELECT a,b,c,d FROM t1, t2, t3, t4 WHERE t1.a = t2.b AND t3.c = t4.d ORDER BY a | 4 + SELECT pg_stat_monitor_reset() | 1 + SELECT query,calls FROM pg_stat_monitor ORDER BY query COLLATE "C" | 1 (3 rows) SELECT pg_stat_monitor_reset(); @@ -69,8 +69,8 @@ SELECT query,calls FROM pg_stat_monitor ORDER BY query COLLATE "C"; query | calls ---------------------------------------------------------------------------------------------------+------- SELECT a,b,c,d FROM t1, t2, t3, t4 WHERE t1.a = t2.b AND t3.c = t4.d ORDER BY a | 1000 - SELECT pg_stat_monitor_reset(); | 1 - SELECT query,calls FROM pg_stat_monitor ORDER BY query COLLATE "C"; | 1 + SELECT pg_stat_monitor_reset() | 1 + SELECT query,calls FROM pg_stat_monitor ORDER BY query COLLATE "C" | 1 do $$ +| 1 declare +| n integer:= 1; +| @@ -80,7 +80,7 @@ SELECT query,calls FROM pg_stat_monitor ORDER BY query COLLATE "C"; exit when n = 1000; +| n := n + 1; +| end loop; +| - end $$; | + end $$ | (4 rows) SELECT pg_stat_monitor_reset(); diff --git a/regression/expected/database.out b/regression/expected/database.out index 378cbea..12c5055 100644 --- a/regression/expected/database.out +++ b/regression/expected/database.out @@ -28,12 +28,12 @@ SELECT * FROM t3,t4 WHERE t3.c = t4.d; \c contrib_regression SELECT datname, query FROM pg_stat_monitor ORDER BY query COLLATE "C"; - datname | query ---------------------+------------------------------------------------------------------------ - db1 | SELECT * FROM t1,t2 WHERE t1.a = t2.b; - db2 | SELECT * FROM t3,t4 WHERE t3.c = t4.d; - contrib_regression | SELECT datname, query FROM pg_stat_monitor ORDER BY query COLLATE "C"; - contrib_regression | SELECT pg_stat_monitor_reset(); + datname | query +--------------------+----------------------------------------------------------------------- + db1 | SELECT * FROM t1,t2 WHERE t1.a = t2.b + db2 | SELECT * FROM t3,t4 WHERE t3.c = t4.d + contrib_regression | SELECT datname, query FROM pg_stat_monitor ORDER BY query COLLATE "C" + contrib_regression | SELECT pg_stat_monitor_reset() (4 rows) SELECT pg_stat_monitor_reset(); diff --git a/regression/expected/error.out b/regression/expected/error.out index ccf7968..2e3d8fa 100644 --- a/regression/expected/error.out +++ b/regression/expected/error.out @@ -21,18 +21,22 @@ RAISE WARNING 'warning message'; END $$; WARNING: warning message SELECT query, elevel, sqlcode, message FROM pg_stat_monitor ORDER BY query COLLATE "C",elevel; - query | elevel | sqlcode | message -------------------------------------------------------------------------------------------------+--------+---------+----------------------------------- - ELECET * FROM unknown; | 20 | 42601 | syntax error at or near "ELECET" - SELECT * FROM unknown; | 20 | 42P01 | relation "unknown" does not exist - SELECT 1/0; | 20 | 22012 | division by zero - SELECT pg_stat_monitor_reset(); | 0 | | - SELECT query, elevel, sqlcode, message FROM pg_stat_monitor ORDER BY query COLLATE "C",elevel; | 0 | | - do $$ +| 19 | 01000 | warning message - BEGIN +| | | - RAISE WARNING 'warning message'; +| | | - END $$; | | | -(6 rows) + query | elevel | sqlcode | message +-----------------------------------------------------------------------------------------------+--------+---------+----------------------------------- + ELECET * FROM unknown; | 20 | 42601 | syntax error at or near "ELECET" + SELECT * FROM unknown; | 20 | 42P01 | relation "unknown" does not exist + SELECT 1/0; | 20 | 22012 | division by zero + SELECT pg_stat_monitor_reset() | 0 | | + SELECT query, elevel, sqlcode, message FROM pg_stat_monitor ORDER BY query COLLATE "C",elevel | 0 | | + do $$ +| 0 | | + BEGIN +| | | + RAISE WARNING 'warning message'; +| | | + END $$ | | | + do $$ +| 19 | 01000 | warning message + BEGIN +| | | + RAISE WARNING 'warning message'; +| | | + END $$; | | | +(7 rows) SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset diff --git a/regression/expected/error_1.out b/regression/expected/error_1.out index 102a92a..7a61786 100644 --- a/regression/expected/error_1.out +++ b/regression/expected/error_1.out @@ -21,18 +21,22 @@ RAISE WARNING 'warning message'; END $$; WARNING: warning message SELECT query, elevel, sqlcode, message FROM pg_stat_monitor ORDER BY query COLLATE "C",elevel; - query | elevel | sqlcode | message -------------------------------------------------------------------------------------------------+--------+---------+----------------------------------- - ELECET * FROM unknown; | 21 | 42601 | syntax error at or near "ELECET" - SELECT * FROM unknown; | 21 | 42P01 | relation "unknown" does not exist - SELECT 1/0; | 21 | 22012 | division by zero - SELECT pg_stat_monitor_reset(); | 0 | | - SELECT query, elevel, sqlcode, message FROM pg_stat_monitor ORDER BY query COLLATE "C",elevel; | 0 | | - do $$ +| 19 | 01000 | warning message - BEGIN +| | | - RAISE WARNING 'warning message'; +| | | - END $$; | | | -(6 rows) + query | elevel | sqlcode | message +-----------------------------------------------------------------------------------------------+--------+---------+----------------------------------- + ELECET * FROM unknown; | 21 | 42601 | syntax error at or near "ELECET" + SELECT * FROM unknown; | 21 | 42P01 | relation "unknown" does not exist + SELECT 1/0; | 21 | 22012 | division by zero + SELECT pg_stat_monitor_reset() | 0 | | + SELECT query, elevel, sqlcode, message FROM pg_stat_monitor ORDER BY query COLLATE "C",elevel | 0 | | + do $$ +| 0 | | + BEGIN +| | | + RAISE WARNING 'warning message'; +| | | + END $$ | | | + do $$ +| 19 | 01000 | warning message + BEGIN +| | | + RAISE WARNING 'warning message'; +| | | + END $$; | | | +(7 rows) SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset diff --git a/regression/expected/relations.out b/regression/expected/relations.out index 9ff9288..6c17a47 100644 --- a/regression/expected/relations.out +++ b/regression/expected/relations.out @@ -37,14 +37,14 @@ SELECT * FROM foo1, foo2, foo3, foo4; (0 rows) SELECT query, relations from pg_stat_monitor ORDER BY query collate "C"; - query | relations ---------------------------------------------------------------------------+--------------------------------------------------- - SELECT * FROM foo1, foo2, foo3, foo4; | {public.foo1,public.foo2,public.foo3,public.foo4} - SELECT * FROM foo1, foo2, foo3; | {public.foo1,public.foo2,public.foo3} - SELECT * FROM foo1, foo2; | {public.foo1,public.foo2} - SELECT * FROM foo1; | {public.foo1} - SELECT pg_stat_monitor_reset(); | - SELECT query, relations from pg_stat_monitor ORDER BY query collate "C"; | {public.pg_stat_monitor*,pg_catalog.pg_database} + query | relations +-------------------------------------------------------------------------+--------------------------------------------------- + SELECT * FROM foo1 | {public.foo1} + SELECT * FROM foo1, foo2 | {public.foo1,public.foo2} + SELECT * FROM foo1, foo2, foo3 | {public.foo1,public.foo2,public.foo3} + SELECT * FROM foo1, foo2, foo3, foo4 | {public.foo1,public.foo2,public.foo3,public.foo4} + SELECT pg_stat_monitor_reset() | + SELECT query, relations from pg_stat_monitor ORDER BY query collate "C" | {public.pg_stat_monitor*,pg_catalog.pg_database} (6 rows) SELECT pg_stat_monitor_reset(); @@ -89,14 +89,14 @@ SELECT * FROM sch1.foo1, sch2.foo2, sch3.foo3, sch4.foo4; (0 rows) SELECT query, relations from pg_stat_monitor ORDER BY query collate "C"; - query | relations ---------------------------------------------------------------------------+-------------------------------------------------- - SELECT * FROM sch1.foo1, sch2.foo2, sch3.foo3, sch4.foo4; | {sch1.foo1,sch2.foo2,sch3.foo3,sch4.foo4} - SELECT * FROM sch1.foo1, sch2.foo2, sch3.foo3; | {sch1.foo1,sch2.foo2,sch3.foo3} - SELECT * FROM sch1.foo1, sch2.foo2; | {sch1.foo1,sch2.foo2} - SELECT * FROM sch1.foo1; | {sch1.foo1} - SELECT pg_stat_monitor_reset(); | - SELECT query, relations from pg_stat_monitor ORDER BY query collate "C"; | {public.pg_stat_monitor*,pg_catalog.pg_database} + query | relations +-------------------------------------------------------------------------+-------------------------------------------------- + SELECT * FROM sch1.foo1 | {sch1.foo1} + SELECT * FROM sch1.foo1, sch2.foo2 | {sch1.foo1,sch2.foo2} + SELECT * FROM sch1.foo1, sch2.foo2, sch3.foo3 | {sch1.foo1,sch2.foo2,sch3.foo3} + SELECT * FROM sch1.foo1, sch2.foo2, sch3.foo3, sch4.foo4 | {sch1.foo1,sch2.foo2,sch3.foo3,sch4.foo4} + SELECT pg_stat_monitor_reset() | + SELECT query, relations from pg_stat_monitor ORDER BY query collate "C" | {public.pg_stat_monitor*,pg_catalog.pg_database} (6 rows) SELECT pg_stat_monitor_reset(); @@ -122,12 +122,12 @@ SELECT * FROM sch1.foo1, sch2.foo2, foo1, foo2; (0 rows) SELECT query, relations from pg_stat_monitor ORDER BY query; - query | relations ---------------------------------------------------------------+-------------------------------------------------- - SELECT * FROM sch1.foo1, foo1; | {sch1.foo1,public.foo1} - SELECT * FROM sch1.foo1, sch2.foo2, foo1, foo2; | {sch1.foo1,sch2.foo2,public.foo1,public.foo2} - SELECT pg_stat_monitor_reset(); | - SELECT query, relations from pg_stat_monitor ORDER BY query; | {public.pg_stat_monitor*,pg_catalog.pg_database} + query | relations +-------------------------------------------------------------+-------------------------------------------------- + SELECT * FROM sch1.foo1, foo1 | {sch1.foo1,public.foo1} + SELECT * FROM sch1.foo1, sch2.foo2, foo1, foo2 | {sch1.foo1,sch2.foo2,public.foo1,public.foo2} + SELECT pg_stat_monitor_reset() | + SELECT query, relations from pg_stat_monitor ORDER BY query | {public.pg_stat_monitor*,pg_catalog.pg_database} (4 rows) SELECT pg_stat_monitor_reset(); @@ -168,14 +168,14 @@ SELECT * FROM v1,v2,v3,v4; (0 rows) SELECT query, relations from pg_stat_monitor ORDER BY query collate "C"; - query | relations ---------------------------------------------------------------------------+----------------------------------------------------------------------------------------------- - SELECT * FROM v1,v2,v3,v4; | {public.v1*,public.foo1,public.v2*,public.foo2,public.v3*,public.foo3,public.v4*,public.foo4} - SELECT * FROM v1,v2,v3; | {public.v1*,public.foo1,public.v2*,public.foo2,public.v3*,public.foo3} - SELECT * FROM v1,v2; | {public.v1*,public.foo1,public.v2*,public.foo2} - SELECT * FROM v1; | {public.v1*,public.foo1} - SELECT pg_stat_monitor_reset(); | - SELECT query, relations from pg_stat_monitor ORDER BY query collate "C"; | {public.pg_stat_monitor*,pg_catalog.pg_database} + query | relations +-------------------------------------------------------------------------+----------------------------------------------------------------------------------------------- + SELECT * FROM v1 | {public.v1*,public.foo1} + SELECT * FROM v1,v2 | {public.v1*,public.foo1,public.v2*,public.foo2} + SELECT * FROM v1,v2,v3 | {public.v1*,public.foo1,public.v2*,public.foo2,public.v3*,public.foo3} + SELECT * FROM v1,v2,v3,v4 | {public.v1*,public.foo1,public.v2*,public.foo2,public.v3*,public.foo3,public.v4*,public.foo4} + SELECT pg_stat_monitor_reset() | + SELECT query, relations from pg_stat_monitor ORDER BY query collate "C" | {public.pg_stat_monitor*,pg_catalog.pg_database} (6 rows) SELECT pg_stat_monitor_reset(); diff --git a/regression/expected/rows.out b/regression/expected/rows.out index 5a0a0df..c68a9d8 100644 --- a/regression/expected/rows.out +++ b/regression/expected/rows.out @@ -8541,14 +8541,14 @@ SELECt * FROM t2 WHERE b % 2 = 0; (2500 rows) SELECT query, rows_retrieved FROM pg_stat_monitor ORDER BY query COLLATE "C"; - query | rows_retrieved --------------------------------------------------------------------------------+---------------- - SELECT * FROM t1 LIMIT $1 | 10 - SELECT * FROM t1; | 1000 - SELECT * FROM t2; | 5000 - SELECT pg_stat_monitor_reset(); | 1 - SELECT query, rows_retrieved FROM pg_stat_monitor ORDER BY query COLLATE "C"; | 0 - SELECt * FROM t2 WHERE b % $1 = $2 | 2500 + query | rows_retrieved +------------------------------------------------------------------------------+---------------- + SELECT * FROM t1 | 1000 + SELECT * FROM t1 LIMIT $1 | 10 + SELECT * FROM t2 | 5000 + SELECT pg_stat_monitor_reset() | 1 + SELECT query, rows_retrieved FROM pg_stat_monitor ORDER BY query COLLATE "C" | 0 + SELECt * FROM t2 WHERE b % $1 = $2 | 2500 (6 rows) SELECT pg_stat_monitor_reset(); diff --git a/regression/expected/rows_1.out b/regression/expected/rows_1.out index d6b3c09..c43d41a 100644 --- a/regression/expected/rows_1.out +++ b/regression/expected/rows_1.out @@ -8541,14 +8541,14 @@ SELECt * FROM t2 WHERE b % 2 = 0; (2500 rows) SELECT query, rows_retrieved FROM pg_stat_monitor ORDER BY query COLLATE "C"; - query | rows_retrieved --------------------------------------------------------------------------------+---------------- - SELECT * FROM t1 LIMIT $1 | 10 - SELECT * FROM t1; | 1000 - SELECT b FROM t2 FOR UPDATE; | 5000 - SELECT pg_stat_monitor_reset(); | 1 - SELECT query, rows_retrieved FROM pg_stat_monitor ORDER BY query COLLATE "C"; | 0 - SELECt * FROM t2 WHERE b % $1 = $2 | 2500 + query | rows_retrieved +------------------------------------------------------------------------------+---------------- + SELECT * FROM t1 | 1000 + SELECT * FROM t1 LIMIT $1 | 10 + SELECT b FROM t2 FOR UPDATE | 5000 + SELECT pg_stat_monitor_reset() | 1 + SELECT query, rows_retrieved FROM pg_stat_monitor ORDER BY query COLLATE "C" | 0 + SELECt * FROM t2 WHERE b % $1 = $2 | 2500 (6 rows) SELECT pg_stat_monitor_reset(); diff --git a/regression/expected/state.out b/regression/expected/state.out index b051091..bac22d1 100644 --- a/regression/expected/state.out +++ b/regression/expected/state.out @@ -14,12 +14,12 @@ SELECT 1; SELECT 1/0; -- divide by zero ERROR: division by zero SELECT query, state_code, state FROM pg_stat_monitor ORDER BY query COLLATE "C"; - query | state_code | state -----------------------------------------------------------------------------------+------------+--------------------- - SELECT $1 | 3 | FINISHED - SELECT 1/0; | 4 | FINISHED WITH ERROR - SELECT pg_stat_monitor_reset(); | 3 | FINISHED - SELECT query, state_code, state FROM pg_stat_monitor ORDER BY query COLLATE "C"; | 2 | ACTIVE + query | state_code | state +---------------------------------------------------------------------------------+------------+--------------------- + SELECT $1 | 3 | FINISHED + SELECT 1/0; | 4 | FINISHED WITH ERROR + SELECT pg_stat_monitor_reset() | 3 | FINISHED + SELECT query, state_code, state FROM pg_stat_monitor ORDER BY query COLLATE "C" | 2 | ACTIVE (4 rows) SELECT pg_stat_monitor_reset(); diff --git a/regression/expected/tags.out b/regression/expected/tags.out index b2cda90..27d37d3 100644 --- a/regression/expected/tags.out +++ b/regression/expected/tags.out @@ -15,8 +15,8 @@ SELECT query, comments FROM pg_stat_monitor ORDER BY query COLLATE "C"; query | comments ---------------------------------------------------------------------------+---------------------------------------------------------- SELECT $1 AS num /* { "application", psql_app, "real_ip", 192.168.1.3) */ | /* { "application", psql_app, "real_ip", 192.168.1.3) */ - SELECT pg_stat_monitor_reset(); | - SELECT query, comments FROM pg_stat_monitor ORDER BY query COLLATE "C"; | + SELECT pg_stat_monitor_reset() | + SELECT query, comments FROM pg_stat_monitor ORDER BY query COLLATE "C" | (3 rows) SELECT pg_stat_monitor_reset(); diff --git a/regression/expected/top_query.out b/regression/expected/top_query.out index 5d810f8..42d4f3b 100644 --- a/regression/expected/top_query.out +++ b/regression/expected/top_query.out @@ -23,23 +23,23 @@ SELECT add2(1,2); (1 row) SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C"; - query | top_query ---------------------------------------------------------------------------+-------------------- - CREATE OR REPLACE FUNCTION add(int, int) RETURNS INTEGER AS +| - $$ +| - BEGIN +| - return (select $1 + $2); +| - END; $$ language plpgsql; | - CREATE OR REPLACE function add2(int, int) RETURNS int as +| - $$ +| - BEGIN +| - return add($1,$2); +| - END; +| - $$ language plpgsql; | - SELECT (select $1 + $2) | SELECT add2($1,$2) - SELECT add2($1,$2) | - SELECT pg_stat_monitor_reset(); | - SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C"; | + query | top_query +-------------------------------------------------------------------------+-------------------- + CREATE OR REPLACE FUNCTION add(int, int) RETURNS INTEGER AS +| + $$ +| + BEGIN +| + return (select $1 + $2); +| + END; $$ language plpgsql | + CREATE OR REPLACE function add2(int, int) RETURNS int as +| + $$ +| + BEGIN +| + return add($1,$2); +| + END; +| + $$ language plpgsql | + SELECT (select $1 + $2) | SELECT add2($1,$2) + SELECT add2($1,$2) | + SELECT pg_stat_monitor_reset() | + SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C" | (6 rows) SELECT pg_stat_monitor_reset(); diff --git a/regression/expected/top_query_1.out b/regression/expected/top_query_1.out index ca738e5..b8e71a8 100644 --- a/regression/expected/top_query_1.out +++ b/regression/expected/top_query_1.out @@ -23,23 +23,23 @@ SELECT add2(1,2); (1 row) SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C"; - query | top_query ---------------------------------------------------------------------------+-------------------- - (select $1 + $2) | SELECT add2($1,$2) - CREATE OR REPLACE FUNCTION add(int, int) RETURNS INTEGER AS +| - $$ +| - BEGIN +| - return (select $1 + $2); +| - END; $$ language plpgsql; | - CREATE OR REPLACE function add2(int, int) RETURNS int as +| - $$ +| - BEGIN +| - return add($1,$2); +| - END; +| - $$ language plpgsql; | - SELECT add2($1,$2) | - SELECT pg_stat_monitor_reset(); | - SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C"; | + query | top_query +-------------------------------------------------------------------------+-------------------- + (select $1 + $2) | SELECT add2($1,$2) + CREATE OR REPLACE FUNCTION add(int, int) RETURNS INTEGER AS +| + $$ +| + BEGIN +| + return (select $1 + $2); +| + END; $$ language plpgsql | + CREATE OR REPLACE function add2(int, int) RETURNS int as +| + $$ +| + BEGIN +| + return add($1,$2); +| + END; +| + $$ language plpgsql | + SELECT add2($1,$2) | + SELECT pg_stat_monitor_reset() | + SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C" | (6 rows) SELECT pg_stat_monitor_reset(); From 6f7f44b744a5831fe7a2daa2615e7bfafc5cd5e7 Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Mon, 6 Dec 2021 13:24:22 -0300 Subject: [PATCH 49/56] PG-286: Fix deadlock. Can't call elog() function from inside the pgsm_log as the pgss_hash lock could be already acquired in exclusive mode, since elog() triggers the psmg_emit_log hook, when it calls pgss_store it will try to acquire the pgss_hash lock again, leading to a deadlock. --- pgsm_errors.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/pgsm_errors.c b/pgsm_errors.c index 95f0197..878712a 100644 --- a/pgsm_errors.c +++ b/pgsm_errors.c @@ -96,9 +96,7 @@ void pgsm_log(PgsmLogSeverity severity, const char *format, ...) LWLockRelease(pgss->errors_lock); /* * We're out of memory, can't track this error message. - * In this case we must fallback to PostgreSQL log facility. */ - elog(WARNING, "pgsm_log_error: "); return; } From b6838049b6a41e5d936aad920dd252f88e3ba774 Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Thu, 9 Dec 2021 14:49:35 -0300 Subject: [PATCH 50/56] PG-293: Add pg_stat_monitor.track GUC. This new GUC allows users to select which statements are tracked by pg_stat_monitor: - 'top': Default, track only top level queries. - 'all': Track top along with sub/nested queries. - 'none': Disable query monitoring. To avoid redudancy, now users disable pg_stat_monitor by setting pg_stat_monitor.track = 'none', similar to pg_stat_statements. This new GUC is an enumeration, so the pg_stat_monitor_settings view was adjusted to add a new column 'options' which lists the possible values for the field. The "value" and "default_value" columns in the pg_stat_monitor_settings view was adjusted to be of type text, so we can better display the enumeration values. Also the boolean types now have their values displayed as either 'yes' or 'no' to easily distinguish them from the integer types. --- guc.c | 50 +++++++++--- pg_stat_monitor--1.0.sql.in | 8 +- pg_stat_monitor.c | 154 ++++++++++++++++++++++++++---------- pg_stat_monitor.h | 44 ++++++++--- 4 files changed, 188 insertions(+), 68 deletions(-) diff --git a/guc.c b/guc.c index 52db4e3..528318f 100644 --- a/guc.c +++ b/guc.c @@ -21,6 +21,7 @@ GucVariable conf[MAX_SETTINGS]; static void DefineIntGUC(GucVariable *conf); static void DefineBoolGUC(GucVariable *conf); +static void DefineEnumGUC(GucVariable *conf, const struct config_enum_entry *options); /* * Define (or redefine) custom GUC variables. @@ -29,6 +30,7 @@ void init_guc(void) { int i = 0; + conf[i] = (GucVariable) { .guc_name = "pg_stat_monitor.pgsm_max", .guc_desc = "Sets the maximum size of shared memory in (MB) used for statement's metadata tracked by pg_stat_monitor.", @@ -53,18 +55,6 @@ init_guc(void) }; DefineIntGUC(&conf[i++]); - conf[i] = (GucVariable) { - .guc_name = "pg_stat_monitor.pgsm_enable", - .guc_desc = "Enable/Disable statistics collector.", - .guc_default = 1, - .guc_min = 0, - .guc_max = 0, - .guc_restart = false, - .guc_unit = 0, - .guc_value = &PGSM_ENABLED - }; - DefineBoolGUC(&conf[i++]); - conf[i] = (GucVariable) { .guc_name = "pg_stat_monitor.pgsm_track_utility", .guc_desc = "Selects whether utility commands are tracked.", @@ -185,6 +175,21 @@ init_guc(void) }; DefineBoolGUC(&conf[i++]); + conf[i] = (GucVariable) { + .guc_name = "pg_stat_monitor.track", + .guc_desc = "Selects which statements are tracked by pg_stat_monitor.", + .n_options = 3, + .guc_default = PGSM_TRACK_TOP, + .guc_min = PSGM_TRACK_NONE, + .guc_max = PGSM_TRACK_ALL, + .guc_restart = false, + .guc_unit = 0, + .guc_value = &PGSM_TRACK + }; + for (int j = 0; j < conf[i].n_options; ++j) { + strlcpy(conf[i].guc_options[j], track_options[j].name, sizeof(conf[i].guc_options[j])); + } + DefineEnumGUC(&conf[i++], track_options); #if PG_VERSION_NUM >= 130000 conf[i] = (GucVariable) { @@ -204,6 +209,7 @@ init_guc(void) static void DefineIntGUC(GucVariable *conf) { + conf->type = PGC_INT; DefineCustomIntVariable(conf->guc_name, conf->guc_desc, NULL, @@ -220,6 +226,7 @@ DefineIntGUC(GucVariable *conf) static void DefineBoolGUC(GucVariable *conf) { + conf->type = PGC_BOOL; DefineCustomBoolVariable(conf->guc_name, conf->guc_desc, NULL, @@ -232,9 +239,26 @@ DefineBoolGUC(GucVariable *conf) NULL); } +static void +DefineEnumGUC(GucVariable *conf, const struct config_enum_entry *options) +{ + conf->type = PGC_ENUM; + DefineCustomEnumVariable(conf->guc_name, + conf->guc_desc, + NULL, + conf->guc_value, + PGSM_TRACK_TOP, + options, + conf->guc_restart ? PGC_POSTMASTER : PGC_USERSET, + 0, + NULL, + NULL, + NULL); +} + GucVariable* get_conf(int i) { - return &conf[i]; + return &conf[i]; } diff --git a/pg_stat_monitor--1.0.sql.in b/pg_stat_monitor--1.0.sql.in index 106106d..8ecbc92 100644 --- a/pg_stat_monitor--1.0.sql.in +++ b/pg_stat_monitor--1.0.sql.in @@ -116,12 +116,13 @@ LANGUAGE SQL PARALLEL SAFE; CREATE FUNCTION pg_stat_monitor_settings( OUT name text, - OUT value INTEGER, - OUT default_value INTEGER, + OUT value text, + OUT default_value text, OUT description text, OUT minimum INTEGER, OUT maximum INTEGER, - OUT restart INTEGER + OUT options text, + OUT restart text ) RETURNS SETOF record AS 'MODULE_PATHNAME', 'pg_stat_monitor_settings' @@ -134,6 +135,7 @@ CREATE VIEW pg_stat_monitor_settings AS SELECT description, minimum, maximum, + options, restart FROM pg_stat_monitor_settings(); diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 0b93662..4e33ad8 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -46,6 +46,12 @@ do \ strlcpy((char *)_str_dst[i], _str_src[i], _len2); \ }while(0) +#define pgsm_enabled(level) \ + (!IsParallelWorker() && \ + (PGSM_TRACK == PGSM_TRACK_ALL || \ + (PGSM_TRACK == PGSM_TRACK_TOP && (level) == 0))) + + /*---- Initicalization Function Declarations ----*/ void _PG_init(void); void _PG_fini(void); @@ -53,7 +59,7 @@ void _PG_fini(void); /*---- Local variables ----*/ /* Current nesting depth of ExecutorRun+ProcessUtility calls */ -static int nested_level = 0; +static int exec_nested_level = 0; #if PG_VERSION_NUM >= 130000 static int plan_nested_level = 0; #endif @@ -356,7 +362,7 @@ pgss_post_parse_analyze(ParseState *pstate, Query *query, JumbleState *jstate) if (!IsSystemInitialized()) return; - if (IsParallelWorker()) + if (!pgsm_enabled(exec_nested_level)) return; /* @@ -421,7 +427,7 @@ pgss_post_parse_analyze(ParseState *pstate, Query *query) if (!IsSystemInitialized()) return; - if (IsParallelWorker()) + if (!pgsm_enabled(exec_nested_level)) return; /* @@ -489,15 +495,13 @@ pgss_ExecutorStart(QueryDesc *queryDesc, int eflags) else standard_ExecutorStart(queryDesc, eflags); - if (IsParallelWorker()) - return; - /* * If query has queryId zero, don't track it. This prevents double * counting of optimizable statements that are directly contained in * utility statements. */ - if (PGSM_ENABLED && queryDesc->plannedstmt->queryId != UINT64CONST(0)) + if (pgsm_enabled(exec_nested_level) && + queryDesc->plannedstmt->queryId != UINT64CONST(0)) { /* * Set up to track total elapsed time in ExecutorRun. Make sure the @@ -552,24 +556,24 @@ static void pgss_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, bool execute_once) { - if (nested_level >=0 && nested_level < max_stack_depth) - nested_queryids[nested_level] = queryDesc->plannedstmt->queryId; - nested_level++; + if (exec_nested_level >=0 && exec_nested_level < max_stack_depth) + nested_queryids[exec_nested_level] = queryDesc->plannedstmt->queryId; + exec_nested_level++; PG_TRY(); { if (prev_ExecutorRun) prev_ExecutorRun(queryDesc, direction, count, execute_once); else standard_ExecutorRun(queryDesc, direction, count, execute_once); - nested_level--; - if (nested_level >=0 && nested_level < max_stack_depth) - nested_queryids[nested_level] = UINT64CONST(0); + exec_nested_level--; + if (exec_nested_level >=0 && exec_nested_level < max_stack_depth) + nested_queryids[exec_nested_level] = UINT64CONST(0); } PG_CATCH(); { - nested_level--; - if (nested_level >=0 && nested_level < max_stack_depth) - nested_queryids[nested_level] = UINT64CONST(0); + exec_nested_level--; + if (exec_nested_level >=0 && exec_nested_level < max_stack_depth) + nested_queryids[exec_nested_level] = UINT64CONST(0); PG_RE_THROW(); } PG_END_TRY(); @@ -592,18 +596,18 @@ pgss_ExecutorFinish_benchmark(QueryDesc *queryDesc) static void pgss_ExecutorFinish(QueryDesc *queryDesc) { - nested_level++; + exec_nested_level++; PG_TRY(); { if (prev_ExecutorFinish) prev_ExecutorFinish(queryDesc); else standard_ExecutorFinish(queryDesc); - nested_level--; + exec_nested_level--; } PG_CATCH(); { - nested_level--; + exec_nested_level--; PG_RE_THROW(); } PG_END_TRY(); @@ -661,7 +665,7 @@ pgss_ExecutorEnd(QueryDesc *queryDesc) MemoryContextSwitchTo(mct); } - if (queryId != UINT64CONST(0) && queryDesc->totaltime && !IsParallelWorker()) + if (queryId != UINT64CONST(0) && queryDesc->totaltime && pgsm_enabled(exec_nested_level)) { /* * Make sure stats accumulation is done. (Note: it's okay if several @@ -778,7 +782,20 @@ pgss_planner_hook(Query *parse, const char *query_string, int cursorOptions, Par { PlannedStmt *result; - if (PGSM_TRACK_PLANNING && query_string && parse->queryId != UINT64CONST(0) && !IsParallelWorker()) + /* + * We can't process the query if no query_string is provided, as + * pgss_store needs it. We also ignore query without queryid, as it would + * be treated as a utility statement, which may not be the case. + * + * Note that planner_hook can be called from the planner itself, so we + * have a specific nesting level for the planner. However, utility + * commands containing optimizable statements can also call the planner, + * same for regular DML (for instance for underlying foreign key queries). + * So testing the planner nesting level only is not enough to detect real + * top level planner call. + */ + if (pgsm_enabled(plan_nested_level + exec_nested_level) && + PGSM_TRACK_PLANNING && query_string && parse->queryId != UINT64CONST(0)) { instr_time start; instr_time duration; @@ -946,7 +963,7 @@ static void pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, * since we are already measuring the statement's costs at the utility * level. */ - if (PGSM_TRACK_UTILITY && !IsParallelWorker()) + if (PGSM_TRACK_UTILITY && pgsm_enabled(exec_nested_level)) pstmt->queryId = UINT64CONST(0); #endif @@ -964,8 +981,8 @@ static void pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, * * Likewise, we don't track execution of DEALLOCATE. */ - if (PGSM_TRACK_UTILITY && PGSM_HANDLED_UTILITY(parsetree) && - !IsParallelWorker()) + if (PGSM_TRACK_UTILITY && pgsm_enabled(exec_nested_level) && + PGSM_HANDLED_UTILITY(parsetree)) { instr_time start; instr_time duration; @@ -1018,7 +1035,7 @@ static void pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, } PG_CATCH(); { - nested_level--; + exec_nested_level--; PG_RE_THROW(); } @@ -1310,10 +1327,10 @@ pgss_update_entry(pgssEntry *entry, e->counters.info.cmd_type = cmd_type; - if(nested_level > 0) + if(exec_nested_level > 0) { - if (nested_level >=0 && nested_level < max_stack_depth) - e->counters.info.parentid = nested_queryids[nested_level - 1]; + if (exec_nested_level >=0 && exec_nested_level < max_stack_depth) + e->counters.info.parentid = nested_queryids[exec_nested_level - 1]; } else { @@ -1429,10 +1446,6 @@ pgss_store(uint64 queryid, static bool found_client_addr = false; static uint client_addr = 0; - /* Monitoring is disabled */ - if (!PGSM_ENABLED) - return; - /* Safety check... */ if (!IsSystemInitialized()) return; @@ -3315,7 +3328,7 @@ pg_stat_monitor_settings(PG_FUNCTION_ARGS) return (Datum) 0; } - if (tupdesc->natts != 7) + if (tupdesc->natts != 8) { pgsm_log_error("pg_stat_monitor_settings: incorrect number of output arguments, required: 7, found %d", tupdesc->natts); return (Datum) 0; @@ -3330,19 +3343,78 @@ pg_stat_monitor_settings(PG_FUNCTION_ARGS) for(i = 0; i < MAX_SETTINGS; i++) { - Datum values[7]; - bool nulls[7]; + Datum values[8]; + bool nulls[8]; int j = 0; + char options[1024] = ""; + GucVariable *conf; + memset(values, 0, sizeof(values)); memset(nulls, 0, sizeof(nulls)); - values[j++] = CStringGetTextDatum(get_conf(i)->guc_name); - values[j++] = Int64GetDatumFast(get_conf(i)->guc_variable); - values[j++] = Int64GetDatumFast(get_conf(i)->guc_default); + conf = get_conf(i); + + values[j++] = CStringGetTextDatum(conf->guc_name); + + /* Handle current and default values. */ + switch (conf->type) + { + case PGC_ENUM: + values[j++] = CStringGetTextDatum(conf->guc_options[conf->guc_variable]); + values[j++] = CStringGetTextDatum(conf->guc_options[conf->guc_default]); + break; + + case PGC_INT: + { + char value[32]; + + sprintf(value, "%d", conf->guc_variable); + values[j++] = CStringGetTextDatum(value); + + sprintf(value, "%d", conf->guc_default); + values[j++] = CStringGetTextDatum(value); + break; + } + + case PGC_BOOL: + values[j++] = CStringGetTextDatum(conf->guc_variable ? "yes" : "no"); + values[j++] = CStringGetTextDatum(conf->guc_default ? "yes" : "no"); + break; + + default: + Assert(false); + } + values[j++] = CStringGetTextDatum(get_conf(i)->guc_desc); - values[j++] = Int64GetDatumFast(get_conf(i)->guc_min); - values[j++] = Int64GetDatumFast(get_conf(i)->guc_max); - values[j++] = Int64GetDatumFast(get_conf(i)->guc_restart); + + /* Minimum and maximum displayed only for integers or real numbers. */ + if (conf->type != PGC_INT) + { + nulls[j++] = true; + nulls[j++] = true; + } + else + { + values[j++] = Int64GetDatumFast(get_conf(i)->guc_min); + values[j++] = Int64GetDatumFast(get_conf(i)->guc_max); + } + + if (conf->type == PGC_ENUM) + { + strcat(options, conf->guc_options[0]); + for (size_t i = 1; i < conf->n_options; ++i) + { + strcat(options, ", "); + strcat(options, conf->guc_options[i]); + } + } + else if (conf->type == PGC_BOOL) + { + strcat(options, "yes, no"); + } + + values[j++] = CStringGetTextDatum(options); + values[j++] = CStringGetTextDatum(get_conf(i)->guc_restart ? "yes" : "no"); tuplestore_putvalues(tupstore, tupdesc, values, nulls); } /* clean up and return the tuplestore */ diff --git a/pg_stat_monitor.h b/pg_stat_monitor.h index 648aa88..eedc702 100644 --- a/pg_stat_monitor.h +++ b/pg_stat_monitor.h @@ -52,6 +52,7 @@ #include "utils/timestamp.h" #include "utils/lsyscache.h" #include "utils/guc.h" +#include "utils/guc_tables.h" #define MAX_BACKEND_PROCESES (MaxBackends + NUM_AUXILIARY_PROCS + max_prepared_xacts) #define IntArrayGetTextDatum(x,y) intarray_get_datum(x,y) @@ -99,8 +100,11 @@ #define MAX_SETTINGS 13 #endif +/* Update this if need a enum GUC with more options. */ +#define MAX_ENUM_OPTIONS 6 typedef struct GucVariables { + enum config_type type; /* PGC_BOOL, PGC_INT, PGC_REAL, PGC_STRING, PGC_ENUM */ int guc_variable; char guc_name[TEXT_LEN]; char guc_desc[TEXT_LEN]; @@ -110,6 +114,8 @@ typedef struct GucVariables int guc_unit; int *guc_value; bool guc_restart; + int n_options; + char guc_options[MAX_ENUM_OPTIONS][32]; } GucVariable; #if PG_VERSION_NUM < 130000 @@ -408,22 +414,38 @@ void set_qbuf(unsigned char *); /* hash_query.c */ void pgss_startup(void); + /*---- GUC variables ----*/ +typedef enum { + PSGM_TRACK_NONE, /* track no statements */ + PGSM_TRACK_TOP, /* only top level statements */ + PGSM_TRACK_ALL /* all statements, including nested ones */ +} PGSMTrackLevel; + +static const struct config_enum_entry track_options[] = +{ + {"none", PSGM_TRACK_NONE, false}, + {"top", PGSM_TRACK_TOP, false}, + {"all", PGSM_TRACK_ALL, false}, + {NULL, 0, false} +}; + #define PGSM_MAX get_conf(0)->guc_variable #define PGSM_QUERY_MAX_LEN get_conf(1)->guc_variable -#define PGSM_ENABLED get_conf(2)->guc_variable -#define PGSM_TRACK_UTILITY get_conf(3)->guc_variable -#define PGSM_NORMALIZED_QUERY get_conf(4)->guc_variable -#define PGSM_MAX_BUCKETS get_conf(5)->guc_variable -#define PGSM_BUCKET_TIME get_conf(6)->guc_variable -#define PGSM_HISTOGRAM_MIN get_conf(7)->guc_variable -#define PGSM_HISTOGRAM_MAX get_conf(8)->guc_variable -#define PGSM_HISTOGRAM_BUCKETS get_conf(9)->guc_variable -#define PGSM_QUERY_SHARED_BUFFER get_conf(10)->guc_variable -#define PGSM_OVERFLOW_TARGET get_conf(11)->guc_variable -#define PGSM_QUERY_PLAN get_conf(12)->guc_variable +#define PGSM_TRACK_UTILITY get_conf(2)->guc_variable +#define PGSM_NORMALIZED_QUERY get_conf(3)->guc_variable +#define PGSM_MAX_BUCKETS get_conf(4)->guc_variable +#define PGSM_BUCKET_TIME get_conf(5)->guc_variable +#define PGSM_HISTOGRAM_MIN get_conf(6)->guc_variable +#define PGSM_HISTOGRAM_MAX get_conf(7)->guc_variable +#define PGSM_HISTOGRAM_BUCKETS get_conf(8)->guc_variable +#define PGSM_QUERY_SHARED_BUFFER get_conf(9)->guc_variable +#define PGSM_OVERFLOW_TARGET get_conf(10)->guc_variable +#define PGSM_QUERY_PLAN get_conf(11)->guc_variable +#define PGSM_TRACK get_conf(12)->guc_variable #define PGSM_TRACK_PLANNING get_conf(13)->guc_variable + /*---- Benchmarking ----*/ #ifdef BENCHMARK /* From 30a328f381b488b08d88e0fdb3ac52ed5a145f1f Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Thu, 9 Dec 2021 14:58:05 -0300 Subject: [PATCH 51/56] PG-293: Update regression tests. cmd_type: Added missing DROP TABLE t2; guc: Adjusted to match the updated settings view, which now display boolean values as 'yes' and 'no', also added the 'options' column to the output. guc_1: Handle PostgreSQL versions <= 12 which don't have the track_planning feature. rows.out: Added missing DROP TABLE t2. Also removed the line 'ERROR: relation "t2" already exists' since we fixed the problem in cmd_type regression. top_query: Handling both track = 'top' and track = 'all' cases. top_query_1: On PostgreSQL >= 14 the sub query from the procedure is stored as (select $1 + $2), whereas on PG <= 13 it is stored as SELECT (select $1 + $2). --- regression/expected/cmd_type.out | 4 ++- regression/expected/guc.out | 32 ++++++++--------- regression/expected/guc_1.out | 30 ++++++++-------- regression/expected/rows.out | 2 +- regression/expected/top_query.out | 56 +++++++++++++++++++++++++++++ regression/expected/top_query_1.out | 56 +++++++++++++++++++++++++++++ regression/sql/cmd_type.sql | 1 + regression/sql/rows.sql | 1 + regression/sql/top_query.sql | 24 +++++++++++++ 9 files changed, 173 insertions(+), 33 deletions(-) diff --git a/regression/expected/cmd_type.out b/regression/expected/cmd_type.out index 086d230..d5dceeb 100644 --- a/regression/expected/cmd_type.out +++ b/regression/expected/cmd_type.out @@ -23,6 +23,7 @@ SELECT b FROM t2 FOR UPDATE; TRUNCATE t1; DROP TABLE t1; +DROP TABLE t2; SELECT query, cmd_type, cmd_type_text FROM pg_stat_monitor ORDER BY query COLLATE "C"; query | cmd_type | cmd_type_text ----------------------------------------------------------------------------------------+----------+--------------- @@ -30,6 +31,7 @@ SELECT query, cmd_type, cmd_type_text FROM pg_stat_monitor ORDER BY query COLLA CREATE TABLE t2 (b INTEGER) | 0 | DELETE FROM t1 | 4 | DELETE DROP TABLE t1 | 0 | + DROP TABLE t2 | 0 | INSERT INTO t1 VALUES($1) | 3 | INSERT SELECT a FROM t1 | 1 | SELECT SELECT b FROM t2 FOR UPDATE | 1 | SELECT @@ -37,7 +39,7 @@ SELECT query, cmd_type, cmd_type_text FROM pg_stat_monitor ORDER BY query COLLA SELECT query, cmd_type, cmd_type_text FROM pg_stat_monitor ORDER BY query COLLATE "C" | 1 | SELECT TRUNCATE t1 | 0 | UPDATE t1 SET a = $1 | 2 | UPDATE -(11 rows) +(12 rows) SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset diff --git a/regression/expected/guc.out b/regression/expected/guc.out index 0da2986..075b3ac 100644 --- a/regression/expected/guc.out +++ b/regression/expected/guc.out @@ -12,22 +12,22 @@ select pg_sleep(.5); (1 row) SELECT * FROM pg_stat_monitor_settings ORDER BY name COLLATE "C"; - name | value | default_value | description | minimum | maximum | restart -------------------------------------------+--------+---------------+----------------------------------------------------------------------------------------------------------+---------+------------+--------- - pg_stat_monitor.pgsm_bucket_time | 60 | 60 | Sets the time in seconds per bucket. | 1 | 2147483647 | 1 - pg_stat_monitor.pgsm_enable | 1 | 1 | Enable/Disable statistics collector. | 0 | 0 | 0 - pg_stat_monitor.pgsm_enable_query_plan | 0 | 0 | Enable/Disable query plan monitoring | 0 | 0 | 0 - pg_stat_monitor.pgsm_histogram_buckets | 10 | 10 | Sets the maximum number of histogram buckets | 2 | 2147483647 | 1 - pg_stat_monitor.pgsm_histogram_max | 100000 | 100000 | Sets the time in millisecond. | 10 | 2147483647 | 1 - pg_stat_monitor.pgsm_histogram_min | 0 | 0 | Sets the time in millisecond. | 0 | 2147483647 | 1 - pg_stat_monitor.pgsm_max | 100 | 100 | Sets the maximum size of shared memory in (MB) used for statement's metadata tracked by pg_stat_monitor. | 1 | 1000 | 1 - pg_stat_monitor.pgsm_max_buckets | 10 | 10 | Sets the maximum number of buckets. | 1 | 10 | 1 - pg_stat_monitor.pgsm_normalized_query | 1 | 1 | Selects whether save query in normalized format. | 0 | 0 | 0 - pg_stat_monitor.pgsm_overflow_target | 1 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | 1 - pg_stat_monitor.pgsm_query_max_len | 1024 | 1024 | Sets the maximum length of query. | 1024 | 2147483647 | 1 - pg_stat_monitor.pgsm_query_shared_buffer | 20 | 20 | Sets the maximum size of shared memory in (MB) used for query tracked by pg_stat_monitor. | 1 | 10000 | 1 - pg_stat_monitor.pgsm_track_planning | 1 | 1 | Selects whether planning statistics are tracked. | 0 | 0 | 0 - pg_stat_monitor.pgsm_track_utility | 1 | 1 | Selects whether utility commands are tracked. | 0 | 0 | 0 + name | value | default_value | description | minimum | maximum | options | restart +------------------------------------------+--------+---------------+----------------------------------------------------------------------------------------------------------+---------+------------+----------------+--------- + pg_stat_monitor.pgsm_bucket_time | 60 | 60 | Sets the time in seconds per bucket. | 1 | 2147483647 | | yes + pg_stat_monitor.pgsm_enable_query_plan | no | no | Enable/Disable query plan monitoring | | | yes, no | no + pg_stat_monitor.pgsm_histogram_buckets | 10 | 10 | Sets the maximum number of histogram buckets | 2 | 2147483647 | | yes + pg_stat_monitor.pgsm_histogram_max | 100000 | 100000 | Sets the time in millisecond. | 10 | 2147483647 | | yes + pg_stat_monitor.pgsm_histogram_min | 0 | 0 | Sets the time in millisecond. | 0 | 2147483647 | | yes + pg_stat_monitor.pgsm_max | 100 | 100 | Sets the maximum size of shared memory in (MB) used for statement's metadata tracked by pg_stat_monitor. | 1 | 1000 | | yes + pg_stat_monitor.pgsm_max_buckets | 10 | 10 | Sets the maximum number of buckets. | 1 | 10 | | yes + pg_stat_monitor.pgsm_normalized_query | yes | yes | Selects whether save query in normalized format. | | | yes, no | no + pg_stat_monitor.pgsm_overflow_target | 1 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | | yes + pg_stat_monitor.pgsm_query_max_len | 1024 | 1024 | Sets the maximum length of query. | 1024 | 2147483647 | | yes + pg_stat_monitor.pgsm_query_shared_buffer | 20 | 20 | Sets the maximum size of shared memory in (MB) used for query tracked by pg_stat_monitor. | 1 | 10000 | | yes + pg_stat_monitor.pgsm_track_planning | yes | yes | Selects whether planning statistics are tracked. | | | yes, no | no + pg_stat_monitor.pgsm_track_utility | yes | yes | Selects whether utility commands are tracked. | | | yes, no | no + pg_stat_monitor.track | top | top | Selects which statements are tracked by pg_stat_monitor. | | | none, top, all | no (14 rows) SELECT pg_stat_monitor_reset(); diff --git a/regression/expected/guc_1.out b/regression/expected/guc_1.out index ef3da0b..f839a94 100644 --- a/regression/expected/guc_1.out +++ b/regression/expected/guc_1.out @@ -12,21 +12,21 @@ select pg_sleep(.5); (1 row) SELECT * FROM pg_stat_monitor_settings ORDER BY name COLLATE "C"; - name | value | default_value | description | minimum | maximum | restart -------------------------------------------+--------+---------------+----------------------------------------------------------------------------------------------------------+---------+------------+--------- - pg_stat_monitor.pgsm_bucket_time | 60 | 60 | Sets the time in seconds per bucket. | 1 | 2147483647 | 1 - pg_stat_monitor.pgsm_enable | 1 | 1 | Enable/Disable statistics collector. | 0 | 0 | 0 - pg_stat_monitor.pgsm_enable_query_plan | 0 | 0 | Enable/Disable query plan monitoring | 0 | 0 | 0 - pg_stat_monitor.pgsm_histogram_buckets | 10 | 10 | Sets the maximum number of histogram buckets | 2 | 2147483647 | 1 - pg_stat_monitor.pgsm_histogram_max | 100000 | 100000 | Sets the time in millisecond. | 10 | 2147483647 | 1 - pg_stat_monitor.pgsm_histogram_min | 0 | 0 | Sets the time in millisecond. | 0 | 2147483647 | 1 - pg_stat_monitor.pgsm_max | 100 | 100 | Sets the maximum size of shared memory in (MB) used for statement's metadata tracked by pg_stat_monitor. | 1 | 1000 | 1 - pg_stat_monitor.pgsm_max_buckets | 10 | 10 | Sets the maximum number of buckets. | 1 | 10 | 1 - pg_stat_monitor.pgsm_normalized_query | 1 | 1 | Selects whether save query in normalized format. | 0 | 0 | 0 - pg_stat_monitor.pgsm_overflow_target | 1 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | 1 - pg_stat_monitor.pgsm_query_max_len | 1024 | 1024 | Sets the maximum length of query. | 1024 | 2147483647 | 1 - pg_stat_monitor.pgsm_query_shared_buffer | 20 | 20 | Sets the maximum size of shared memory in (MB) used for query tracked by pg_stat_monitor. | 1 | 10000 | 1 - pg_stat_monitor.pgsm_track_utility | 1 | 1 | Selects whether utility commands are tracked. | 0 | 0 | 0 + name | value | default_value | description | minimum | maximum | options | restart +------------------------------------------+--------+---------------+----------------------------------------------------------------------------------------------------------+---------+------------+----------------+--------- + pg_stat_monitor.pgsm_bucket_time | 60 | 60 | Sets the time in seconds per bucket. | 1 | 2147483647 | | yes + pg_stat_monitor.pgsm_enable_query_plan | no | no | Enable/Disable query plan monitoring | | | yes, no | no + pg_stat_monitor.pgsm_histogram_buckets | 10 | 10 | Sets the maximum number of histogram buckets | 2 | 2147483647 | | yes + pg_stat_monitor.pgsm_histogram_max | 100000 | 100000 | Sets the time in millisecond. | 10 | 2147483647 | | yes + pg_stat_monitor.pgsm_histogram_min | 0 | 0 | Sets the time in millisecond. | 0 | 2147483647 | | yes + pg_stat_monitor.pgsm_max | 100 | 100 | Sets the maximum size of shared memory in (MB) used for statement's metadata tracked by pg_stat_monitor. | 1 | 1000 | | yes + pg_stat_monitor.pgsm_max_buckets | 10 | 10 | Sets the maximum number of buckets. | 1 | 10 | | yes + pg_stat_monitor.pgsm_normalized_query | yes | yes | Selects whether save query in normalized format. | | | yes, no | no + pg_stat_monitor.pgsm_overflow_target | 1 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | | yes + pg_stat_monitor.pgsm_query_max_len | 1024 | 1024 | Sets the maximum length of query. | 1024 | 2147483647 | | yes + pg_stat_monitor.pgsm_query_shared_buffer | 20 | 20 | Sets the maximum size of shared memory in (MB) used for query tracked by pg_stat_monitor. | 1 | 10000 | | yes + pg_stat_monitor.pgsm_track_utility | yes | yes | Selects whether utility commands are tracked. | | | yes, no | no + pg_stat_monitor.track | top | top | Selects which statements are tracked by pg_stat_monitor. | | | none, top, all | no (13 rows) SELECT pg_stat_monitor_reset(); diff --git a/regression/expected/rows.out b/regression/expected/rows.out index c68a9d8..fe4cb94 100644 --- a/regression/expected/rows.out +++ b/regression/expected/rows.out @@ -1,7 +1,6 @@ CREATE EXTENSION pg_stat_monitor; CREATE TABLE t1(a int); CREATE TABLE t2(b int); -ERROR: relation "t2" already exists INSERT INTO t1 VALUES(generate_series(1,1000)); INSERT INTO t2 VALUES(generate_series(1,5000)); SELECT pg_stat_monitor_reset(); @@ -8558,4 +8557,5 @@ SELECT pg_stat_monitor_reset(); (1 row) DROP TABLE t1; +DROP TABLE t2; DROP EXTENSION pg_stat_monitor; diff --git a/regression/expected/top_query.out b/regression/expected/top_query.out index 42d4f3b..e980297 100644 --- a/regression/expected/top_query.out +++ b/regression/expected/top_query.out @@ -22,6 +22,55 @@ SELECT add2(1,2); 3 (1 row) +SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C"; + query | top_query +-------------------------------------------------------------------------+----------- + CREATE OR REPLACE FUNCTION add(int, int) RETURNS INTEGER AS +| + $$ +| + BEGIN +| + return (select $1 + $2); +| + END; $$ language plpgsql | + CREATE OR REPLACE function add2(int, int) RETURNS int as +| + $$ +| + BEGIN +| + return add($1,$2); +| + END; +| + $$ language plpgsql | + SELECT add2($1,$2) | + SELECT pg_stat_monitor_reset() | + SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C" | +(5 rows) + +ALTER SYSTEM SET pg_stat_monitor.track TO 'all'; +SELECT pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + +SELECT pg_stat_monitor_reset(); + pg_stat_monitor_reset +----------------------- + +(1 row) + +CREATE OR REPLACE FUNCTION add(int, int) RETURNS INTEGER AS +$$ +BEGIN + return (select $1 + $2); +END; $$ language plpgsql; +CREATE OR REPLACE function add2(int, int) RETURNS int as +$$ +BEGIN + return add($1,$2); +END; +$$ language plpgsql; +SELECT add2(1,2); + add2 +------ + 3 +(1 row) + SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C"; query | top_query -------------------------------------------------------------------------+-------------------- @@ -42,6 +91,13 @@ SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C"; SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C" | (6 rows) +ALTER SYSTEM SET pg_stat_monitor.track TO 'top'; +SELECT pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset ----------------------- diff --git a/regression/expected/top_query_1.out b/regression/expected/top_query_1.out index b8e71a8..59a19e5 100644 --- a/regression/expected/top_query_1.out +++ b/regression/expected/top_query_1.out @@ -22,6 +22,55 @@ SELECT add2(1,2); 3 (1 row) +SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C"; + query | top_query +-------------------------------------------------------------------------+----------- + CREATE OR REPLACE FUNCTION add(int, int) RETURNS INTEGER AS +| + $$ +| + BEGIN +| + return (select $1 + $2); +| + END; $$ language plpgsql | + CREATE OR REPLACE function add2(int, int) RETURNS int as +| + $$ +| + BEGIN +| + return add($1,$2); +| + END; +| + $$ language plpgsql | + SELECT add2($1,$2) | + SELECT pg_stat_monitor_reset() | + SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C" | +(5 rows) + +ALTER SYSTEM SET pg_stat_monitor.track TO 'all'; +SELECT pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + +SELECT pg_stat_monitor_reset(); + pg_stat_monitor_reset +----------------------- + +(1 row) + +CREATE OR REPLACE FUNCTION add(int, int) RETURNS INTEGER AS +$$ +BEGIN + return (select $1 + $2); +END; $$ language plpgsql; +CREATE OR REPLACE function add2(int, int) RETURNS int as +$$ +BEGIN + return add($1,$2); +END; +$$ language plpgsql; +SELECT add2(1,2); + add2 +------ + 3 +(1 row) + SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C"; query | top_query -------------------------------------------------------------------------+-------------------- @@ -42,6 +91,13 @@ SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C"; SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C" | (6 rows) +ALTER SYSTEM SET pg_stat_monitor.track TO 'top'; +SELECT pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset ----------------------- diff --git a/regression/sql/cmd_type.sql b/regression/sql/cmd_type.sql index 5f1fa1b..9fb2fa6 100644 --- a/regression/sql/cmd_type.sql +++ b/regression/sql/cmd_type.sql @@ -10,6 +10,7 @@ DELETE FROM t1; SELECT b FROM t2 FOR UPDATE; TRUNCATE t1; DROP TABLE t1; +DROP TABLE t2; SELECT query, cmd_type, cmd_type_text FROM pg_stat_monitor ORDER BY query COLLATE "C"; SELECT pg_stat_monitor_reset(); diff --git a/regression/sql/rows.sql b/regression/sql/rows.sql index 93ae9c1..ffd449c 100644 --- a/regression/sql/rows.sql +++ b/regression/sql/rows.sql @@ -16,4 +16,5 @@ SELECT query, rows_retrieved FROM pg_stat_monitor ORDER BY query COLLATE "C"; SELECT pg_stat_monitor_reset(); DROP TABLE t1; +DROP TABLE t2; DROP EXTENSION pg_stat_monitor; diff --git a/regression/sql/top_query.sql b/regression/sql/top_query.sql index 3642f99..16aa2ce 100644 --- a/regression/sql/top_query.sql +++ b/regression/sql/top_query.sql @@ -1,5 +1,6 @@ CREATE EXTENSION pg_stat_monitor; SELECT pg_stat_monitor_reset(); + CREATE OR REPLACE FUNCTION add(int, int) RETURNS INTEGER AS $$ BEGIN @@ -15,5 +16,28 @@ $$ language plpgsql; SELECT add2(1,2); SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C"; + +ALTER SYSTEM SET pg_stat_monitor.track TO 'all'; +SELECT pg_reload_conf(); SELECT pg_stat_monitor_reset(); + +CREATE OR REPLACE FUNCTION add(int, int) RETURNS INTEGER AS +$$ +BEGIN + return (select $1 + $2); +END; $$ language plpgsql; + +CREATE OR REPLACE function add2(int, int) RETURNS int as +$$ +BEGIN + return add($1,$2); +END; +$$ language plpgsql; + +SELECT add2(1,2); +SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C"; +ALTER SYSTEM SET pg_stat_monitor.track TO 'top'; +SELECT pg_reload_conf(); +SELECT pg_stat_monitor_reset(); + DROP EXTENSION pg_stat_monitor; From 60427959308dd6610133f7ec04cb7e498c455b8b Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Thu, 9 Dec 2021 15:55:30 -0300 Subject: [PATCH 52/56] PG-293: Add pg_stat_monitor.extract_comments GUC. This new GUC allows the user to enable/disable extracting query comments. --- guc.c | 12 ++++++++++++ pg_stat_monitor.c | 7 +++++-- pg_stat_monitor.h | 7 ++++--- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/guc.c b/guc.c index 528318f..b7f5e08 100644 --- a/guc.c +++ b/guc.c @@ -191,6 +191,18 @@ init_guc(void) } DefineEnumGUC(&conf[i++], track_options); + conf[i] = (GucVariable) { + .guc_name = "pg_stat_monitor.extract_comments", + .guc_desc = "Enable/Disable extracting comments from queries.", + .guc_default = 0, + .guc_min = 0, + .guc_max = 0, + .guc_restart = false, + .guc_unit = 0, + .guc_value = &PGSM_EXTRACT_COMMENTS + }; + DefineBoolGUC(&conf[i++]); + #if PG_VERSION_NUM >= 130000 conf[i] = (GucVariable) { .guc_name = "pg_stat_monitor.pgsm_track_planning", diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 4e33ad8..0e6a4ac 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -1496,8 +1496,6 @@ pgss_store(uint64 queryid, planid = plan_info ? plan_info->planid : 0; appid = djb2_hash((unsigned char *)application_name, application_name_len); - extract_query_comments(query, comments, sizeof(comments)); - prev_bucket_id = pg_atomic_read_u64(&pgss->current_wbucket); bucketid = get_next_wbucket(pgss); @@ -1608,6 +1606,10 @@ pgss_store(uint64 queryid, } if (jstate == NULL) + { + if (PGSM_EXTRACT_COMMENTS) + extract_query_comments(query, comments, sizeof(comments)); + pgss_update_entry(entry, /* entry */ bucketid, /* bucketid */ queryid, /* queryid */ @@ -1625,6 +1627,7 @@ pgss_store(uint64 queryid, kind, /* kind */ application_name, application_name_len); + } LWLockRelease(pgss->lock); if (norm_query) diff --git a/pg_stat_monitor.h b/pg_stat_monitor.h index eedc702..8cb78f9 100644 --- a/pg_stat_monitor.h +++ b/pg_stat_monitor.h @@ -95,9 +95,9 @@ #define SQLCODE_LEN 20 #if PG_VERSION_NUM >= 130000 -#define MAX_SETTINGS 14 +#define MAX_SETTINGS 15 #else -#define MAX_SETTINGS 13 +#define MAX_SETTINGS 14 #endif /* Update this if need a enum GUC with more options. */ @@ -443,7 +443,8 @@ static const struct config_enum_entry track_options[] = #define PGSM_OVERFLOW_TARGET get_conf(10)->guc_variable #define PGSM_QUERY_PLAN get_conf(11)->guc_variable #define PGSM_TRACK get_conf(12)->guc_variable -#define PGSM_TRACK_PLANNING get_conf(13)->guc_variable +#define PGSM_EXTRACT_COMMENTS get_conf(13)->guc_variable +#define PGSM_TRACK_PLANNING get_conf(14)->guc_variable /*---- Benchmarking ----*/ From a702f24465460ac9e496b12be9482542c40da656 Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Thu, 9 Dec 2021 15:56:35 -0300 Subject: [PATCH 53/56] PG-293: Update regression tests (extract_comments). guc: Add the new GUC variable to the output. tags: Handle both cases, enable/disable extracting query comments. --- regression/expected/guc.out | 3 ++- regression/expected/guc_1.out | 3 ++- regression/expected/tags.out | 40 +++++++++++++++++++++++++++++++++++ regression/sql/tags.sql | 8 +++++++ 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/regression/expected/guc.out b/regression/expected/guc.out index 075b3ac..90bb7b9 100644 --- a/regression/expected/guc.out +++ b/regression/expected/guc.out @@ -14,6 +14,7 @@ select pg_sleep(.5); SELECT * FROM pg_stat_monitor_settings ORDER BY name COLLATE "C"; name | value | default_value | description | minimum | maximum | options | restart ------------------------------------------+--------+---------------+----------------------------------------------------------------------------------------------------------+---------+------------+----------------+--------- + pg_stat_monitor.extract_comments | no | no | Enable/Disable extracting comments from queries. | | | yes, no | no pg_stat_monitor.pgsm_bucket_time | 60 | 60 | Sets the time in seconds per bucket. | 1 | 2147483647 | | yes pg_stat_monitor.pgsm_enable_query_plan | no | no | Enable/Disable query plan monitoring | | | yes, no | no pg_stat_monitor.pgsm_histogram_buckets | 10 | 10 | Sets the maximum number of histogram buckets | 2 | 2147483647 | | yes @@ -28,7 +29,7 @@ SELECT * FROM pg_stat_monitor_settings ORDER BY name COLLATE "C"; pg_stat_monitor.pgsm_track_planning | yes | yes | Selects whether planning statistics are tracked. | | | yes, no | no pg_stat_monitor.pgsm_track_utility | yes | yes | Selects whether utility commands are tracked. | | | yes, no | no pg_stat_monitor.track | top | top | Selects which statements are tracked by pg_stat_monitor. | | | none, top, all | no -(14 rows) +(15 rows) SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset diff --git a/regression/expected/guc_1.out b/regression/expected/guc_1.out index f839a94..ed62b3c 100644 --- a/regression/expected/guc_1.out +++ b/regression/expected/guc_1.out @@ -14,6 +14,7 @@ select pg_sleep(.5); SELECT * FROM pg_stat_monitor_settings ORDER BY name COLLATE "C"; name | value | default_value | description | minimum | maximum | options | restart ------------------------------------------+--------+---------------+----------------------------------------------------------------------------------------------------------+---------+------------+----------------+--------- + pg_stat_monitor.extract_comments | no | no | Enable/Disable extracting comments from queries. | | | yes, no | no pg_stat_monitor.pgsm_bucket_time | 60 | 60 | Sets the time in seconds per bucket. | 1 | 2147483647 | | yes pg_stat_monitor.pgsm_enable_query_plan | no | no | Enable/Disable query plan monitoring | | | yes, no | no pg_stat_monitor.pgsm_histogram_buckets | 10 | 10 | Sets the maximum number of histogram buckets | 2 | 2147483647 | | yes @@ -27,7 +28,7 @@ SELECT * FROM pg_stat_monitor_settings ORDER BY name COLLATE "C"; pg_stat_monitor.pgsm_query_shared_buffer | 20 | 20 | Sets the maximum size of shared memory in (MB) used for query tracked by pg_stat_monitor. | 1 | 10000 | | yes pg_stat_monitor.pgsm_track_utility | yes | yes | Selects whether utility commands are tracked. | | | yes, no | no pg_stat_monitor.track | top | top | Selects which statements are tracked by pg_stat_monitor. | | | none, top, all | no -(13 rows) +(14 rows) SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset diff --git a/regression/expected/tags.out b/regression/expected/tags.out index 27d37d3..8ca4ccf 100644 --- a/regression/expected/tags.out +++ b/regression/expected/tags.out @@ -11,6 +11,39 @@ SELECT 1 AS num /* { "application", psql_app, "real_ip", 192.168.1.3) */; 1 (1 row) +SELECT query, comments FROM pg_stat_monitor ORDER BY query COLLATE "C"; + query | comments +---------------------------------------------------------------------------+---------- + SELECT $1 AS num /* { "application", psql_app, "real_ip", 192.168.1.3) */ | + SELECT pg_stat_monitor_reset() | + SELECT query, comments FROM pg_stat_monitor ORDER BY query COLLATE "C" | +(3 rows) + +ALTER SYSTEM SET pg_stat_monitor.extract_comments TO 'yes'; +SELECT pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + +select pg_sleep(1); + pg_sleep +---------- + +(1 row) + +SELECT pg_stat_monitor_reset(); + pg_stat_monitor_reset +----------------------- + +(1 row) + +SELECT 1 AS num /* { "application", psql_app, "real_ip", 192.168.1.3) */; + num +----- + 1 +(1 row) + SELECT query, comments FROM pg_stat_monitor ORDER BY query COLLATE "C"; query | comments ---------------------------------------------------------------------------+---------------------------------------------------------- @@ -19,6 +52,13 @@ SELECT query, comments FROM pg_stat_monitor ORDER BY query COLLATE "C"; SELECT query, comments FROM pg_stat_monitor ORDER BY query COLLATE "C" | (3 rows) +ALTER SYSTEM SET pg_stat_monitor.extract_comments TO 'no'; +SELECT pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset ----------------------- diff --git a/regression/sql/tags.sql b/regression/sql/tags.sql index 8325598..8912080 100644 --- a/regression/sql/tags.sql +++ b/regression/sql/tags.sql @@ -2,5 +2,13 @@ CREATE EXTENSION pg_stat_monitor; SELECT pg_stat_monitor_reset(); SELECT 1 AS num /* { "application", psql_app, "real_ip", 192.168.1.3) */; SELECT query, comments FROM pg_stat_monitor ORDER BY query COLLATE "C"; +ALTER SYSTEM SET pg_stat_monitor.extract_comments TO 'yes'; +SELECT pg_reload_conf(); +select pg_sleep(1); +SELECT pg_stat_monitor_reset(); +SELECT 1 AS num /* { "application", psql_app, "real_ip", 192.168.1.3) */; +SELECT query, comments FROM pg_stat_monitor ORDER BY query COLLATE "C"; +ALTER SYSTEM SET pg_stat_monitor.extract_comments TO 'no'; +SELECT pg_reload_conf(); SELECT pg_stat_monitor_reset(); DROP EXTENSION pg_stat_monitor; From fd1691626c7b752370635c9593e457fd800f3c39 Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Mon, 20 Dec 2021 11:28:29 -0300 Subject: [PATCH 54/56] PG-293: Disable pgsm_track_planning. This GUC must be disabled by default, it incurss a small performance penalty in the PostgreSQL TPS, users can enable it at anytime if they wish to. --- guc.c | 2 +- regression/expected/guc.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/guc.c b/guc.c index b7f5e08..6612384 100644 --- a/guc.c +++ b/guc.c @@ -207,7 +207,7 @@ init_guc(void) conf[i] = (GucVariable) { .guc_name = "pg_stat_monitor.pgsm_track_planning", .guc_desc = "Selects whether planning statistics are tracked.", - .guc_default = 1, + .guc_default = 0, .guc_min = 0, .guc_max = 0, .guc_restart = false, diff --git a/regression/expected/guc.out b/regression/expected/guc.out index 90bb7b9..953d783 100644 --- a/regression/expected/guc.out +++ b/regression/expected/guc.out @@ -26,7 +26,7 @@ SELECT * FROM pg_stat_monitor_settings ORDER BY name COLLATE "C"; pg_stat_monitor.pgsm_overflow_target | 1 | 1 | Sets the overflow target for pg_stat_monitor | 0 | 1 | | yes pg_stat_monitor.pgsm_query_max_len | 1024 | 1024 | Sets the maximum length of query. | 1024 | 2147483647 | | yes pg_stat_monitor.pgsm_query_shared_buffer | 20 | 20 | Sets the maximum size of shared memory in (MB) used for query tracked by pg_stat_monitor. | 1 | 10000 | | yes - pg_stat_monitor.pgsm_track_planning | yes | yes | Selects whether planning statistics are tracked. | | | yes, no | no + pg_stat_monitor.pgsm_track_planning | no | no | Selects whether planning statistics are tracked. | | | yes, no | no pg_stat_monitor.pgsm_track_utility | yes | yes | Selects whether utility commands are tracked. | | | yes, no | no pg_stat_monitor.track | top | top | Selects which statements are tracked by pg_stat_monitor. | | | none, top, all | no (15 rows) From 57839c7664c608ea8bf3bc9fa1132f29656a9e65 Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Tue, 21 Dec 2021 11:06:06 -0300 Subject: [PATCH 55/56] PG-295: Fix top_query regression test. The issue is that between changing GUC "track" from track='top' to track='all' the queries are executing using previous state of track='top', to fix that we sleep 1 second after calling pg_reload_conf() to ensure that queries will run with new settings. --- regression/expected/top_query.out | 12 ++++++++++++ regression/expected/top_query_1.out | 12 ++++++++++++ regression/sql/top_query.sql | 2 ++ 3 files changed, 26 insertions(+) diff --git a/regression/expected/top_query.out b/regression/expected/top_query.out index e980297..6b118ff 100644 --- a/regression/expected/top_query.out +++ b/regression/expected/top_query.out @@ -48,6 +48,12 @@ SELECT pg_reload_conf(); t (1 row) +SELECT pg_sleep(1); + pg_sleep +---------- + +(1 row) + SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset ----------------------- @@ -98,6 +104,12 @@ SELECT pg_reload_conf(); t (1 row) +SELECT pg_sleep(1); + pg_sleep +---------- + +(1 row) + SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset ----------------------- diff --git a/regression/expected/top_query_1.out b/regression/expected/top_query_1.out index 59a19e5..a98455b 100644 --- a/regression/expected/top_query_1.out +++ b/regression/expected/top_query_1.out @@ -48,6 +48,12 @@ SELECT pg_reload_conf(); t (1 row) +SELECT pg_sleep(1); + pg_sleep +---------- + +(1 row) + SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset ----------------------- @@ -98,6 +104,12 @@ SELECT pg_reload_conf(); t (1 row) +SELECT pg_sleep(1); + pg_sleep +---------- + +(1 row) + SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset ----------------------- diff --git a/regression/sql/top_query.sql b/regression/sql/top_query.sql index 16aa2ce..b25ca05 100644 --- a/regression/sql/top_query.sql +++ b/regression/sql/top_query.sql @@ -19,6 +19,7 @@ SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C"; ALTER SYSTEM SET pg_stat_monitor.track TO 'all'; SELECT pg_reload_conf(); +SELECT pg_sleep(1); SELECT pg_stat_monitor_reset(); CREATE OR REPLACE FUNCTION add(int, int) RETURNS INTEGER AS @@ -38,6 +39,7 @@ SELECT add2(1,2); SELECT query, top_query FROM pg_stat_monitor ORDER BY query COLLATE "C"; ALTER SYSTEM SET pg_stat_monitor.track TO 'top'; SELECT pg_reload_conf(); +SELECT pg_sleep(1); SELECT pg_stat_monitor_reset(); DROP EXTENSION pg_stat_monitor; From eb4087be4e8cce8eb19d893d9a47975dd19039a0 Mon Sep 17 00:00:00 2001 From: Diego Fronza Date: Wed, 22 Dec 2021 12:46:58 -0300 Subject: [PATCH 56/56] PG-291: Fix query call count. The issue with wrong query call count was taking place during transition to a new bucket, the process is shortly describe bellow: 1. Scan for pending queries in previous bucket. 2. Add pending queries to the new bucket id. 3. Remove pending queries from previous bucket id. The problem is that when switching to a new bucket, we reset query statistics for a given entry being processed, so, for example, if the pending query had a call count of 10 (9 of which were finished, 10th is the pending one), if we move this query to the new bucket, the entry will have its stats reseted, clearing the query call count to zero. To solve the problem, whenever a pending query is detected, if the entry has a call count > 1, we mark it as finished, and don't remove it from the previous bucket in order to keep its statistics, then we move just the pending query (10th in the example) to the new bucket id. Another issue is that when moving a entry to a new bucket, we missed copying the query position from the previous entry, which is used to locate the query text in the query buffer: hash_entry_dealloc():291 new_entry->query_pos = old_entry->query_pos; --- hash_query.c | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/hash_query.c b/hash_query.c index 5074d5f..888c111 100644 --- a/hash_query.c +++ b/hash_query.c @@ -225,9 +225,21 @@ hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_bu pgssEntry *bkp_entry = malloc(sizeof(pgssEntry)); if (!bkp_entry) { - /* No memory, remove pending query entry from the previous bucket. */ pgsm_log_error("hash_entry_dealloc: out of memory"); - entry = hash_search(pgss_hash, &entry->key, HASH_REMOVE, NULL); + /* + * No memory, If the entry has calls > 1 then we change the state to finished, + * as the pending query will likely finish execution during the new bucket + * time window. The pending query will vanish in this case, can't list it + * until it completes. + * + * If there is only one call to the query and it's pending, remove the + * entry from the previous bucket and allow it to finish in the new bucket, + * in order to avoid the query living in the old bucket forever. + */ + if (entry->counters.calls.calls > 1) + entry->counters.state = PGSS_FINISHED; + else + entry = hash_search(pgss_hash, &entry->key, HASH_REMOVE, NULL); continue; } @@ -240,8 +252,20 @@ hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_bu /* Add the entry to a list of nodes to be processed later. */ pending_entries = lappend(pending_entries, bkp_entry); - /* Finally remove the pending query from the expired bucket id. */ - entry = hash_search(pgss_hash, &entry->key, HASH_REMOVE, NULL); + /* + * If the entry has calls > 1 then we change the state to finished in + * the previous bucket, as the pending query will likely finish execution + * during the new bucket time window. Can't remove it from the previous bucket + * as it may have many calls and we would lose the query statistics. + * + * If there is only one call to the query and it's pending, remove the entry + * from the previous bucket and allow it to finish in the new bucket, + * in order to avoid the query living in the old bucket forever. + */ + if (entry->counters.calls.calls > 1) + entry->counters.state = PGSS_FINISHED; + else + entry = hash_search(pgss_hash, &entry->key, HASH_REMOVE, NULL); } } } @@ -264,6 +288,7 @@ hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_bu new_entry->counters = old_entry->counters; SpinLockInit(&new_entry->mutex); new_entry->encoding = old_entry->encoding; + new_entry->query_pos = old_entry->query_pos; } free(old_entry);