external/pg_stat_monitor
1
0
Fork 0
mirror of https://github.com/percona/pg_stat_monitor.git synced 2026-02-04 05:56:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

PG-588: Some queries are not being normalised.

Browse Source 
This bug uncovered serious issues with how the data was being stored by PSGM.
So it require a complete redesign.

pg_stat_monitor now stores the data locally within the backend process's local
memory. The data is only stored when the query completes. This reduces the
number of lock acquisitions that were previously needed during various stages
of the execution. Also, this avoids data loss in case the current bucket
changes during execution. Also, the unavailability of jumble state during later
stages of executions was causing pg_stat_monitor to save non-normalized query.
This was a major problem as well.

pg_stat_monitor specific memory context is implemented. It is used for saving
data locally. The context memory callback helps us clear the locally saved data
so that we do not store it multiple times in the shared hash.

As part of this major rewrite, pgss reference in function and variable names
is changed to pgsm. Memory footprint for the entries is reduced, data types
are corrected where needed, and we've removed unused variables, functions and
macros.

This patch was mutually created by:
Co-authored-by: Hamid Akhtar <hamid.akhtar@percona.com>
Co-authored-by: Muhammad Usama <muhammad.usama@percona.com>
This commit is contained in:
Hamid Akhtar
2023-02-22 19:14:42 +05:00
parent 837bacdf3a
commit de66ef0fce
27 changed files with 1287 additions and 1049 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
regression/expected/cmd_type.out
View File

@@ -23,6 +23,7 @@ SELECT b FROM t2 FOR UPDATE;
TRUNCATE t1;
DROP TABLE t1;
DROP TABLE t2;
SELECT query, cmd_type, cmd_type_text FROM pg_stat_monitor ORDER BY query COLLATE "C";
query | cmd_type | cmd_type_text
--------------------------------+----------+---------------
@@ -30,13 +31,14 @@ SELECT query, cmd_type, cmd_type_text FROM pg_stat_monitor ORDER BY query COLLA
CREATE TABLE t2 (b INTEGER) | 0 |
DELETE FROM t1 | 4 | DELETE
DROP TABLE t1 | 0 |
DROP TABLE t2 | 0 |
INSERT INTO t1 VALUES(1) | 3 | INSERT
SELECT a FROM t1 | 1 | SELECT
SELECT b FROM t2 FOR UPDATE | 1 | SELECT
SELECT pg_stat_monitor_reset() | 1 | SELECT
TRUNCATE t1 | 0 |
UPDATE t1 SET a = 2 | 2 | UPDATE
(10 rows)
(11 rows)
SELECT pg_stat_monitor_reset();
pg_stat_monitor_reset

8
regression/expected/database.out
View File

@@ -27,13 +27,15 @@ SELECT * FROM t3,t4 WHERE t3.c = t4.d;
(0 rows)
\c contrib_regression
DROP DATABASE db2;
SELECT datname, query FROM pg_stat_monitor ORDER BY query COLLATE "C";
datname | query
--------------------+---------------------------------------
contrib_regression | DROP DATABASE db2
db1 | SELECT * FROM t1,t2 WHERE t1.a = t2.b
db2 | SELECT * FROM t3,t4 WHERE t3.c = t4.d
contrib_regression | SELECT pg_stat_monitor_reset()
(3 rows)
(4 rows)
SELECT pg_stat_monitor_reset();
pg_stat_monitor_reset
@@ -44,10 +46,6 @@ SELECT pg_stat_monitor_reset();
\c db1
DROP TABLE t1;
DROP TABLE t2;
\c db2
DROP TABLE t3;
DROP TABLE t4;
\c contrib_regression
DROP DATABASE db1;
DROP DATABASE db2;
DROP EXTENSION pg_stat_monitor;

42
regression/expected/error_2.out Normal file
View File

@@ -0,0 +1,42 @@
CREATE EXTENSION pg_stat_monitor;
SELECT pg_stat_monitor_reset();
pg_stat_monitor_reset
-----------------------
(1 row)
SELECT 1/0; -- divide by zero
ERROR: division by zero
SELECT * FROM unknown; -- unknown table
ERROR: relation "unknown" does not exist
LINE 1: SELECT * FROM unknown;
^
ELECET * FROM unknown; -- syntax error
ERROR: syntax error at or near "ELECET"
LINE 1: ELECET * FROM unknown;
^
do $$
BEGIN
RAISE WARNING 'warning message';
END $$;
WARNING: warning message
SELECT query, elevel, sqlcode, message FROM pg_stat_monitor ORDER BY query COLLATE "C",elevel;
query | elevel | sqlcode | message
----------------------------------+--------+---------+-----------------------------------
ELECET * FROM unknown; | 20 | 42601 | syntax error at or near "ELECET"
SELECT * FROM unknown; | 20 | 42P01 | relation "unknown" does not exist
SELECT 1/0; | 20 | 22012 | division by zero
SELECT pg_stat_monitor_reset() | 0 | |
do $$ +| 0 | 01000 | warning message
BEGIN +| | |
RAISE WARNING 'warning message';+| | |
END $$; | | |
(5 rows)
SELECT pg_stat_monitor_reset();
pg_stat_monitor_reset
-----------------------
(1 row)
DROP EXTENSION pg_stat_monitor;

19
regression/expected/functions.out
View File

@@ -1,4 +1,5 @@
DROP ROLE IF EXISTS su;
NOTICE: role "su" does not exist, skipping
CREATE USER su WITH SUPERUSER;
SET ROLE su;
CREATE EXTENSION pg_stat_monitor;
@@ -29,13 +30,19 @@ SELECT routine_schema, routine_name, routine_type, data_type FROM information_sc
SET ROLE u1;
SELECT routine_schema, routine_name, routine_type, data_type FROM information_schema.routines WHERE routine_schema = 'public' ORDER BY routine_name COLLATE "C";
routine_schema | routine_name | routine_type | data_type
----------------+-------------------------+--------------+-----------
public | histogram | FUNCTION | record
public | pg_stat_monitor_reset | FUNCTION | void
public | pg_stat_monitor_version | FUNCTION | text
(3 rows)
routine_schema | routine_name | routine_type | data_type
----------------+--------------------------+--------------+-----------
public | decode_error_level | FUNCTION | text
public | get_cmd_type | FUNCTION | text
public | get_histogram_timings | FUNCTION | text
public | histogram | FUNCTION | record
public | pg_stat_monitor_internal | FUNCTION | record
public | pg_stat_monitor_version | FUNCTION | text
public | range | FUNCTION | ARRAY
(7 rows)
SET ROLE su;
DROP USER u1;
DROP EXTENSION pg_stat_monitor;
DROP USER su;
ERROR: current user cannot be dropped

4
regression/expected/guc.out
View File

@@ -24,8 +24,8 @@ COLLATE "C";
pg_stat_monitor.pgsm_histogram_buckets | 20 | | postmaster | integer | default | 2 | 50 | | 20 | 20 | f
pg_stat_monitor.pgsm_histogram_max | 100000 | | postmaster | integer | default | 10 | 2147483647 | | 100000 | 100000 | f
pg_stat_monitor.pgsm_histogram_min | 1 | | postmaster | integer | default | 0 | 2147483647 | | 1 | 1 | f
pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | default | 1 | 1000 | | 100 | 100 | f
pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 10 | | 10 | 10 | f
pg_stat_monitor.pgsm_max | 256 | MB | postmaster | integer | default | 10 | 10240 | | 256 | 256 | f
pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 20000 | | 10 | 10 | f
pg_stat_monitor.pgsm_normalized_query | off | | user | bool | default | | | | off | off | f
pg_stat_monitor.pgsm_overflow_target | 1 | | postmaster | integer | default | 0 | 1 | | 1 | 1 | f
pg_stat_monitor.pgsm_query_max_len | 2048 | | postmaster | integer | default | 1024 | 2147483647 | | 2048 | 2048 | f

4
regression/expected/guc_1.out
View File

@@ -24,8 +24,8 @@ COLLATE "C";
pg_stat_monitor.pgsm_histogram_buckets | 20 | | postmaster | integer | default | 2 | 50 | | 20 | 20 | f
pg_stat_monitor.pgsm_histogram_max | 100000 | | postmaster | integer | default | 10 | 2147483647 | | 100000 | 100000 | f
pg_stat_monitor.pgsm_histogram_min | 1 | | postmaster | integer | default | 0 | 2147483647 | | 1 | 1 | f
pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | default | 1 | 1000 | | 100 | 100 | f
pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 10 | | 10 | 10 | f
pg_stat_monitor.pgsm_max | 256 | MB | postmaster | integer | default | 10 | 10240 | | 256 | 256 | f
pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 20000 | | 10 | 10 | f
pg_stat_monitor.pgsm_normalized_query | off | | user | bool | default | | | | off | off | f
pg_stat_monitor.pgsm_overflow_target | 1 | | postmaster | integer | default | 0 | 1 | | 1 | 1 | f
pg_stat_monitor.pgsm_query_max_len | 2048 | | postmaster | integer | default | 1024 | 2147483647 | | 2048 | 2048 | f

40
regression/expected/pgsm_query_id.out
View File

@@ -40,6 +40,22 @@ SELECT * FROM t2;
---
(0 rows)
-- Check that spaces and comments do not generate a different pgsm_query_id
SELECT * FROM t2 --WHATEVER;
;
b
---
(0 rows)
SELECT * FROM t2 /* ...
...
More comments to check for spaces.
*/
;
b
---
(0 rows)
\c db2
SELECT * FROM t1;
a
@@ -57,18 +73,18 @@ SELECT * FROM t3;
(0 rows)
\c contrib_regression
SELECT datname, pgsm_query_id, query FROM pg_stat_monitor ORDER BY pgsm_query_id, query, datname;
datname | pgsm_query_id | query
--------------------+----------------------+--------------------------------
db2 | -5029137034974447432 | SELECT * FROM t3
contrib_regression | 689150021118383254 | SELECT pg_stat_monitor_reset()
db1 | 1897482803466821995 | SELECT * FROM t2
db1 | 1988437669671417938 | SELECT * FROM t1
db2 | 1988437669671417938 | SELECT * FROM t1
db1 | 2864453209316739369 | select $1 + $2
db2 | 2864453209316739369 | select $1 + $2
db1 | 8140395000078788481 | SELECT *, ADD(1, 2) FROM t1
db2 | 8140395000078788481 | SELECT *, ADD(1, 2) FROM t1
SELECT datname, pgsm_query_id, query, calls FROM pg_stat_monitor ORDER BY pgsm_query_id, query, datname;
datname | pgsm_query_id | query | calls
--------------------+----------------------+--------------------------------+-------
db2 | -5029137034974447432 | SELECT * FROM t3 | 1
contrib_regression | 689150021118383254 | SELECT pg_stat_monitor_reset() | 1
db1 | 1897482803466821995 | SELECT * FROM t2 | 3
db1 | 1988437669671417938 | SELECT * FROM t1 | 1
db2 | 1988437669671417938 | SELECT * FROM t1 | 1
db1 | 2864453209316739369 | select $1 + $2 | 1
db2 | 2864453209316739369 | select $1 + $2 | 1
db1 | 8140395000078788481 | SELECT *, ADD(1, 2) FROM t1 | 1
db2 | 8140395000078788481 | SELECT *, ADD(1, 2) FROM t1 | 1
(9 rows)
SELECT pg_stat_monitor_reset();

2
regression/expected/rows.out
View File

@@ -1,7 +1,6 @@
CREATE EXTENSION pg_stat_monitor;
CREATE TABLE t1(a int);
CREATE TABLE t2(b int);
ERROR: relation "t2" already exists
INSERT INTO t1 VALUES(generate_series(1,1000));
INSERT INTO t2 VALUES(generate_series(1,5000));
SELECT pg_stat_monitor_reset();
@@ -8557,4 +8556,5 @@ SELECT pg_stat_monitor_reset();
(1 row)
DROP TABLE t1;
DROP TABLE t2;
DROP EXTENSION pg_stat_monitor;

65
regression/expected/user.out Normal file
View File

@@ -0,0 +1,65 @@
CREATE USER su WITH SUPERUSER;
ERROR: role "su" already exists
SET ROLE su;
CREATE EXTENSION pg_stat_monitor;
CREATE USER u1;
CREATE USER u2;
GRANT ALL ON SCHEMA public TO u1;
GRANT ALL ON SCHEMA public TO u2;
SET ROLE su;
SELECT pg_stat_monitor_reset();
pg_stat_monitor_reset
-----------------------
(1 row)
SET ROLE u1;
SELECT pg_stat_monitor_reset();
ERROR: permission denied for function pg_stat_monitor_reset
CREATE TABLE t1 (a int);
SELECT * FROM t1;
a
---
(0 rows)
SET ROLE u2;
CREATE TABLE t2 (a int);
SELECT * FROM t2;
a
---
(0 rows)
DROP TABLE t2;
SET ROLE su;
DROP OWNED BY u2;
DROP USER u2;
SELECT username, query FROM pg_stat_monitor ORDER BY username, query COLLATE "C";
username | query
----------+---------------------------------
su | DROP OWNED BY u2
su | DROP USER u2
su | SELECT pg_stat_monitor_reset()
su | SET ROLE su
u1 | CREATE TABLE t1 (a int)
u1 | SELECT * FROM t1
u1 | SELECT pg_stat_monitor_reset();
u1 | SET ROLE u1
u2 | CREATE TABLE t2 (a int)
u2 | DROP TABLE t2
u2 | SELECT * FROM t2
u2 | SET ROLE u2
(12 rows)
SELECT pg_stat_monitor_reset();
pg_stat_monitor_reset
-----------------------
(1 row)
DROP TABLE t1;
DROP OWNED BY u1;
DROP USER u1;
DROP EXTENSION pg_stat_monitor;
SET ROLE NONE;
DROP OWNED BY su;
DROP USER su;

1
regression/sql/cmd_type.sql
View File

@@ -10,6 +10,7 @@ DELETE FROM t1;
SELECT b FROM t2 FOR UPDATE;
TRUNCATE t1;
DROP TABLE t1;
DROP TABLE t2;
SELECT query, cmd_type, cmd_type_text FROM pg_stat_monitor ORDER BY query COLLATE "C";
SELECT pg_stat_monitor_reset();

7
regression/sql/database.sql
View File

@@ -20,6 +20,8 @@ SELECT * FROM t1,t2 WHERE t1.a = t2.b;
SELECT * FROM t3,t4 WHERE t3.c = t4.d;
\c contrib_regression
DROP DATABASE db2;
SELECT datname, query FROM pg_stat_monitor ORDER BY query COLLATE "C";
SELECT pg_stat_monitor_reset();
@@ -27,11 +29,6 @@ SELECT pg_stat_monitor_reset();
DROP TABLE t1;
DROP TABLE t2;
\c db2
DROP TABLE t3;
DROP TABLE t4;
\c contrib_regression
DROP DATABASE db1;
DROP DATABASE db2;
DROP EXTENSION pg_stat_monitor;

1
regression/sql/functions.sql
View File

@@ -14,3 +14,4 @@ SELECT routine_schema, routine_name, routine_type, data_type FROM information_sc
SET ROLE su;
DROP USER u1;
DROP EXTENSION pg_stat_monitor;
DROP USER su;

10
regression/sql/pgsm_query_id.sql
View File

@@ -29,6 +29,14 @@ SELECT pg_stat_monitor_reset();
SELECT * FROM t1;
SELECT *, ADD(1, 2) FROM t1;
SELECT * FROM t2;
-- Check that spaces and comments do not generate a different pgsm_query_id
SELECT * FROM t2 --WHATEVER;
;
SELECT * FROM t2 /* ...
...
More comments to check for spaces.
*/
;
\c db2
SELECT * FROM t1;
@@ -36,7 +44,7 @@ SELECT *, ADD(1, 2) FROM t1;
SELECT * FROM t3;
\c contrib_regression
SELECT datname, pgsm_query_id, query FROM pg_stat_monitor ORDER BY pgsm_query_id, query, datname;
SELECT datname, pgsm_query_id, query, calls FROM pg_stat_monitor ORDER BY pgsm_query_id, query, datname;
SELECT pg_stat_monitor_reset();
\c db1

1
regression/sql/rows.sql
View File

@@ -16,4 +16,5 @@ SELECT query, rows FROM pg_stat_monitor ORDER BY query COLLATE "C";
SELECT pg_stat_monitor_reset();
DROP TABLE t1;
DROP TABLE t2;
DROP EXTENSION pg_stat_monitor;

22
regression/sql/user.sql
View File

@@ -6,25 +6,37 @@ CREATE EXTENSION pg_stat_monitor;
CREATE USER u1;
CREATE USER u2;
SET ROLE su;
GRANT ALL ON SCHEMA public TO u1;
GRANT ALL ON SCHEMA public TO u2;
SET ROLE su;
SELECT pg_stat_monitor_reset();
SET ROLE u1;
SELECT pg_stat_monitor_reset();
CREATE TABLE t1 (a int);
SELECT * FROM t1;
SET ROLE u2;
CREATE TABLE t2 (a int);
SELECT * FROM t2;
DROP TABLE t2;
SET ROLE su;
SELECT userid, query FROM pg_stat_monitor ORDER BY query COLLATE "C";
DROP OWNED BY u2;
DROP USER u2;
SELECT username, query FROM pg_stat_monitor ORDER BY username, query COLLATE "C";
SELECT pg_stat_monitor_reset();
DROP TABLE t1;
DROP TABLE t2;
DROP OWNED BY u1;
DROP USER u1;
DROP USER u2;
DROP EXTENSION pg_stat_monitor;
SET ROLE NONE;
DROP OWNED BY su;
DROP USER su;