PG-588: Some queries are not being normalised.

This bug uncovered serious issues with how the data was being stored by PSGM.
So it require a complete redesign.

pg_stat_monitor now stores the data locally within the backend process's local
memory. The data is only stored when the query completes. This reduces the
number of lock acquisitions that were previously needed during various stages
of the execution. Also, this avoids data loss in case the current bucket
changes during execution. Also, the unavailability of jumble state during later
stages of executions was causing pg_stat_monitor to save non-normalized query.
This was a major problem as well.

pg_stat_monitor specific memory context is implemented. It is used for saving
data locally. The context memory callback helps us clear the locally saved data
so that we do not store it multiple times in the shared hash.

As part of this major rewrite, pgss reference in function and variable names
is changed to pgsm. Memory footprint for the entries is reduced, data types
are corrected where needed, and we've removed unused variables, functions and
macros.

This patch was mutually created by:
Co-authored-by: Hamid Akhtar <hamid.akhtar@percona.com>
Co-authored-by: Muhammad Usama <muhammad.usama@percona.com>

t/018_column_names.pl

@@ -36,7 +36,7 @@ my %pg_versions_pgsm_columns = ( 15 => "application_name,blk_read_time," .
     "shared_blks_written,sqlcode,stddev_exec_time,stddev_plan_time," .
     "temp_blk_read_time,temp_blk_write_time,temp_blks_read,temp_blks_written," .
     "top_query,top_queryid,toplevel,total_exec_time,total_plan_time," .
-    "user,userid,wal_bytes,wal_fpi,wal_records",
+    "userid,username,wal_bytes,wal_fpi,wal_records",
  14 => "application_name,blk_read_time," .
     "blk_write_time,bucket,bucket_done,bucket_start_time,calls," .
     "client_ip,cmd_type,cmd_type_text,comments,cpu_sys_time,cpu_user_time," .
@@ -47,7 +47,7 @@ my %pg_versions_pgsm_columns = ( 15 => "application_name,blk_read_time," .
     "rows,shared_blks_dirtied,shared_blks_hit,shared_blks_read," .
     "shared_blks_written,sqlcode,stddev_exec_time,stddev_plan_time," .
     "temp_blks_read,temp_blks_written,top_query,top_queryid,toplevel," .
-    "total_exec_time,total_plan_time,user,userid,wal_bytes,wal_fpi,wal_records",
+    "total_exec_time,total_plan_time,userid,username,wal_bytes,wal_fpi,wal_records",
  13 => "application_name,blk_read_time," .
     "blk_write_time,bucket,bucket_done,bucket_start_time,calls," .
     "client_ip,cmd_type,cmd_type_text,comments,cpu_sys_time,cpu_user_time," .
@@ -58,7 +58,7 @@ my %pg_versions_pgsm_columns = ( 15 => "application_name,blk_read_time," .
     "rows,shared_blks_dirtied,shared_blks_hit,shared_blks_read," .
     "shared_blks_written,sqlcode,stddev_exec_time,stddev_plan_time," .
     "temp_blks_read,temp_blks_written,top_query,top_queryid,toplevel," .
-    "total_exec_time,total_plan_time,user,userid,wal_bytes,wal_fpi,wal_records",
+    "total_exec_time,total_plan_time,userid,username,wal_bytes,wal_fpi,wal_records",
  12 => "application_name,blk_read_time,blk_write_time,bucket,bucket_done," .
     "bucket_start_time,calls,client_ip,cmd_type,cmd_type_text,comments," .
     "cpu_sys_time,cpu_user_time,datname,dbid,elevel,local_blks_dirtied," .
@@ -66,7 +66,7 @@ my %pg_versions_pgsm_columns = ( 15 => "application_name,blk_read_time," .
     "message,min_time,pgsm_query_id,planid,query,query_plan,queryid,relations,resp_calls," .
     "rows,shared_blks_dirtied,shared_blks_hit,shared_blks_read," .
     "shared_blks_written,sqlcode,stddev_time,temp_blks_read,temp_blks_written," .
-    "top_query,top_queryid,total_time,user,userid"
+    "top_query,top_queryid,total_time,userid,username"
  );
 
 # Start server

t/expected/001_settings_default.out

@@ -14,8 +14,8 @@ SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals
  pg_stat_monitor.pgsm_histogram_buckets   | 20      |      | postmaster | integer | default | 2       | 50         |                | 20       | 20        | f
  pg_stat_monitor.pgsm_histogram_max       | 100000  |      | postmaster | integer | default | 10      | 2147483647 |                | 100000   | 100000    | f
  pg_stat_monitor.pgsm_histogram_min       | 1       |      | postmaster | integer | default | 0       | 2147483647 |                | 1        | 1         | f
- pg_stat_monitor.pgsm_max                 | 100     | MB   | postmaster | integer | default | 1       | 1000       |                | 100      | 100       | f
- pg_stat_monitor.pgsm_max_buckets         | 10      |      | postmaster | integer | default | 1       | 10         |                | 10       | 10        | f
+ pg_stat_monitor.pgsm_max                 | 256     | MB   | postmaster | integer | default | 10      | 10240      |                | 256      | 256       | f
+ pg_stat_monitor.pgsm_max_buckets         | 10      |      | postmaster | integer | default | 1       | 20000      |                | 10       | 10        | f
  pg_stat_monitor.pgsm_normalized_query    | off     |      | user       | bool    | default |         |            |                | off      | off       | f
  pg_stat_monitor.pgsm_overflow_target     | 1       |      | postmaster | integer | default | 0       | 1          |                | 1        | 1         | f
  pg_stat_monitor.pgsm_query_max_len       | 2048    |      | postmaster | integer | default | 1024    | 2147483647 |                | 2048     | 2048      | f
@@ -41,8 +41,8 @@ SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals
  pg_stat_monitor.pgsm_histogram_buckets   | 20      |      | postmaster | integer | default | 2       | 50         |                | 20       | 20        | f
  pg_stat_monitor.pgsm_histogram_max       | 100000  |      | postmaster | integer | default | 10      | 2147483647 |                | 100000   | 100000    | f
  pg_stat_monitor.pgsm_histogram_min       | 1       |      | postmaster | integer | default | 0       | 2147483647 |                | 1        | 1         | f
- pg_stat_monitor.pgsm_max                 | 100     | MB   | postmaster | integer | default | 1       | 1000       |                | 100      | 100       | f
- pg_stat_monitor.pgsm_max_buckets         | 10      |      | postmaster | integer | default | 1       | 10         |                | 10       | 10        | f
+ pg_stat_monitor.pgsm_max                 | 256     | MB   | postmaster | integer | default | 10      | 10240      |                | 256      | 256       | f
+ pg_stat_monitor.pgsm_max_buckets         | 10      |      | postmaster | integer | default | 1       | 20000      |                | 10       | 10        | f
  pg_stat_monitor.pgsm_normalized_query    | off     |      | user       | bool    | default |         |            |                | off      | off       | f
  pg_stat_monitor.pgsm_overflow_target     | 1       |      | postmaster | integer | default | 0       | 1          |                | 1        | 1         | f
  pg_stat_monitor.pgsm_query_max_len       | 2048    |      | postmaster | integer | default | 1024    | 2147483647 |                | 2048     | 2048      | f

t/expected/001_settings_default.out.12

@@ -14,8 +14,8 @@ SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals
  pg_stat_monitor.pgsm_histogram_buckets   | 20      |      | postmaster | integer | default | 2       | 50         |                | 20       | 20        | f
  pg_stat_monitor.pgsm_histogram_max       | 100000  |      | postmaster | integer | default | 10      | 2147483647 |                | 100000   | 100000    | f
  pg_stat_monitor.pgsm_histogram_min       | 1       |      | postmaster | integer | default | 0       | 2147483647 |                | 1        | 1         | f
- pg_stat_monitor.pgsm_max                 | 100     | MB   | postmaster | integer | default | 1       | 1000       |                | 100      | 100       | f
- pg_stat_monitor.pgsm_max_buckets         | 10      |      | postmaster | integer | default | 1       | 10         |                | 10       | 10        | f
+ pg_stat_monitor.pgsm_max                 | 256     | MB   | postmaster | integer | default | 10      | 10240      |                | 256      | 256       | f
+ pg_stat_monitor.pgsm_max_buckets         | 10      |      | postmaster | integer | default | 1       | 20000      |                | 10       | 10        | f
  pg_stat_monitor.pgsm_normalized_query    | off     |      | user       | bool    | default |         |            |                | off      | off       | f
  pg_stat_monitor.pgsm_overflow_target     | 1       |      | postmaster | integer | default | 0       | 1          |                | 1        | 1         | f
  pg_stat_monitor.pgsm_query_max_len       | 2048    |      | postmaster | integer | default | 1024    | 2147483647 |                | 2048     | 2048      | f
@@ -40,8 +40,8 @@ SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals
  pg_stat_monitor.pgsm_histogram_buckets   | 20      |      | postmaster | integer | default | 2       | 50         |                | 20       | 20        | f
  pg_stat_monitor.pgsm_histogram_max       | 100000  |      | postmaster | integer | default | 10      | 2147483647 |                | 100000   | 100000    | f
  pg_stat_monitor.pgsm_histogram_min       | 1       |      | postmaster | integer | default | 0       | 2147483647 |                | 1        | 1         | f
- pg_stat_monitor.pgsm_max                 | 100     | MB   | postmaster | integer | default | 1       | 1000       |                | 100      | 100       | f
- pg_stat_monitor.pgsm_max_buckets         | 10      |      | postmaster | integer | default | 1       | 10         |                | 10       | 10        | f
+ pg_stat_monitor.pgsm_max                 | 256     | MB   | postmaster | integer | default | 10      | 10240      |                | 256      | 256       | f
+ pg_stat_monitor.pgsm_max_buckets         | 10      |      | postmaster | integer | default | 1       | 20000      |                | 10       | 10        | f
  pg_stat_monitor.pgsm_normalized_query    | off     |      | user       | bool    | default |         |            |                | off      | off       | f
  pg_stat_monitor.pgsm_overflow_target     | 1       |      | postmaster | integer | default | 0       | 1          |                | 1        | 1         | f
  pg_stat_monitor.pgsm_query_max_len       | 2048    |      | postmaster | integer | default | 1024    | 2147483647 |                | 2048     | 2048      | f

t/expected/012_settings_pgsm_max_buckets.out

@@ -8,7 +8,7 @@ SELECT pg_stat_monitor_reset();
 SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets';
                name               | setting | unit |  context   | vartype |       source       | min_val | max_val | enumvals | boot_val | reset_val | pending_restart 
 ----------------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+-----------------
- pg_stat_monitor.pgsm_max_buckets | 1       |      | postmaster | integer | configuration file | 1       | 10      |          | 10       | 1         | f
+ pg_stat_monitor.pgsm_max_buckets | 1       |      | postmaster | integer | configuration file | 1       | 20000   |          | 10       | 1         | f
 (1 row)
 
 SELECT pg_stat_monitor_reset();
@@ -20,7 +20,7 @@ SELECT pg_stat_monitor_reset();
 SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets';
                name               | setting | unit |  context   | vartype |       source       | min_val | max_val | enumvals | boot_val | reset_val | pending_restart 
 ----------------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+-----------------
- pg_stat_monitor.pgsm_max_buckets | 2       |      | postmaster | integer | configuration file | 1       | 10      |          | 10       | 2         | f
+ pg_stat_monitor.pgsm_max_buckets | 2       |      | postmaster | integer | configuration file | 1       | 20000   |          | 10       | 2         | f
 (1 row)
 
 SELECT pg_stat_monitor_reset();
@@ -32,7 +32,7 @@ SELECT pg_stat_monitor_reset();
 SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets';
                name               | setting | unit |  context   | vartype |       source       | min_val | max_val | enumvals | boot_val | reset_val | pending_restart 
 ----------------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+-----------------
- pg_stat_monitor.pgsm_max_buckets | 5       |      | postmaster | integer | configuration file | 1       | 10      |          | 10       | 5         | f
+ pg_stat_monitor.pgsm_max_buckets | 5       |      | postmaster | integer | configuration file | 1       | 20000   |          | 10       | 5         | f
 (1 row)
 
 SELECT pg_stat_monitor_reset();
@@ -44,7 +44,19 @@ SELECT pg_stat_monitor_reset();
 SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets';
                name               | setting | unit |  context   | vartype |       source       | min_val | max_val | enumvals | boot_val | reset_val | pending_restart 
 ----------------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+-----------------
- pg_stat_monitor.pgsm_max_buckets | 10      |      | postmaster | integer | configuration file | 1       | 10      |          | 10       | 10        | f
+ pg_stat_monitor.pgsm_max_buckets | 10      |      | postmaster | integer | configuration file | 1       | 20000   |          | 10       | 10        | f
+(1 row)
+
+SELECT pg_stat_monitor_reset();
+ pg_stat_monitor_reset 
+-----------------------
+ 
+(1 row)
+
+SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets';
+               name               | setting | unit |  context   | vartype |       source       | min_val | max_val | enumvals | boot_val | reset_val | pending_restart 
+----------------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+-----------------
+ pg_stat_monitor.pgsm_max_buckets | 11      |      | postmaster | integer | configuration file | 1       | 20000   |          | 10       | 11        | f
 (1 row)
 
 SELECT pg_stat_monitor_reset();
@@ -56,19 +68,7 @@ SELECT pg_stat_monitor_reset();
 SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets';
                name               | setting | unit |  context   | vartype | source  | min_val | max_val | enumvals | boot_val | reset_val | pending_restart 
 ----------------------------------+---------+------+------------+---------+---------+---------+---------+----------+----------+-----------+-----------------
- pg_stat_monitor.pgsm_max_buckets | 10      |      | postmaster | integer | default | 1       | 10      |          | 10       | 10        | f
-(1 row)
-
-SELECT pg_stat_monitor_reset();
- pg_stat_monitor_reset 
------------------------
- 
-(1 row)
-
-SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets';
-               name               | setting | unit |  context   | vartype | source  | min_val | max_val | enumvals | boot_val | reset_val | pending_restart 
-----------------------------------+---------+------+------------+---------+---------+---------+---------+----------+----------+-----------+-----------------
- pg_stat_monitor.pgsm_max_buckets | 10      |      | postmaster | integer | default | 1       | 10      |          | 10       | 10        | f
+ pg_stat_monitor.pgsm_max_buckets | 10      |      | postmaster | integer | default | 1       | 20000   |          | 10       | 10        | f
 (1 row)
 
 DROP EXTENSION pg_stat_monitor;

t/expected/016_settings_pgsm_max.out

@@ -8,7 +8,7 @@ SELECT pg_stat_monitor_reset();
 SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max';
            name           | setting | unit |  context   | vartype |       source       | min_val | max_val | enumvals | boot_val | reset_val | pending_restart 
 --------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+-----------------
- pg_stat_monitor.pgsm_max | 1000    | MB   | postmaster | integer | configuration file | 1       | 1000    |          | 100      | 1000      | f
+ pg_stat_monitor.pgsm_max | 1000    | MB   | postmaster | integer | configuration file | 10      | 10240   |          | 256      | 1000      | f
 (1 row)
 
 SELECT pg_stat_monitor_reset();
@@ -20,7 +20,7 @@ SELECT pg_stat_monitor_reset();
 SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max';
            name           | setting | unit |  context   | vartype |       source       | min_val | max_val | enumvals | boot_val | reset_val | pending_restart 
 --------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+-----------------
- pg_stat_monitor.pgsm_max | 500     | MB   | postmaster | integer | configuration file | 1       | 1000    |          | 100      | 500       | f
+ pg_stat_monitor.pgsm_max | 500     | MB   | postmaster | integer | configuration file | 10      | 10240   |          | 256      | 500       | f
 (1 row)
 
 SELECT pg_stat_monitor_reset();
@@ -32,7 +32,7 @@ SELECT pg_stat_monitor_reset();
 SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max';
            name           | setting | unit |  context   | vartype |       source       | min_val | max_val | enumvals | boot_val | reset_val | pending_restart 
 --------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+-----------------
- pg_stat_monitor.pgsm_max | 100     | MB   | postmaster | integer | configuration file | 1       | 1000    |          | 100      | 100       | f
+ pg_stat_monitor.pgsm_max | 100     | MB   | postmaster | integer | configuration file | 10      | 10240   |          | 256      | 100       | f
 (1 row)
 
 SELECT pg_stat_monitor_reset();
@@ -44,7 +44,7 @@ SELECT pg_stat_monitor_reset();
 SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max';
            name           | setting | unit |  context   | vartype |       source       | min_val | max_val | enumvals | boot_val | reset_val | pending_restart 
 --------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+-----------------
- pg_stat_monitor.pgsm_max | 10      | MB   | postmaster | integer | configuration file | 1       | 1000    |          | 100      | 10        | f
+ pg_stat_monitor.pgsm_max | 10      | MB   | postmaster | integer | configuration file | 10      | 10240   |          | 256      | 10        | f
 (1 row)
 
 SELECT pg_stat_monitor_reset();
@@ -56,7 +56,7 @@ SELECT pg_stat_monitor_reset();
 SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max';
            name           | setting | unit |  context   | vartype | source  | min_val | max_val | enumvals | boot_val | reset_val | pending_restart 
 --------------------------+---------+------+------------+---------+---------+---------+---------+----------+----------+-----------+-----------------
- pg_stat_monitor.pgsm_max | 100     | MB   | postmaster | integer | default | 1       | 1000    |          | 100      | 100       | f
+ pg_stat_monitor.pgsm_max | 256     | MB   | postmaster | integer | default | 10      | 10240   |          | 256      | 256       | f
 (1 row)
 
 DROP EXTENSION pg_stat_monitor;