From de66ef0fce03ed72833c7091837e921701edecd3 Mon Sep 17 00:00:00 2001 From: Hamid Akhtar Date: Wed, 22 Feb 2023 19:14:42 +0500 Subject: [PATCH] PG-588: Some queries are not being normalised. This bug uncovered serious issues with how the data was being stored by PSGM. So it require a complete redesign. pg_stat_monitor now stores the data locally within the backend process's local memory. The data is only stored when the query completes. This reduces the number of lock acquisitions that were previously needed during various stages of the execution. Also, this avoids data loss in case the current bucket changes during execution. Also, the unavailability of jumble state during later stages of executions was causing pg_stat_monitor to save non-normalized query. This was a major problem as well. pg_stat_monitor specific memory context is implemented. It is used for saving data locally. The context memory callback helps us clear the locally saved data so that we do not store it multiple times in the shared hash. As part of this major rewrite, pgss reference in function and variable names is changed to pgsm. Memory footprint for the entries is reduced, data types are corrected where needed, and we've removed unused variables, functions and macros. This patch was mutually created by: Co-authored-by: Hamid Akhtar Co-authored-by: Muhammad Usama --- Makefile | 2 +- guc.c | 8 +- hash_query.c | 266 +-- pg_stat_monitor--1.0--2.0.sql | 32 +- pg_stat_monitor--2.0.sql | 34 +- pg_stat_monitor.c | 1542 ++++++++++-------- pg_stat_monitor.h | 154 +- regression/expected/cmd_type.out | 4 +- regression/expected/database.out | 8 +- regression/expected/error_2.out | 42 + regression/expected/functions.out | 19 +- regression/expected/guc.out | 4 +- regression/expected/guc_1.out | 4 +- regression/expected/pgsm_query_id.out | 40 +- regression/expected/rows.out | 2 +- regression/expected/user.out | 65 + regression/sql/cmd_type.sql | 1 + regression/sql/database.sql | 7 +- regression/sql/functions.sql | 1 + regression/sql/pgsm_query_id.sql | 10 +- regression/sql/rows.sql | 1 + regression/sql/user.sql | 22 +- t/018_column_names.pl | 8 +- t/expected/001_settings_default.out | 8 +- t/expected/001_settings_default.out.12 | 8 +- t/expected/012_settings_pgsm_max_buckets.out | 34 +- t/expected/016_settings_pgsm_max.out | 10 +- 27 files changed, 1287 insertions(+), 1049 deletions(-) create mode 100644 regression/expected/error_2.out create mode 100644 regression/expected/user.out mode change 100755 => 100644 t/expected/001_settings_default.out.12 diff --git a/Makefile b/Makefile index fe567a8..c305a4c 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ LDFLAGS_SL += $(filter -lm, $(LIBS)) TAP_TESTS = 1 REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/pg_stat_monitor/pg_stat_monitor.conf --inputdir=regression -REGRESS = basic version guc pgsm_query_id functions counters relations database error_insert application_name application_name_unique top_query cmd_type error rows tags +REGRESS = basic version guc pgsm_query_id functions counters relations database error_insert application_name application_name_unique top_query cmd_type error rows tags user # Disabled because these tests require "shared_preload_libraries=pg_stat_statements", # which typical installcheck users do not have (e.g. buildfarm clients). diff --git a/guc.c b/guc.c index 6acbe0e..f7f8bb9 100644 --- a/guc.c +++ b/guc.c @@ -41,9 +41,9 @@ init_guc(void) { .guc_name = "pg_stat_monitor.pgsm_max", .guc_desc = "Sets the maximum size of shared memory in (MB) used for statement's metadata tracked by pg_stat_monitor.", - .guc_default = 100, - .guc_min = 1, - .guc_max = 1000, + .guc_default = 256, + .guc_min = 10, + .guc_max = 10240, .guc_restart = true, .guc_unit = GUC_UNIT_MB, .guc_value = &PGSM_MAX @@ -95,7 +95,7 @@ init_guc(void) .guc_desc = "Sets the maximum number of buckets.", .guc_default = 10, .guc_min = 1, - .guc_max = 10, + .guc_max = 20000, .guc_restart = true, .guc_unit = 0, .guc_value = &PGSM_MAX_BUCKETS diff --git a/hash_query.c b/hash_query.c index 8cc3bf8..6a626dd 100644 --- a/hash_query.c +++ b/hash_query.c @@ -19,14 +19,15 @@ #include "pg_stat_monitor.h" static pgsmLocalState pgsmStateLocal; -static PGSM_HASH_TABLE_HANDLE pgsm_create_bucket_hash(pgssSharedState *pgss, dsa_area *dsa); +static PGSM_HASH_TABLE_HANDLE pgsm_create_bucket_hash(pgsmSharedState *pgsm, dsa_area *dsa); static Size pgsm_get_shared_area_size(void); +static void InitializeSharedState(pgsmSharedState *pgsm); #if USE_DYNAMIC_HASH /* parameter for the shared hash */ static dshash_parameters dsh_params = { - sizeof(pgssHashKey), - sizeof(pgssEntry), + sizeof(pgsmHashKey), + sizeof(pgsmEntry), dshash_memcmp, dshash_memhash }; @@ -54,12 +55,12 @@ pgsm_query_area_size(void) Size pgsm_ShmemSize(void) { - Size sz = MAXALIGN(sizeof(pgssSharedState)); + Size sz = MAXALIGN(sizeof(pgsmSharedState)); sz = add_size(sz, MAX_QUERY_BUF); #if USE_DYNAMIC_HASH sz = add_size(sz, MAX_BUCKETS_MEM); #else - sz = add_size(sz, hash_estimate_size(MAX_BUCKET_ENTRIES, sizeof(pgssEntry))); + sz = add_size(sz, hash_estimate_size(MAX_BUCKET_ENTRIES, sizeof(pgsmEntry))); #endif return MAXALIGN(sz); } @@ -77,47 +78,48 @@ pgsm_get_shared_area_size(void) #if USE_DYNAMIC_HASH sz = pgsm_ShmemSize(); #else - sz = MAXALIGN(sizeof(pgssSharedState)); + sz = MAXALIGN(sizeof(pgsmSharedState)); sz = add_size(sz, pgsm_query_area_size()); #endif return sz; } void -pgss_startup(void) +pgsm_startup(void) { bool found = false; - pgssSharedState *pgss; + pgsmSharedState *pgsm; /* reset in case this is a restart within the postmaster */ pgsmStateLocal.dsa = NULL; pgsmStateLocal.shared_hash = NULL; - pgsmStateLocal.shared_pgssState = NULL; + pgsmStateLocal.shared_pgsmState = NULL; /* * Create or attach to the shared memory state, including hash table */ LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); - pgss = ShmemInitStruct("pg_stat_monitor", pgsm_get_shared_area_size(), &found); + pgsm = ShmemInitStruct("pg_stat_monitor", pgsm_get_shared_area_size(), &found); if (!found) { /* First time through ... */ dsa_area *dsa; - char *p = (char *) pgss; + char *p = (char *) pgsm; - pgss->lock = &(GetNamedLWLockTranche("pg_stat_monitor"))->lock; - SpinLockInit(&pgss->mutex); - ResetSharedState(pgss); - /* the allocation of pgssSharedState itself */ - p += MAXALIGN(sizeof(pgssSharedState)); - pgss->raw_dsa_area = p; - dsa = dsa_create_in_place(pgss->raw_dsa_area, + pgsm->pgsm_oom = false; + pgsm->lock = &(GetNamedLWLockTranche("pg_stat_monitor"))->lock; + SpinLockInit(&pgsm->mutex); + InitializeSharedState(pgsm); + /* the allocation of pgsmSharedState itself */ + p += MAXALIGN(sizeof(pgsmSharedState)); + pgsm->raw_dsa_area = p; + dsa = dsa_create_in_place(pgsm->raw_dsa_area, pgsm_query_area_size(), LWLockNewTrancheId(), 0); dsa_pin(dsa); dsa_set_size_limit(dsa, pgsm_query_area_size()); - pgss->hash_handle = pgsm_create_bucket_hash(pgss,dsa); + pgsm->hash_handle = pgsm_create_bucket_hash(pgsm,dsa); /* If overflow is enabled, set the DSA size to unlimited, * and allow the DSA to grow beyond the shared memory space @@ -125,7 +127,7 @@ pgss_startup(void) if (PGSM_OVERFLOW_TARGET == OVERFLOW_TARGET_DISK) dsa_set_size_limit(dsa, -1); - pgsmStateLocal.shared_pgssState = pgss; + pgsmStateLocal.shared_pgsmState = pgsm; /* * Postmaster will never access the dsa again, thus free it's local * references. @@ -143,29 +145,41 @@ pgss_startup(void) * If we're in the postmaster (or a standalone backend...), set up a shmem * exit hook to dump the statistics to disk. */ - on_shmem_exit(pgss_shmem_shutdown, (Datum) 0); + on_shmem_exit(pgsm_shmem_shutdown, (Datum) 0); } +static void +InitializeSharedState(pgsmSharedState *pgsm) +{ + pg_atomic_init_u64(&pgsm->current_wbucket, 0); + pg_atomic_init_u64(&pgsm->prev_bucket_sec, 0); + memset(&pgsm->bucket_entry, 0, MAX_BUCKETS * sizeof(uint64)); + pgsm->pgsm_mem_cxt = AllocSetContextCreate(TopMemoryContext, + "pg_stat_monitor local store", + ALLOCSET_DEFAULT_SIZES); +} + + /* * Create the classic or dshahs hash table for storing the query statistics. */ static PGSM_HASH_TABLE_HANDLE -pgsm_create_bucket_hash(pgssSharedState *pgss, dsa_area *dsa) +pgsm_create_bucket_hash(pgsmSharedState *pgsm, dsa_area *dsa) { PGSM_HASH_TABLE_HANDLE bucket_hash; #if USE_DYNAMIC_HASH dshash_table *dsh; - pgss->hash_tranche_id = LWLockNewTrancheId(); - dsh_params.tranche_id = pgss->hash_tranche_id; + pgsm->hash_tranche_id = LWLockNewTrancheId(); + dsh_params.tranche_id = pgsm->hash_tranche_id; dsh = dshash_create(dsa, &dsh_params, 0); bucket_hash = dshash_get_hash_table_handle(dsh); dshash_detach(dsh); #else HASHCTL info; memset(&info, 0, sizeof(info)); - info.keysize = sizeof(pgssHashKey); - info.entrysize = sizeof(pgssEntry); + info.keysize = sizeof(pgsmHashKey); + info.entrysize = sizeof(pgsmEntry); bucket_hash = ShmemInitHash("pg_stat_monitor: bucket hashtable", MAX_BUCKET_ENTRIES, MAX_BUCKET_ENTRIES, &info, HASH_ELEM | HASH_BLOBS); #endif return bucket_hash; @@ -193,7 +207,7 @@ pgsm_attach_shmem(void) */ oldcontext = MemoryContextSwitchTo(TopMemoryContext); - pgsmStateLocal.dsa = dsa_attach_in_place(pgsmStateLocal.shared_pgssState->raw_dsa_area, + pgsmStateLocal.dsa = dsa_attach_in_place(pgsmStateLocal.shared_pgsmState->raw_dsa_area, NULL); /* pin the attached area to keep the area attached until end of * session or explicit detach. @@ -201,11 +215,11 @@ pgsm_attach_shmem(void) dsa_pin_mapping(pgsmStateLocal.dsa); #if USE_DYNAMIC_HASH - dsh_params.tranche_id = pgsmStateLocal.shared_pgssState->hash_tranche_id; + dsh_params.tranche_id = pgsmStateLocal.shared_pgsmState->hash_tranche_id; pgsmStateLocal.shared_hash = dshash_attach(pgsmStateLocal.dsa, &dsh_params, - pgsmStateLocal.shared_pgssState->hash_handle, 0); + pgsmStateLocal.shared_pgsmState->hash_handle, 0); #else - pgsmStateLocal.shared_hash = pgsmStateLocal.shared_pgssState->hash_handle; + pgsmStateLocal.shared_hash = pgsmStateLocal.shared_pgsmState->hash_handle; #endif MemoryContextSwitchTo(oldcontext); @@ -219,17 +233,17 @@ get_dsa_area_for_query_text(void) } PGSM_HASH_TABLE* -get_pgssHash(void) +get_pgsmHash(void) { pgsm_attach_shmem(); return pgsmStateLocal.shared_hash; } -pgssSharedState * +pgsmSharedState * pgsm_get_ss(void) { pgsm_attach_shmem(); - return pgsmStateLocal.shared_pgssState; + return pgsmStateLocal.shared_pgsmState; } @@ -240,35 +254,36 @@ pgsm_get_ss(void) * other processes running when this is called. */ void -pgss_shmem_shutdown(int code, Datum arg) +pgsm_shmem_shutdown(int code, Datum arg) { /* Don't try to dump during a crash. */ - elog(LOG,"pgss_shmem_shutdown"); + elog(LOG,"[pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated."); + if (code) return; - pgsmStateLocal.shared_pgssState = NULL; + pgsmStateLocal.shared_pgsmState = NULL; /* Safety check ... shouldn't get here unless shmem is set up. */ if (!IsHashInitialize()) return; } -pgssEntry * -hash_entry_alloc(pgssSharedState *pgss, pgssHashKey *key, int encoding) +pgsmEntry * +hash_entry_alloc(pgsmSharedState *pgsm, pgsmHashKey *key, int encoding) { - pgssEntry *entry = NULL; + pgsmEntry *entry = NULL; bool found = false; /* Find or create an entry with desired hash code */ - entry = (pgssEntry*) pgsm_hash_find_or_insert(pgsmStateLocal.shared_hash, key, &found); + entry = (pgsmEntry*) pgsm_hash_find_or_insert(pgsmStateLocal.shared_hash, key, &found); if (entry == NULL) - elog(DEBUG1, "hash_entry_alloc: OUT OF MEMORY"); + elog(DEBUG1, "[pg_stat_monitor] hash_entry_alloc: OUT OF MEMORY."); else if (!found) { - pgss->bucket_entry[pg_atomic_read_u64(&pgss->current_wbucket)]++; + pgsm->bucket_entry[pg_atomic_read_u64(&pgsm->current_wbucket)]++; /* New entry, initialize it */ /* reset the statistics */ memset(&entry->counters, 0, sizeof(Counters)); - entry->query_pos = InvalidDsaPointer; + entry->query_text.query_pos = InvalidDsaPointer; entry->counters.info.parent_query = InvalidDsaPointer; /* set the appropriate initial usage count */ @@ -288,23 +303,21 @@ hash_entry_alloc(pgssSharedState *pgss, pgssHashKey *key, int encoding) /* * Prepare resources for using the new bucket: * - Deallocate finished hash table entries in new_bucket_id (entries whose - * state is PGSS_FINISHED or PGSS_FINISHED). + * state is PGSM_EXEC or PGSM_ERROR). * - Clear query buffer for new_bucket_id. * - If old_bucket_id != -1, move all pending hash table entries in * old_bucket_id to the new bucket id, also move pending queries from the * previous query buffer (query_buffer[old_bucket_id]) to the new one * (query_buffer[new_bucket_id]). * - * Caller must hold an exclusive lock on pgss->lock. + * Caller must hold an exclusive lock on pgsm->lock. */ void hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_buffer) { PGSM_HASH_SEQ_STATUS hstat; - pgssEntry *entry = NULL; + pgsmEntry *entry = NULL; /* Store pending query ids from the previous bucket. */ - List *pending_entries = NIL; - ListCell *pending_entry; if (!pgsmStateLocal.shared_hash) return; @@ -321,11 +334,10 @@ hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_bu * in new_bucket_id if it has finished already. */ if (new_bucket_id < 0 || - (entry->key.bucket_id == new_bucket_id && - (entry->counters.state == PGSS_FINISHED || entry->counters.state == PGSS_ERROR))) + (entry->key.bucket_id == new_bucket_id )) { dsa_pointer parent_qdsa = entry->counters.info.parent_query; - pdsa = entry->query_pos; + pdsa = entry->query_text.query_pos; pgsm_hash_delete_current(&hstat, pgsmStateLocal.shared_hash, &entry->key); @@ -334,153 +346,23 @@ hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_bu if (DsaPointerIsValid(parent_qdsa)) dsa_free(pgsmStateLocal.dsa, parent_qdsa); - continue; - } - /* - * If we detect a pending query residing in the previous bucket id, we - * add it to a list of pending elements to be moved to the new bucket - * id. Can't update the hash table while iterating it inside this - * loop, as this may introduce all sort of problems. - */ - if (old_bucket_id != -1 && entry->key.bucket_id == old_bucket_id) - { - if (entry->counters.state == PGSS_PARSE || - entry->counters.state == PGSS_PLAN || - entry->counters.state == PGSS_EXEC) - { - pgssEntry *bkp_entry = malloc(sizeof(pgssEntry)); - - if (!bkp_entry) - { - elog(DEBUG1, "hash_entry_dealloc: out of memory"); - - /* - * No memory, If the entry has calls > 1 then we change - * the state to finished, as the pending query will likely - * finish execution during the new bucket time window. The - * pending query will vanish in this case, can't list it - * until it completes. - * - * If there is only one call to the query and it's - * pending, remove the entry from the previous bucket and - * allow it to finish in the new bucket, in order to avoid - * the query living in the old bucket forever. - */ - if (entry->counters.calls.calls > 1) - entry->counters.state = PGSS_FINISHED; - else - { - pdsa = entry->query_pos; - pgsm_hash_delete_current(&hstat, pgsmStateLocal.shared_hash, &entry->key); - if (DsaPointerIsValid(pdsa)) - dsa_free(pgsmStateLocal.dsa, pdsa); - } - continue; - } - - /* Save key/data from the previous entry. */ - memcpy(bkp_entry, entry, sizeof(pgssEntry)); - - /* Update key to use the new bucket id. */ - bkp_entry->key.bucket_id = new_bucket_id; - - /* Add the entry to a list of nodes to be processed later. */ - pending_entries = lappend(pending_entries, bkp_entry); - - /* - * If the entry has calls > 1 then we change the state to - * finished in the previous bucket, as the pending query will - * likely finish execution during the new bucket time window. - * Can't remove it from the previous bucket as it may have - * many calls and we would lose the query statistics. - * - * If there is only one call to the query and it's pending, - * remove the entry from the previous bucket and allow it to - * finish in the new bucket, in order to avoid the query - * living in the old bucket forever. - */ - if (entry->counters.calls.calls > 1) - entry->counters.state = PGSS_FINISHED; - else - { - pdsa = entry->query_pos; - pgsm_hash_delete_current(&hstat, pgsmStateLocal.shared_hash, &entry->key); - /* We should not delete the Query in DSA here - * as the same will get reused when the entry gets inserted into new bucket - */ - } - } + pgsmStateLocal.shared_pgsmState->pgsm_oom = false; } } pgsm_hash_seq_term(&hstat); - /* - * Iterate over the list of pending queries in order to add them back to - * the hash table with the updated bucket id. - */ - foreach(pending_entry, pending_entries) - { - bool found = false; - pgssEntry *new_entry; - pgssEntry *old_entry = (pgssEntry *) lfirst(pending_entry); - - - PGSM_DISABLE_ERROR_CAPUTRE(); - { - new_entry = (pgssEntry*) pgsm_hash_find_or_insert(pgsmStateLocal.shared_hash, &old_entry->key, &found); - }PGSM_END_DISABLE_ERROR_CAPTURE(); - - if (new_entry == NULL) - elog(DEBUG1, "%s", "pg_stat_monitor: out of memory"); - else if (!found) - { - /* Restore counters and other data. */ - new_entry->counters = old_entry->counters; - SpinLockInit(&new_entry->mutex); - new_entry->encoding = old_entry->encoding; - new_entry->query_pos = old_entry->query_pos; - } - #if USE_DYNAMIC_HASH - if(new_entry) - dshash_release_lock(pgsmStateLocal.shared_hash, new_entry); - #endif - free(old_entry); - } - list_free(pending_entries); -} - -/* - * Release all entries. - */ -void -hash_entry_reset() -{ - pgssSharedState *pgss = pgsm_get_ss(); - PGSM_HASH_SEQ_STATUS hstat; - pgssEntry *entry; - - LWLockAcquire(pgss->lock, LW_EXCLUSIVE); - - pgsm_hash_seq_init(&hstat, pgsmStateLocal.shared_hash, true); - - while ((entry = pgsm_hash_seq_next(&hstat)) != NULL) - { - dsa_pointer pdsa = entry->query_pos; - pgsm_hash_delete_current(&hstat, pgsmStateLocal.shared_hash, &entry->key); - if (DsaPointerIsValid(pdsa)) - dsa_free(pgsmStateLocal.dsa, pdsa); - } - - pgsm_hash_seq_term(&hstat); - - pg_atomic_write_u64(&pgss->current_wbucket, 0); - LWLockRelease(pgss->lock); } bool IsHashInitialize(void) { - return (pgsmStateLocal.shared_pgssState != NULL); + return (pgsmStateLocal.shared_pgsmState != NULL); +} + +bool +IsSystemOOM(void) +{ + return (IsHashInitialize() && pgsmStateLocal.shared_pgsmState->pgsm_oom); } /* @@ -489,7 +371,7 @@ IsHashInitialize(void) */ void * -pgsm_hash_find_or_insert(PGSM_HASH_TABLE *shared_hash, pgssHashKey *key, bool* found) +pgsm_hash_find_or_insert(PGSM_HASH_TABLE *shared_hash, pgsmHashKey *key, bool* found) { #if USE_DYNAMIC_HASH void *entry; @@ -501,7 +383,7 @@ pgsm_hash_find_or_insert(PGSM_HASH_TABLE *shared_hash, pgssHashKey *key, bool* f } void * -pgsm_hash_find(PGSM_HASH_TABLE *shared_hash, pgssHashKey *key, bool* found) +pgsm_hash_find(PGSM_HASH_TABLE *shared_hash, pgsmHashKey *key, bool* found) { #if USE_DYNAMIC_HASH return dshash_find(shared_hash, key, false); @@ -546,4 +428,4 @@ pgsm_hash_delete_current(PGSM_HASH_SEQ_STATUS *hstat, PGSM_HASH_TABLE *shared_ha #else hash_search(shared_hash, key, HASH_REMOVE, NULL); #endif -} \ No newline at end of file +} diff --git a/pg_stat_monitor--1.0--2.0.sql b/pg_stat_monitor--1.0--2.0.sql index a502a9c..de114ba 100644 --- a/pg_stat_monitor--1.0--2.0.sql +++ b/pg_stat_monitor--1.0--2.0.sql @@ -12,7 +12,9 @@ CREATE FUNCTION pg_stat_monitor_internal( IN showtext boolean, OUT bucket int8, -- 0 OUT userid oid, + OUT username text, OUT dbid oid, + OUT datname text, OUT client_ip int8, OUT queryid int8, -- 4 @@ -113,7 +115,7 @@ CREATE VIEW pg_stat_monitor AS SELECT bucket, bucket_start_time AS bucket_start_time, userid, - userid::regrole AS user, + username, dbid, datname, '0.0.0.0'::inet + client_ip AS client_ip, @@ -155,7 +157,7 @@ CREATE VIEW pg_stat_monitor AS SELECT cpu_user_time, cpu_sys_time, bucket_done -FROM pg_stat_monitor_internal(TRUE) p, pg_database d WHERE dbid = oid +FROM pg_stat_monitor_internal(TRUE) ORDER BY bucket_start_time; RETURN 0; END; @@ -169,7 +171,7 @@ CREATE VIEW pg_stat_monitor AS SELECT bucket, bucket_start_time AS bucket_start_time, userid, - userid::regrole AS user, + username, dbid, datname, '0.0.0.0'::inet + client_ip AS client_ip, @@ -222,7 +224,7 @@ CREATE VIEW pg_stat_monitor AS SELECT max_plan_time, mean_plan_time, stddev_plan_time -FROM pg_stat_monitor_internal(TRUE) p, pg_database d WHERE dbid = oid +FROM pg_stat_monitor_internal(TRUE) ORDER BY bucket_start_time; RETURN 0; END; @@ -235,7 +237,7 @@ CREATE VIEW pg_stat_monitor AS SELECT bucket, bucket_start_time AS bucket_start_time, userid, - userid::regrole AS user, + username, dbid, datname, '0.0.0.0'::inet + client_ip AS client_ip, @@ -288,7 +290,7 @@ CREATE VIEW pg_stat_monitor AS SELECT max_plan_time, mean_plan_time, stddev_plan_time -FROM pg_stat_monitor_internal(TRUE) p, pg_database d WHERE dbid = oid +FROM pg_stat_monitor_internal(TRUE) ORDER BY bucket_start_time; RETURN 0; END; @@ -301,7 +303,7 @@ CREATE VIEW pg_stat_monitor AS SELECT bucket, bucket_start_time AS bucket_start_time, userid, - userid::regrole AS user, + username, dbid, datname, '0.0.0.0'::inet + client_ip AS client_ip, @@ -367,7 +369,7 @@ CREATE VIEW pg_stat_monitor AS SELECT jit_emission_count, jit_emission_time -FROM pg_stat_monitor_internal(TRUE) p, pg_database d WHERE dbid = oid +FROM pg_stat_monitor_internal(TRUE) ORDER BY bucket_start_time; RETURN 0; END; @@ -395,15 +397,19 @@ $$ $$ LANGUAGE plpgsql; SELECT pgsm_create_view(); -REVOKE ALL ON FUNCTION range FROM PUBLIC; -REVOKE ALL ON FUNCTION get_cmd_type FROM PUBLIC; -REVOKE ALL ON FUNCTION decode_error_level FROM PUBLIC; -REVOKE ALL ON FUNCTION pg_stat_monitor_internal FROM PUBLIC; -REVOKE ALL ON FUNCTION get_histogram_timings FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_view FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_11_view FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_13_view FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_14_view FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_15_view FROM PUBLIC; +GRANT EXECUTE ON FUNCTION range TO PUBLIC; +GRANT EXECUTE ON FUNCTION decode_error_level TO PUBLIC; +GRANT EXECUTE ON FUNCTION get_histogram_timings TO PUBLIC; +GRANT EXECUTE ON FUNCTION get_cmd_type TO PUBLIC; +GRANT EXECUTE ON FUNCTION pg_stat_monitor_internal TO PUBLIC; + GRANT SELECT ON pg_stat_monitor TO PUBLIC; + +-- Reset is only available to super user +REVOKE ALL ON FUNCTION pg_stat_monitor_reset FROM PUBLIC; diff --git a/pg_stat_monitor--2.0.sql b/pg_stat_monitor--2.0.sql index 7bc2836..e9957c1 100644 --- a/pg_stat_monitor--2.0.sql +++ b/pg_stat_monitor--2.0.sql @@ -84,8 +84,10 @@ CREATE FUNCTION pg_stat_monitor_internal( IN showtext boolean, OUT bucket int8, -- 0 OUT userid oid, + OUT username text, OUT dbid oid, - OUT client_ip int8, + OUT datname text, + OUT client_ip int4, OUT queryid int8, -- 4 OUT planid int8, @@ -169,7 +171,7 @@ CREATE VIEW pg_stat_monitor AS SELECT bucket, bucket_start_time AS bucket_start_time, userid, - userid::regrole AS user, + username, dbid, datname, '0.0.0.0'::inet + client_ip AS client_ip, @@ -211,7 +213,7 @@ CREATE VIEW pg_stat_monitor AS SELECT cpu_user_time, cpu_sys_time, bucket_done -FROM pg_stat_monitor_internal(TRUE) p, pg_database d WHERE dbid = oid +FROM pg_stat_monitor_internal(TRUE) ORDER BY bucket_start_time; RETURN 0; END; @@ -225,7 +227,7 @@ CREATE VIEW pg_stat_monitor AS SELECT bucket, bucket_start_time AS bucket_start_time, userid, - userid::regrole AS user, + username, dbid, datname, '0.0.0.0'::inet + client_ip AS client_ip, @@ -278,7 +280,7 @@ CREATE VIEW pg_stat_monitor AS SELECT max_plan_time, mean_plan_time, stddev_plan_time -FROM pg_stat_monitor_internal(TRUE) p, pg_database d WHERE dbid = oid +FROM pg_stat_monitor_internal(TRUE) ORDER BY bucket_start_time; RETURN 0; END; @@ -291,7 +293,7 @@ CREATE VIEW pg_stat_monitor AS SELECT bucket, bucket_start_time AS bucket_start_time, userid, - userid::regrole AS user, + username, dbid, datname, '0.0.0.0'::inet + client_ip AS client_ip, @@ -344,7 +346,7 @@ CREATE VIEW pg_stat_monitor AS SELECT max_plan_time, mean_plan_time, stddev_plan_time -FROM pg_stat_monitor_internal(TRUE) p, pg_database d WHERE dbid = oid +FROM pg_stat_monitor_internal(TRUE) ORDER BY bucket_start_time; RETURN 0; END; @@ -357,7 +359,7 @@ CREATE VIEW pg_stat_monitor AS SELECT bucket, bucket_start_time AS bucket_start_time, userid, - userid::regrole AS user, + username, dbid, datname, '0.0.0.0'::inet + client_ip AS client_ip, @@ -423,7 +425,7 @@ CREATE VIEW pg_stat_monitor AS SELECT jit_emission_count, jit_emission_time -FROM pg_stat_monitor_internal(TRUE) p, pg_database d WHERE dbid = oid +FROM pg_stat_monitor_internal(TRUE) ORDER BY bucket_start_time; RETURN 0; END; @@ -451,15 +453,19 @@ $$ $$ LANGUAGE plpgsql; SELECT pgsm_create_view(); -REVOKE ALL ON FUNCTION range FROM PUBLIC; -REVOKE ALL ON FUNCTION get_cmd_type FROM PUBLIC; -REVOKE ALL ON FUNCTION decode_error_level FROM PUBLIC; -REVOKE ALL ON FUNCTION pg_stat_monitor_internal FROM PUBLIC; -REVOKE ALL ON FUNCTION get_histogram_timings FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_view FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_11_view FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_13_view FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_14_view FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_15_view FROM PUBLIC; +GRANT EXECUTE ON FUNCTION range TO PUBLIC; +GRANT EXECUTE ON FUNCTION decode_error_level TO PUBLIC; +GRANT EXECUTE ON FUNCTION get_histogram_timings TO PUBLIC; +GRANT EXECUTE ON FUNCTION get_cmd_type TO PUBLIC; +GRANT EXECUTE ON FUNCTION pg_stat_monitor_internal TO PUBLIC; + GRANT SELECT ON pg_stat_monitor TO PUBLIC; + +-- Reset is only available to super user +REVOKE ALL ON FUNCTION pg_stat_monitor_reset FROM PUBLIC; diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 5cd43f4..7de8864 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -17,9 +17,11 @@ #include "postgres.h" #include "access/parallel.h" +#include "nodes/pg_list.h" #include "utils/guc.h" #include #include "pgstat.h" +#include "commands/dbcommands.h" #include "commands/explain.h" #include "pg_stat_monitor.h" @@ -38,8 +40,8 @@ PG_MODULE_MAGIC; /* Number of output arguments (columns) for various API versions */ #define PG_STAT_MONITOR_COLS_V1_0 52 -#define PG_STAT_MONITOR_COLS_V2_0 62 -#define PG_STAT_MONITOR_COLS 62 /* maximum of above */ +#define PG_STAT_MONITOR_COLS_V2_0 64 +#define PG_STAT_MONITOR_COLS PG_STAT_MONITOR_COLS_V2_0 /* maximum of above */ #define PGSM_TEXT_FILE PGSTAT_STAT_PERMANENT_DIRECTORY "pg_stat_monitor_query" @@ -63,10 +65,6 @@ do \ /*---- Initicalization Function Declarations ----*/ void _PG_init(void); -void _PG_fini(void); - - -/*---- Local variables ----*/ /* Current nesting depth of ExecutorRun+ProcessUtility calls */ static int exec_nested_level = 0; @@ -85,6 +83,7 @@ static int hist_bucket_count_total; /* The array to store outer layer query id*/ uint64 *nested_queryids; char **nested_query_txts; +List *lentries = NIL; /* Regex object used to extract query comments. */ static regex_t preg_query_comments; @@ -97,7 +96,7 @@ static struct rusage rusage_end; /* Query buffer, store queries' text. */ -static char *pgss_explain(QueryDesc *queryDesc); +static char *pgsm_explain(QueryDesc *queryDesc); static void extract_query_comments(const char *query, char *comments, size_t max_len); static void histogram_bucket_timings(int index, int64 *b_start, int64 *b_end); @@ -110,7 +109,7 @@ static void request_additional_shared_resources(void); /* Saved hook values in case of unload */ #if PG_VERSION_NUM >= 150000 -static void pgss_shmem_request(void); +static void pgsm_shmem_request(void); static shmem_request_hook_type prev_shmem_request_hook = NULL; #endif #if PG_VERSION_NUM >= 130000 @@ -137,33 +136,33 @@ PG_FUNCTION_INFO_V1(get_histogram_timings); PG_FUNCTION_INFO_V1(pg_stat_monitor_hook_stats); static uint pg_get_client_addr(bool *ok); -static int pg_get_application_name(char *application_name, bool *ok); +static int pg_get_application_name(char *name, int buff_size); static PgBackendStatus *pg_get_backend_status(void); static Datum intarray_get_datum(int32 arr[], int len); #if PG_VERSION_NUM < 140000 -DECLARE_HOOK(void pgss_post_parse_analyze, ParseState *pstate, Query *query); +DECLARE_HOOK(void pgsm_post_parse_analyze, ParseState *pstate, Query *query); #else -DECLARE_HOOK(void pgss_post_parse_analyze, ParseState *pstate, Query *query, JumbleState *jstate); +DECLARE_HOOK(void pgsm_post_parse_analyze, ParseState *pstate, Query *query, JumbleState *jstate); #endif -DECLARE_HOOK(void pgss_ExecutorStart, QueryDesc *queryDesc, int eflags); -DECLARE_HOOK(void pgss_ExecutorRun, QueryDesc *queryDesc, ScanDirection direction, uint64 count, bool execute_once); -DECLARE_HOOK(void pgss_ExecutorFinish, QueryDesc *queryDesc); -DECLARE_HOOK(void pgss_ExecutorEnd, QueryDesc *queryDesc); -DECLARE_HOOK(bool pgss_ExecutorCheckPerms, List *rt, bool abort); +DECLARE_HOOK(void pgsm_ExecutorStart, QueryDesc *queryDesc, int eflags); +DECLARE_HOOK(void pgsm_ExecutorRun, QueryDesc *queryDesc, ScanDirection direction, uint64 count, bool execute_once); +DECLARE_HOOK(void pgsm_ExecutorFinish, QueryDesc *queryDesc); +DECLARE_HOOK(void pgsm_ExecutorEnd, QueryDesc *queryDesc); +DECLARE_HOOK(bool pgsm_ExecutorCheckPerms, List *rt, bool abort); #if PG_VERSION_NUM >= 140000 -DECLARE_HOOK(PlannedStmt *pgss_planner_hook, Query *parse, const char *query_string, int cursorOptions, ParamListInfo boundParams); -DECLARE_HOOK(void pgss_ProcessUtility, PlannedStmt *pstmt, const char *queryString, +DECLARE_HOOK(PlannedStmt *pgsm_planner_hook, Query *parse, const char *query_string, int cursorOptions, ParamListInfo boundParams); +DECLARE_HOOK(void pgsm_ProcessUtility, PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *qc); #elif PG_VERSION_NUM >= 130000 -DECLARE_HOOK(PlannedStmt *pgss_planner_hook, Query *parse, const char *query_string, int cursorOptions, ParamListInfo boundParams); -DECLARE_HOOK(void pgss_ProcessUtility, PlannedStmt *pstmt, const char *queryString, +DECLARE_HOOK(PlannedStmt *pgsm_planner_hook, Query *parse, const char *query_string, int cursorOptions, ParamListInfo boundParams); +DECLARE_HOOK(void pgsm_ProcessUtility, PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, @@ -171,36 +170,50 @@ DECLARE_HOOK(void pgss_ProcessUtility, PlannedStmt *pstmt, const char *queryStri #else static void BufferUsageAccumDiff(BufferUsage *bufusage, BufferUsage *pgBufferUsage, BufferUsage *bufusage_start); -DECLARE_HOOK(void pgss_ProcessUtility, PlannedStmt *pstmt, const char *queryString, +DECLARE_HOOK(void pgsm_ProcessUtility, PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, char *completionTag); #endif -static uint64 pgss_hash_string(const char *str, int len); +static uint64 pgsm_hash_string(const char *str, int len); char *unpack_sql_state(int sql_state); #define PGSM_HANDLED_UTILITY(n) (!IsA(n, ExecuteStmt) && \ !IsA(n, PrepareStmt) && \ !IsA(n, DeallocateStmt)) -static void pgss_store_error(uint64 queryid, const char *query, ErrorData *edata); -static void pgss_store(uint64 queryid, - const char *query, - int query_location, - int query_len, - PlanInfo * plan_info, - CmdType cmd_type, - SysInfo * sys_info, - ErrorInfo * error_info, - double total_time, - uint64 rows, - BufferUsage *bufusage, - WalUsage *walusage, - const struct JitInstrumentation *jitusage, - JumbleState *jstate, - pgssStoreKind kind); +static pgsmEntry *pgsm_create_hash_entry(uint64 bucket_id, uint64 queryid, PlanInfo *plan_info); +static void pgsm_add_to_list(pgsmEntry *entry, char *query_text, int query_len); +static pgsmEntry* pgsm_get_entry_for_query(uint64 queryid, PlanInfo *plan_info, const char* query_text, int query_len, bool create); +static uint64 get_pgsm_query_id_hash(const char *norm_query, int len); + +static void pgsm_cleanup_callback(void *arg); +static void pgsm_store_error(const char *query, ErrorData *edata); + +/*---- Local variables ----*/ +MemoryContextCallback mem_cxt_reset_callback = + { + .func = pgsm_cleanup_callback, + .arg = NULL + }; +volatile bool callback_setup = false; + +static void pgsm_update_entry(pgsmEntry *entry, + const char *query, + PlanInfo * plan_info, + SysInfo * sys_info, + ErrorInfo * error_info, + double plan_total_time, + double exec_total_time, + uint64 rows, + BufferUsage *bufusage, + WalUsage *walusage, + const struct JitInstrumentation *jitusage, + bool reset, + pgsmStoreKind kind); +static void pgsm_store(pgsmEntry *entry); static void pg_stat_monitor_internal(FunctionCallInfo fcinfo, pgsmVersion api_version, @@ -219,6 +232,7 @@ static void RecordConstLocation(JumbleState *jstate, int location); * relevant part of the string. */ static const char *CleanQuerytext(const char *query, int *location, int *len); +static uint64 get_query_id(JumbleState *jstate, Query *query); #endif static char *generate_normalized_query(JumbleState *jstate, const char *query, @@ -226,17 +240,8 @@ static char *generate_normalized_query(JumbleState *jstate, const char *query, static void fill_in_constant_lengths(JumbleState *jstate, const char *query, int query_loc); static int comp_location(const void *a, const void *b); -static uint64 get_next_wbucket(pgssSharedState *pgss); +static uint64 get_next_wbucket(pgsmSharedState *pgsm); -#if PG_VERSION_NUM < 140000 -static uint64 get_query_id(JumbleState *jstate, Query *query); -#endif - -/* Daniel J. Bernstein's hash algorithm: see http://www.cse.yorku.ca/~oz/hash.html */ -static uint64 djb2_hash(unsigned char *str, size_t len); - -/* Same as above, but stores the calculated string length into *out_len (small optimization) */ -static uint64 djb2_hash_str(unsigned char *str, int *out_len); /* * Module load callback */ @@ -246,7 +251,7 @@ _PG_init(void) { int rc; - elog(DEBUG2, "pg_stat_monitor: %s()", __FUNCTION__); + elog(DEBUG2, "[pg_stat_monitor] pg_stat_monitor: %s().", __FUNCTION__); /* * In order to create our shared memory area, we have to be loaded via @@ -323,7 +328,7 @@ _PG_init(void) rc = regcomp(&preg_query_comments, "/\\*([^*]|[\r\n]|(\\*+([^*/]|[\r\n])))*\\*+/", REG_EXTENDED); if (rc != 0) { - elog(ERROR, "pg_stat_monitor: query comments regcomp() failed, return code=(%d)\n", rc); + elog(ERROR, "[pg_stat_monitor] _PG_init: query comments regcomp() failed, return code=(%d).", rc); } /* @@ -331,32 +336,32 @@ _PG_init(void) */ #if PG_VERSION_NUM >= 150000 prev_shmem_request_hook = shmem_request_hook; - shmem_request_hook = pgss_shmem_request; + shmem_request_hook = pgsm_shmem_request; #else request_additional_shared_resources(); #endif prev_shmem_startup_hook = shmem_startup_hook; - shmem_startup_hook = pgss_shmem_startup; + shmem_startup_hook = pgsm_shmem_startup; prev_post_parse_analyze_hook = post_parse_analyze_hook; - post_parse_analyze_hook = HOOK(pgss_post_parse_analyze); + post_parse_analyze_hook = HOOK(pgsm_post_parse_analyze); prev_ExecutorStart = ExecutorStart_hook; - ExecutorStart_hook = HOOK(pgss_ExecutorStart); + ExecutorStart_hook = HOOK(pgsm_ExecutorStart); prev_ExecutorRun = ExecutorRun_hook; - ExecutorRun_hook = HOOK(pgss_ExecutorRun); + ExecutorRun_hook = HOOK(pgsm_ExecutorRun); prev_ExecutorFinish = ExecutorFinish_hook; - ExecutorFinish_hook = HOOK(pgss_ExecutorFinish); + ExecutorFinish_hook = HOOK(pgsm_ExecutorFinish); prev_ExecutorEnd = ExecutorEnd_hook; - ExecutorEnd_hook = HOOK(pgss_ExecutorEnd); + ExecutorEnd_hook = HOOK(pgsm_ExecutorEnd); prev_ProcessUtility = ProcessUtility_hook; - ProcessUtility_hook = HOOK(pgss_ProcessUtility); + ProcessUtility_hook = HOOK(pgsm_ProcessUtility); #if PG_VERSION_NUM >= 130000 planner_hook_next = planner_hook; - planner_hook = HOOK(pgss_planner_hook); + planner_hook = HOOK(pgsm_planner_hook); #endif prev_emit_log_hook = emit_log_hook; emit_log_hook = HOOK(pgsm_emit_log_hook); prev_ExecutorCheckPerms_hook = ExecutorCheckPerms_hook; - ExecutorCheckPerms_hook = HOOK(pgss_ExecutorCheckPerms); + ExecutorCheckPerms_hook = HOOK(pgsm_ExecutorCheckPerms); nested_queryids = (uint64 *) malloc(sizeof(uint64) * max_stack_depth); nested_query_txts = (char **) malloc(sizeof(char*) * max_stack_depth); @@ -364,30 +369,6 @@ _PG_init(void) system_init = true; } -/* - * Module unload callback - */ -/* cppcheck-suppress unusedFunction */ -void -_PG_fini(void) -{ - system_init = false; - shmem_startup_hook = prev_shmem_startup_hook; - post_parse_analyze_hook = prev_post_parse_analyze_hook; - ExecutorStart_hook = prev_ExecutorStart; - ExecutorRun_hook = prev_ExecutorRun; - ExecutorFinish_hook = prev_ExecutorFinish; - ExecutorEnd_hook = prev_ExecutorEnd; - ProcessUtility_hook = prev_ProcessUtility; - emit_log_hook = prev_emit_log_hook; - - free(nested_queryids); - free(nested_query_txts); - regfree(&preg_query_comments); - - hash_entry_reset(); -} - /* * shmem_startup hook: allocate or attach to shared memory, * then load any pre-existing statistics from file. @@ -395,12 +376,12 @@ _PG_fini(void) * (even if empty) while the module is enabled. */ void -pgss_shmem_startup(void) +pgsm_shmem_startup(void) { if (prev_shmem_startup_hook) prev_shmem_startup_hook(); - pgss_startup(); + pgsm_startup(); } static void @@ -409,7 +390,7 @@ request_additional_shared_resources(void) /* * Request additional shared resources. (These are no-ops if we're not in * the postmaster process.) We'll allocate or attach to the shared - * resources in pgss_shmem_startup(). + * resources in pgsm_shmem_startup(). */ RequestAddinShmemSpace(pgsm_ShmemSize() + HOOK_STATS_SIZE); RequestNamedLWLockTranche("pg_stat_monitor", 1); @@ -426,10 +407,10 @@ pg_stat_monitor_version(PG_FUNCTION_ARGS) #if PG_VERSION_NUM >= 150000 /* * shmem_request hook: request additional shared resources. We'll allocate or - * attach to the shared resources in pgss_shmem_startup(). + * attach to the shared resources in pgsm_shmem_startup(). */ static void -pgss_shmem_request(void) +pgsm_shmem_request(void) { if (prev_shmem_request_hook) prev_shmem_request_hook(); @@ -437,20 +418,32 @@ pgss_shmem_request(void) } #endif -#if PG_VERSION_NUM >= 140000 -/* - * Post-parse-analysis hook: mark query with a queryId - */ static void -pgss_post_parse_analyze(ParseState *pstate, Query *query, JumbleState *jstate) +pgsm_post_parse_analyze_internal(ParseState *pstate, Query *query, JumbleState *jstate) { - if (prev_post_parse_analyze_hook) - prev_post_parse_analyze_hook(pstate, query, jstate); + pgsmEntry *entry; + const char *query_text; + char *norm_query = NULL; + int norm_query_len; + int location; + int query_len; /* Safety check... */ if (!IsSystemInitialized()) return; + if (callback_setup == false) + { + /* If MessageContext is valid setup a callback to cleanup + * our local stats list when the MessagContext gets reset + */ + if (MemoryContextIsValid(MessageContext)) + { + MemoryContextRegisterResetCallback(MessageContext, &mem_cxt_reset_callback); + callback_setup = true; + } + } + if (!pgsm_enabled(exec_nested_level)) return; @@ -463,67 +456,17 @@ pgss_post_parse_analyze(ParseState *pstate, Query *query, JumbleState *jstate) { if (PGSM_TRACK_UTILITY && !PGSM_HANDLED_UTILITY(query->utilityStmt)) query->queryId = UINT64CONST(0); + return; } /* - * If query jumbling were able to identify any ignorable constants, we - * immediately create a hash table entry for the query, so that we can - * record the normalized form of the query string. If there were no such - * constants, the normalized string would be the same as the query text - * anyway, so there's no need for an early entry. + * Let's calculate queryid for versions 13 and below. We don't have to check + * that jstate is valid, it always will be for these versions. */ - if (jstate && jstate->clocations_count > 0) - pgss_store(query->queryId, /* query id */ - pstate->p_sourcetext, /* query */ - query->stmt_location, /* query location */ - query->stmt_len, /* query length */ - NULL, /* PlanInfo */ - query->commandType, /* CmdType */ - NULL, /* SysInfo */ - NULL, /* ErrorInfo */ - 0, /* totaltime */ - 0, /* rows */ - NULL, /* bufusage */ - NULL, /* walusage */ - NULL, /* jitusage */ - jstate, /* JumbleState */ - PGSS_PARSE); /* pgssStoreKind */ -} -#else - -/* - * Post-parse-analysis hook: mark query with a queryId - */ -static void -pgss_post_parse_analyze(ParseState *pstate, Query *query) -{ - JumbleState jstate; - - if (prev_post_parse_analyze_hook) - prev_post_parse_analyze_hook(pstate, query); - - /* Safety check... */ - if (!IsSystemInitialized()) - return; - if (!pgsm_enabled(exec_nested_level)) - return; - - /* - * Utility statements get queryId zero. We do this even in cases where - * the statement contains an optimizable statement for which a queryId - * could be derived (such as EXPLAIN or DECLARE CURSOR). For such cases, - * runtime control will first go through ProcessUtility and then the - * executor, and we don't want the executor hooks to do anything, since we - * are already measuring the statement's costs at the utility level. - */ - if (query->utilityStmt) - { - query->queryId = UINT64CONST(0); - return; - } - - query->queryId = get_query_id(&jstate, query); +#if PG_VERSION_NUM < 140000 + query->queryId = get_query_id(jstate, query); +#endif /* * If we are unlucky enough to get a hash of zero, use 1 instead, to @@ -532,22 +475,93 @@ pgss_post_parse_analyze(ParseState *pstate, Query *query) if (query->queryId == UINT64CONST(0)) query->queryId = UINT64CONST(1); - if (jstate.clocations_count > 0) - pgss_store(query->queryId, /* query id */ - pstate->p_sourcetext, /* query */ - query->stmt_location, /* query location */ - query->stmt_len, /* query length */ - NULL, /* PlanInfo */ - query->commandType, /* CmdType */ - NULL, /* SysInfo */ - NULL, /* ErrorInfo */ - 0, /* totaltime */ - 0, /* rows */ - NULL, /* bufusage */ - NULL, /* walusage */ - NULL, /* jitusage */ - &jstate, /* JumbleState */ - PGSS_PARSE); /* pgssStoreKind */ + /* + * Let's save the normalized query so that we can save the data without in + * hash later on without the need of jstate which wouldn't be available. + */ + query_text = pstate->p_sourcetext; + location = query->stmt_location; + query_len = query->stmt_len; + + /* We should always have a valid query. */ + query_text = CleanQuerytext(query_text, &location, &query_len); + Assert(query_text); + + norm_query_len = query_len; + + /* Generate a normalized query */ + if (jstate && jstate->clocations_count > 0) + { + norm_query = generate_normalized_query(jstate, + query_text, /* query */ + location, /* query location */ + &norm_query_len, + GetDatabaseEncoding()); + + Assert(norm_query); + } + + /* + * At this point, we don't know which bucket this query will land in, so passing + * 0. The store function MUST later update it based on the current bucket value. + * The correct bucket value will be needed then to search the hash table, or create + * the appropriate entry. + */ + entry = pgsm_create_hash_entry(0, query->queryId, NULL); + + /* Update other member that are not counters, so that we don't have to worry about these. */ + entry->pgsm_query_id = get_pgsm_query_id_hash(norm_query ? norm_query : query_text, norm_query_len); + entry->counters.info.cmd_type = query->commandType; + + /* + * Add the query text and entry to the local list. + * + * Preserve the normalized query if needed and we got a valid one. + * Otherwise, store the actual query so that we don't have to check + * what query to store when saving into the hash. + * + * In case of query_text, request the function to duplicate it so that + * it is put in the relevant memory context. + */ + if (PGSM_NORMALIZED_QUERY && norm_query) + pgsm_add_to_list(entry, norm_query, norm_query_len); + else + { + pgsm_add_to_list(entry, (char *)query_text, query_len); + } + + /* Check that we've not exceeded max_stack_depth */ + Assert(list_length(lentries) <= max_stack_depth); + + if (norm_query) + pfree(norm_query); +} + +#if PG_VERSION_NUM >= 140000 +/* + * Post-parse-analysis hook: mark query with a queryId + */ +static void +pgsm_post_parse_analyze(ParseState *pstate, Query *query, JumbleState *jstate) +{ + if (prev_post_parse_analyze_hook) + prev_post_parse_analyze_hook(pstate, query, jstate); + + pgsm_post_parse_analyze_internal(pstate, query, jstate); +} +#else +/* + * Post-parse-analysis hook: mark query with a queryId + */ +static void +pgsm_post_parse_analyze(ParseState *pstate, Query *query) +{ + JumbleState jstate; + + if (prev_post_parse_analyze_hook) + prev_post_parse_analyze_hook(pstate, query); + + pgsm_post_parse_analyze_internal(pstate, query, &jstate); } #endif @@ -555,10 +569,10 @@ pgss_post_parse_analyze(ParseState *pstate, Query *query) * ExecutorStart hook: start up tracking if needed */ static void -pgss_ExecutorStart(QueryDesc *queryDesc, int eflags) +pgsm_ExecutorStart(QueryDesc *queryDesc, int eflags) { if (getrusage(RUSAGE_SELF, &rusage_start) != 0) - elog(DEBUG1, "pgss_ExecutorStart: failed to execute getrusage"); + elog(DEBUG1, "[pg_stat_monitor] pgsm_ExecutorStart: failed to execute getrusage."); if (prev_ExecutorStart) prev_ExecutorStart(queryDesc, eflags); @@ -598,7 +612,7 @@ pgss_ExecutorStart(QueryDesc *queryDesc, int eflags) * ExecutorRun hook: all we need do is track nesting depth */ static void -pgss_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, +pgsm_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, bool execute_once) { if (exec_nested_level >= 0 && exec_nested_level < max_stack_depth) @@ -642,9 +656,10 @@ pgss_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, * ExecutorFinish hook: all we need do is track nesting depth */ static void -pgss_ExecutorFinish(QueryDesc *queryDesc) +pgsm_ExecutorFinish(QueryDesc *queryDesc) { exec_nested_level++; + PG_TRY(); { if (prev_ExecutorFinish) @@ -659,10 +674,11 @@ pgss_ExecutorFinish(QueryDesc *queryDesc) PG_RE_THROW(); } PG_END_TRY(); + } static char * -pgss_explain(QueryDesc *queryDesc) +pgsm_explain(QueryDesc *queryDesc) { ExplainState *es = NewExplainState(); @@ -685,26 +701,34 @@ pgss_explain(QueryDesc *queryDesc) * ExecutorEnd hook: store results if needed */ static void -pgss_ExecutorEnd(QueryDesc *queryDesc) +pgsm_ExecutorEnd(QueryDesc *queryDesc) { uint64 queryId = queryDesc->plannedstmt->queryId; SysInfo sys_info; PlanInfo plan_info; PlanInfo *plan_ptr = NULL; + pgsmEntry *entry = NULL; /* Extract the plan information in case of SELECT statement */ if (queryDesc->operation == CMD_SELECT && PGSM_QUERY_PLAN) { - MemoryContext mct = MemoryContextSwitchTo(TopMemoryContext); - - plan_info.plan_len = snprintf(plan_info.plan_text, PLAN_TEXT_LEN, "%s", pgss_explain(queryDesc)); - plan_info.planid = pgss_hash_string(plan_info.plan_text, plan_info.plan_len); + plan_info.plan_len = snprintf(plan_info.plan_text, PLAN_TEXT_LEN, "%s", pgsm_explain(queryDesc)); + plan_info.planid = pgsm_hash_string(plan_info.plan_text, plan_info.plan_len); plan_ptr = &plan_info; - MemoryContextSwitchTo(mct); } if (queryId != UINT64CONST(0) && queryDesc->totaltime && pgsm_enabled(exec_nested_level)) { + entry = pgsm_get_entry_for_query(queryId, plan_ptr, (char *)queryDesc->sourceText, strlen(queryDesc->sourceText), true); + if(!entry) + { + elog(DEBUG2,"[pg_stat_monitor] pgsm_ExecutorEnd: Failed to find entry for [%lu] %s.",queryId, queryDesc->sourceText); + return; + } + + if (entry->key.planid == 0) + entry->key.planid = (plan_ptr) ? plan_ptr->planid : 0; + /* * Make sure stats accumulation is done. (Note: it's okay if several * levels of hook all do this.) @@ -715,46 +739,48 @@ pgss_ExecutorEnd(QueryDesc *queryDesc) sys_info.stime = 0; if (getrusage(RUSAGE_SELF, &rusage_end) != 0) - elog(DEBUG1, "pg_stat_monitor: failed to execute getrusage"); + elog(DEBUG1, "[pg_stat_monitor] pgsm_ExecutorEnd: Failed to execute getrusage."); else { sys_info.utime = time_diff(rusage_end.ru_utime, rusage_start.ru_utime); sys_info.stime = time_diff(rusage_end.ru_stime, rusage_start.ru_stime); } - pgss_store(queryId, /* query id */ - queryDesc->sourceText, /* query text */ - queryDesc->plannedstmt->stmt_location, /* query location */ - queryDesc->plannedstmt->stmt_len, /* query length */ - plan_ptr, /* PlanInfo */ - queryDesc->operation, /* CmdType */ - &sys_info, /* SysInfo */ - NULL, /* ErrorInfo */ - queryDesc->totaltime->total * 1000.0, /* totaltime */ - queryDesc->estate->es_processed, /* rows */ - &queryDesc->totaltime->bufusage, /* bufusage */ + pgsm_update_entry(entry, /* entry */ + NULL, /* query */ + plan_ptr, /* PlanInfo */ + &sys_info, /* SysInfo */ + NULL, /* ErrorInfo */ + 0, /* plan_total_time */ + queryDesc->totaltime->total * 1000.0, /* exec_total_time */ + queryDesc->estate->es_processed, /* rows */ + &queryDesc->totaltime->bufusage, /* bufusage */ #if PG_VERSION_NUM >= 130000 - &queryDesc->totaltime->walusage, /* walusage */ + &queryDesc->totaltime->walusage, /* walusage */ #else - NULL, + NULL, #endif #if PG_VERSION_NUM >= 150000 - queryDesc->estate->es_jit ? &queryDesc->estate->es_jit->instr : NULL, + queryDesc->estate->es_jit ? &queryDesc->estate->es_jit->instr : NULL, /* jitusage */ #else - NULL, + NULL, #endif - NULL, - PGSS_FINISHED); /* pgssStoreKind */ + false, /* reset */ + PGSM_EXEC); /* kind */ + + pgsm_store(entry); } + if (prev_ExecutorEnd) prev_ExecutorEnd(queryDesc); else standard_ExecutorEnd(queryDesc); + num_relations = 0; } static bool -pgss_ExecutorCheckPerms(List *rt, bool abort) +pgsm_ExecutorCheckPerms(List *rt, bool abort) { ListCell *lr = NULL; int i = 0; @@ -805,14 +831,16 @@ pgss_ExecutorCheckPerms(List *rt, bool abort) #if PG_VERSION_NUM >= 130000 static PlannedStmt * -pgss_planner_hook(Query *parse, const char *query_string, int cursorOptions, ParamListInfo boundParams) +pgsm_planner_hook(Query *parse, const char *query_string, int cursorOptions, ParamListInfo boundParams) { PlannedStmt *result; + pgsmEntry *entry = NULL; + /* * We can't process the query if no query_string is provided, as - * pgss_store needs it. We also ignore query without queryid, as it would - * be treated as a utility statement, which may not be the case. + * pgsm_store needs it. We also ignore query without queryid, + * as it would be treated as a utility statement, which may not be the case. * * Note that planner_hook can be called from the planner itself, so we * have a specific nesting level for the planner. However, utility @@ -821,6 +849,10 @@ pgss_planner_hook(Query *parse, const char *query_string, int cursorOptions, Par * So testing the planner nesting level only is not enough to detect real * top level planner call. */ + if (MemoryContextIsValid(MessageContext)) + entry = pgsm_get_entry_for_query(parse->queryId, NULL, query_string, strlen(query_string), true); + + if (pgsm_enabled(plan_nested_level + exec_nested_level) && PGSM_TRACK_PLANNING && query_string && parse->queryId != UINT64CONST(0)) { @@ -872,21 +904,22 @@ pgss_planner_hook(Query *parse, const char *query_string, int cursorOptions, Par /* calc differences of WAL counters. */ memset(&walusage, 0, sizeof(WalUsage)); WalUsageAccumDiff(&walusage, &pgWalUsage, &walusage_start); - pgss_store(parse->queryId, /* query id */ - query_string, /* query */ - parse->stmt_location, /* query location */ - parse->stmt_len, /* query length */ - NULL, /* PlanInfo */ - parse->commandType, /* CmdType */ - NULL, /* SysInfo */ - NULL, /* ErrorInfo */ - INSTR_TIME_GET_MILLISEC(duration), /* totaltime */ - 0, /* rows */ - &bufusage, /* bufusage */ - &walusage, /* walusage */ - NULL, /* JumbleState */ - NULL, - PGSS_PLAN); /* pgssStoreKind */ + + /* The plan details are captured when the query finishes */ + if(entry) + pgsm_update_entry(entry, /* entry */ + NULL, /* query */ + NULL, /* PlanInfo */ + NULL, /* SysInfo */ + NULL, /* ErrorInfo */ + INSTR_TIME_GET_MILLISEC(duration), /* plan_total_time */ + 0, /* exec_total_time */ + 0, /* rows */ + &bufusage, /* bufusage */ + &walusage, /* walusage */ + NULL, /* jitusage */ + false, /* reset */ + PGSM_PLAN); /* kind */ } else { @@ -897,22 +930,25 @@ pgss_planner_hook(Query *parse, const char *query_string, int cursorOptions, Par * since it modifies the first argument (Query *), the second call * would trigger an assertion failure. */ + plan_nested_level++; + if (planner_hook_next) result = planner_hook_next(parse, query_string, cursorOptions, boundParams); else result = standard_planner(parse, query_string, cursorOptions, boundParams); + plan_nested_level--; + } return result; } #endif - /* * ProcessUtility hook */ #if PG_VERSION_NUM >= 140000 static void -pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, +pgsm_ProcessUtility(PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, @@ -921,7 +957,7 @@ pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, #elif PG_VERSION_NUM >= 130000 static void -pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, +pgsm_ProcessUtility(PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, @@ -929,7 +965,7 @@ pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, #else static void -pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, +pgsm_ProcessUtility(PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, @@ -938,9 +974,12 @@ pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, { Node *parsetree = pstmt->utilityStmt; uint64 queryId = 0; - SysInfo sys_info; -#if PG_VERSION_NUM >= 140000 +#if PG_VERSION_NUM < 140000 + int len = strlen(queryString); + + queryId = pgsm_hash_string(queryString, len); +#else queryId = pstmt->queryId; /* @@ -973,9 +1012,14 @@ pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, if (PGSM_TRACK_UTILITY && pgsm_enabled(exec_nested_level) && PGSM_HANDLED_UTILITY(parsetree)) { + pgsmEntry *entry; + char *query_text; + int location; + int query_len; instr_time start; instr_time duration; uint64 rows; + SysInfo sys_info; BufferUsage bufusage; BufferUsage bufusage_start = pgBufferUsage; #if PG_VERSION_NUM >= 130000 @@ -984,7 +1028,7 @@ pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, #endif if (getrusage(RUSAGE_SELF, &rusage_start) != 0) - elog(DEBUG1, "pg_stat_monitor: failed to execute getrusage"); + elog(DEBUG1, "[pg_stat_monitor] pgsm_ProcessUtility: Failed to execute getrusage."); INSTR_TIME_SET_CURRENT(start); exec_nested_level++; @@ -1041,7 +1085,7 @@ pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, PG_END_TRY(); if (getrusage(RUSAGE_SELF, &rusage_end) != 0) - elog(DEBUG1, "pg_stat_monitor: failed to execute getrusage"); + elog(DEBUG1, "[pg_stat_monitor] pgsm_ProcessUtility: Failed to execute getrusage."); else { sys_info.utime = time_diff(rusage_end.ru_utime, rusage_start.ru_utime); @@ -1076,26 +1120,42 @@ pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, /* calc differences of buffer counters. */ memset(&bufusage, 0, sizeof(BufferUsage)); BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &bufusage_start); - pgss_store( - queryId, /* query ID */ - queryString, /* query text */ - pstmt->stmt_location, /* query location */ - pstmt->stmt_len, /* query length */ - NULL, /* PlanInfo */ - 0, /* CmdType */ - &sys_info, /* SysInfo */ - NULL, /* ErrorInfo */ - INSTR_TIME_GET_MILLISEC(duration), /* total_time */ - rows, /* rows */ - &bufusage, /* bufusage */ + + /* Create an entry for this query */ + entry = pgsm_create_hash_entry(0, queryId, NULL); + + location = pstmt->stmt_location; + query_len = pstmt->stmt_len; + query_text = (char *)CleanQuerytext(queryString, &location, &query_len); + + entry->pgsm_query_id = get_pgsm_query_id_hash(query_text, query_len); + entry->counters.info.cmd_type = 0; + + pgsm_add_to_list(entry, query_text, query_len); + + /* Check that we've not exceeded max_stack_depth */ + Assert(list_length(lentries) <= max_stack_depth); + + /* The plan details are captured when the query finishes */ + pgsm_update_entry(entry, /* entry */ + (char *)query_text, /* query */ + NULL, /* PlanInfo */ + &sys_info, /* SysInfo */ + NULL, /* ErrorInfo */ + 0, /* plan_total_time */ + INSTR_TIME_GET_MILLISEC(duration), /* exec_total_time */ + rows, /* rows */ + &bufusage, /* bufusage */ #if PG_VERSION_NUM >= 130000 - &walusage, /* walusage */ + &walusage, /* walusage */ #else - NULL, /* walusage, NULL for PG <= 12 */ + NULL, #endif - NULL, - NULL, /* JumbleState */ - PGSS_FINISHED); /* pgssStoreKind */ + NULL, /* jitusage */ + false, /* reset */ + PGSM_EXEC); /* kind */ + + pgsm_store(entry); } else { @@ -1166,7 +1226,7 @@ BufferUsageAccumDiff(BufferUsage *bufusage, BufferUsage *pgBufferUsage, BufferUs * utility statements. */ static uint64 -pgss_hash_string(const char *str, int len) +pgsm_hash_string(const char *str, int len) { return DatumGetUInt64(hash_any_extended((const unsigned char *) str, len, 0)); @@ -1195,17 +1255,30 @@ pg_get_backend_status(void) return NULL; } +/* + * The caller should allocate max_len memory to name including terminating null. + * The function returns the length of the string. + */ static int -pg_get_application_name(char *application_name, bool *ok) +pg_get_application_name(char *name, int buff_size) { - PgBackendStatus *beentry = pg_get_backend_status(); + PgBackendStatus *beentry; - if (!beentry) - return snprintf(application_name, APPLICATIONNAME_LEN, "%s", "unknown"); + /* Try to read application name from GUC directly */ + if (application_name && *application_name) + snprintf(name, buff_size, "%s", application_name); + else + { + beentry = pg_get_backend_status(); - *ok = true; + if (!beentry) + snprintf(name, buff_size, "%s", "unknown"); + else + snprintf(name, buff_size, "%s", beentry->st_appname); + } - return snprintf(application_name, APPLICATIONNAME_LEN, "%s", beentry->st_appname); + /* Return length so that others don't have to calculate */ + return strlen(name); } static uint @@ -1237,141 +1310,166 @@ pg_get_client_addr(bool *ok) } static void -pgss_update_entry(pgssEntry *entry, - uint64 bucketid, - uint64 queryid, +pgsm_update_entry(pgsmEntry *entry, const char *query, - const char *comments, PlanInfo * plan_info, - CmdType cmd_type, SysInfo * sys_info, ErrorInfo * error_info, - double total_time, + double plan_total_time, + double exec_total_time, uint64 rows, BufferUsage *bufusage, WalUsage *walusage, const struct JitInstrumentation *jitusage, bool reset, - pgssStoreKind kind, - const char *app_name, - size_t app_name_len) + pgsmStoreKind kind) { int index; double old_mean; int message_len = error_info ? strlen(error_info->message) : 0; - int comments_len = comments ? strlen(comments) : 0; int sqlcode_len = error_info ? strlen(error_info->sqlcode) : 0; int plan_text_len = plan_info ? plan_info->plan_len : 0; + char app_name[APPLICATIONNAME_LEN] = ""; + int app_name_len = 0; + /* Start collecting data for next bucket and reset all counters */ + if (reset) + memset(&entry->counters, 0, sizeof(Counters)); /* volatile block */ { - volatile pgssEntry *e = (volatile pgssEntry *) entry; + volatile pgsmEntry *e = (volatile pgsmEntry *)entry; - SpinLockAcquire(&e->mutex); - /* Start collecting data for next bucket and reset all counters */ - if (reset) - memset(&entry->counters, 0, sizeof(Counters)); + if (kind == PGSM_STORE) + SpinLockAcquire(&e->mutex); - if (comments_len > 0) - _snprintf(e->counters.info.comments, comments, comments_len + 1, COMMENTS_LEN); - e->counters.state = kind; - if (kind == PGSS_PLAN) + /* Extract comments if enabled and only when the query has completed with or without error */ + if (PGSM_EXTRACT_COMMENTS && query && kind == PGSM_STORE) + { + char comments[512] = {0}; + int comments_len; + + extract_query_comments(query, comments, sizeof(comments)); + comments_len = strlen(comments); + + if (comments_len > 0) + _snprintf(e->counters.info.comments, comments, comments_len + 1, COMMENTS_LEN); + } + + if (kind == PGSM_PLAN || kind == PGSM_STORE) { if (e->counters.plancalls.calls == 0) e->counters.plancalls.usage = USAGE_INIT; + e->counters.plancalls.calls += 1; - e->counters.plantime.total_time += total_time; + e->counters.plantime.total_time += plan_total_time; if (e->counters.plancalls.calls == 1) { - e->counters.plantime.min_time = total_time; - e->counters.plantime.max_time = total_time; - e->counters.plantime.mean_time = total_time; + e->counters.plantime.min_time = plan_total_time; + e->counters.plantime.max_time = plan_total_time; + e->counters.plantime.mean_time = plan_total_time; } + else + { + /* Increment the counts, except when jstate is not NULL */ + old_mean = e->counters.plantime.mean_time; - /* Increment the counts, except when jstate is not NULL */ - old_mean = e->counters.plantime.mean_time; - e->counters.plantime.mean_time += (total_time - old_mean) / e->counters.plancalls.calls; - e->counters.plantime.sum_var_time += (total_time - old_mean) * (total_time - e->counters.plantime.mean_time); + e->counters.plantime.mean_time += (plan_total_time - old_mean) / e->counters.plancalls.calls; + e->counters.plantime.sum_var_time += (plan_total_time - old_mean) * (plan_total_time - e->counters.plantime.mean_time); - /* calculate min and max time */ - if (e->counters.plantime.min_time > total_time) - e->counters.plantime.min_time = total_time; - if (e->counters.plantime.max_time < total_time) - e->counters.plantime.max_time = total_time; + /* calculate min and max time */ + if (e->counters.plantime.min_time > plan_total_time) + e->counters.plantime.min_time = plan_total_time; + + if (e->counters.plantime.max_time < plan_total_time) + e->counters.plantime.max_time = plan_total_time; + } } - else if (kind == PGSS_FINISHED) + + if (kind == PGSM_EXEC || kind == PGSM_STORE) { if (e->counters.calls.calls == 0) e->counters.calls.usage = USAGE_INIT; + e->counters.calls.calls += 1; - e->counters.time.total_time += total_time; + e->counters.time.total_time += exec_total_time; if (e->counters.calls.calls == 1) { - e->counters.time.min_time = total_time; - e->counters.time.max_time = total_time; - e->counters.time.mean_time = total_time; + e->counters.time.min_time = exec_total_time; + e->counters.time.max_time = exec_total_time; + e->counters.time.mean_time = exec_total_time; + } + else + { + /* Increment the counts, except when jstate is not NULL */ + old_mean = e->counters.time.mean_time; + e->counters.time.mean_time += (exec_total_time - old_mean) / e->counters.calls.calls; + e->counters.time.sum_var_time += (exec_total_time - old_mean) * (exec_total_time - e->counters.time.mean_time); + + /* calculate min and max time */ + if (e->counters.time.min_time > exec_total_time) + e->counters.time.min_time = exec_total_time; + + if (e->counters.time.max_time < exec_total_time) + e->counters.time.max_time = exec_total_time; } - /* Increment the counts, except when jstate is not NULL */ - old_mean = e->counters.time.mean_time; - e->counters.time.mean_time += (total_time - old_mean) / e->counters.calls.calls; - e->counters.time.sum_var_time += (total_time - old_mean) * (total_time - e->counters.time.mean_time); - - /* calculate min and max time */ - if (e->counters.time.min_time > total_time) - e->counters.time.min_time = total_time; - if (e->counters.time.max_time < total_time) - e->counters.time.max_time = total_time; - - index = get_histogram_bucket(total_time); + index = get_histogram_bucket(exec_total_time); e->counters.resp_calls[index]++; } if (plan_text_len > 0 && !e->counters.planinfo.plan_text[0]) - _snprintf(e->counters.planinfo.plan_text, plan_info->plan_text, plan_text_len + 1, PLAN_TEXT_LEN); - - if (app_name_len > 0 && !e->counters.info.application_name[0]) - _snprintf(e->counters.info.application_name, app_name, app_name_len + 1, APPLICATIONNAME_LEN); - - e->counters.info.num_relations = num_relations; - _snprintf2(e->counters.info.relations, relations, num_relations, REL_LEN); - - e->counters.info.cmd_type = cmd_type; - - if (exec_nested_level > 0) { - if (exec_nested_level >= 0 && exec_nested_level < max_stack_depth) + e->counters.planinfo.planid = plan_info->planid; + e->counters.planinfo.plan_len = plan_text_len; + _snprintf(e->counters.planinfo.plan_text, plan_info->plan_text, plan_text_len + 1, PLAN_TEXT_LEN); + } + + /* Only should process this once when storing the data */ + if (kind == PGSM_STORE) + { + app_name_len = pg_get_application_name(app_name, APPLICATIONNAME_LEN); + + if (app_name_len > 0 && !e->counters.info.application_name[0]) + _snprintf(e->counters.info.application_name, app_name, app_name_len + 1, APPLICATIONNAME_LEN); + + e->counters.info.num_relations = num_relations; + _snprintf2(e->counters.info.relations, relations, num_relations, REL_LEN); + + if (exec_nested_level > 0) { - int parent_query_len = nested_query_txts[exec_nested_level - 1]? - strlen(nested_query_txts[exec_nested_level - 1]): 0; - e->counters.info.parentid = nested_queryids[exec_nested_level - 1]; - e->counters.info.parent_query = InvalidDsaPointer; - /* If we have a parent query, store it in the raw dsa area */ - if (parent_query_len > 0) + if (exec_nested_level >= 0 && exec_nested_level < max_stack_depth) { - char *qry_buff; - dsa_area *query_dsa_area = get_dsa_area_for_query_text(); - /* Use dsa_allocate_extended with DSA_ALLOC_NO_OOM flag, as we don't want to get an - * error if memory allocation fails.*/ - dsa_pointer qry = dsa_allocate_extended(query_dsa_area, parent_query_len+1, DSA_ALLOC_NO_OOM | DSA_ALLOC_ZERO); - if (DsaPointerIsValid(qry)) + int parent_query_len = nested_query_txts[exec_nested_level - 1]? + strlen(nested_query_txts[exec_nested_level - 1]): 0; + e->counters.info.parentid = nested_queryids[exec_nested_level - 1]; + e->counters.info.parent_query = InvalidDsaPointer; + /* If we have a parent query, store it in the raw dsa area */ + if (parent_query_len > 0) { - qry_buff = dsa_get_address(query_dsa_area, qry); - memcpy(qry_buff, nested_query_txts[exec_nested_level - 1], parent_query_len); - qry_buff[parent_query_len] = 0; - /* store the dsa pointer for parent query text */ - e->counters.info.parent_query = qry; + char *qry_buff; + dsa_area *query_dsa_area = get_dsa_area_for_query_text(); + /* Use dsa_allocate_extended with DSA_ALLOC_NO_OOM flag, as we don't want to get an + * error if memory allocation fails.*/ + dsa_pointer qry = dsa_allocate_extended(query_dsa_area, parent_query_len+1, DSA_ALLOC_NO_OOM | DSA_ALLOC_ZERO); + if (DsaPointerIsValid(qry)) + { + qry_buff = dsa_get_address(query_dsa_area, qry); + memcpy(qry_buff, nested_query_txts[exec_nested_level - 1], parent_query_len); + qry_buff[parent_query_len] = 0; + /* store the dsa pointer for parent query text */ + e->counters.info.parent_query = qry; + } } } } - } - else - { - e->counters.info.parentid = UINT64CONST(0); - e->counters.info.parent_query = InvalidDsaPointer; + else + { + e->counters.info.parentid = UINT64CONST(0); + e->counters.info.parent_query = InvalidDsaPointer; + } } if (error_info) @@ -1380,7 +1478,9 @@ pgss_update_entry(pgssEntry *entry, _snprintf(e->counters.error.sqlcode, error_info->sqlcode, sqlcode_len, SQLCODE_LEN); _snprintf(e->counters.error.message, error_info->message, message_len, ERROR_MESSAGE_LEN); } + e->counters.calls.rows += rows; + if (bufusage) { e->counters.blocks.shared_blks_hit += bufusage->shared_blks_hit; @@ -1399,8 +1499,18 @@ pgss_update_entry(pgssEntry *entry, e->counters.blocks.temp_blk_read_time += INSTR_TIME_GET_MILLISEC(bufusage->temp_blk_read_time); e->counters.blocks.temp_blk_write_time += INSTR_TIME_GET_MILLISEC(bufusage->temp_blk_write_time); #endif + + memcpy((void *)&e->counters.blocks.instr_blk_read_time, &bufusage->blk_read_time, sizeof(instr_time)); + memcpy((void *)&e->counters.blocks.instr_blk_write_time, &bufusage->blk_write_time, sizeof(instr_time)); + + #if PG_VERSION_NUM >= 150000 + memcpy((void *)&e->counters.blocks.instr_temp_blk_read_time, &bufusage->temp_blk_read_time, sizeof(bufusage->temp_blk_read_time)); + memcpy((void *)&e->counters.blocks.instr_temp_blk_write_time, &bufusage->temp_blk_write_time, sizeof(bufusage->temp_blk_write_time)); + #endif } - e->counters.calls.usage += USAGE_EXEC(total_time); + + e->counters.calls.usage += USAGE_EXEC(exec_total_time + plan_total_time); + if (sys_info) { e->counters.sysinfo.utime += sys_info->utime; @@ -1428,39 +1538,173 @@ pgss_update_entry(pgssEntry *entry, if (INSTR_TIME_GET_MILLISEC(jitusage->emission_counter)) e->counters.jitinfo.jit_emission_count++; e->counters.jitinfo.jit_emission_time += INSTR_TIME_GET_MILLISEC(jitusage->emission_counter); + + memcpy((void *)&e->counters.jitinfo.instr_generation_counter, &jitusage->generation_counter, sizeof(instr_time)); + memcpy((void *)&e->counters.jitinfo.instr_inlining_counter, &jitusage->inlining_counter, sizeof(instr_time)); + memcpy((void *)&e->counters.jitinfo.instr_optimization_counter, &jitusage->optimization_counter, sizeof(instr_time)); + memcpy((void *)&e->counters.jitinfo.instr_emission_counter, &jitusage->emission_counter, sizeof(instr_time)); } - SpinLockRelease(&e->mutex); + + if (kind == PGSM_STORE) + SpinLockRelease(&e->mutex); } } static void -pgss_store_error(uint64 queryid, - const char *query, - ErrorData *edata) +pgsm_store_error(const char *query, ErrorData *edata) { - ErrorInfo error_info; + pgsmEntry *entry; + uint64 queryid = 0; + int len = strlen(query); - error_info.elevel = edata->elevel; - snprintf(error_info.message, ERROR_MESSAGE_LEN, "%s", edata->message); - snprintf(error_info.sqlcode, SQLCODE_LEN, "%s", unpack_sql_state(edata->sqlerrcode)); + if (!query || len == 0) + return; - pgss_store(queryid, /* query id */ - query, /* query text */ - 0, /* query location */ - strlen(query), /* query length */ - NULL, /* PlanInfo */ - 0, /* CmdType */ - NULL, /* SysInfo */ - &error_info, /* ErrorInfo */ - 0, /* total_time */ - 0, /* rows */ - NULL, /* bufusage */ - NULL, /* walusage */ - NULL, /* JumbleState */ - NULL, - PGSS_ERROR); /* pgssStoreKind */ + len = strlen(query); + + queryid = pgsm_hash_string(query, len); + + entry = pgsm_create_hash_entry(0, queryid, NULL); + entry->query_text.query_pointer = pnstrdup(query, len); + + entry->counters.error.elevel = edata->elevel; + snprintf(entry->counters.error.message, ERROR_MESSAGE_LEN, "%s", edata->message); + snprintf(entry->counters.error.sqlcode, SQLCODE_LEN, "%s", unpack_sql_state(edata->sqlerrcode)); + + pgsm_store(entry); } +static void +pgsm_add_to_list(pgsmEntry *entry, char *query_text, int query_len) +{ + /* Switch to pgsm memory context */ + MemoryContext oldctx = MemoryContextSwitchTo(pgsm_get_ss()->pgsm_mem_cxt); + entry->query_text.query_pointer = pnstrdup(query_text, query_len); + lentries = lappend(lentries, entry); + MemoryContextSwitchTo(oldctx); +} + +static pgsmEntry* +pgsm_get_entry_for_query(uint64 queryid, PlanInfo *plan_info, const char* query_text, int query_len, bool create) +{ + pgsmEntry *entry = NULL; + ListCell *lc = NULL; + + /* First bet is on the last entry */ + if (lentries == NIL && !create) + return NULL; + + if (lentries) + { + entry = (pgsmEntry *)llast(lentries); + if(entry->key.queryid == queryid) + return entry; + + foreach(lc, lentries) + { + entry = lfirst(lc); + if(entry->key.queryid == queryid) + return entry; + } + } + if (create && query_text) + { + /* + * At this point, we don't know which bucket this query will land in, so passing + * 0. The store function MUST later update it based on the current bucket value. + * The correct bucket value will be needed then to search the hash table, or create + * the appropriate entry. + */ + entry = pgsm_create_hash_entry(0, queryid, plan_info); + + /* Update other member that are not counters, so that we don't have to worry about these. */ + entry->pgsm_query_id = get_pgsm_query_id_hash(query_text, query_len); + pgsm_add_to_list(entry, (char *)query_text, query_len); + } + + return entry; +} + +static void +pgsm_cleanup_callback(void *arg) +{ + /* Reset the memory context holding the list */ + MemoryContextReset(pgsm_get_ss()->pgsm_mem_cxt); + lentries = NIL; + callback_setup = false; +} +/* + * Function encapsulating some external calls for filling up the hash key data structure. + * The bucket_id may not be known at this stage. So pass any value that you may wish. + */ +static pgsmEntry * +pgsm_create_hash_entry(uint64 bucket_id, uint64 queryid, PlanInfo *plan_info) +{ + pgsmEntry *entry; + int sec_ctx; + bool found_client_addr = false; + char app_name[APPLICATIONNAME_LEN] = ""; + char *app_name_ptr = app_name; + int app_name_len = 0; + MemoryContext oldctx; + char *datname = NULL; + char *username = NULL; + + /* Create an entry in the pgsm memory context */ + oldctx = MemoryContextSwitchTo(pgsm_get_ss()->pgsm_mem_cxt); + entry = palloc0(sizeof(pgsmEntry)); + + /* + * Get the user ID. Let's use this instead of GetUserID as this + * won't throw an assertion in case of an error. + */ + GetUserIdAndSecContext((Oid *) &entry->key.userid, &sec_ctx); + + /* Get the application name and set appid */ + app_name_len = pg_get_application_name(app_name, APPLICATIONNAME_LEN); + entry->key.appid = pgsm_hash_string((const char *)app_name_ptr, app_name_len); + + /* client address */ + entry->key.ip = pg_get_client_addr(&found_client_addr); + + /* PlanID, if there is one */ + entry->key.planid = plan_info ? plan_info->planid : 0; + + /* Set remaining data */ + entry->key.dbid = MyDatabaseId; + entry->key.queryid = queryid; + entry->key.bucket_id = bucket_id; + +#if PG_VERSION_NUM < 140000 + entry->key.toplevel = 1; +#else + entry->key.toplevel = ((exec_nested_level + plan_nested_level) == 0); +#endif + + if (IsTransactionState()) + { + datname = get_database_name(entry->key.dbid); + username = GetUserNameFromId(entry->key.userid, true); + } + + if (!datname) + datname = pnstrdup("", sizeof(entry->datname) - 1); + + if (!username) + username = pnstrdup("", sizeof(entry->username) - 1); + + snprintf(entry->datname, sizeof(entry->datname), "%s", datname); + snprintf(entry->username, sizeof(entry->username), "%s", username); + + pfree(datname); + pfree(username); + + MemoryContextSwitchTo(oldctx); + + return entry; +} + + /* * Store some statistics for a statement. * @@ -1472,179 +1716,50 @@ pgss_store_error(uint64 queryid, * query string. total_time, rows, bufusage are ignored in this case. */ static void -pgss_store(uint64 queryid, - const char *query, - int query_location, - int query_len, - PlanInfo * plan_info, - CmdType cmd_type, - SysInfo * sys_info, - ErrorInfo * error_info, - double total_time, - uint64 rows, - BufferUsage *bufusage, - WalUsage *walusage, - const struct JitInstrumentation *jitusage, - JumbleState *jstate, - pgssStoreKind kind) +pgsm_store(pgsmEntry *entry) { - pgssHashKey key; - pgssEntry *entry; - pgssSharedState *pgss; - char *app_name_ptr; - char app_name[APPLICATIONNAME_LEN] = ""; - int app_name_len = 0; - bool reset = false; - uint64 pgsm_query_id = 0; + pgsmEntry *shared_hash_entry; + pgsmSharedState *pgsm; + bool found; uint64 bucketid; uint64 prev_bucket_id; - uint64 userid; - uint64 planid; - uint64 appid = 0; - int norm_query_len = 0; - char *norm_query = NULL; - char comments[512] = ""; - bool found_app_name = false; - bool found_client_addr = false; - uint client_addr = 0; - bool found; + bool reset = false; /* Only used in update function - HAMID */ + char *query; + int query_len; + BufferUsage bufusage; + WalUsage walusage; + JitInstrumentation jitusage; /* Safety check... */ if (!IsSystemInitialized()) return; - pgss = pgsm_get_ss(); + pgsm = pgsm_get_ss(); -#if PG_VERSION_NUM >= 140000 - - /* - * Nothing to do if compute_query_id isn't enabled and no other module - * computed a query identifier. - */ - if (queryid == UINT64CONST(0)) - return; -#endif - - query = CleanQuerytext(query, &query_location, &query_len); - -#if PG_VERSION_NUM < 140000 - - /* - * For utility statements, we just hash the query string to get an ID. - */ - if (queryid == UINT64CONST(0)) - { - queryid = pgss_hash_string(query, query_len); - - /* - * If we are unlucky enough to get a hash of zero(invalid), use - * queryID as 2 instead, queryID 1 is already in use for normal - * statements. - */ - if (queryid == UINT64CONST(0)) - queryid = UINT64CONST(2); - } -#endif - - Assert(query != NULL); - if (kind == PGSS_ERROR) - { - int sec_ctx; - - GetUserIdAndSecContext((Oid *) &userid, &sec_ctx); - } - else - userid = GetUserId(); - - /* Try to read application name from GUC directly */ - if (application_name && *application_name) - { - app_name_ptr = application_name; - appid = djb2_hash_str((unsigned char *) application_name, &app_name_len); - } - else - { - app_name_len = pg_get_application_name(app_name, &found_app_name); - if (found_app_name) - appid = djb2_hash((unsigned char *) app_name, app_name_len); - app_name_ptr = app_name; - } - - if (!found_client_addr) - client_addr = pg_get_client_addr(&found_client_addr); - - planid = plan_info ? plan_info->planid : 0; - - /* Extract comments if enabled. */ - if (PGSM_EXTRACT_COMMENTS) - extract_query_comments(query, comments, sizeof(comments)); - - prev_bucket_id = pg_atomic_read_u64(&pgss->current_wbucket); - bucketid = get_next_wbucket(pgss); + /* We should lock the hash table here what if the bucket is removed; e.g. reset is called - HAMID */ + prev_bucket_id = pg_atomic_read_u64(&pgsm->current_wbucket); + bucketid = get_next_wbucket(pgsm); if (bucketid != prev_bucket_id) reset = true; - key.bucket_id = bucketid; - key.userid = userid; - key.dbid = MyDatabaseId; - key.queryid = queryid; - key.ip = client_addr; - key.planid = planid; - key.appid = appid; -#if PG_VERSION_NUM < 140000 - key.toplevel = 1; -#else - key.toplevel = ((exec_nested_level + plan_nested_level) == 0); -#endif + entry->key.bucket_id = bucketid; + query = entry->query_text.query_pointer; + query_len = strlen(query); - LWLockAcquire(pgss->lock, LW_SHARED); + /* + * Acquire a share lock to start with. We'd have to acquire exclusive + * if we need ot create the entry. + */ + LWLockAcquire(pgsm->lock, LW_SHARED); + shared_hash_entry = (pgsmEntry *) pgsm_hash_find(get_pgsmHash(), &entry->key, &found); - entry = (pgssEntry *) pgsm_hash_find(get_pgssHash(), &key, &found); - if (!entry) + if (!shared_hash_entry) { dsa_pointer dsa_query_pointer; dsa_area *query_dsa_area; char *query_buff; - /* - * Create a new, normalized query string if caller asked. We don't - * need to hold the lock while doing this work. (Note: in any case, - * it's possible that someone else creates a duplicate hashtable entry - * in the interval where we don't hold the lock below. That case is - * handled by entry_alloc. - */ - if (jstate) - { - norm_query_len = query_len; - - LWLockRelease(pgss->lock); - norm_query = generate_normalized_query(jstate, query, - query_location, - &norm_query_len, - GetDatabaseEncoding()); - LWLockAcquire(pgss->lock, LW_SHARED); - - pgsm_query_id = pgss_hash_string(norm_query, norm_query_len); - - /* Free up norm_query if we don't intend to show normalized version in the view */ - if (PGSM_NORMALIZED_QUERY) - { - query_len = norm_query_len; - } - else - { - if (norm_query) - pfree(norm_query); - - norm_query = NULL; - } - } - else - { - pgsm_query_id = pgss_hash_string(query, query_len); - } - /* New query, truncate length if necessary. */ if (query_len > PGSM_QUERY_MAX_LEN) query_len = PGSM_QUERY_MAX_LEN; @@ -1654,27 +1769,28 @@ pgss_store(uint64 queryid, dsa_query_pointer = dsa_allocate_extended(query_dsa_area, query_len+1, DSA_ALLOC_NO_OOM | DSA_ALLOC_ZERO); if (!DsaPointerIsValid(dsa_query_pointer)) { - LWLockRelease(pgss->lock); - if (norm_query) - pfree(norm_query); + LWLockRelease(pgsm->lock); return; } + /* Get the memory address from DSA pointer and copy the query text in local variable */ query_buff = dsa_get_address(query_dsa_area, dsa_query_pointer); - memcpy(query_buff, norm_query ? norm_query : query, query_len); - /* OK to create a new hashtable entry */ + memcpy(query_buff, query, query_len); + LWLockRelease(pgsm->lock); + LWLockAcquire(pgsm->lock, LW_EXCLUSIVE); + + /* OK to create a new hashtable entry */ PGSM_DISABLE_ERROR_CAPUTRE(); { PG_TRY(); { - entry = hash_entry_alloc(pgss, &key, GetDatabaseEncoding()); + shared_hash_entry = hash_entry_alloc(pgsm, &entry->key, GetDatabaseEncoding()); } PG_CATCH(); { - LWLockRelease(pgss->lock); - if (norm_query) - pfree(norm_query); + LWLockRelease(pgsm->lock); + if (DsaPointerIsValid(dsa_query_pointer)) dsa_free(query_dsa_area, dsa_query_pointer); PG_RE_THROW(); @@ -1682,50 +1798,91 @@ pgss_store(uint64 queryid, PG_END_TRY(); }PGSM_END_DISABLE_ERROR_CAPTURE(); - if (entry == NULL) + if (shared_hash_entry == NULL) { - LWLockRelease(pgss->lock); - if (norm_query) - pfree(norm_query); + /* Out of memory; report only if the state has changed now. Otherwise we risk filling up the log file with these message. */ + if (!IsSystemOOM()) + { + pgsm->pgsm_oom = true; + + ereport(WARNING, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("[pg_stat_monitor] pgsm_store: Hash table is out of memory and can no longer store queries!"), + errdetail("You may reset the view or when the buckets are deallocated, pg_stat_monitor will resume saving " \ + "queries. Alternatively, try increasing the value of pg_stat_monitor.pgsm_max."))); + } + + LWLockRelease(pgsm->lock); + if (DsaPointerIsValid(dsa_query_pointer)) dsa_free(query_dsa_area, dsa_query_pointer); + return; } - entry->query_pos = dsa_query_pointer; - entry->pgsm_query_id = pgsm_query_id; - } - else - { - #if USE_DYNAMIC_HASH - if(entry) - dshash_release_lock(get_pgssHash(), entry); - #endif + /* If we already have the pointer set, free this one */ + if (DsaPointerIsValid(shared_hash_entry->query_text.query_pos)) + dsa_free(query_dsa_area, dsa_query_pointer); + else + shared_hash_entry->query_text.query_pos = dsa_query_pointer; + + shared_hash_entry->pgsm_query_id = entry->pgsm_query_id; + shared_hash_entry->encoding = entry->encoding; + shared_hash_entry->counters.info.cmd_type = entry->counters.info.cmd_type; + + snprintf(shared_hash_entry->datname, sizeof(shared_hash_entry->datname), "%s", entry->datname); + snprintf(shared_hash_entry->username, sizeof(shared_hash_entry->username), "%s", entry->username); } - if (jstate == NULL) - pgss_update_entry(entry, /* entry */ - bucketid, /* bucketid */ - queryid, /* queryid */ - query, /* query */ - comments, /* comments */ - plan_info, /* PlanInfo */ - cmd_type, /* CmdType */ - sys_info, /* SysInfo */ - error_info, /* ErrorInfo */ - total_time, /* total_time */ - rows, /* rows */ - bufusage, /* bufusage */ - walusage, /* walusage */ - jitusage, - reset, /* reset */ - kind, /* kind */ - app_name_ptr, - app_name_len); + /* bufusage */ + bufusage.shared_blks_hit = entry->counters.blocks.shared_blks_hit; + bufusage.shared_blks_read = entry->counters.blocks.shared_blks_read; + bufusage.shared_blks_dirtied = entry->counters.blocks.shared_blks_dirtied; + bufusage.shared_blks_written = entry->counters.blocks.shared_blks_written; + bufusage.local_blks_hit = entry->counters.blocks.local_blks_hit; + bufusage.local_blks_read = entry->counters.blocks.local_blks_read; + bufusage.local_blks_dirtied = entry->counters.blocks.local_blks_dirtied; + bufusage.local_blks_written = entry->counters.blocks.local_blks_written; + bufusage.temp_blks_read = entry->counters.blocks.temp_blks_read; + bufusage.temp_blks_written = entry->counters.blocks.temp_blks_written; - LWLockRelease(pgss->lock); - if (norm_query) - pfree(norm_query); + memcpy(&bufusage.blk_read_time, &entry->counters.blocks.instr_blk_read_time, sizeof(instr_time)); + memcpy(&bufusage.blk_write_time, &entry->counters.blocks.instr_blk_write_time, sizeof(instr_time)); + + #if PG_VERSION_NUM >= 150000 + memcpy(&bufusage.temp_blk_read_time, &entry->counters.blocks.instr_temp_blk_read_time, sizeof(instr_time)); + memcpy(&bufusage.temp_blk_write_time, &entry->counters.blocks.instr_temp_blk_write_time, sizeof(instr_time)); + #endif + + /* walusage */ + walusage.wal_records = entry->counters.walusage.wal_records; + walusage.wal_fpi = entry->counters.walusage.wal_fpi; + walusage.wal_bytes = entry->counters.walusage.wal_bytes; + + /* jit */ + jitusage.created_functions = entry->counters.jitinfo.jit_functions; + memcpy(&jitusage.generation_counter, &entry->counters.jitinfo.instr_generation_counter, sizeof(instr_time)); + memcpy(&jitusage.inlining_counter, &entry->counters.jitinfo.instr_inlining_counter, sizeof(instr_time)); + memcpy(&jitusage.optimization_counter, &entry->counters.jitinfo.instr_optimization_counter, sizeof(instr_time)); + memcpy(&jitusage.emission_counter, &entry->counters.jitinfo.instr_emission_counter, sizeof(instr_time)); + + + pgsm_update_entry(shared_hash_entry, /* entry */ + query, /* query */ + &entry->counters.planinfo, /* PlanInfo */ + &entry->counters.sysinfo, /* SysInfo */ + &entry->counters.error, /* ErrorInfo */ + entry->counters.plantime.total_time, /* plan_total_time */ + entry->counters.time.total_time, /* exec_total_time */ + entry->counters.calls.rows, /* rows */ + &bufusage, /* bufusage */ + &walusage, /* walusage */ + &jitusage, /* jitusage */ + reset, /* reset */ + PGSM_STORE); + + memset(&entry->counters, 0, sizeof(entry->counters)); + LWLockRelease(pgsm->lock); } /* @@ -1734,7 +1891,7 @@ pgss_store(uint64 queryid, Datum pg_stat_monitor_reset(PG_FUNCTION_ARGS) { - pgssSharedState *pgss; + pgsmSharedState *pgsm; /* Safety check... */ if (!IsSystemInitialized()) @@ -1742,11 +1899,11 @@ pg_stat_monitor_reset(PG_FUNCTION_ARGS) (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("pg_stat_monitor: must be loaded via shared_preload_libraries"))); - pgss = pgsm_get_ss(); - LWLockAcquire(pgss->lock, LW_EXCLUSIVE); + pgsm = pgsm_get_ss(); + LWLockAcquire(pgsm->lock, LW_EXCLUSIVE); hash_entry_dealloc(-1, -1, NULL); - LWLockRelease(pgss->lock); + LWLockRelease(pgsm->lock); PG_RETURN_VOID(); } @@ -1780,9 +1937,9 @@ IsBucketValid(uint64 bucketid) long secs; int microsecs; TimestampTz current_tz = GetCurrentTimestamp(); - pgssSharedState *pgss = pgsm_get_ss(); + pgsmSharedState *pgsm = pgsm_get_ss(); - TimestampDifference(pgss->bucket_start_time[bucketid], current_tz,&secs, µsecs); + TimestampDifference(pgsm->bucket_start_time[bucketid], current_tz,&secs, µsecs); if (secs > (PGSM_BUCKET_TIME * PGSM_MAX_BUCKETS)) return false; @@ -1801,36 +1958,40 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, MemoryContext per_query_ctx; MemoryContext oldcontext; PGSM_HASH_SEQ_STATUS hstat; - pgssEntry *entry; - pgssSharedState *pgss; - char *query_txt = NULL; - char *parent_query_txt = NULL; + pgsmEntry *entry; + pgsmSharedState *pgsm; int expected_columns = (api_version >= PGSM_V2_0)?PG_STAT_MONITOR_COLS_V2_0:PG_STAT_MONITOR_COLS_V1_0; /* Disallow old api usage */ if (api_version < PGSM_V2_0) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("pg_stat_monitor: API version not supported"), - errhint("upgrade pg_stat_monitor extension"))); + errmsg("[pg_stat_monitor] pg_stat_monitor_internal: API version not supported."), + errhint("Upgrade pg_stat_monitor extension"))); /* Safety check... */ if (!IsSystemInitialized()) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("pg_stat_monitor: must be loaded via shared_preload_libraries"))); + errmsg("[pg_stat_monitor] pg_stat_monitor_internal: Must be loaded via shared_preload_libraries."))); + + /* Out of memory? */ + if (IsSystemOOM()) + ereport(WARNING, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("[pg_stat_monitor] pg_stat_monitor_internal: Hash table is out of memory and can no longer store queries!"), + errdetail("You may reset the view or when the buckets are deallocated, pg_stat_monitor will resume saving " \ + "queries. Alternatively, try increasing the value of pg_stat_monitor.pgsm_max."))); /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("pg_stat_monitor: set-valued function called in context that cannot accept a set"))); + errmsg("[pg_stat_monitor] pg_stat_monitor_internal: Set-valued function called in context that cannot accept a set."))); if (!(rsinfo->allowedModes & SFRM_Materialize)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("pg_stat_monitor: materialize mode required, but it is not " \ - "allowed in this context"))); - - pgss = pgsm_get_ss(); + errmsg("[pg_stat_monitor] pg_stat_monitor_internal: Materialize mode required, but it is not " \ + "allowed in this context."))); /* Switch into long-lived context to construct returned data structures */ per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; @@ -1838,10 +1999,10 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, /* Build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "pg_stat_monitor: return type must be a row type"); + elog(ERROR, "[pg_stat_monitor] pg_stat_monitor_internal: Return type must be a row type."); if (tupdesc->natts != expected_columns) - elog(ERROR, "pg_stat_monitor: incorrect number of output arguments, required %d", tupdesc->natts); + elog(ERROR, "[pg_stat_monitor] pg_stat_monitor_internal: Incorrect number of output arguments, received %d, required %d.", tupdesc->natts, expected_columns); tupstore = tuplestore_begin_heap(true, false, work_mem); rsinfo->returnMode = SFRM_Materialize; @@ -1850,9 +2011,9 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, MemoryContextSwitchTo(oldcontext); - LWLockAcquire(pgss->lock, LW_SHARED); - - pgsm_hash_seq_init(&hstat, get_pgssHash(), false); + pgsm = pgsm_get_ss(); + LWLockAcquire(pgsm->lock, LW_SHARED); + pgsm_hash_seq_init(&hstat, get_pgsmHash(), false); while ((entry = pgsm_hash_seq_next(&hstat)) != NULL) { @@ -1863,25 +2024,27 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, double stddev; uint64 queryid = entry->key.queryid; int64 bucketid = entry->key.bucket_id; - uint64 dbid = entry->key.dbid; - uint64 userid = entry->key.userid; - int64 ip = entry->key.ip; + Oid dbid = entry->key.dbid; + Oid userid = entry->key.userid; + uint32 ip = entry->key.ip; uint64 planid = entry->key.planid; uint64 pgsm_query_id = entry->pgsm_query_id; dsa_area *query_dsa_area; char *query_ptr; + char *query_txt = NULL; + char *parent_query_txt = NULL; + + bool toplevel = entry->key.toplevel; #if PG_VERSION_NUM < 140000 - bool toplevel = 1; bool is_allowed_role = is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_ALL_STATS); #else bool is_allowed_role = is_member_of_role(GetUserId(), ROLE_PG_READ_ALL_STATS); - bool toplevel = entry->key.toplevel; #endif /* Load the query text from dsa area */ - if (DsaPointerIsValid(entry->query_pos)) + if (DsaPointerIsValid(entry->query_text.query_pos)) { query_dsa_area = get_dsa_area_for_query_text(); - query_ptr = dsa_get_address(query_dsa_area, entry->query_pos); + query_ptr = dsa_get_address(query_dsa_area, entry->query_text.query_pos); query_txt = pstrdup(query_ptr); } else @@ -1889,7 +2052,7 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, /* copy counters to a local variable to keep locking time short */ { - volatile pgssEntry *e = (volatile pgssEntry *) entry; + volatile pgsmEntry *e = (volatile pgsmEntry *) entry; SpinLockAcquire(&e->mutex); tmp = e->counters; @@ -1905,13 +2068,8 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, if (!IsBucketValid(bucketid)) { - if (tmp.state == PGSS_FINISHED) - continue; - } - - /* Skip queries such as, $1, $2 := $3, etc. */ - if (tmp.state == PGSS_PARSE || tmp.state == PGSS_PLAN) continue; + } /* read the parent query text if any */ if (tmp.info.parentid != UINT64CONST(0)) @@ -1931,15 +2089,21 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, /* userid at column number 1 */ values[i++] = ObjectIdGetDatum(userid); + /* userid at column number 1 */ + values[i++] = CStringGetTextDatum(entry->username); + /* dbid at column number 2 */ values[i++] = ObjectIdGetDatum(dbid); + /* userid at column number 1 */ + values[i++] = CStringGetTextDatum(entry->datname); + /* * ip address at column number 3, Superusers or members of * pg_read_all_stats members are allowed */ if (is_allowed_role || userid == GetUserId()) - values[i++] = Int64GetDatumFast(ip); + values[i++] = UInt32GetDatum(ip); else nulls[i++] = true; @@ -1989,10 +2153,6 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, values[i++] = UInt64GetDatum(pgsm_query_id); - /* state at column number 8 for V1.0 API*/ - if (api_version <= PGSM_V1_0) - values[i++] = Int64GetDatumFast(tmp.state); - /* parentid at column number 9 */ if (tmp.info.parentid != UINT64CONST(0)) { @@ -2015,8 +2175,8 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, if (tmp.info.num_relations > 0) { int j; - char *text_str = palloc0(1024); - char *tmp_str = palloc0(1024); + char *text_str = palloc0(TOTAL_RELS_LENGTH); + char *tmp_str = palloc0(TOTAL_RELS_LENGTH); bool first = true; /* @@ -2062,7 +2222,7 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, values[i++] = CStringGetTextDatum(tmp.error.message); /* bucket_start_time at column number 15 */ - values[i++] = TimestampTzGetDatum(pgss->bucket_start_time[entry->key.bucket_id]); + values[i++] = TimestampTzGetDatum(pgsm->bucket_start_time[entry->key.bucket_id]); if (tmp.calls.calls == 0) { @@ -2138,12 +2298,8 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, values[i++] = Int64GetDatumFast(tmp.blocks.temp_blks_written); values[i++] = Float8GetDatumFast(tmp.blocks.blk_read_time); values[i++] = Float8GetDatumFast(tmp.blocks.blk_write_time); - - if (api_version >= PGSM_V2_0) - { - values[i++] = Float8GetDatumFast(tmp.blocks.temp_blk_read_time); - values[i++] = Float8GetDatumFast(tmp.blocks.temp_blk_write_time); - } + values[i++] = Float8GetDatumFast(tmp.blocks.temp_blk_read_time); + values[i++] = Float8GetDatumFast(tmp.blocks.temp_blk_write_time); /* resp_calls at column number 41 */ values[i++] = IntArrayGetTextDatum(tmp.resp_calls, hist_bucket_count_total); @@ -2179,39 +2335,36 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, else nulls[i++] = true; - if (api_version >= PGSM_V2_0) - { - values[i++] = Int64GetDatumFast(tmp.jitinfo.jit_functions); - values[i++] = Float8GetDatumFast(tmp.jitinfo.jit_generation_time); - values[i++] = Int64GetDatumFast(tmp.jitinfo.jit_inlining_count); - values[i++] = Float8GetDatumFast(tmp.jitinfo.jit_inlining_time); - values[i++] = Int64GetDatumFast(tmp.jitinfo.jit_optimization_count); - values[i++] = Float8GetDatumFast(tmp.jitinfo.jit_optimization_time); - values[i++] = Int64GetDatumFast(tmp.jitinfo.jit_emission_count); - values[i++] = Float8GetDatumFast(tmp.jitinfo.jit_emission_time); - } - + values[i++] = Int64GetDatumFast(tmp.jitinfo.jit_functions); + values[i++] = Float8GetDatumFast(tmp.jitinfo.jit_generation_time); + values[i++] = Int64GetDatumFast(tmp.jitinfo.jit_inlining_count); + values[i++] = Float8GetDatumFast(tmp.jitinfo.jit_inlining_time); + values[i++] = Int64GetDatumFast(tmp.jitinfo.jit_optimization_count); + values[i++] = Float8GetDatumFast(tmp.jitinfo.jit_optimization_time); + values[i++] = Int64GetDatumFast(tmp.jitinfo.jit_emission_count); + values[i++] = Float8GetDatumFast(tmp.jitinfo.jit_emission_time); } values[i++] = BoolGetDatum(toplevel); - values[i++] = BoolGetDatum(pg_atomic_read_u64(&pgss->current_wbucket) != bucketid); + values[i++] = BoolGetDatum(pg_atomic_read_u64(&pgsm->current_wbucket) != bucketid); /* clean up and return the tuplestore */ tuplestore_putvalues(tupstore, tupdesc, values, nulls); + + if(query_txt) + pfree(query_txt); + if(parent_query_txt) + pfree(parent_query_txt); } /* clean up and return the tuplestore */ pgsm_hash_seq_term(&hstat); - LWLockRelease(pgss->lock); + LWLockRelease(pgsm->lock); - if(query_txt) - pfree(query_txt); - if(parent_query_txt) - pfree(parent_query_txt); tuplestore_donestoring(tupstore); } static uint64 -get_next_wbucket(pgssSharedState *pgss) +get_next_wbucket(pgsmSharedState *pgsm) { struct timeval tv; uint64 current_bucket_sec; @@ -2221,7 +2374,7 @@ get_next_wbucket(pgssSharedState *pgss) bool update_bucket = false; gettimeofday(&tv, NULL); - current_bucket_sec = pg_atomic_read_u64(&pgss->prev_bucket_sec); + current_bucket_sec = pg_atomic_read_u64(&pgsm->prev_bucket_sec); /* * If current bucket expired we loop attempting to update prev_bucket_sec. @@ -2240,13 +2393,13 @@ get_next_wbucket(pgssSharedState *pgss) */ while ((tv.tv_sec - (uint)current_bucket_sec) >= ((uint)PGSM_BUCKET_TIME)) { - if (pg_atomic_compare_exchange_u64(&pgss->prev_bucket_sec, ¤t_bucket_sec, (uint64)tv.tv_sec)) + if (pg_atomic_compare_exchange_u64(&pgsm->prev_bucket_sec, ¤t_bucket_sec, (uint64)tv.tv_sec)) { update_bucket = true; break; } - current_bucket_sec = pg_atomic_read_u64(&pgss->prev_bucket_sec); + current_bucket_sec = pg_atomic_read_u64(&pgsm->prev_bucket_sec); } if (update_bucket) @@ -2255,25 +2408,99 @@ get_next_wbucket(pgssSharedState *pgss) new_bucket_id = (tv.tv_sec / PGSM_BUCKET_TIME) % PGSM_MAX_BUCKETS; /* Update bucket id and retrieve the previous one. */ - prev_bucket_id = pg_atomic_exchange_u64(&pgss->current_wbucket, new_bucket_id); + prev_bucket_id = pg_atomic_exchange_u64(&pgsm->current_wbucket, new_bucket_id); - LWLockAcquire(pgss->lock, LW_EXCLUSIVE); + LWLockAcquire(pgsm->lock, LW_EXCLUSIVE); hash_entry_dealloc(new_bucket_id, prev_bucket_id, NULL); - LWLockRelease(pgss->lock); + LWLockRelease(pgsm->lock); /* Allign the value in prev_bucket_sec to the bucket start time */ tv.tv_sec = (tv.tv_sec) - (tv.tv_sec % PGSM_BUCKET_TIME); - pg_atomic_exchange_u64(&pgss->prev_bucket_sec, (uint64)tv.tv_sec); + pg_atomic_exchange_u64(&pgsm->prev_bucket_sec, (uint64)tv.tv_sec); - pgss->bucket_start_time[new_bucket_id] = (TimestampTz) tv.tv_sec - + pgsm->bucket_start_time[new_bucket_id] = (TimestampTz) tv.tv_sec - ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY); - pgss->bucket_start_time[new_bucket_id] = pgss->bucket_start_time[new_bucket_id] * USECS_PER_SEC; + pgsm->bucket_start_time[new_bucket_id] = pgsm->bucket_start_time[new_bucket_id] * USECS_PER_SEC; return new_bucket_id; } - return pg_atomic_read_u64(&pgss->current_wbucket); + return pg_atomic_read_u64(&pgsm->current_wbucket); +} + +/* + * This function expects a NORMALIZED query as the input. + * It iterates oer the normalized query skipping comments and + * multiple spaces. All spaces are converted to ' ' so that we + * the calculation is independent of the space type whether + * newline, tab, or any other type. Trailing and leading spaces + * are also removed before calculating the hash. + */ +uint64 +get_pgsm_query_id_hash(const char *norm_query, int norm_len) +{ + char *query = palloc(norm_len + 1); + char *q_iter = query; + char *norm_q_iter = (char *)norm_query; + uint64 pgsm_query_id = 0; + + while (norm_q_iter && *norm_q_iter && norm_q_iter < (norm_query + norm_len)) + { + /* Skip multiline comments, + 1 is safe even if we've reach end of string */ + if (*norm_q_iter == '/' && *(norm_q_iter + 1) == '*') + { + while (*norm_q_iter && *norm_q_iter != '*' && *(norm_q_iter + 1) != '/') + norm_q_iter++; + + /* Skip the '/' if the current character is valid. */ + if (*norm_q_iter) + norm_q_iter++; + } + + /* Skip single line comments, + 1 is safe even if we've reach end of string */ + if (*norm_q_iter == '-' && *(norm_q_iter + 1) == '-') + { + while (*norm_q_iter && *norm_q_iter != '\n') + norm_q_iter++; + } + + /* Skip white spaces */ + if (scanner_isspace(*norm_q_iter)) + { + while (scanner_isspace(*++norm_q_iter)); + + /* + * Let's replace it with a simple space. -1 is safe as we + * are making sure we are not at the start of the string. + */ + if (q_iter != query && !scanner_isspace(*(q_iter - 1))) + *q_iter++ = ' '; + + continue; + } + + *q_iter++ = *norm_q_iter++; + } + + /* Ensure we have a terminating zero at the end */ + *q_iter = '\0'; + + /* Get rid of trailing spaces */ + while (q_iter > query && *q_iter == '\0') + { + q_iter--; + + /* Continue reducing the string size if space is found. */ + if (scanner_isspace(*q_iter)) + *q_iter = '\0'; + } + + /* Calcuate the hash. */ + pgsm_query_id = pgsm_hash_string(query, strlen(query)); + + pfree(query); + return pgsm_query_id; } #if PG_VERSION_NUM < 140000 @@ -2300,7 +2527,7 @@ AppendJumble(JumbleState *jstate, const unsigned char *item, Size size) { uint64 start_hash; - start_hash = pgss_hash_string((char *)jumble, JUMBLE_SIZE); + start_hash = pgsm_hash_string((char *)jumble, JUMBLE_SIZE); memcpy(jumble, &start_hash, sizeof(start_hash)); jumble_len = sizeof(start_hash); } @@ -2408,7 +2635,7 @@ JumbleRangeTable(JumbleState *jstate, List *rtable, CmdType cmd_type) APP_JUMB_STRING(rte->enrname); break; default: - elog(ERROR, "unrecognized RTE kind: %d", (int) rte->rtekind); + elog(ERROR, "[pg_stat_monitor] JumbleRangeTable: unrecognized RTE kind: %d.", (int) rte->rtekind); break; } } @@ -2909,7 +3136,7 @@ JumbleExpr(JumbleState *jstate, Node *node) break; default: /* Only a warning, since we can stumble along anyway */ - elog(INFO, "unrecognized node type: %d", + elog(INFO, "[pg_stat_monitor] JumbleExpr: unrecognized node type: %d.", (int) nodeTag(node)); break; } @@ -3273,16 +3500,10 @@ pgsm_emit_log_hook(ErrorData *edata) if (MyProc == NULL) goto exit; - if (PGSM_ERROR_CAPTURE_ENABLED && - (edata->elevel == ERROR || edata->elevel == WARNING || edata->elevel == INFO || edata->elevel == DEBUG1)) + /* Do not store */ + if (PGSM_ERROR_CAPTURE_ENABLED && edata->elevel >= WARNING && IsSystemOOM() == false) { - uint64 queryid = 0; - - if (debug_query_string) - queryid = pgss_hash_string(debug_query_string, strlen(debug_query_string)); - - pgss_store_error(queryid, - debug_query_string ? debug_query_string : "", + pgsm_store_error(debug_query_string ? debug_query_string : "", edata); } exit: @@ -3296,7 +3517,6 @@ IsSystemInitialized(void) return (system_init && IsHashInitialize()); } - static double time_diff(struct timeval end, struct timeval start) { @@ -3488,38 +3708,8 @@ get_query_id(JumbleState *jstate, Query *query) /* Compute query ID and mark the Query node with it */ JumbleQuery(jstate, query); - queryid = pgss_hash_string((const char *)jstate->jumble, jstate->jumble_len); + queryid = pgsm_hash_string((const char *)jstate->jumble, jstate->jumble_len); return queryid; } #endif -static uint64 -djb2_hash(unsigned char *str, size_t len) -{ - uint64 hash = 5381LLU; - - while (len--) - hash = ((hash << 5) + hash) ^ *str++; - /* hash(i - 1) * 33 ^ str[i] */ - - return hash; -} - -static uint64 -djb2_hash_str(unsigned char *str, int *out_len) -{ - uint64 hash = 5381LLU; - unsigned char *start = str; - unsigned char c; - - while ((c = *str) != '\0') - { - hash = ((hash << 5) + hash) ^ c; - /* hash(i - 1) * 33 ^ str[i] */ - ++str; - } - - *out_len = str - start; - - return hash; -} diff --git a/pg_stat_monitor.h b/pg_stat_monitor.h index 6bcf089..a65af74 100644 --- a/pg_stat_monitor.h +++ b/pg_stat_monitor.h @@ -57,6 +57,9 @@ #include "utils/lsyscache.h" #include "utils/guc.h" #include "utils/guc_tables.h" +#include "utils/memutils.h" +#include "utils/palloc.h" + #define MAX_BACKEND_PROCESES (MaxBackends + NUM_AUXILIARY_PROCS + max_prepared_xacts) #define IntArrayGetTextDatum(x,y) intarray_get_datum(x,y) @@ -64,7 +67,6 @@ /* XXX: Should USAGE_EXEC reflect execution time and/or buffer usage? */ #define USAGE_EXEC(duration) (1.0) #define USAGE_INIT (1.0) /* including initial planning */ -#define ASSUMED_MEDIAN_INIT (10.0) /* initial assumed median usage */ #define ASSUMED_LENGTH_INIT 1024 /* initial assumed mean query length */ #define USAGE_DECREASE_FACTOR (0.99) /* decreased every entry_dealloc */ #define STICKY_DECREASE_FACTOR (0.50) /* factor for sticky entries */ @@ -74,16 +76,16 @@ #define MAX_RESPONSE_BUCKET 50 #define INVALID_BUCKET_ID -1 -#define MAX_REL_LEN 255 #define MAX_BUCKETS 10 #define TEXT_LEN 255 #define ERROR_MESSAGE_LEN 100 +#define REL_TYPENAME_LEN 64 #define REL_LST 10 -#define REL_LEN 1000 +#define REL_LEN 132 /* REL_TYPENAME_LEN * 2 (relname + schema) + 1 (for view indication) + 1 and dot and string terminator */ #define CMD_LST 10 #define CMD_LEN 20 -#define APPLICATIONNAME_LEN 100 -#define COMMENTS_LEN 512 +#define APPLICATIONNAME_LEN NAMEDATALEN +#define COMMENTS_LEN 256 #define PGSM_OVER_FLOW_MAX 10 #define PLAN_TEXT_LEN 1024 /* the assumption of query max nested level */ @@ -91,12 +93,13 @@ #define MAX_QUERY_BUF (PGSM_QUERY_SHARED_BUFFER * 1024 * 1024) #define MAX_BUCKETS_MEM (PGSM_MAX * 1024 * 1024) -#define BUCKETS_MEM_OVERFLOW() ((hash_get_num_entries(pgss_hash) * sizeof(pgssEntry)) >= MAX_BUCKETS_MEM) -#define MAX_BUCKET_ENTRIES (MAX_BUCKETS_MEM / sizeof(pgssEntry)) +#define BUCKETS_MEM_OVERFLOW() ((hash_get_num_entries(pgsm_hash) * sizeof(pgsmEntry)) >= MAX_BUCKETS_MEM) +#define MAX_BUCKET_ENTRIES (MAX_BUCKETS_MEM / sizeof(pgsmEntry)) #define QUERY_BUFFER_OVERFLOW(x,y) ((x + y + sizeof(uint64) + sizeof(uint64)) > MAX_QUERY_BUF) #define QUERY_MARGIN 100 #define MIN_QUERY_LEN 10 #define SQLCODE_LEN 20 +#define TOTAL_RELS_LENGTH (REL_LST * REL_LEN) #if PG_VERSION_NUM >= 130000 #define MAX_SETTINGS 15 @@ -196,23 +199,23 @@ typedef enum OVERFLOW_TARGET OVERFLOW_TARGET_DISK } OVERFLOW_TARGET; -typedef enum pgssStoreKind +typedef enum pgsmStoreKind { - PGSS_INVALID = -1, + PGSM_INVALID = -1, /* - * PGSS_PLAN and PGSS_EXEC must be respectively 0 and 1 as they're used to + * PGSM_PLAN and PGSM_EXEC must be respectively 0 and 1 as they're used to * reference the underlying values in the arrays in the Counters struct, - * and this order is required in pg_stat_statements_internal(). + * and this order is required in pg_stat_monitor_internal(). */ - PGSS_PARSE = 0, - PGSS_PLAN, - PGSS_EXEC, - PGSS_FINISHED, - PGSS_ERROR, + PGSM_PARSE = 0, + PGSM_PLAN, + PGSM_EXEC, + PGSM_STORE, + PGSM_ERROR, - PGSS_NUMKIND /* Must be last value of this enum */ -} pgssStoreKind; + PGSM_NUMKIND /* Must be last value of this enum */ +} pgsmStoreKind; /* the assumption of query max nested level */ #define DEFAULT_MAX_NESTED_LEVEL 10 @@ -247,17 +250,17 @@ typedef struct PlanInfo size_t plan_len; /* strlen(plan_text) */ } PlanInfo; -typedef struct pgssHashKey +typedef struct pgsmHashKey { uint64 bucket_id; /* bucket number */ uint64 queryid; /* query identifier */ - uint64 userid; /* user OID */ - uint64 dbid; /* database OID */ - uint64 ip; /* client ip address */ uint64 planid; /* plan identifier */ uint64 appid; /* hash of application name */ - uint64 toplevel; /* query executed at top level */ -} pgssHashKey; + Oid userid; /* user OID */ + Oid dbid; /* database OID */ + uint32 ip; /* client ip address */ + bool toplevel; /* query executed at top level */ +} pgsmHashKey; typedef struct QueryInfo { @@ -307,6 +310,15 @@ typedef struct Blocks double temp_blk_read_time; /* time spent reading temp blocks, in msec */ double temp_blk_write_time; /* time spent writing temp blocks, in * msec */ + + /* + * Variables for local entry. The values to be passed to pgsm_update_entry + * from pgsm_store. + */ + instr_time instr_blk_read_time; /* time spent reading blocks */ + instr_time instr_blk_write_time; /* time spent writing blocks */ + instr_time instr_temp_blk_read_time; /* time spent reading temp blocks */ + instr_time instr_temp_blk_write_time; /* time spent writing temp blocks */ } Blocks; typedef struct JitInfo @@ -322,12 +334,21 @@ typedef struct JitInfo int64 jit_emission_count; /* number of times emission time has been * > 0 */ double jit_emission_time; /* total time to emit jit code */ + + /* + * Variables for local entry. The values to be passed to pgsm_update_entry + * from pgsm_store. + */ + instr_time instr_generation_counter; /* generation counter */ + instr_time instr_inlining_counter; /* inlining counter */ + instr_time instr_optimization_counter; /* optimization counter */ + instr_time instr_emission_counter; /* emission counter */ } JitInfo; typedef struct SysInfo { - float utime; /* user cpu time */ - float stime; /* system cpu time */ + double utime; /* user cpu time */ + double stime; /* system cpu time */ } SysInfo; typedef struct Wal_Usage @@ -339,7 +360,6 @@ typedef struct Wal_Usage typedef struct Counters { - uint64 bucket_id; /* bucket id */ Calls calls; QueryInfo info; CallTime time; @@ -355,7 +375,6 @@ typedef struct Counters Wal_Usage walusage; int resp_calls[MAX_RESPONSE_BUCKET]; /* execution time's in * msec */ - int64 state; /* query state */ } Counters; /* Some global structure to get the cpu usage, really don't like the idea of global variable */ @@ -363,32 +382,33 @@ typedef struct Counters /* * Statistics per statement */ -typedef struct pgssEntry +typedef struct pgsmEntry { - pgssHashKey key; /* hash key of entry - MUST BE FIRST */ - uint64 pgsm_query_id; /* pgsm generate normalized query hash */ - Counters counters; /* the statistics for this query */ - int encoding; /* query text encoding */ - slock_t mutex; /* protects the counters only */ - dsa_pointer query_pos; /* query location within query buffer */ -} pgssEntry; + pgsmHashKey key; /* hash key of entry - MUST BE FIRST */ + uint64 pgsm_query_id; /* pgsm generate normalized query hash */ + char datname[NAMEDATALEN]; /* database name */ + char username[NAMEDATALEN]; /* user name */ + Counters counters; /* the statistics for this query */ + int encoding; /* query text encoding */ + slock_t mutex; /* protects the counters only */ + union + { + dsa_pointer query_pos; /* query location within query buffer */ + char* query_pointer; + }query_text; +} pgsmEntry; /* * Global shared state */ -typedef struct pgssSharedState +typedef struct pgsmSharedState { LWLock *lock; /* protects hashtable search/modification */ - double cur_median_usage; /* current median usage in hashtable */ slock_t mutex; /* protects following fields only: */ - Size extent; /* current extent of query file */ - int64 n_writers; /* number of active writers to query file */ pg_atomic_uint64 current_wbucket; pg_atomic_uint64 prev_bucket_sec; uint64 bucket_entry[MAX_BUCKETS]; TimestampTz bucket_start_time[MAX_BUCKETS]; /* start time of the bucket */ - LWLock *errors_lock; /* protects errors hashtable - * search/modification */ int hash_tranche_id; void *raw_dsa_area; /* DSA area pointer to store query texts. * dshash also lives in this memory when @@ -398,28 +418,22 @@ typedef struct pgssSharedState * classic shared memory hash or dshash * (if we are using USE_DYNAMIC_HASH) */ -} pgssSharedState; + MemoryContext pgsm_mem_cxt; + /* context to store stats in local + * memory until they are pushed to shared hash + */ + bool pgsm_oom; +} pgsmSharedState; typedef struct pgsmLocalState { - pgssSharedState *shared_pgssState; + pgsmSharedState *shared_pgsmState; dsa_area *dsa; /* local dsa area for backend attached to the * dsa area created by postmaster at startup. */ PGSM_HASH_TABLE *shared_hash; }pgsmLocalState; -#define ResetSharedState(x) \ -do { \ - x->cur_median_usage = ASSUMED_MEDIAN_INIT; \ - x->cur_median_usage = ASSUMED_MEDIAN_INIT; \ - x->n_writers = 0; \ - pg_atomic_init_u64(&x->current_wbucket, 0); \ - pg_atomic_init_u64(&x->prev_bucket_sec, 0); \ - memset(&x->bucket_entry, 0, MAX_BUCKETS * sizeof(uint64)); \ -} while(0) - - #if PG_VERSION_NUM < 140000 /* * Struct for tracking locations/lengths of constants during normalization @@ -456,40 +470,30 @@ typedef struct JumbleState } JumbleState; #endif -/* Links to shared memory state */ - -bool SaveQueryText(uint64 bucketid, - uint64 queryid, - unsigned char *buf, - const char *query, - uint64 query_len, - size_t *query_pos); - /* guc.c */ void init_guc(void); GucVariable *get_conf(int i); /* hash_create.c */ dsa_area *get_dsa_area_for_query_text(void); -PGSM_HASH_TABLE *get_pgssHash(void); +PGSM_HASH_TABLE *get_pgsmHash(void); void pgsm_attach_shmem(void); bool IsHashInitialize(void); -void pgss_shmem_startup(void); -void pgss_shmem_shutdown(int code, Datum arg); +bool IsSystemOOM(void); +void pgsm_shmem_startup(void); +void pgsm_shmem_shutdown(int code, Datum arg); int pgsm_get_bucket_size(void); -pgssSharedState *pgsm_get_ss(void); -void hash_entry_reset(void); -void hash_query_entryies_reset(void); +pgsmSharedState *pgsm_get_ss(void); void hash_query_entries(); void hash_query_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_buffer[]); void hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_buffer); -pgssEntry *hash_entry_alloc(pgssSharedState *pgss, pgssHashKey *key, int encoding); +pgsmEntry *hash_entry_alloc(pgsmSharedState *pgsm, pgsmHashKey *key, int encoding); Size pgsm_ShmemSize(void); -void pgss_startup(void); +void pgsm_startup(void); /* hash_query.c */ -void pgss_startup(void); +void pgsm_startup(void); /*---- GUC variables ----*/ typedef enum @@ -528,8 +532,8 @@ static const struct config_enum_entry track_options[] = #define HOOK_STATS_SIZE 0 #endif -void *pgsm_hash_find_or_insert(PGSM_HASH_TABLE *shared_hash, pgssHashKey *key, bool* found); -void *pgsm_hash_find(PGSM_HASH_TABLE *shared_hash, pgssHashKey *key, bool* found); +void *pgsm_hash_find_or_insert(PGSM_HASH_TABLE *shared_hash, pgsmHashKey *key, bool* found); +void *pgsm_hash_find(PGSM_HASH_TABLE *shared_hash, pgsmHashKey *key, bool* found); void pgsm_hash_seq_init(PGSM_HASH_SEQ_STATUS *hstat, PGSM_HASH_TABLE *shared_hash, bool lock); void *pgsm_hash_seq_next(PGSM_HASH_SEQ_STATUS *hstat); void pgsm_hash_seq_term(PGSM_HASH_SEQ_STATUS *hstat); diff --git a/regression/expected/cmd_type.out b/regression/expected/cmd_type.out index 2981c8f..b006af1 100644 --- a/regression/expected/cmd_type.out +++ b/regression/expected/cmd_type.out @@ -23,6 +23,7 @@ SELECT b FROM t2 FOR UPDATE; TRUNCATE t1; DROP TABLE t1; +DROP TABLE t2; SELECT query, cmd_type, cmd_type_text FROM pg_stat_monitor ORDER BY query COLLATE "C"; query | cmd_type | cmd_type_text --------------------------------+----------+--------------- @@ -30,13 +31,14 @@ SELECT query, cmd_type, cmd_type_text FROM pg_stat_monitor ORDER BY query COLLA CREATE TABLE t2 (b INTEGER) | 0 | DELETE FROM t1 | 4 | DELETE DROP TABLE t1 | 0 | + DROP TABLE t2 | 0 | INSERT INTO t1 VALUES(1) | 3 | INSERT SELECT a FROM t1 | 1 | SELECT SELECT b FROM t2 FOR UPDATE | 1 | SELECT SELECT pg_stat_monitor_reset() | 1 | SELECT TRUNCATE t1 | 0 | UPDATE t1 SET a = 2 | 2 | UPDATE -(10 rows) +(11 rows) SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset diff --git a/regression/expected/database.out b/regression/expected/database.out index 7062272..bd377f6 100644 --- a/regression/expected/database.out +++ b/regression/expected/database.out @@ -27,13 +27,15 @@ SELECT * FROM t3,t4 WHERE t3.c = t4.d; (0 rows) \c contrib_regression +DROP DATABASE db2; SELECT datname, query FROM pg_stat_monitor ORDER BY query COLLATE "C"; datname | query --------------------+--------------------------------------- + contrib_regression | DROP DATABASE db2 db1 | SELECT * FROM t1,t2 WHERE t1.a = t2.b db2 | SELECT * FROM t3,t4 WHERE t3.c = t4.d contrib_regression | SELECT pg_stat_monitor_reset() -(3 rows) +(4 rows) SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset @@ -44,10 +46,6 @@ SELECT pg_stat_monitor_reset(); \c db1 DROP TABLE t1; DROP TABLE t2; -\c db2 -DROP TABLE t3; -DROP TABLE t4; \c contrib_regression DROP DATABASE db1; -DROP DATABASE db2; DROP EXTENSION pg_stat_monitor; diff --git a/regression/expected/error_2.out b/regression/expected/error_2.out new file mode 100644 index 0000000..c89920b --- /dev/null +++ b/regression/expected/error_2.out @@ -0,0 +1,42 @@ +CREATE EXTENSION pg_stat_monitor; +SELECT pg_stat_monitor_reset(); + pg_stat_monitor_reset +----------------------- + +(1 row) + +SELECT 1/0; -- divide by zero +ERROR: division by zero +SELECT * FROM unknown; -- unknown table +ERROR: relation "unknown" does not exist +LINE 1: SELECT * FROM unknown; + ^ +ELECET * FROM unknown; -- syntax error +ERROR: syntax error at or near "ELECET" +LINE 1: ELECET * FROM unknown; + ^ +do $$ +BEGIN +RAISE WARNING 'warning message'; +END $$; +WARNING: warning message +SELECT query, elevel, sqlcode, message FROM pg_stat_monitor ORDER BY query COLLATE "C",elevel; + query | elevel | sqlcode | message +----------------------------------+--------+---------+----------------------------------- + ELECET * FROM unknown; | 20 | 42601 | syntax error at or near "ELECET" + SELECT * FROM unknown; | 20 | 42P01 | relation "unknown" does not exist + SELECT 1/0; | 20 | 22012 | division by zero + SELECT pg_stat_monitor_reset() | 0 | | + do $$ +| 0 | 01000 | warning message + BEGIN +| | | + RAISE WARNING 'warning message';+| | | + END $$; | | | +(5 rows) + +SELECT pg_stat_monitor_reset(); + pg_stat_monitor_reset +----------------------- + +(1 row) + +DROP EXTENSION pg_stat_monitor; diff --git a/regression/expected/functions.out b/regression/expected/functions.out index aa22a3a..b924c8d 100644 --- a/regression/expected/functions.out +++ b/regression/expected/functions.out @@ -1,4 +1,5 @@ DROP ROLE IF EXISTS su; +NOTICE: role "su" does not exist, skipping CREATE USER su WITH SUPERUSER; SET ROLE su; CREATE EXTENSION pg_stat_monitor; @@ -29,13 +30,19 @@ SELECT routine_schema, routine_name, routine_type, data_type FROM information_sc SET ROLE u1; SELECT routine_schema, routine_name, routine_type, data_type FROM information_schema.routines WHERE routine_schema = 'public' ORDER BY routine_name COLLATE "C"; - routine_schema | routine_name | routine_type | data_type -----------------+-------------------------+--------------+----------- - public | histogram | FUNCTION | record - public | pg_stat_monitor_reset | FUNCTION | void - public | pg_stat_monitor_version | FUNCTION | text -(3 rows) + routine_schema | routine_name | routine_type | data_type +----------------+--------------------------+--------------+----------- + public | decode_error_level | FUNCTION | text + public | get_cmd_type | FUNCTION | text + public | get_histogram_timings | FUNCTION | text + public | histogram | FUNCTION | record + public | pg_stat_monitor_internal | FUNCTION | record + public | pg_stat_monitor_version | FUNCTION | text + public | range | FUNCTION | ARRAY +(7 rows) SET ROLE su; DROP USER u1; DROP EXTENSION pg_stat_monitor; +DROP USER su; +ERROR: current user cannot be dropped diff --git a/regression/expected/guc.out b/regression/expected/guc.out index c0702a9..e62dc2d 100644 --- a/regression/expected/guc.out +++ b/regression/expected/guc.out @@ -24,8 +24,8 @@ COLLATE "C"; pg_stat_monitor.pgsm_histogram_buckets | 20 | | postmaster | integer | default | 2 | 50 | | 20 | 20 | f pg_stat_monitor.pgsm_histogram_max | 100000 | | postmaster | integer | default | 10 | 2147483647 | | 100000 | 100000 | f pg_stat_monitor.pgsm_histogram_min | 1 | | postmaster | integer | default | 0 | 2147483647 | | 1 | 1 | f - pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | default | 1 | 1000 | | 100 | 100 | f - pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 10 | | 10 | 10 | f + pg_stat_monitor.pgsm_max | 256 | MB | postmaster | integer | default | 10 | 10240 | | 256 | 256 | f + pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 20000 | | 10 | 10 | f pg_stat_monitor.pgsm_normalized_query | off | | user | bool | default | | | | off | off | f pg_stat_monitor.pgsm_overflow_target | 1 | | postmaster | integer | default | 0 | 1 | | 1 | 1 | f pg_stat_monitor.pgsm_query_max_len | 2048 | | postmaster | integer | default | 1024 | 2147483647 | | 2048 | 2048 | f diff --git a/regression/expected/guc_1.out b/regression/expected/guc_1.out index e83f603..087cf5d 100644 --- a/regression/expected/guc_1.out +++ b/regression/expected/guc_1.out @@ -24,8 +24,8 @@ COLLATE "C"; pg_stat_monitor.pgsm_histogram_buckets | 20 | | postmaster | integer | default | 2 | 50 | | 20 | 20 | f pg_stat_monitor.pgsm_histogram_max | 100000 | | postmaster | integer | default | 10 | 2147483647 | | 100000 | 100000 | f pg_stat_monitor.pgsm_histogram_min | 1 | | postmaster | integer | default | 0 | 2147483647 | | 1 | 1 | f - pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | default | 1 | 1000 | | 100 | 100 | f - pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 10 | | 10 | 10 | f + pg_stat_monitor.pgsm_max | 256 | MB | postmaster | integer | default | 10 | 10240 | | 256 | 256 | f + pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 20000 | | 10 | 10 | f pg_stat_monitor.pgsm_normalized_query | off | | user | bool | default | | | | off | off | f pg_stat_monitor.pgsm_overflow_target | 1 | | postmaster | integer | default | 0 | 1 | | 1 | 1 | f pg_stat_monitor.pgsm_query_max_len | 2048 | | postmaster | integer | default | 1024 | 2147483647 | | 2048 | 2048 | f diff --git a/regression/expected/pgsm_query_id.out b/regression/expected/pgsm_query_id.out index 09897f7..be1009f 100644 --- a/regression/expected/pgsm_query_id.out +++ b/regression/expected/pgsm_query_id.out @@ -40,6 +40,22 @@ SELECT * FROM t2; --- (0 rows) +-- Check that spaces and comments do not generate a different pgsm_query_id +SELECT * FROM t2 --WHATEVER; +; + b +--- +(0 rows) + +SELECT * FROM t2 /* ... +... +More comments to check for spaces. +*/ + ; + b +--- +(0 rows) + \c db2 SELECT * FROM t1; a @@ -57,18 +73,18 @@ SELECT * FROM t3; (0 rows) \c contrib_regression -SELECT datname, pgsm_query_id, query FROM pg_stat_monitor ORDER BY pgsm_query_id, query, datname; - datname | pgsm_query_id | query ---------------------+----------------------+-------------------------------- - db2 | -5029137034974447432 | SELECT * FROM t3 - contrib_regression | 689150021118383254 | SELECT pg_stat_monitor_reset() - db1 | 1897482803466821995 | SELECT * FROM t2 - db1 | 1988437669671417938 | SELECT * FROM t1 - db2 | 1988437669671417938 | SELECT * FROM t1 - db1 | 2864453209316739369 | select $1 + $2 - db2 | 2864453209316739369 | select $1 + $2 - db1 | 8140395000078788481 | SELECT *, ADD(1, 2) FROM t1 - db2 | 8140395000078788481 | SELECT *, ADD(1, 2) FROM t1 +SELECT datname, pgsm_query_id, query, calls FROM pg_stat_monitor ORDER BY pgsm_query_id, query, datname; + datname | pgsm_query_id | query | calls +--------------------+----------------------+--------------------------------+------- + db2 | -5029137034974447432 | SELECT * FROM t3 | 1 + contrib_regression | 689150021118383254 | SELECT pg_stat_monitor_reset() | 1 + db1 | 1897482803466821995 | SELECT * FROM t2 | 3 + db1 | 1988437669671417938 | SELECT * FROM t1 | 1 + db2 | 1988437669671417938 | SELECT * FROM t1 | 1 + db1 | 2864453209316739369 | select $1 + $2 | 1 + db2 | 2864453209316739369 | select $1 + $2 | 1 + db1 | 8140395000078788481 | SELECT *, ADD(1, 2) FROM t1 | 1 + db2 | 8140395000078788481 | SELECT *, ADD(1, 2) FROM t1 | 1 (9 rows) SELECT pg_stat_monitor_reset(); diff --git a/regression/expected/rows.out b/regression/expected/rows.out index e06cd5d..6350ecd 100644 --- a/regression/expected/rows.out +++ b/regression/expected/rows.out @@ -1,7 +1,6 @@ CREATE EXTENSION pg_stat_monitor; CREATE TABLE t1(a int); CREATE TABLE t2(b int); -ERROR: relation "t2" already exists INSERT INTO t1 VALUES(generate_series(1,1000)); INSERT INTO t2 VALUES(generate_series(1,5000)); SELECT pg_stat_monitor_reset(); @@ -8557,4 +8556,5 @@ SELECT pg_stat_monitor_reset(); (1 row) DROP TABLE t1; +DROP TABLE t2; DROP EXTENSION pg_stat_monitor; diff --git a/regression/expected/user.out b/regression/expected/user.out new file mode 100644 index 0000000..a3eb20f --- /dev/null +++ b/regression/expected/user.out @@ -0,0 +1,65 @@ +CREATE USER su WITH SUPERUSER; +ERROR: role "su" already exists +SET ROLE su; +CREATE EXTENSION pg_stat_monitor; +CREATE USER u1; +CREATE USER u2; +GRANT ALL ON SCHEMA public TO u1; +GRANT ALL ON SCHEMA public TO u2; +SET ROLE su; +SELECT pg_stat_monitor_reset(); + pg_stat_monitor_reset +----------------------- + +(1 row) + +SET ROLE u1; +SELECT pg_stat_monitor_reset(); +ERROR: permission denied for function pg_stat_monitor_reset +CREATE TABLE t1 (a int); +SELECT * FROM t1; + a +--- +(0 rows) + +SET ROLE u2; +CREATE TABLE t2 (a int); +SELECT * FROM t2; + a +--- +(0 rows) + +DROP TABLE t2; +SET ROLE su; +DROP OWNED BY u2; +DROP USER u2; +SELECT username, query FROM pg_stat_monitor ORDER BY username, query COLLATE "C"; + username | query +----------+--------------------------------- + su | DROP OWNED BY u2 + su | DROP USER u2 + su | SELECT pg_stat_monitor_reset() + su | SET ROLE su + u1 | CREATE TABLE t1 (a int) + u1 | SELECT * FROM t1 + u1 | SELECT pg_stat_monitor_reset(); + u1 | SET ROLE u1 + u2 | CREATE TABLE t2 (a int) + u2 | DROP TABLE t2 + u2 | SELECT * FROM t2 + u2 | SET ROLE u2 +(12 rows) + +SELECT pg_stat_monitor_reset(); + pg_stat_monitor_reset +----------------------- + +(1 row) + +DROP TABLE t1; +DROP OWNED BY u1; +DROP USER u1; +DROP EXTENSION pg_stat_monitor; +SET ROLE NONE; +DROP OWNED BY su; +DROP USER su; diff --git a/regression/sql/cmd_type.sql b/regression/sql/cmd_type.sql index 5f1fa1b..9fb2fa6 100644 --- a/regression/sql/cmd_type.sql +++ b/regression/sql/cmd_type.sql @@ -10,6 +10,7 @@ DELETE FROM t1; SELECT b FROM t2 FOR UPDATE; TRUNCATE t1; DROP TABLE t1; +DROP TABLE t2; SELECT query, cmd_type, cmd_type_text FROM pg_stat_monitor ORDER BY query COLLATE "C"; SELECT pg_stat_monitor_reset(); diff --git a/regression/sql/database.sql b/regression/sql/database.sql index 0431004..0e38754 100644 --- a/regression/sql/database.sql +++ b/regression/sql/database.sql @@ -20,6 +20,8 @@ SELECT * FROM t1,t2 WHERE t1.a = t2.b; SELECT * FROM t3,t4 WHERE t3.c = t4.d; \c contrib_regression +DROP DATABASE db2; + SELECT datname, query FROM pg_stat_monitor ORDER BY query COLLATE "C"; SELECT pg_stat_monitor_reset(); @@ -27,11 +29,6 @@ SELECT pg_stat_monitor_reset(); DROP TABLE t1; DROP TABLE t2; -\c db2 -DROP TABLE t3; -DROP TABLE t4; - \c contrib_regression DROP DATABASE db1; -DROP DATABASE db2; DROP EXTENSION pg_stat_monitor; diff --git a/regression/sql/functions.sql b/regression/sql/functions.sql index d16453b..75f0ad9 100644 --- a/regression/sql/functions.sql +++ b/regression/sql/functions.sql @@ -14,3 +14,4 @@ SELECT routine_schema, routine_name, routine_type, data_type FROM information_sc SET ROLE su; DROP USER u1; DROP EXTENSION pg_stat_monitor; +DROP USER su; diff --git a/regression/sql/pgsm_query_id.sql b/regression/sql/pgsm_query_id.sql index c26296e..eb5bebc 100644 --- a/regression/sql/pgsm_query_id.sql +++ b/regression/sql/pgsm_query_id.sql @@ -29,6 +29,14 @@ SELECT pg_stat_monitor_reset(); SELECT * FROM t1; SELECT *, ADD(1, 2) FROM t1; SELECT * FROM t2; +-- Check that spaces and comments do not generate a different pgsm_query_id +SELECT * FROM t2 --WHATEVER; +; +SELECT * FROM t2 /* ... +... +More comments to check for spaces. +*/ + ; \c db2 SELECT * FROM t1; @@ -36,7 +44,7 @@ SELECT *, ADD(1, 2) FROM t1; SELECT * FROM t3; \c contrib_regression -SELECT datname, pgsm_query_id, query FROM pg_stat_monitor ORDER BY pgsm_query_id, query, datname; +SELECT datname, pgsm_query_id, query, calls FROM pg_stat_monitor ORDER BY pgsm_query_id, query, datname; SELECT pg_stat_monitor_reset(); \c db1 diff --git a/regression/sql/rows.sql b/regression/sql/rows.sql index 02f0f70..7a09398 100644 --- a/regression/sql/rows.sql +++ b/regression/sql/rows.sql @@ -16,4 +16,5 @@ SELECT query, rows FROM pg_stat_monitor ORDER BY query COLLATE "C"; SELECT pg_stat_monitor_reset(); DROP TABLE t1; +DROP TABLE t2; DROP EXTENSION pg_stat_monitor; diff --git a/regression/sql/user.sql b/regression/sql/user.sql index 03237d2..1e3086d 100644 --- a/regression/sql/user.sql +++ b/regression/sql/user.sql @@ -6,25 +6,37 @@ CREATE EXTENSION pg_stat_monitor; CREATE USER u1; CREATE USER u2; -SET ROLE su; +GRANT ALL ON SCHEMA public TO u1; +GRANT ALL ON SCHEMA public TO u2; +SET ROLE su; SELECT pg_stat_monitor_reset(); + SET ROLE u1; +SELECT pg_stat_monitor_reset(); CREATE TABLE t1 (a int); SELECT * FROM t1; SET ROLE u2; CREATE TABLE t2 (a int); SELECT * FROM t2; +DROP TABLE t2; SET ROLE su; -SELECT userid, query FROM pg_stat_monitor ORDER BY query COLLATE "C"; + +DROP OWNED BY u2; +DROP USER u2; + +SELECT username, query FROM pg_stat_monitor ORDER BY username, query COLLATE "C"; SELECT pg_stat_monitor_reset(); DROP TABLE t1; -DROP TABLE t2; - +DROP OWNED BY u1; DROP USER u1; -DROP USER u2; DROP EXTENSION pg_stat_monitor; + +SET ROLE NONE; + +DROP OWNED BY su; +DROP USER su; diff --git a/t/018_column_names.pl b/t/018_column_names.pl index 584b9d8..a43780c 100644 --- a/t/018_column_names.pl +++ b/t/018_column_names.pl @@ -36,7 +36,7 @@ my %pg_versions_pgsm_columns = ( 15 => "application_name,blk_read_time," . "shared_blks_written,sqlcode,stddev_exec_time,stddev_plan_time," . "temp_blk_read_time,temp_blk_write_time,temp_blks_read,temp_blks_written," . "top_query,top_queryid,toplevel,total_exec_time,total_plan_time," . - "user,userid,wal_bytes,wal_fpi,wal_records", + "userid,username,wal_bytes,wal_fpi,wal_records", 14 => "application_name,blk_read_time," . "blk_write_time,bucket,bucket_done,bucket_start_time,calls," . "client_ip,cmd_type,cmd_type_text,comments,cpu_sys_time,cpu_user_time," . @@ -47,7 +47,7 @@ my %pg_versions_pgsm_columns = ( 15 => "application_name,blk_read_time," . "rows,shared_blks_dirtied,shared_blks_hit,shared_blks_read," . "shared_blks_written,sqlcode,stddev_exec_time,stddev_plan_time," . "temp_blks_read,temp_blks_written,top_query,top_queryid,toplevel," . - "total_exec_time,total_plan_time,user,userid,wal_bytes,wal_fpi,wal_records", + "total_exec_time,total_plan_time,userid,username,wal_bytes,wal_fpi,wal_records", 13 => "application_name,blk_read_time," . "blk_write_time,bucket,bucket_done,bucket_start_time,calls," . "client_ip,cmd_type,cmd_type_text,comments,cpu_sys_time,cpu_user_time," . @@ -58,7 +58,7 @@ my %pg_versions_pgsm_columns = ( 15 => "application_name,blk_read_time," . "rows,shared_blks_dirtied,shared_blks_hit,shared_blks_read," . "shared_blks_written,sqlcode,stddev_exec_time,stddev_plan_time," . "temp_blks_read,temp_blks_written,top_query,top_queryid,toplevel," . - "total_exec_time,total_plan_time,user,userid,wal_bytes,wal_fpi,wal_records", + "total_exec_time,total_plan_time,userid,username,wal_bytes,wal_fpi,wal_records", 12 => "application_name,blk_read_time,blk_write_time,bucket,bucket_done," . "bucket_start_time,calls,client_ip,cmd_type,cmd_type_text,comments," . "cpu_sys_time,cpu_user_time,datname,dbid,elevel,local_blks_dirtied," . @@ -66,7 +66,7 @@ my %pg_versions_pgsm_columns = ( 15 => "application_name,blk_read_time," . "message,min_time,pgsm_query_id,planid,query,query_plan,queryid,relations,resp_calls," . "rows,shared_blks_dirtied,shared_blks_hit,shared_blks_read," . "shared_blks_written,sqlcode,stddev_time,temp_blks_read,temp_blks_written," . - "top_query,top_queryid,total_time,user,userid" + "top_query,top_queryid,total_time,userid,username" ); # Start server diff --git a/t/expected/001_settings_default.out b/t/expected/001_settings_default.out index 498bb39..57ac4c4 100644 --- a/t/expected/001_settings_default.out +++ b/t/expected/001_settings_default.out @@ -14,8 +14,8 @@ SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals pg_stat_monitor.pgsm_histogram_buckets | 20 | | postmaster | integer | default | 2 | 50 | | 20 | 20 | f pg_stat_monitor.pgsm_histogram_max | 100000 | | postmaster | integer | default | 10 | 2147483647 | | 100000 | 100000 | f pg_stat_monitor.pgsm_histogram_min | 1 | | postmaster | integer | default | 0 | 2147483647 | | 1 | 1 | f - pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | default | 1 | 1000 | | 100 | 100 | f - pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 10 | | 10 | 10 | f + pg_stat_monitor.pgsm_max | 256 | MB | postmaster | integer | default | 10 | 10240 | | 256 | 256 | f + pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 20000 | | 10 | 10 | f pg_stat_monitor.pgsm_normalized_query | off | | user | bool | default | | | | off | off | f pg_stat_monitor.pgsm_overflow_target | 1 | | postmaster | integer | default | 0 | 1 | | 1 | 1 | f pg_stat_monitor.pgsm_query_max_len | 2048 | | postmaster | integer | default | 1024 | 2147483647 | | 2048 | 2048 | f @@ -41,8 +41,8 @@ SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals pg_stat_monitor.pgsm_histogram_buckets | 20 | | postmaster | integer | default | 2 | 50 | | 20 | 20 | f pg_stat_monitor.pgsm_histogram_max | 100000 | | postmaster | integer | default | 10 | 2147483647 | | 100000 | 100000 | f pg_stat_monitor.pgsm_histogram_min | 1 | | postmaster | integer | default | 0 | 2147483647 | | 1 | 1 | f - pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | default | 1 | 1000 | | 100 | 100 | f - pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 10 | | 10 | 10 | f + pg_stat_monitor.pgsm_max | 256 | MB | postmaster | integer | default | 10 | 10240 | | 256 | 256 | f + pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 20000 | | 10 | 10 | f pg_stat_monitor.pgsm_normalized_query | off | | user | bool | default | | | | off | off | f pg_stat_monitor.pgsm_overflow_target | 1 | | postmaster | integer | default | 0 | 1 | | 1 | 1 | f pg_stat_monitor.pgsm_query_max_len | 2048 | | postmaster | integer | default | 1024 | 2147483647 | | 2048 | 2048 | f diff --git a/t/expected/001_settings_default.out.12 b/t/expected/001_settings_default.out.12 old mode 100755 new mode 100644 index d77c7e4..157d687 --- a/t/expected/001_settings_default.out.12 +++ b/t/expected/001_settings_default.out.12 @@ -14,8 +14,8 @@ SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals pg_stat_monitor.pgsm_histogram_buckets | 20 | | postmaster | integer | default | 2 | 50 | | 20 | 20 | f pg_stat_monitor.pgsm_histogram_max | 100000 | | postmaster | integer | default | 10 | 2147483647 | | 100000 | 100000 | f pg_stat_monitor.pgsm_histogram_min | 1 | | postmaster | integer | default | 0 | 2147483647 | | 1 | 1 | f - pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | default | 1 | 1000 | | 100 | 100 | f - pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 10 | | 10 | 10 | f + pg_stat_monitor.pgsm_max | 256 | MB | postmaster | integer | default | 10 | 10240 | | 256 | 256 | f + pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 20000 | | 10 | 10 | f pg_stat_monitor.pgsm_normalized_query | off | | user | bool | default | | | | off | off | f pg_stat_monitor.pgsm_overflow_target | 1 | | postmaster | integer | default | 0 | 1 | | 1 | 1 | f pg_stat_monitor.pgsm_query_max_len | 2048 | | postmaster | integer | default | 1024 | 2147483647 | | 2048 | 2048 | f @@ -40,8 +40,8 @@ SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals pg_stat_monitor.pgsm_histogram_buckets | 20 | | postmaster | integer | default | 2 | 50 | | 20 | 20 | f pg_stat_monitor.pgsm_histogram_max | 100000 | | postmaster | integer | default | 10 | 2147483647 | | 100000 | 100000 | f pg_stat_monitor.pgsm_histogram_min | 1 | | postmaster | integer | default | 0 | 2147483647 | | 1 | 1 | f - pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | default | 1 | 1000 | | 100 | 100 | f - pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 10 | | 10 | 10 | f + pg_stat_monitor.pgsm_max | 256 | MB | postmaster | integer | default | 10 | 10240 | | 256 | 256 | f + pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 20000 | | 10 | 10 | f pg_stat_monitor.pgsm_normalized_query | off | | user | bool | default | | | | off | off | f pg_stat_monitor.pgsm_overflow_target | 1 | | postmaster | integer | default | 0 | 1 | | 1 | 1 | f pg_stat_monitor.pgsm_query_max_len | 2048 | | postmaster | integer | default | 1024 | 2147483647 | | 2048 | 2048 | f diff --git a/t/expected/012_settings_pgsm_max_buckets.out b/t/expected/012_settings_pgsm_max_buckets.out index 3942aae..2560281 100644 --- a/t/expected/012_settings_pgsm_max_buckets.out +++ b/t/expected/012_settings_pgsm_max_buckets.out @@ -8,7 +8,7 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart ----------------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max_buckets | 1 | | postmaster | integer | configuration file | 1 | 10 | | 10 | 1 | f + pg_stat_monitor.pgsm_max_buckets | 1 | | postmaster | integer | configuration file | 1 | 20000 | | 10 | 1 | f (1 row) SELECT pg_stat_monitor_reset(); @@ -20,7 +20,7 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart ----------------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max_buckets | 2 | | postmaster | integer | configuration file | 1 | 10 | | 10 | 2 | f + pg_stat_monitor.pgsm_max_buckets | 2 | | postmaster | integer | configuration file | 1 | 20000 | | 10 | 2 | f (1 row) SELECT pg_stat_monitor_reset(); @@ -32,7 +32,7 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart ----------------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max_buckets | 5 | | postmaster | integer | configuration file | 1 | 10 | | 10 | 5 | f + pg_stat_monitor.pgsm_max_buckets | 5 | | postmaster | integer | configuration file | 1 | 20000 | | 10 | 5 | f (1 row) SELECT pg_stat_monitor_reset(); @@ -44,7 +44,19 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart ----------------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | configuration file | 1 | 10 | | 10 | 10 | f + pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | configuration file | 1 | 20000 | | 10 | 10 | f +(1 row) + +SELECT pg_stat_monitor_reset(); + pg_stat_monitor_reset +----------------------- + +(1 row) + +SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets'; + name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart +----------------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+----------------- + pg_stat_monitor.pgsm_max_buckets | 11 | | postmaster | integer | configuration file | 1 | 20000 | | 10 | 11 | f (1 row) SELECT pg_stat_monitor_reset(); @@ -56,19 +68,7 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart ----------------------------------+---------+------+------------+---------+---------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 10 | | 10 | 10 | f -(1 row) - -SELECT pg_stat_monitor_reset(); - pg_stat_monitor_reset ------------------------ - -(1 row) - -SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets'; - name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart -----------------------------------+---------+------+------------+---------+---------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 10 | | 10 | 10 | f + pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 20000 | | 10 | 10 | f (1 row) DROP EXTENSION pg_stat_monitor; diff --git a/t/expected/016_settings_pgsm_max.out b/t/expected/016_settings_pgsm_max.out index 4f7e090..111ad5f 100644 --- a/t/expected/016_settings_pgsm_max.out +++ b/t/expected/016_settings_pgsm_max.out @@ -8,7 +8,7 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart --------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max | 1000 | MB | postmaster | integer | configuration file | 1 | 1000 | | 100 | 1000 | f + pg_stat_monitor.pgsm_max | 1000 | MB | postmaster | integer | configuration file | 10 | 10240 | | 256 | 1000 | f (1 row) SELECT pg_stat_monitor_reset(); @@ -20,7 +20,7 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart --------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max | 500 | MB | postmaster | integer | configuration file | 1 | 1000 | | 100 | 500 | f + pg_stat_monitor.pgsm_max | 500 | MB | postmaster | integer | configuration file | 10 | 10240 | | 256 | 500 | f (1 row) SELECT pg_stat_monitor_reset(); @@ -32,7 +32,7 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart --------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | configuration file | 1 | 1000 | | 100 | 100 | f + pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | configuration file | 10 | 10240 | | 256 | 100 | f (1 row) SELECT pg_stat_monitor_reset(); @@ -44,7 +44,7 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart --------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max | 10 | MB | postmaster | integer | configuration file | 1 | 1000 | | 100 | 10 | f + pg_stat_monitor.pgsm_max | 10 | MB | postmaster | integer | configuration file | 10 | 10240 | | 256 | 10 | f (1 row) SELECT pg_stat_monitor_reset(); @@ -56,7 +56,7 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart --------------------------+---------+------+------------+---------+---------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | default | 1 | 1000 | | 100 | 100 | f + pg_stat_monitor.pgsm_max | 256 | MB | postmaster | integer | default | 10 | 10240 | | 256 | 256 | f (1 row) DROP EXTENSION pg_stat_monitor;