diff --git a/Makefile b/Makefile index fe567a8..c305a4c 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ LDFLAGS_SL += $(filter -lm, $(LIBS)) TAP_TESTS = 1 REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/pg_stat_monitor/pg_stat_monitor.conf --inputdir=regression -REGRESS = basic version guc pgsm_query_id functions counters relations database error_insert application_name application_name_unique top_query cmd_type error rows tags +REGRESS = basic version guc pgsm_query_id functions counters relations database error_insert application_name application_name_unique top_query cmd_type error rows tags user # Disabled because these tests require "shared_preload_libraries=pg_stat_statements", # which typical installcheck users do not have (e.g. buildfarm clients). diff --git a/guc.c b/guc.c index 6acbe0e..f7f8bb9 100644 --- a/guc.c +++ b/guc.c @@ -41,9 +41,9 @@ init_guc(void) { .guc_name = "pg_stat_monitor.pgsm_max", .guc_desc = "Sets the maximum size of shared memory in (MB) used for statement's metadata tracked by pg_stat_monitor.", - .guc_default = 100, - .guc_min = 1, - .guc_max = 1000, + .guc_default = 256, + .guc_min = 10, + .guc_max = 10240, .guc_restart = true, .guc_unit = GUC_UNIT_MB, .guc_value = &PGSM_MAX @@ -95,7 +95,7 @@ init_guc(void) .guc_desc = "Sets the maximum number of buckets.", .guc_default = 10, .guc_min = 1, - .guc_max = 10, + .guc_max = 20000, .guc_restart = true, .guc_unit = 0, .guc_value = &PGSM_MAX_BUCKETS diff --git a/hash_query.c b/hash_query.c index 8cc3bf8..6a626dd 100644 --- a/hash_query.c +++ b/hash_query.c @@ -19,14 +19,15 @@ #include "pg_stat_monitor.h" static pgsmLocalState pgsmStateLocal; -static PGSM_HASH_TABLE_HANDLE pgsm_create_bucket_hash(pgssSharedState *pgss, dsa_area *dsa); +static PGSM_HASH_TABLE_HANDLE pgsm_create_bucket_hash(pgsmSharedState *pgsm, dsa_area *dsa); static Size pgsm_get_shared_area_size(void); +static void InitializeSharedState(pgsmSharedState *pgsm); #if USE_DYNAMIC_HASH /* parameter for the shared hash */ static dshash_parameters dsh_params = { - sizeof(pgssHashKey), - sizeof(pgssEntry), + sizeof(pgsmHashKey), + sizeof(pgsmEntry), dshash_memcmp, dshash_memhash }; @@ -54,12 +55,12 @@ pgsm_query_area_size(void) Size pgsm_ShmemSize(void) { - Size sz = MAXALIGN(sizeof(pgssSharedState)); + Size sz = MAXALIGN(sizeof(pgsmSharedState)); sz = add_size(sz, MAX_QUERY_BUF); #if USE_DYNAMIC_HASH sz = add_size(sz, MAX_BUCKETS_MEM); #else - sz = add_size(sz, hash_estimate_size(MAX_BUCKET_ENTRIES, sizeof(pgssEntry))); + sz = add_size(sz, hash_estimate_size(MAX_BUCKET_ENTRIES, sizeof(pgsmEntry))); #endif return MAXALIGN(sz); } @@ -77,47 +78,48 @@ pgsm_get_shared_area_size(void) #if USE_DYNAMIC_HASH sz = pgsm_ShmemSize(); #else - sz = MAXALIGN(sizeof(pgssSharedState)); + sz = MAXALIGN(sizeof(pgsmSharedState)); sz = add_size(sz, pgsm_query_area_size()); #endif return sz; } void -pgss_startup(void) +pgsm_startup(void) { bool found = false; - pgssSharedState *pgss; + pgsmSharedState *pgsm; /* reset in case this is a restart within the postmaster */ pgsmStateLocal.dsa = NULL; pgsmStateLocal.shared_hash = NULL; - pgsmStateLocal.shared_pgssState = NULL; + pgsmStateLocal.shared_pgsmState = NULL; /* * Create or attach to the shared memory state, including hash table */ LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); - pgss = ShmemInitStruct("pg_stat_monitor", pgsm_get_shared_area_size(), &found); + pgsm = ShmemInitStruct("pg_stat_monitor", pgsm_get_shared_area_size(), &found); if (!found) { /* First time through ... */ dsa_area *dsa; - char *p = (char *) pgss; + char *p = (char *) pgsm; - pgss->lock = &(GetNamedLWLockTranche("pg_stat_monitor"))->lock; - SpinLockInit(&pgss->mutex); - ResetSharedState(pgss); - /* the allocation of pgssSharedState itself */ - p += MAXALIGN(sizeof(pgssSharedState)); - pgss->raw_dsa_area = p; - dsa = dsa_create_in_place(pgss->raw_dsa_area, + pgsm->pgsm_oom = false; + pgsm->lock = &(GetNamedLWLockTranche("pg_stat_monitor"))->lock; + SpinLockInit(&pgsm->mutex); + InitializeSharedState(pgsm); + /* the allocation of pgsmSharedState itself */ + p += MAXALIGN(sizeof(pgsmSharedState)); + pgsm->raw_dsa_area = p; + dsa = dsa_create_in_place(pgsm->raw_dsa_area, pgsm_query_area_size(), LWLockNewTrancheId(), 0); dsa_pin(dsa); dsa_set_size_limit(dsa, pgsm_query_area_size()); - pgss->hash_handle = pgsm_create_bucket_hash(pgss,dsa); + pgsm->hash_handle = pgsm_create_bucket_hash(pgsm,dsa); /* If overflow is enabled, set the DSA size to unlimited, * and allow the DSA to grow beyond the shared memory space @@ -125,7 +127,7 @@ pgss_startup(void) if (PGSM_OVERFLOW_TARGET == OVERFLOW_TARGET_DISK) dsa_set_size_limit(dsa, -1); - pgsmStateLocal.shared_pgssState = pgss; + pgsmStateLocal.shared_pgsmState = pgsm; /* * Postmaster will never access the dsa again, thus free it's local * references. @@ -143,29 +145,41 @@ pgss_startup(void) * If we're in the postmaster (or a standalone backend...), set up a shmem * exit hook to dump the statistics to disk. */ - on_shmem_exit(pgss_shmem_shutdown, (Datum) 0); + on_shmem_exit(pgsm_shmem_shutdown, (Datum) 0); } +static void +InitializeSharedState(pgsmSharedState *pgsm) +{ + pg_atomic_init_u64(&pgsm->current_wbucket, 0); + pg_atomic_init_u64(&pgsm->prev_bucket_sec, 0); + memset(&pgsm->bucket_entry, 0, MAX_BUCKETS * sizeof(uint64)); + pgsm->pgsm_mem_cxt = AllocSetContextCreate(TopMemoryContext, + "pg_stat_monitor local store", + ALLOCSET_DEFAULT_SIZES); +} + + /* * Create the classic or dshahs hash table for storing the query statistics. */ static PGSM_HASH_TABLE_HANDLE -pgsm_create_bucket_hash(pgssSharedState *pgss, dsa_area *dsa) +pgsm_create_bucket_hash(pgsmSharedState *pgsm, dsa_area *dsa) { PGSM_HASH_TABLE_HANDLE bucket_hash; #if USE_DYNAMIC_HASH dshash_table *dsh; - pgss->hash_tranche_id = LWLockNewTrancheId(); - dsh_params.tranche_id = pgss->hash_tranche_id; + pgsm->hash_tranche_id = LWLockNewTrancheId(); + dsh_params.tranche_id = pgsm->hash_tranche_id; dsh = dshash_create(dsa, &dsh_params, 0); bucket_hash = dshash_get_hash_table_handle(dsh); dshash_detach(dsh); #else HASHCTL info; memset(&info, 0, sizeof(info)); - info.keysize = sizeof(pgssHashKey); - info.entrysize = sizeof(pgssEntry); + info.keysize = sizeof(pgsmHashKey); + info.entrysize = sizeof(pgsmEntry); bucket_hash = ShmemInitHash("pg_stat_monitor: bucket hashtable", MAX_BUCKET_ENTRIES, MAX_BUCKET_ENTRIES, &info, HASH_ELEM | HASH_BLOBS); #endif return bucket_hash; @@ -193,7 +207,7 @@ pgsm_attach_shmem(void) */ oldcontext = MemoryContextSwitchTo(TopMemoryContext); - pgsmStateLocal.dsa = dsa_attach_in_place(pgsmStateLocal.shared_pgssState->raw_dsa_area, + pgsmStateLocal.dsa = dsa_attach_in_place(pgsmStateLocal.shared_pgsmState->raw_dsa_area, NULL); /* pin the attached area to keep the area attached until end of * session or explicit detach. @@ -201,11 +215,11 @@ pgsm_attach_shmem(void) dsa_pin_mapping(pgsmStateLocal.dsa); #if USE_DYNAMIC_HASH - dsh_params.tranche_id = pgsmStateLocal.shared_pgssState->hash_tranche_id; + dsh_params.tranche_id = pgsmStateLocal.shared_pgsmState->hash_tranche_id; pgsmStateLocal.shared_hash = dshash_attach(pgsmStateLocal.dsa, &dsh_params, - pgsmStateLocal.shared_pgssState->hash_handle, 0); + pgsmStateLocal.shared_pgsmState->hash_handle, 0); #else - pgsmStateLocal.shared_hash = pgsmStateLocal.shared_pgssState->hash_handle; + pgsmStateLocal.shared_hash = pgsmStateLocal.shared_pgsmState->hash_handle; #endif MemoryContextSwitchTo(oldcontext); @@ -219,17 +233,17 @@ get_dsa_area_for_query_text(void) } PGSM_HASH_TABLE* -get_pgssHash(void) +get_pgsmHash(void) { pgsm_attach_shmem(); return pgsmStateLocal.shared_hash; } -pgssSharedState * +pgsmSharedState * pgsm_get_ss(void) { pgsm_attach_shmem(); - return pgsmStateLocal.shared_pgssState; + return pgsmStateLocal.shared_pgsmState; } @@ -240,35 +254,36 @@ pgsm_get_ss(void) * other processes running when this is called. */ void -pgss_shmem_shutdown(int code, Datum arg) +pgsm_shmem_shutdown(int code, Datum arg) { /* Don't try to dump during a crash. */ - elog(LOG,"pgss_shmem_shutdown"); + elog(LOG,"[pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated."); + if (code) return; - pgsmStateLocal.shared_pgssState = NULL; + pgsmStateLocal.shared_pgsmState = NULL; /* Safety check ... shouldn't get here unless shmem is set up. */ if (!IsHashInitialize()) return; } -pgssEntry * -hash_entry_alloc(pgssSharedState *pgss, pgssHashKey *key, int encoding) +pgsmEntry * +hash_entry_alloc(pgsmSharedState *pgsm, pgsmHashKey *key, int encoding) { - pgssEntry *entry = NULL; + pgsmEntry *entry = NULL; bool found = false; /* Find or create an entry with desired hash code */ - entry = (pgssEntry*) pgsm_hash_find_or_insert(pgsmStateLocal.shared_hash, key, &found); + entry = (pgsmEntry*) pgsm_hash_find_or_insert(pgsmStateLocal.shared_hash, key, &found); if (entry == NULL) - elog(DEBUG1, "hash_entry_alloc: OUT OF MEMORY"); + elog(DEBUG1, "[pg_stat_monitor] hash_entry_alloc: OUT OF MEMORY."); else if (!found) { - pgss->bucket_entry[pg_atomic_read_u64(&pgss->current_wbucket)]++; + pgsm->bucket_entry[pg_atomic_read_u64(&pgsm->current_wbucket)]++; /* New entry, initialize it */ /* reset the statistics */ memset(&entry->counters, 0, sizeof(Counters)); - entry->query_pos = InvalidDsaPointer; + entry->query_text.query_pos = InvalidDsaPointer; entry->counters.info.parent_query = InvalidDsaPointer; /* set the appropriate initial usage count */ @@ -288,23 +303,21 @@ hash_entry_alloc(pgssSharedState *pgss, pgssHashKey *key, int encoding) /* * Prepare resources for using the new bucket: * - Deallocate finished hash table entries in new_bucket_id (entries whose - * state is PGSS_FINISHED or PGSS_FINISHED). + * state is PGSM_EXEC or PGSM_ERROR). * - Clear query buffer for new_bucket_id. * - If old_bucket_id != -1, move all pending hash table entries in * old_bucket_id to the new bucket id, also move pending queries from the * previous query buffer (query_buffer[old_bucket_id]) to the new one * (query_buffer[new_bucket_id]). * - * Caller must hold an exclusive lock on pgss->lock. + * Caller must hold an exclusive lock on pgsm->lock. */ void hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_buffer) { PGSM_HASH_SEQ_STATUS hstat; - pgssEntry *entry = NULL; + pgsmEntry *entry = NULL; /* Store pending query ids from the previous bucket. */ - List *pending_entries = NIL; - ListCell *pending_entry; if (!pgsmStateLocal.shared_hash) return; @@ -321,11 +334,10 @@ hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_bu * in new_bucket_id if it has finished already. */ if (new_bucket_id < 0 || - (entry->key.bucket_id == new_bucket_id && - (entry->counters.state == PGSS_FINISHED || entry->counters.state == PGSS_ERROR))) + (entry->key.bucket_id == new_bucket_id )) { dsa_pointer parent_qdsa = entry->counters.info.parent_query; - pdsa = entry->query_pos; + pdsa = entry->query_text.query_pos; pgsm_hash_delete_current(&hstat, pgsmStateLocal.shared_hash, &entry->key); @@ -334,153 +346,23 @@ hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_bu if (DsaPointerIsValid(parent_qdsa)) dsa_free(pgsmStateLocal.dsa, parent_qdsa); - continue; - } - /* - * If we detect a pending query residing in the previous bucket id, we - * add it to a list of pending elements to be moved to the new bucket - * id. Can't update the hash table while iterating it inside this - * loop, as this may introduce all sort of problems. - */ - if (old_bucket_id != -1 && entry->key.bucket_id == old_bucket_id) - { - if (entry->counters.state == PGSS_PARSE || - entry->counters.state == PGSS_PLAN || - entry->counters.state == PGSS_EXEC) - { - pgssEntry *bkp_entry = malloc(sizeof(pgssEntry)); - - if (!bkp_entry) - { - elog(DEBUG1, "hash_entry_dealloc: out of memory"); - - /* - * No memory, If the entry has calls > 1 then we change - * the state to finished, as the pending query will likely - * finish execution during the new bucket time window. The - * pending query will vanish in this case, can't list it - * until it completes. - * - * If there is only one call to the query and it's - * pending, remove the entry from the previous bucket and - * allow it to finish in the new bucket, in order to avoid - * the query living in the old bucket forever. - */ - if (entry->counters.calls.calls > 1) - entry->counters.state = PGSS_FINISHED; - else - { - pdsa = entry->query_pos; - pgsm_hash_delete_current(&hstat, pgsmStateLocal.shared_hash, &entry->key); - if (DsaPointerIsValid(pdsa)) - dsa_free(pgsmStateLocal.dsa, pdsa); - } - continue; - } - - /* Save key/data from the previous entry. */ - memcpy(bkp_entry, entry, sizeof(pgssEntry)); - - /* Update key to use the new bucket id. */ - bkp_entry->key.bucket_id = new_bucket_id; - - /* Add the entry to a list of nodes to be processed later. */ - pending_entries = lappend(pending_entries, bkp_entry); - - /* - * If the entry has calls > 1 then we change the state to - * finished in the previous bucket, as the pending query will - * likely finish execution during the new bucket time window. - * Can't remove it from the previous bucket as it may have - * many calls and we would lose the query statistics. - * - * If there is only one call to the query and it's pending, - * remove the entry from the previous bucket and allow it to - * finish in the new bucket, in order to avoid the query - * living in the old bucket forever. - */ - if (entry->counters.calls.calls > 1) - entry->counters.state = PGSS_FINISHED; - else - { - pdsa = entry->query_pos; - pgsm_hash_delete_current(&hstat, pgsmStateLocal.shared_hash, &entry->key); - /* We should not delete the Query in DSA here - * as the same will get reused when the entry gets inserted into new bucket - */ - } - } + pgsmStateLocal.shared_pgsmState->pgsm_oom = false; } } pgsm_hash_seq_term(&hstat); - /* - * Iterate over the list of pending queries in order to add them back to - * the hash table with the updated bucket id. - */ - foreach(pending_entry, pending_entries) - { - bool found = false; - pgssEntry *new_entry; - pgssEntry *old_entry = (pgssEntry *) lfirst(pending_entry); - - - PGSM_DISABLE_ERROR_CAPUTRE(); - { - new_entry = (pgssEntry*) pgsm_hash_find_or_insert(pgsmStateLocal.shared_hash, &old_entry->key, &found); - }PGSM_END_DISABLE_ERROR_CAPTURE(); - - if (new_entry == NULL) - elog(DEBUG1, "%s", "pg_stat_monitor: out of memory"); - else if (!found) - { - /* Restore counters and other data. */ - new_entry->counters = old_entry->counters; - SpinLockInit(&new_entry->mutex); - new_entry->encoding = old_entry->encoding; - new_entry->query_pos = old_entry->query_pos; - } - #if USE_DYNAMIC_HASH - if(new_entry) - dshash_release_lock(pgsmStateLocal.shared_hash, new_entry); - #endif - free(old_entry); - } - list_free(pending_entries); -} - -/* - * Release all entries. - */ -void -hash_entry_reset() -{ - pgssSharedState *pgss = pgsm_get_ss(); - PGSM_HASH_SEQ_STATUS hstat; - pgssEntry *entry; - - LWLockAcquire(pgss->lock, LW_EXCLUSIVE); - - pgsm_hash_seq_init(&hstat, pgsmStateLocal.shared_hash, true); - - while ((entry = pgsm_hash_seq_next(&hstat)) != NULL) - { - dsa_pointer pdsa = entry->query_pos; - pgsm_hash_delete_current(&hstat, pgsmStateLocal.shared_hash, &entry->key); - if (DsaPointerIsValid(pdsa)) - dsa_free(pgsmStateLocal.dsa, pdsa); - } - - pgsm_hash_seq_term(&hstat); - - pg_atomic_write_u64(&pgss->current_wbucket, 0); - LWLockRelease(pgss->lock); } bool IsHashInitialize(void) { - return (pgsmStateLocal.shared_pgssState != NULL); + return (pgsmStateLocal.shared_pgsmState != NULL); +} + +bool +IsSystemOOM(void) +{ + return (IsHashInitialize() && pgsmStateLocal.shared_pgsmState->pgsm_oom); } /* @@ -489,7 +371,7 @@ IsHashInitialize(void) */ void * -pgsm_hash_find_or_insert(PGSM_HASH_TABLE *shared_hash, pgssHashKey *key, bool* found) +pgsm_hash_find_or_insert(PGSM_HASH_TABLE *shared_hash, pgsmHashKey *key, bool* found) { #if USE_DYNAMIC_HASH void *entry; @@ -501,7 +383,7 @@ pgsm_hash_find_or_insert(PGSM_HASH_TABLE *shared_hash, pgssHashKey *key, bool* f } void * -pgsm_hash_find(PGSM_HASH_TABLE *shared_hash, pgssHashKey *key, bool* found) +pgsm_hash_find(PGSM_HASH_TABLE *shared_hash, pgsmHashKey *key, bool* found) { #if USE_DYNAMIC_HASH return dshash_find(shared_hash, key, false); @@ -546,4 +428,4 @@ pgsm_hash_delete_current(PGSM_HASH_SEQ_STATUS *hstat, PGSM_HASH_TABLE *shared_ha #else hash_search(shared_hash, key, HASH_REMOVE, NULL); #endif -} \ No newline at end of file +} diff --git a/pg_stat_monitor--1.0--2.0.sql b/pg_stat_monitor--1.0--2.0.sql index a502a9c..de114ba 100644 --- a/pg_stat_monitor--1.0--2.0.sql +++ b/pg_stat_monitor--1.0--2.0.sql @@ -12,7 +12,9 @@ CREATE FUNCTION pg_stat_monitor_internal( IN showtext boolean, OUT bucket int8, -- 0 OUT userid oid, + OUT username text, OUT dbid oid, + OUT datname text, OUT client_ip int8, OUT queryid int8, -- 4 @@ -113,7 +115,7 @@ CREATE VIEW pg_stat_monitor AS SELECT bucket, bucket_start_time AS bucket_start_time, userid, - userid::regrole AS user, + username, dbid, datname, '0.0.0.0'::inet + client_ip AS client_ip, @@ -155,7 +157,7 @@ CREATE VIEW pg_stat_monitor AS SELECT cpu_user_time, cpu_sys_time, bucket_done -FROM pg_stat_monitor_internal(TRUE) p, pg_database d WHERE dbid = oid +FROM pg_stat_monitor_internal(TRUE) ORDER BY bucket_start_time; RETURN 0; END; @@ -169,7 +171,7 @@ CREATE VIEW pg_stat_monitor AS SELECT bucket, bucket_start_time AS bucket_start_time, userid, - userid::regrole AS user, + username, dbid, datname, '0.0.0.0'::inet + client_ip AS client_ip, @@ -222,7 +224,7 @@ CREATE VIEW pg_stat_monitor AS SELECT max_plan_time, mean_plan_time, stddev_plan_time -FROM pg_stat_monitor_internal(TRUE) p, pg_database d WHERE dbid = oid +FROM pg_stat_monitor_internal(TRUE) ORDER BY bucket_start_time; RETURN 0; END; @@ -235,7 +237,7 @@ CREATE VIEW pg_stat_monitor AS SELECT bucket, bucket_start_time AS bucket_start_time, userid, - userid::regrole AS user, + username, dbid, datname, '0.0.0.0'::inet + client_ip AS client_ip, @@ -288,7 +290,7 @@ CREATE VIEW pg_stat_monitor AS SELECT max_plan_time, mean_plan_time, stddev_plan_time -FROM pg_stat_monitor_internal(TRUE) p, pg_database d WHERE dbid = oid +FROM pg_stat_monitor_internal(TRUE) ORDER BY bucket_start_time; RETURN 0; END; @@ -301,7 +303,7 @@ CREATE VIEW pg_stat_monitor AS SELECT bucket, bucket_start_time AS bucket_start_time, userid, - userid::regrole AS user, + username, dbid, datname, '0.0.0.0'::inet + client_ip AS client_ip, @@ -367,7 +369,7 @@ CREATE VIEW pg_stat_monitor AS SELECT jit_emission_count, jit_emission_time -FROM pg_stat_monitor_internal(TRUE) p, pg_database d WHERE dbid = oid +FROM pg_stat_monitor_internal(TRUE) ORDER BY bucket_start_time; RETURN 0; END; @@ -395,15 +397,19 @@ $$ $$ LANGUAGE plpgsql; SELECT pgsm_create_view(); -REVOKE ALL ON FUNCTION range FROM PUBLIC; -REVOKE ALL ON FUNCTION get_cmd_type FROM PUBLIC; -REVOKE ALL ON FUNCTION decode_error_level FROM PUBLIC; -REVOKE ALL ON FUNCTION pg_stat_monitor_internal FROM PUBLIC; -REVOKE ALL ON FUNCTION get_histogram_timings FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_view FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_11_view FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_13_view FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_14_view FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_15_view FROM PUBLIC; +GRANT EXECUTE ON FUNCTION range TO PUBLIC; +GRANT EXECUTE ON FUNCTION decode_error_level TO PUBLIC; +GRANT EXECUTE ON FUNCTION get_histogram_timings TO PUBLIC; +GRANT EXECUTE ON FUNCTION get_cmd_type TO PUBLIC; +GRANT EXECUTE ON FUNCTION pg_stat_monitor_internal TO PUBLIC; + GRANT SELECT ON pg_stat_monitor TO PUBLIC; + +-- Reset is only available to super user +REVOKE ALL ON FUNCTION pg_stat_monitor_reset FROM PUBLIC; diff --git a/pg_stat_monitor--2.0.sql b/pg_stat_monitor--2.0.sql index 7bc2836..e9957c1 100644 --- a/pg_stat_monitor--2.0.sql +++ b/pg_stat_monitor--2.0.sql @@ -84,8 +84,10 @@ CREATE FUNCTION pg_stat_monitor_internal( IN showtext boolean, OUT bucket int8, -- 0 OUT userid oid, + OUT username text, OUT dbid oid, - OUT client_ip int8, + OUT datname text, + OUT client_ip int4, OUT queryid int8, -- 4 OUT planid int8, @@ -169,7 +171,7 @@ CREATE VIEW pg_stat_monitor AS SELECT bucket, bucket_start_time AS bucket_start_time, userid, - userid::regrole AS user, + username, dbid, datname, '0.0.0.0'::inet + client_ip AS client_ip, @@ -211,7 +213,7 @@ CREATE VIEW pg_stat_monitor AS SELECT cpu_user_time, cpu_sys_time, bucket_done -FROM pg_stat_monitor_internal(TRUE) p, pg_database d WHERE dbid = oid +FROM pg_stat_monitor_internal(TRUE) ORDER BY bucket_start_time; RETURN 0; END; @@ -225,7 +227,7 @@ CREATE VIEW pg_stat_monitor AS SELECT bucket, bucket_start_time AS bucket_start_time, userid, - userid::regrole AS user, + username, dbid, datname, '0.0.0.0'::inet + client_ip AS client_ip, @@ -278,7 +280,7 @@ CREATE VIEW pg_stat_monitor AS SELECT max_plan_time, mean_plan_time, stddev_plan_time -FROM pg_stat_monitor_internal(TRUE) p, pg_database d WHERE dbid = oid +FROM pg_stat_monitor_internal(TRUE) ORDER BY bucket_start_time; RETURN 0; END; @@ -291,7 +293,7 @@ CREATE VIEW pg_stat_monitor AS SELECT bucket, bucket_start_time AS bucket_start_time, userid, - userid::regrole AS user, + username, dbid, datname, '0.0.0.0'::inet + client_ip AS client_ip, @@ -344,7 +346,7 @@ CREATE VIEW pg_stat_monitor AS SELECT max_plan_time, mean_plan_time, stddev_plan_time -FROM pg_stat_monitor_internal(TRUE) p, pg_database d WHERE dbid = oid +FROM pg_stat_monitor_internal(TRUE) ORDER BY bucket_start_time; RETURN 0; END; @@ -357,7 +359,7 @@ CREATE VIEW pg_stat_monitor AS SELECT bucket, bucket_start_time AS bucket_start_time, userid, - userid::regrole AS user, + username, dbid, datname, '0.0.0.0'::inet + client_ip AS client_ip, @@ -423,7 +425,7 @@ CREATE VIEW pg_stat_monitor AS SELECT jit_emission_count, jit_emission_time -FROM pg_stat_monitor_internal(TRUE) p, pg_database d WHERE dbid = oid +FROM pg_stat_monitor_internal(TRUE) ORDER BY bucket_start_time; RETURN 0; END; @@ -451,15 +453,19 @@ $$ $$ LANGUAGE plpgsql; SELECT pgsm_create_view(); -REVOKE ALL ON FUNCTION range FROM PUBLIC; -REVOKE ALL ON FUNCTION get_cmd_type FROM PUBLIC; -REVOKE ALL ON FUNCTION decode_error_level FROM PUBLIC; -REVOKE ALL ON FUNCTION pg_stat_monitor_internal FROM PUBLIC; -REVOKE ALL ON FUNCTION get_histogram_timings FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_view FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_11_view FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_13_view FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_14_view FROM PUBLIC; REVOKE ALL ON FUNCTION pgsm_create_15_view FROM PUBLIC; +GRANT EXECUTE ON FUNCTION range TO PUBLIC; +GRANT EXECUTE ON FUNCTION decode_error_level TO PUBLIC; +GRANT EXECUTE ON FUNCTION get_histogram_timings TO PUBLIC; +GRANT EXECUTE ON FUNCTION get_cmd_type TO PUBLIC; +GRANT EXECUTE ON FUNCTION pg_stat_monitor_internal TO PUBLIC; + GRANT SELECT ON pg_stat_monitor TO PUBLIC; + +-- Reset is only available to super user +REVOKE ALL ON FUNCTION pg_stat_monitor_reset FROM PUBLIC; diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 5cd43f4..7de8864 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -17,9 +17,11 @@ #include "postgres.h" #include "access/parallel.h" +#include "nodes/pg_list.h" #include "utils/guc.h" #include #include "pgstat.h" +#include "commands/dbcommands.h" #include "commands/explain.h" #include "pg_stat_monitor.h" @@ -38,8 +40,8 @@ PG_MODULE_MAGIC; /* Number of output arguments (columns) for various API versions */ #define PG_STAT_MONITOR_COLS_V1_0 52 -#define PG_STAT_MONITOR_COLS_V2_0 62 -#define PG_STAT_MONITOR_COLS 62 /* maximum of above */ +#define PG_STAT_MONITOR_COLS_V2_0 64 +#define PG_STAT_MONITOR_COLS PG_STAT_MONITOR_COLS_V2_0 /* maximum of above */ #define PGSM_TEXT_FILE PGSTAT_STAT_PERMANENT_DIRECTORY "pg_stat_monitor_query" @@ -63,10 +65,6 @@ do \ /*---- Initicalization Function Declarations ----*/ void _PG_init(void); -void _PG_fini(void); - - -/*---- Local variables ----*/ /* Current nesting depth of ExecutorRun+ProcessUtility calls */ static int exec_nested_level = 0; @@ -85,6 +83,7 @@ static int hist_bucket_count_total; /* The array to store outer layer query id*/ uint64 *nested_queryids; char **nested_query_txts; +List *lentries = NIL; /* Regex object used to extract query comments. */ static regex_t preg_query_comments; @@ -97,7 +96,7 @@ static struct rusage rusage_end; /* Query buffer, store queries' text. */ -static char *pgss_explain(QueryDesc *queryDesc); +static char *pgsm_explain(QueryDesc *queryDesc); static void extract_query_comments(const char *query, char *comments, size_t max_len); static void histogram_bucket_timings(int index, int64 *b_start, int64 *b_end); @@ -110,7 +109,7 @@ static void request_additional_shared_resources(void); /* Saved hook values in case of unload */ #if PG_VERSION_NUM >= 150000 -static void pgss_shmem_request(void); +static void pgsm_shmem_request(void); static shmem_request_hook_type prev_shmem_request_hook = NULL; #endif #if PG_VERSION_NUM >= 130000 @@ -137,33 +136,33 @@ PG_FUNCTION_INFO_V1(get_histogram_timings); PG_FUNCTION_INFO_V1(pg_stat_monitor_hook_stats); static uint pg_get_client_addr(bool *ok); -static int pg_get_application_name(char *application_name, bool *ok); +static int pg_get_application_name(char *name, int buff_size); static PgBackendStatus *pg_get_backend_status(void); static Datum intarray_get_datum(int32 arr[], int len); #if PG_VERSION_NUM < 140000 -DECLARE_HOOK(void pgss_post_parse_analyze, ParseState *pstate, Query *query); +DECLARE_HOOK(void pgsm_post_parse_analyze, ParseState *pstate, Query *query); #else -DECLARE_HOOK(void pgss_post_parse_analyze, ParseState *pstate, Query *query, JumbleState *jstate); +DECLARE_HOOK(void pgsm_post_parse_analyze, ParseState *pstate, Query *query, JumbleState *jstate); #endif -DECLARE_HOOK(void pgss_ExecutorStart, QueryDesc *queryDesc, int eflags); -DECLARE_HOOK(void pgss_ExecutorRun, QueryDesc *queryDesc, ScanDirection direction, uint64 count, bool execute_once); -DECLARE_HOOK(void pgss_ExecutorFinish, QueryDesc *queryDesc); -DECLARE_HOOK(void pgss_ExecutorEnd, QueryDesc *queryDesc); -DECLARE_HOOK(bool pgss_ExecutorCheckPerms, List *rt, bool abort); +DECLARE_HOOK(void pgsm_ExecutorStart, QueryDesc *queryDesc, int eflags); +DECLARE_HOOK(void pgsm_ExecutorRun, QueryDesc *queryDesc, ScanDirection direction, uint64 count, bool execute_once); +DECLARE_HOOK(void pgsm_ExecutorFinish, QueryDesc *queryDesc); +DECLARE_HOOK(void pgsm_ExecutorEnd, QueryDesc *queryDesc); +DECLARE_HOOK(bool pgsm_ExecutorCheckPerms, List *rt, bool abort); #if PG_VERSION_NUM >= 140000 -DECLARE_HOOK(PlannedStmt *pgss_planner_hook, Query *parse, const char *query_string, int cursorOptions, ParamListInfo boundParams); -DECLARE_HOOK(void pgss_ProcessUtility, PlannedStmt *pstmt, const char *queryString, +DECLARE_HOOK(PlannedStmt *pgsm_planner_hook, Query *parse, const char *query_string, int cursorOptions, ParamListInfo boundParams); +DECLARE_HOOK(void pgsm_ProcessUtility, PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *qc); #elif PG_VERSION_NUM >= 130000 -DECLARE_HOOK(PlannedStmt *pgss_planner_hook, Query *parse, const char *query_string, int cursorOptions, ParamListInfo boundParams); -DECLARE_HOOK(void pgss_ProcessUtility, PlannedStmt *pstmt, const char *queryString, +DECLARE_HOOK(PlannedStmt *pgsm_planner_hook, Query *parse, const char *query_string, int cursorOptions, ParamListInfo boundParams); +DECLARE_HOOK(void pgsm_ProcessUtility, PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, @@ -171,36 +170,50 @@ DECLARE_HOOK(void pgss_ProcessUtility, PlannedStmt *pstmt, const char *queryStri #else static void BufferUsageAccumDiff(BufferUsage *bufusage, BufferUsage *pgBufferUsage, BufferUsage *bufusage_start); -DECLARE_HOOK(void pgss_ProcessUtility, PlannedStmt *pstmt, const char *queryString, +DECLARE_HOOK(void pgsm_ProcessUtility, PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, char *completionTag); #endif -static uint64 pgss_hash_string(const char *str, int len); +static uint64 pgsm_hash_string(const char *str, int len); char *unpack_sql_state(int sql_state); #define PGSM_HANDLED_UTILITY(n) (!IsA(n, ExecuteStmt) && \ !IsA(n, PrepareStmt) && \ !IsA(n, DeallocateStmt)) -static void pgss_store_error(uint64 queryid, const char *query, ErrorData *edata); -static void pgss_store(uint64 queryid, - const char *query, - int query_location, - int query_len, - PlanInfo * plan_info, - CmdType cmd_type, - SysInfo * sys_info, - ErrorInfo * error_info, - double total_time, - uint64 rows, - BufferUsage *bufusage, - WalUsage *walusage, - const struct JitInstrumentation *jitusage, - JumbleState *jstate, - pgssStoreKind kind); +static pgsmEntry *pgsm_create_hash_entry(uint64 bucket_id, uint64 queryid, PlanInfo *plan_info); +static void pgsm_add_to_list(pgsmEntry *entry, char *query_text, int query_len); +static pgsmEntry* pgsm_get_entry_for_query(uint64 queryid, PlanInfo *plan_info, const char* query_text, int query_len, bool create); +static uint64 get_pgsm_query_id_hash(const char *norm_query, int len); + +static void pgsm_cleanup_callback(void *arg); +static void pgsm_store_error(const char *query, ErrorData *edata); + +/*---- Local variables ----*/ +MemoryContextCallback mem_cxt_reset_callback = + { + .func = pgsm_cleanup_callback, + .arg = NULL + }; +volatile bool callback_setup = false; + +static void pgsm_update_entry(pgsmEntry *entry, + const char *query, + PlanInfo * plan_info, + SysInfo * sys_info, + ErrorInfo * error_info, + double plan_total_time, + double exec_total_time, + uint64 rows, + BufferUsage *bufusage, + WalUsage *walusage, + const struct JitInstrumentation *jitusage, + bool reset, + pgsmStoreKind kind); +static void pgsm_store(pgsmEntry *entry); static void pg_stat_monitor_internal(FunctionCallInfo fcinfo, pgsmVersion api_version, @@ -219,6 +232,7 @@ static void RecordConstLocation(JumbleState *jstate, int location); * relevant part of the string. */ static const char *CleanQuerytext(const char *query, int *location, int *len); +static uint64 get_query_id(JumbleState *jstate, Query *query); #endif static char *generate_normalized_query(JumbleState *jstate, const char *query, @@ -226,17 +240,8 @@ static char *generate_normalized_query(JumbleState *jstate, const char *query, static void fill_in_constant_lengths(JumbleState *jstate, const char *query, int query_loc); static int comp_location(const void *a, const void *b); -static uint64 get_next_wbucket(pgssSharedState *pgss); +static uint64 get_next_wbucket(pgsmSharedState *pgsm); -#if PG_VERSION_NUM < 140000 -static uint64 get_query_id(JumbleState *jstate, Query *query); -#endif - -/* Daniel J. Bernstein's hash algorithm: see http://www.cse.yorku.ca/~oz/hash.html */ -static uint64 djb2_hash(unsigned char *str, size_t len); - -/* Same as above, but stores the calculated string length into *out_len (small optimization) */ -static uint64 djb2_hash_str(unsigned char *str, int *out_len); /* * Module load callback */ @@ -246,7 +251,7 @@ _PG_init(void) { int rc; - elog(DEBUG2, "pg_stat_monitor: %s()", __FUNCTION__); + elog(DEBUG2, "[pg_stat_monitor] pg_stat_monitor: %s().", __FUNCTION__); /* * In order to create our shared memory area, we have to be loaded via @@ -323,7 +328,7 @@ _PG_init(void) rc = regcomp(&preg_query_comments, "/\\*([^*]|[\r\n]|(\\*+([^*/]|[\r\n])))*\\*+/", REG_EXTENDED); if (rc != 0) { - elog(ERROR, "pg_stat_monitor: query comments regcomp() failed, return code=(%d)\n", rc); + elog(ERROR, "[pg_stat_monitor] _PG_init: query comments regcomp() failed, return code=(%d).", rc); } /* @@ -331,32 +336,32 @@ _PG_init(void) */ #if PG_VERSION_NUM >= 150000 prev_shmem_request_hook = shmem_request_hook; - shmem_request_hook = pgss_shmem_request; + shmem_request_hook = pgsm_shmem_request; #else request_additional_shared_resources(); #endif prev_shmem_startup_hook = shmem_startup_hook; - shmem_startup_hook = pgss_shmem_startup; + shmem_startup_hook = pgsm_shmem_startup; prev_post_parse_analyze_hook = post_parse_analyze_hook; - post_parse_analyze_hook = HOOK(pgss_post_parse_analyze); + post_parse_analyze_hook = HOOK(pgsm_post_parse_analyze); prev_ExecutorStart = ExecutorStart_hook; - ExecutorStart_hook = HOOK(pgss_ExecutorStart); + ExecutorStart_hook = HOOK(pgsm_ExecutorStart); prev_ExecutorRun = ExecutorRun_hook; - ExecutorRun_hook = HOOK(pgss_ExecutorRun); + ExecutorRun_hook = HOOK(pgsm_ExecutorRun); prev_ExecutorFinish = ExecutorFinish_hook; - ExecutorFinish_hook = HOOK(pgss_ExecutorFinish); + ExecutorFinish_hook = HOOK(pgsm_ExecutorFinish); prev_ExecutorEnd = ExecutorEnd_hook; - ExecutorEnd_hook = HOOK(pgss_ExecutorEnd); + ExecutorEnd_hook = HOOK(pgsm_ExecutorEnd); prev_ProcessUtility = ProcessUtility_hook; - ProcessUtility_hook = HOOK(pgss_ProcessUtility); + ProcessUtility_hook = HOOK(pgsm_ProcessUtility); #if PG_VERSION_NUM >= 130000 planner_hook_next = planner_hook; - planner_hook = HOOK(pgss_planner_hook); + planner_hook = HOOK(pgsm_planner_hook); #endif prev_emit_log_hook = emit_log_hook; emit_log_hook = HOOK(pgsm_emit_log_hook); prev_ExecutorCheckPerms_hook = ExecutorCheckPerms_hook; - ExecutorCheckPerms_hook = HOOK(pgss_ExecutorCheckPerms); + ExecutorCheckPerms_hook = HOOK(pgsm_ExecutorCheckPerms); nested_queryids = (uint64 *) malloc(sizeof(uint64) * max_stack_depth); nested_query_txts = (char **) malloc(sizeof(char*) * max_stack_depth); @@ -364,30 +369,6 @@ _PG_init(void) system_init = true; } -/* - * Module unload callback - */ -/* cppcheck-suppress unusedFunction */ -void -_PG_fini(void) -{ - system_init = false; - shmem_startup_hook = prev_shmem_startup_hook; - post_parse_analyze_hook = prev_post_parse_analyze_hook; - ExecutorStart_hook = prev_ExecutorStart; - ExecutorRun_hook = prev_ExecutorRun; - ExecutorFinish_hook = prev_ExecutorFinish; - ExecutorEnd_hook = prev_ExecutorEnd; - ProcessUtility_hook = prev_ProcessUtility; - emit_log_hook = prev_emit_log_hook; - - free(nested_queryids); - free(nested_query_txts); - regfree(&preg_query_comments); - - hash_entry_reset(); -} - /* * shmem_startup hook: allocate or attach to shared memory, * then load any pre-existing statistics from file. @@ -395,12 +376,12 @@ _PG_fini(void) * (even if empty) while the module is enabled. */ void -pgss_shmem_startup(void) +pgsm_shmem_startup(void) { if (prev_shmem_startup_hook) prev_shmem_startup_hook(); - pgss_startup(); + pgsm_startup(); } static void @@ -409,7 +390,7 @@ request_additional_shared_resources(void) /* * Request additional shared resources. (These are no-ops if we're not in * the postmaster process.) We'll allocate or attach to the shared - * resources in pgss_shmem_startup(). + * resources in pgsm_shmem_startup(). */ RequestAddinShmemSpace(pgsm_ShmemSize() + HOOK_STATS_SIZE); RequestNamedLWLockTranche("pg_stat_monitor", 1); @@ -426,10 +407,10 @@ pg_stat_monitor_version(PG_FUNCTION_ARGS) #if PG_VERSION_NUM >= 150000 /* * shmem_request hook: request additional shared resources. We'll allocate or - * attach to the shared resources in pgss_shmem_startup(). + * attach to the shared resources in pgsm_shmem_startup(). */ static void -pgss_shmem_request(void) +pgsm_shmem_request(void) { if (prev_shmem_request_hook) prev_shmem_request_hook(); @@ -437,20 +418,32 @@ pgss_shmem_request(void) } #endif -#if PG_VERSION_NUM >= 140000 -/* - * Post-parse-analysis hook: mark query with a queryId - */ static void -pgss_post_parse_analyze(ParseState *pstate, Query *query, JumbleState *jstate) +pgsm_post_parse_analyze_internal(ParseState *pstate, Query *query, JumbleState *jstate) { - if (prev_post_parse_analyze_hook) - prev_post_parse_analyze_hook(pstate, query, jstate); + pgsmEntry *entry; + const char *query_text; + char *norm_query = NULL; + int norm_query_len; + int location; + int query_len; /* Safety check... */ if (!IsSystemInitialized()) return; + if (callback_setup == false) + { + /* If MessageContext is valid setup a callback to cleanup + * our local stats list when the MessagContext gets reset + */ + if (MemoryContextIsValid(MessageContext)) + { + MemoryContextRegisterResetCallback(MessageContext, &mem_cxt_reset_callback); + callback_setup = true; + } + } + if (!pgsm_enabled(exec_nested_level)) return; @@ -463,67 +456,17 @@ pgss_post_parse_analyze(ParseState *pstate, Query *query, JumbleState *jstate) { if (PGSM_TRACK_UTILITY && !PGSM_HANDLED_UTILITY(query->utilityStmt)) query->queryId = UINT64CONST(0); + return; } /* - * If query jumbling were able to identify any ignorable constants, we - * immediately create a hash table entry for the query, so that we can - * record the normalized form of the query string. If there were no such - * constants, the normalized string would be the same as the query text - * anyway, so there's no need for an early entry. + * Let's calculate queryid for versions 13 and below. We don't have to check + * that jstate is valid, it always will be for these versions. */ - if (jstate && jstate->clocations_count > 0) - pgss_store(query->queryId, /* query id */ - pstate->p_sourcetext, /* query */ - query->stmt_location, /* query location */ - query->stmt_len, /* query length */ - NULL, /* PlanInfo */ - query->commandType, /* CmdType */ - NULL, /* SysInfo */ - NULL, /* ErrorInfo */ - 0, /* totaltime */ - 0, /* rows */ - NULL, /* bufusage */ - NULL, /* walusage */ - NULL, /* jitusage */ - jstate, /* JumbleState */ - PGSS_PARSE); /* pgssStoreKind */ -} -#else - -/* - * Post-parse-analysis hook: mark query with a queryId - */ -static void -pgss_post_parse_analyze(ParseState *pstate, Query *query) -{ - JumbleState jstate; - - if (prev_post_parse_analyze_hook) - prev_post_parse_analyze_hook(pstate, query); - - /* Safety check... */ - if (!IsSystemInitialized()) - return; - if (!pgsm_enabled(exec_nested_level)) - return; - - /* - * Utility statements get queryId zero. We do this even in cases where - * the statement contains an optimizable statement for which a queryId - * could be derived (such as EXPLAIN or DECLARE CURSOR). For such cases, - * runtime control will first go through ProcessUtility and then the - * executor, and we don't want the executor hooks to do anything, since we - * are already measuring the statement's costs at the utility level. - */ - if (query->utilityStmt) - { - query->queryId = UINT64CONST(0); - return; - } - - query->queryId = get_query_id(&jstate, query); +#if PG_VERSION_NUM < 140000 + query->queryId = get_query_id(jstate, query); +#endif /* * If we are unlucky enough to get a hash of zero, use 1 instead, to @@ -532,22 +475,93 @@ pgss_post_parse_analyze(ParseState *pstate, Query *query) if (query->queryId == UINT64CONST(0)) query->queryId = UINT64CONST(1); - if (jstate.clocations_count > 0) - pgss_store(query->queryId, /* query id */ - pstate->p_sourcetext, /* query */ - query->stmt_location, /* query location */ - query->stmt_len, /* query length */ - NULL, /* PlanInfo */ - query->commandType, /* CmdType */ - NULL, /* SysInfo */ - NULL, /* ErrorInfo */ - 0, /* totaltime */ - 0, /* rows */ - NULL, /* bufusage */ - NULL, /* walusage */ - NULL, /* jitusage */ - &jstate, /* JumbleState */ - PGSS_PARSE); /* pgssStoreKind */ + /* + * Let's save the normalized query so that we can save the data without in + * hash later on without the need of jstate which wouldn't be available. + */ + query_text = pstate->p_sourcetext; + location = query->stmt_location; + query_len = query->stmt_len; + + /* We should always have a valid query. */ + query_text = CleanQuerytext(query_text, &location, &query_len); + Assert(query_text); + + norm_query_len = query_len; + + /* Generate a normalized query */ + if (jstate && jstate->clocations_count > 0) + { + norm_query = generate_normalized_query(jstate, + query_text, /* query */ + location, /* query location */ + &norm_query_len, + GetDatabaseEncoding()); + + Assert(norm_query); + } + + /* + * At this point, we don't know which bucket this query will land in, so passing + * 0. The store function MUST later update it based on the current bucket value. + * The correct bucket value will be needed then to search the hash table, or create + * the appropriate entry. + */ + entry = pgsm_create_hash_entry(0, query->queryId, NULL); + + /* Update other member that are not counters, so that we don't have to worry about these. */ + entry->pgsm_query_id = get_pgsm_query_id_hash(norm_query ? norm_query : query_text, norm_query_len); + entry->counters.info.cmd_type = query->commandType; + + /* + * Add the query text and entry to the local list. + * + * Preserve the normalized query if needed and we got a valid one. + * Otherwise, store the actual query so that we don't have to check + * what query to store when saving into the hash. + * + * In case of query_text, request the function to duplicate it so that + * it is put in the relevant memory context. + */ + if (PGSM_NORMALIZED_QUERY && norm_query) + pgsm_add_to_list(entry, norm_query, norm_query_len); + else + { + pgsm_add_to_list(entry, (char *)query_text, query_len); + } + + /* Check that we've not exceeded max_stack_depth */ + Assert(list_length(lentries) <= max_stack_depth); + + if (norm_query) + pfree(norm_query); +} + +#if PG_VERSION_NUM >= 140000 +/* + * Post-parse-analysis hook: mark query with a queryId + */ +static void +pgsm_post_parse_analyze(ParseState *pstate, Query *query, JumbleState *jstate) +{ + if (prev_post_parse_analyze_hook) + prev_post_parse_analyze_hook(pstate, query, jstate); + + pgsm_post_parse_analyze_internal(pstate, query, jstate); +} +#else +/* + * Post-parse-analysis hook: mark query with a queryId + */ +static void +pgsm_post_parse_analyze(ParseState *pstate, Query *query) +{ + JumbleState jstate; + + if (prev_post_parse_analyze_hook) + prev_post_parse_analyze_hook(pstate, query); + + pgsm_post_parse_analyze_internal(pstate, query, &jstate); } #endif @@ -555,10 +569,10 @@ pgss_post_parse_analyze(ParseState *pstate, Query *query) * ExecutorStart hook: start up tracking if needed */ static void -pgss_ExecutorStart(QueryDesc *queryDesc, int eflags) +pgsm_ExecutorStart(QueryDesc *queryDesc, int eflags) { if (getrusage(RUSAGE_SELF, &rusage_start) != 0) - elog(DEBUG1, "pgss_ExecutorStart: failed to execute getrusage"); + elog(DEBUG1, "[pg_stat_monitor] pgsm_ExecutorStart: failed to execute getrusage."); if (prev_ExecutorStart) prev_ExecutorStart(queryDesc, eflags); @@ -598,7 +612,7 @@ pgss_ExecutorStart(QueryDesc *queryDesc, int eflags) * ExecutorRun hook: all we need do is track nesting depth */ static void -pgss_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, +pgsm_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, bool execute_once) { if (exec_nested_level >= 0 && exec_nested_level < max_stack_depth) @@ -642,9 +656,10 @@ pgss_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, * ExecutorFinish hook: all we need do is track nesting depth */ static void -pgss_ExecutorFinish(QueryDesc *queryDesc) +pgsm_ExecutorFinish(QueryDesc *queryDesc) { exec_nested_level++; + PG_TRY(); { if (prev_ExecutorFinish) @@ -659,10 +674,11 @@ pgss_ExecutorFinish(QueryDesc *queryDesc) PG_RE_THROW(); } PG_END_TRY(); + } static char * -pgss_explain(QueryDesc *queryDesc) +pgsm_explain(QueryDesc *queryDesc) { ExplainState *es = NewExplainState(); @@ -685,26 +701,34 @@ pgss_explain(QueryDesc *queryDesc) * ExecutorEnd hook: store results if needed */ static void -pgss_ExecutorEnd(QueryDesc *queryDesc) +pgsm_ExecutorEnd(QueryDesc *queryDesc) { uint64 queryId = queryDesc->plannedstmt->queryId; SysInfo sys_info; PlanInfo plan_info; PlanInfo *plan_ptr = NULL; + pgsmEntry *entry = NULL; /* Extract the plan information in case of SELECT statement */ if (queryDesc->operation == CMD_SELECT && PGSM_QUERY_PLAN) { - MemoryContext mct = MemoryContextSwitchTo(TopMemoryContext); - - plan_info.plan_len = snprintf(plan_info.plan_text, PLAN_TEXT_LEN, "%s", pgss_explain(queryDesc)); - plan_info.planid = pgss_hash_string(plan_info.plan_text, plan_info.plan_len); + plan_info.plan_len = snprintf(plan_info.plan_text, PLAN_TEXT_LEN, "%s", pgsm_explain(queryDesc)); + plan_info.planid = pgsm_hash_string(plan_info.plan_text, plan_info.plan_len); plan_ptr = &plan_info; - MemoryContextSwitchTo(mct); } if (queryId != UINT64CONST(0) && queryDesc->totaltime && pgsm_enabled(exec_nested_level)) { + entry = pgsm_get_entry_for_query(queryId, plan_ptr, (char *)queryDesc->sourceText, strlen(queryDesc->sourceText), true); + if(!entry) + { + elog(DEBUG2,"[pg_stat_monitor] pgsm_ExecutorEnd: Failed to find entry for [%lu] %s.",queryId, queryDesc->sourceText); + return; + } + + if (entry->key.planid == 0) + entry->key.planid = (plan_ptr) ? plan_ptr->planid : 0; + /* * Make sure stats accumulation is done. (Note: it's okay if several * levels of hook all do this.) @@ -715,46 +739,48 @@ pgss_ExecutorEnd(QueryDesc *queryDesc) sys_info.stime = 0; if (getrusage(RUSAGE_SELF, &rusage_end) != 0) - elog(DEBUG1, "pg_stat_monitor: failed to execute getrusage"); + elog(DEBUG1, "[pg_stat_monitor] pgsm_ExecutorEnd: Failed to execute getrusage."); else { sys_info.utime = time_diff(rusage_end.ru_utime, rusage_start.ru_utime); sys_info.stime = time_diff(rusage_end.ru_stime, rusage_start.ru_stime); } - pgss_store(queryId, /* query id */ - queryDesc->sourceText, /* query text */ - queryDesc->plannedstmt->stmt_location, /* query location */ - queryDesc->plannedstmt->stmt_len, /* query length */ - plan_ptr, /* PlanInfo */ - queryDesc->operation, /* CmdType */ - &sys_info, /* SysInfo */ - NULL, /* ErrorInfo */ - queryDesc->totaltime->total * 1000.0, /* totaltime */ - queryDesc->estate->es_processed, /* rows */ - &queryDesc->totaltime->bufusage, /* bufusage */ + pgsm_update_entry(entry, /* entry */ + NULL, /* query */ + plan_ptr, /* PlanInfo */ + &sys_info, /* SysInfo */ + NULL, /* ErrorInfo */ + 0, /* plan_total_time */ + queryDesc->totaltime->total * 1000.0, /* exec_total_time */ + queryDesc->estate->es_processed, /* rows */ + &queryDesc->totaltime->bufusage, /* bufusage */ #if PG_VERSION_NUM >= 130000 - &queryDesc->totaltime->walusage, /* walusage */ + &queryDesc->totaltime->walusage, /* walusage */ #else - NULL, + NULL, #endif #if PG_VERSION_NUM >= 150000 - queryDesc->estate->es_jit ? &queryDesc->estate->es_jit->instr : NULL, + queryDesc->estate->es_jit ? &queryDesc->estate->es_jit->instr : NULL, /* jitusage */ #else - NULL, + NULL, #endif - NULL, - PGSS_FINISHED); /* pgssStoreKind */ + false, /* reset */ + PGSM_EXEC); /* kind */ + + pgsm_store(entry); } + if (prev_ExecutorEnd) prev_ExecutorEnd(queryDesc); else standard_ExecutorEnd(queryDesc); + num_relations = 0; } static bool -pgss_ExecutorCheckPerms(List *rt, bool abort) +pgsm_ExecutorCheckPerms(List *rt, bool abort) { ListCell *lr = NULL; int i = 0; @@ -805,14 +831,16 @@ pgss_ExecutorCheckPerms(List *rt, bool abort) #if PG_VERSION_NUM >= 130000 static PlannedStmt * -pgss_planner_hook(Query *parse, const char *query_string, int cursorOptions, ParamListInfo boundParams) +pgsm_planner_hook(Query *parse, const char *query_string, int cursorOptions, ParamListInfo boundParams) { PlannedStmt *result; + pgsmEntry *entry = NULL; + /* * We can't process the query if no query_string is provided, as - * pgss_store needs it. We also ignore query without queryid, as it would - * be treated as a utility statement, which may not be the case. + * pgsm_store needs it. We also ignore query without queryid, + * as it would be treated as a utility statement, which may not be the case. * * Note that planner_hook can be called from the planner itself, so we * have a specific nesting level for the planner. However, utility @@ -821,6 +849,10 @@ pgss_planner_hook(Query *parse, const char *query_string, int cursorOptions, Par * So testing the planner nesting level only is not enough to detect real * top level planner call. */ + if (MemoryContextIsValid(MessageContext)) + entry = pgsm_get_entry_for_query(parse->queryId, NULL, query_string, strlen(query_string), true); + + if (pgsm_enabled(plan_nested_level + exec_nested_level) && PGSM_TRACK_PLANNING && query_string && parse->queryId != UINT64CONST(0)) { @@ -872,21 +904,22 @@ pgss_planner_hook(Query *parse, const char *query_string, int cursorOptions, Par /* calc differences of WAL counters. */ memset(&walusage, 0, sizeof(WalUsage)); WalUsageAccumDiff(&walusage, &pgWalUsage, &walusage_start); - pgss_store(parse->queryId, /* query id */ - query_string, /* query */ - parse->stmt_location, /* query location */ - parse->stmt_len, /* query length */ - NULL, /* PlanInfo */ - parse->commandType, /* CmdType */ - NULL, /* SysInfo */ - NULL, /* ErrorInfo */ - INSTR_TIME_GET_MILLISEC(duration), /* totaltime */ - 0, /* rows */ - &bufusage, /* bufusage */ - &walusage, /* walusage */ - NULL, /* JumbleState */ - NULL, - PGSS_PLAN); /* pgssStoreKind */ + + /* The plan details are captured when the query finishes */ + if(entry) + pgsm_update_entry(entry, /* entry */ + NULL, /* query */ + NULL, /* PlanInfo */ + NULL, /* SysInfo */ + NULL, /* ErrorInfo */ + INSTR_TIME_GET_MILLISEC(duration), /* plan_total_time */ + 0, /* exec_total_time */ + 0, /* rows */ + &bufusage, /* bufusage */ + &walusage, /* walusage */ + NULL, /* jitusage */ + false, /* reset */ + PGSM_PLAN); /* kind */ } else { @@ -897,22 +930,25 @@ pgss_planner_hook(Query *parse, const char *query_string, int cursorOptions, Par * since it modifies the first argument (Query *), the second call * would trigger an assertion failure. */ + plan_nested_level++; + if (planner_hook_next) result = planner_hook_next(parse, query_string, cursorOptions, boundParams); else result = standard_planner(parse, query_string, cursorOptions, boundParams); + plan_nested_level--; + } return result; } #endif - /* * ProcessUtility hook */ #if PG_VERSION_NUM >= 140000 static void -pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, +pgsm_ProcessUtility(PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, @@ -921,7 +957,7 @@ pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, #elif PG_VERSION_NUM >= 130000 static void -pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, +pgsm_ProcessUtility(PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, @@ -929,7 +965,7 @@ pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, #else static void -pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, +pgsm_ProcessUtility(PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, @@ -938,9 +974,12 @@ pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, { Node *parsetree = pstmt->utilityStmt; uint64 queryId = 0; - SysInfo sys_info; -#if PG_VERSION_NUM >= 140000 +#if PG_VERSION_NUM < 140000 + int len = strlen(queryString); + + queryId = pgsm_hash_string(queryString, len); +#else queryId = pstmt->queryId; /* @@ -973,9 +1012,14 @@ pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, if (PGSM_TRACK_UTILITY && pgsm_enabled(exec_nested_level) && PGSM_HANDLED_UTILITY(parsetree)) { + pgsmEntry *entry; + char *query_text; + int location; + int query_len; instr_time start; instr_time duration; uint64 rows; + SysInfo sys_info; BufferUsage bufusage; BufferUsage bufusage_start = pgBufferUsage; #if PG_VERSION_NUM >= 130000 @@ -984,7 +1028,7 @@ pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, #endif if (getrusage(RUSAGE_SELF, &rusage_start) != 0) - elog(DEBUG1, "pg_stat_monitor: failed to execute getrusage"); + elog(DEBUG1, "[pg_stat_monitor] pgsm_ProcessUtility: Failed to execute getrusage."); INSTR_TIME_SET_CURRENT(start); exec_nested_level++; @@ -1041,7 +1085,7 @@ pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, PG_END_TRY(); if (getrusage(RUSAGE_SELF, &rusage_end) != 0) - elog(DEBUG1, "pg_stat_monitor: failed to execute getrusage"); + elog(DEBUG1, "[pg_stat_monitor] pgsm_ProcessUtility: Failed to execute getrusage."); else { sys_info.utime = time_diff(rusage_end.ru_utime, rusage_start.ru_utime); @@ -1076,26 +1120,42 @@ pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, /* calc differences of buffer counters. */ memset(&bufusage, 0, sizeof(BufferUsage)); BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &bufusage_start); - pgss_store( - queryId, /* query ID */ - queryString, /* query text */ - pstmt->stmt_location, /* query location */ - pstmt->stmt_len, /* query length */ - NULL, /* PlanInfo */ - 0, /* CmdType */ - &sys_info, /* SysInfo */ - NULL, /* ErrorInfo */ - INSTR_TIME_GET_MILLISEC(duration), /* total_time */ - rows, /* rows */ - &bufusage, /* bufusage */ + + /* Create an entry for this query */ + entry = pgsm_create_hash_entry(0, queryId, NULL); + + location = pstmt->stmt_location; + query_len = pstmt->stmt_len; + query_text = (char *)CleanQuerytext(queryString, &location, &query_len); + + entry->pgsm_query_id = get_pgsm_query_id_hash(query_text, query_len); + entry->counters.info.cmd_type = 0; + + pgsm_add_to_list(entry, query_text, query_len); + + /* Check that we've not exceeded max_stack_depth */ + Assert(list_length(lentries) <= max_stack_depth); + + /* The plan details are captured when the query finishes */ + pgsm_update_entry(entry, /* entry */ + (char *)query_text, /* query */ + NULL, /* PlanInfo */ + &sys_info, /* SysInfo */ + NULL, /* ErrorInfo */ + 0, /* plan_total_time */ + INSTR_TIME_GET_MILLISEC(duration), /* exec_total_time */ + rows, /* rows */ + &bufusage, /* bufusage */ #if PG_VERSION_NUM >= 130000 - &walusage, /* walusage */ + &walusage, /* walusage */ #else - NULL, /* walusage, NULL for PG <= 12 */ + NULL, #endif - NULL, - NULL, /* JumbleState */ - PGSS_FINISHED); /* pgssStoreKind */ + NULL, /* jitusage */ + false, /* reset */ + PGSM_EXEC); /* kind */ + + pgsm_store(entry); } else { @@ -1166,7 +1226,7 @@ BufferUsageAccumDiff(BufferUsage *bufusage, BufferUsage *pgBufferUsage, BufferUs * utility statements. */ static uint64 -pgss_hash_string(const char *str, int len) +pgsm_hash_string(const char *str, int len) { return DatumGetUInt64(hash_any_extended((const unsigned char *) str, len, 0)); @@ -1195,17 +1255,30 @@ pg_get_backend_status(void) return NULL; } +/* + * The caller should allocate max_len memory to name including terminating null. + * The function returns the length of the string. + */ static int -pg_get_application_name(char *application_name, bool *ok) +pg_get_application_name(char *name, int buff_size) { - PgBackendStatus *beentry = pg_get_backend_status(); + PgBackendStatus *beentry; - if (!beentry) - return snprintf(application_name, APPLICATIONNAME_LEN, "%s", "unknown"); + /* Try to read application name from GUC directly */ + if (application_name && *application_name) + snprintf(name, buff_size, "%s", application_name); + else + { + beentry = pg_get_backend_status(); - *ok = true; + if (!beentry) + snprintf(name, buff_size, "%s", "unknown"); + else + snprintf(name, buff_size, "%s", beentry->st_appname); + } - return snprintf(application_name, APPLICATIONNAME_LEN, "%s", beentry->st_appname); + /* Return length so that others don't have to calculate */ + return strlen(name); } static uint @@ -1237,141 +1310,166 @@ pg_get_client_addr(bool *ok) } static void -pgss_update_entry(pgssEntry *entry, - uint64 bucketid, - uint64 queryid, +pgsm_update_entry(pgsmEntry *entry, const char *query, - const char *comments, PlanInfo * plan_info, - CmdType cmd_type, SysInfo * sys_info, ErrorInfo * error_info, - double total_time, + double plan_total_time, + double exec_total_time, uint64 rows, BufferUsage *bufusage, WalUsage *walusage, const struct JitInstrumentation *jitusage, bool reset, - pgssStoreKind kind, - const char *app_name, - size_t app_name_len) + pgsmStoreKind kind) { int index; double old_mean; int message_len = error_info ? strlen(error_info->message) : 0; - int comments_len = comments ? strlen(comments) : 0; int sqlcode_len = error_info ? strlen(error_info->sqlcode) : 0; int plan_text_len = plan_info ? plan_info->plan_len : 0; + char app_name[APPLICATIONNAME_LEN] = ""; + int app_name_len = 0; + /* Start collecting data for next bucket and reset all counters */ + if (reset) + memset(&entry->counters, 0, sizeof(Counters)); /* volatile block */ { - volatile pgssEntry *e = (volatile pgssEntry *) entry; + volatile pgsmEntry *e = (volatile pgsmEntry *)entry; - SpinLockAcquire(&e->mutex); - /* Start collecting data for next bucket and reset all counters */ - if (reset) - memset(&entry->counters, 0, sizeof(Counters)); + if (kind == PGSM_STORE) + SpinLockAcquire(&e->mutex); - if (comments_len > 0) - _snprintf(e->counters.info.comments, comments, comments_len + 1, COMMENTS_LEN); - e->counters.state = kind; - if (kind == PGSS_PLAN) + /* Extract comments if enabled and only when the query has completed with or without error */ + if (PGSM_EXTRACT_COMMENTS && query && kind == PGSM_STORE) + { + char comments[512] = {0}; + int comments_len; + + extract_query_comments(query, comments, sizeof(comments)); + comments_len = strlen(comments); + + if (comments_len > 0) + _snprintf(e->counters.info.comments, comments, comments_len + 1, COMMENTS_LEN); + } + + if (kind == PGSM_PLAN || kind == PGSM_STORE) { if (e->counters.plancalls.calls == 0) e->counters.plancalls.usage = USAGE_INIT; + e->counters.plancalls.calls += 1; - e->counters.plantime.total_time += total_time; + e->counters.plantime.total_time += plan_total_time; if (e->counters.plancalls.calls == 1) { - e->counters.plantime.min_time = total_time; - e->counters.plantime.max_time = total_time; - e->counters.plantime.mean_time = total_time; + e->counters.plantime.min_time = plan_total_time; + e->counters.plantime.max_time = plan_total_time; + e->counters.plantime.mean_time = plan_total_time; } + else + { + /* Increment the counts, except when jstate is not NULL */ + old_mean = e->counters.plantime.mean_time; - /* Increment the counts, except when jstate is not NULL */ - old_mean = e->counters.plantime.mean_time; - e->counters.plantime.mean_time += (total_time - old_mean) / e->counters.plancalls.calls; - e->counters.plantime.sum_var_time += (total_time - old_mean) * (total_time - e->counters.plantime.mean_time); + e->counters.plantime.mean_time += (plan_total_time - old_mean) / e->counters.plancalls.calls; + e->counters.plantime.sum_var_time += (plan_total_time - old_mean) * (plan_total_time - e->counters.plantime.mean_time); - /* calculate min and max time */ - if (e->counters.plantime.min_time > total_time) - e->counters.plantime.min_time = total_time; - if (e->counters.plantime.max_time < total_time) - e->counters.plantime.max_time = total_time; + /* calculate min and max time */ + if (e->counters.plantime.min_time > plan_total_time) + e->counters.plantime.min_time = plan_total_time; + + if (e->counters.plantime.max_time < plan_total_time) + e->counters.plantime.max_time = plan_total_time; + } } - else if (kind == PGSS_FINISHED) + + if (kind == PGSM_EXEC || kind == PGSM_STORE) { if (e->counters.calls.calls == 0) e->counters.calls.usage = USAGE_INIT; + e->counters.calls.calls += 1; - e->counters.time.total_time += total_time; + e->counters.time.total_time += exec_total_time; if (e->counters.calls.calls == 1) { - e->counters.time.min_time = total_time; - e->counters.time.max_time = total_time; - e->counters.time.mean_time = total_time; + e->counters.time.min_time = exec_total_time; + e->counters.time.max_time = exec_total_time; + e->counters.time.mean_time = exec_total_time; + } + else + { + /* Increment the counts, except when jstate is not NULL */ + old_mean = e->counters.time.mean_time; + e->counters.time.mean_time += (exec_total_time - old_mean) / e->counters.calls.calls; + e->counters.time.sum_var_time += (exec_total_time - old_mean) * (exec_total_time - e->counters.time.mean_time); + + /* calculate min and max time */ + if (e->counters.time.min_time > exec_total_time) + e->counters.time.min_time = exec_total_time; + + if (e->counters.time.max_time < exec_total_time) + e->counters.time.max_time = exec_total_time; } - /* Increment the counts, except when jstate is not NULL */ - old_mean = e->counters.time.mean_time; - e->counters.time.mean_time += (total_time - old_mean) / e->counters.calls.calls; - e->counters.time.sum_var_time += (total_time - old_mean) * (total_time - e->counters.time.mean_time); - - /* calculate min and max time */ - if (e->counters.time.min_time > total_time) - e->counters.time.min_time = total_time; - if (e->counters.time.max_time < total_time) - e->counters.time.max_time = total_time; - - index = get_histogram_bucket(total_time); + index = get_histogram_bucket(exec_total_time); e->counters.resp_calls[index]++; } if (plan_text_len > 0 && !e->counters.planinfo.plan_text[0]) - _snprintf(e->counters.planinfo.plan_text, plan_info->plan_text, plan_text_len + 1, PLAN_TEXT_LEN); - - if (app_name_len > 0 && !e->counters.info.application_name[0]) - _snprintf(e->counters.info.application_name, app_name, app_name_len + 1, APPLICATIONNAME_LEN); - - e->counters.info.num_relations = num_relations; - _snprintf2(e->counters.info.relations, relations, num_relations, REL_LEN); - - e->counters.info.cmd_type = cmd_type; - - if (exec_nested_level > 0) { - if (exec_nested_level >= 0 && exec_nested_level < max_stack_depth) + e->counters.planinfo.planid = plan_info->planid; + e->counters.planinfo.plan_len = plan_text_len; + _snprintf(e->counters.planinfo.plan_text, plan_info->plan_text, plan_text_len + 1, PLAN_TEXT_LEN); + } + + /* Only should process this once when storing the data */ + if (kind == PGSM_STORE) + { + app_name_len = pg_get_application_name(app_name, APPLICATIONNAME_LEN); + + if (app_name_len > 0 && !e->counters.info.application_name[0]) + _snprintf(e->counters.info.application_name, app_name, app_name_len + 1, APPLICATIONNAME_LEN); + + e->counters.info.num_relations = num_relations; + _snprintf2(e->counters.info.relations, relations, num_relations, REL_LEN); + + if (exec_nested_level > 0) { - int parent_query_len = nested_query_txts[exec_nested_level - 1]? - strlen(nested_query_txts[exec_nested_level - 1]): 0; - e->counters.info.parentid = nested_queryids[exec_nested_level - 1]; - e->counters.info.parent_query = InvalidDsaPointer; - /* If we have a parent query, store it in the raw dsa area */ - if (parent_query_len > 0) + if (exec_nested_level >= 0 && exec_nested_level < max_stack_depth) { - char *qry_buff; - dsa_area *query_dsa_area = get_dsa_area_for_query_text(); - /* Use dsa_allocate_extended with DSA_ALLOC_NO_OOM flag, as we don't want to get an - * error if memory allocation fails.*/ - dsa_pointer qry = dsa_allocate_extended(query_dsa_area, parent_query_len+1, DSA_ALLOC_NO_OOM | DSA_ALLOC_ZERO); - if (DsaPointerIsValid(qry)) + int parent_query_len = nested_query_txts[exec_nested_level - 1]? + strlen(nested_query_txts[exec_nested_level - 1]): 0; + e->counters.info.parentid = nested_queryids[exec_nested_level - 1]; + e->counters.info.parent_query = InvalidDsaPointer; + /* If we have a parent query, store it in the raw dsa area */ + if (parent_query_len > 0) { - qry_buff = dsa_get_address(query_dsa_area, qry); - memcpy(qry_buff, nested_query_txts[exec_nested_level - 1], parent_query_len); - qry_buff[parent_query_len] = 0; - /* store the dsa pointer for parent query text */ - e->counters.info.parent_query = qry; + char *qry_buff; + dsa_area *query_dsa_area = get_dsa_area_for_query_text(); + /* Use dsa_allocate_extended with DSA_ALLOC_NO_OOM flag, as we don't want to get an + * error if memory allocation fails.*/ + dsa_pointer qry = dsa_allocate_extended(query_dsa_area, parent_query_len+1, DSA_ALLOC_NO_OOM | DSA_ALLOC_ZERO); + if (DsaPointerIsValid(qry)) + { + qry_buff = dsa_get_address(query_dsa_area, qry); + memcpy(qry_buff, nested_query_txts[exec_nested_level - 1], parent_query_len); + qry_buff[parent_query_len] = 0; + /* store the dsa pointer for parent query text */ + e->counters.info.parent_query = qry; + } } } } - } - else - { - e->counters.info.parentid = UINT64CONST(0); - e->counters.info.parent_query = InvalidDsaPointer; + else + { + e->counters.info.parentid = UINT64CONST(0); + e->counters.info.parent_query = InvalidDsaPointer; + } } if (error_info) @@ -1380,7 +1478,9 @@ pgss_update_entry(pgssEntry *entry, _snprintf(e->counters.error.sqlcode, error_info->sqlcode, sqlcode_len, SQLCODE_LEN); _snprintf(e->counters.error.message, error_info->message, message_len, ERROR_MESSAGE_LEN); } + e->counters.calls.rows += rows; + if (bufusage) { e->counters.blocks.shared_blks_hit += bufusage->shared_blks_hit; @@ -1399,8 +1499,18 @@ pgss_update_entry(pgssEntry *entry, e->counters.blocks.temp_blk_read_time += INSTR_TIME_GET_MILLISEC(bufusage->temp_blk_read_time); e->counters.blocks.temp_blk_write_time += INSTR_TIME_GET_MILLISEC(bufusage->temp_blk_write_time); #endif + + memcpy((void *)&e->counters.blocks.instr_blk_read_time, &bufusage->blk_read_time, sizeof(instr_time)); + memcpy((void *)&e->counters.blocks.instr_blk_write_time, &bufusage->blk_write_time, sizeof(instr_time)); + + #if PG_VERSION_NUM >= 150000 + memcpy((void *)&e->counters.blocks.instr_temp_blk_read_time, &bufusage->temp_blk_read_time, sizeof(bufusage->temp_blk_read_time)); + memcpy((void *)&e->counters.blocks.instr_temp_blk_write_time, &bufusage->temp_blk_write_time, sizeof(bufusage->temp_blk_write_time)); + #endif } - e->counters.calls.usage += USAGE_EXEC(total_time); + + e->counters.calls.usage += USAGE_EXEC(exec_total_time + plan_total_time); + if (sys_info) { e->counters.sysinfo.utime += sys_info->utime; @@ -1428,39 +1538,173 @@ pgss_update_entry(pgssEntry *entry, if (INSTR_TIME_GET_MILLISEC(jitusage->emission_counter)) e->counters.jitinfo.jit_emission_count++; e->counters.jitinfo.jit_emission_time += INSTR_TIME_GET_MILLISEC(jitusage->emission_counter); + + memcpy((void *)&e->counters.jitinfo.instr_generation_counter, &jitusage->generation_counter, sizeof(instr_time)); + memcpy((void *)&e->counters.jitinfo.instr_inlining_counter, &jitusage->inlining_counter, sizeof(instr_time)); + memcpy((void *)&e->counters.jitinfo.instr_optimization_counter, &jitusage->optimization_counter, sizeof(instr_time)); + memcpy((void *)&e->counters.jitinfo.instr_emission_counter, &jitusage->emission_counter, sizeof(instr_time)); } - SpinLockRelease(&e->mutex); + + if (kind == PGSM_STORE) + SpinLockRelease(&e->mutex); } } static void -pgss_store_error(uint64 queryid, - const char *query, - ErrorData *edata) +pgsm_store_error(const char *query, ErrorData *edata) { - ErrorInfo error_info; + pgsmEntry *entry; + uint64 queryid = 0; + int len = strlen(query); - error_info.elevel = edata->elevel; - snprintf(error_info.message, ERROR_MESSAGE_LEN, "%s", edata->message); - snprintf(error_info.sqlcode, SQLCODE_LEN, "%s", unpack_sql_state(edata->sqlerrcode)); + if (!query || len == 0) + return; - pgss_store(queryid, /* query id */ - query, /* query text */ - 0, /* query location */ - strlen(query), /* query length */ - NULL, /* PlanInfo */ - 0, /* CmdType */ - NULL, /* SysInfo */ - &error_info, /* ErrorInfo */ - 0, /* total_time */ - 0, /* rows */ - NULL, /* bufusage */ - NULL, /* walusage */ - NULL, /* JumbleState */ - NULL, - PGSS_ERROR); /* pgssStoreKind */ + len = strlen(query); + + queryid = pgsm_hash_string(query, len); + + entry = pgsm_create_hash_entry(0, queryid, NULL); + entry->query_text.query_pointer = pnstrdup(query, len); + + entry->counters.error.elevel = edata->elevel; + snprintf(entry->counters.error.message, ERROR_MESSAGE_LEN, "%s", edata->message); + snprintf(entry->counters.error.sqlcode, SQLCODE_LEN, "%s", unpack_sql_state(edata->sqlerrcode)); + + pgsm_store(entry); } +static void +pgsm_add_to_list(pgsmEntry *entry, char *query_text, int query_len) +{ + /* Switch to pgsm memory context */ + MemoryContext oldctx = MemoryContextSwitchTo(pgsm_get_ss()->pgsm_mem_cxt); + entry->query_text.query_pointer = pnstrdup(query_text, query_len); + lentries = lappend(lentries, entry); + MemoryContextSwitchTo(oldctx); +} + +static pgsmEntry* +pgsm_get_entry_for_query(uint64 queryid, PlanInfo *plan_info, const char* query_text, int query_len, bool create) +{ + pgsmEntry *entry = NULL; + ListCell *lc = NULL; + + /* First bet is on the last entry */ + if (lentries == NIL && !create) + return NULL; + + if (lentries) + { + entry = (pgsmEntry *)llast(lentries); + if(entry->key.queryid == queryid) + return entry; + + foreach(lc, lentries) + { + entry = lfirst(lc); + if(entry->key.queryid == queryid) + return entry; + } + } + if (create && query_text) + { + /* + * At this point, we don't know which bucket this query will land in, so passing + * 0. The store function MUST later update it based on the current bucket value. + * The correct bucket value will be needed then to search the hash table, or create + * the appropriate entry. + */ + entry = pgsm_create_hash_entry(0, queryid, plan_info); + + /* Update other member that are not counters, so that we don't have to worry about these. */ + entry->pgsm_query_id = get_pgsm_query_id_hash(query_text, query_len); + pgsm_add_to_list(entry, (char *)query_text, query_len); + } + + return entry; +} + +static void +pgsm_cleanup_callback(void *arg) +{ + /* Reset the memory context holding the list */ + MemoryContextReset(pgsm_get_ss()->pgsm_mem_cxt); + lentries = NIL; + callback_setup = false; +} +/* + * Function encapsulating some external calls for filling up the hash key data structure. + * The bucket_id may not be known at this stage. So pass any value that you may wish. + */ +static pgsmEntry * +pgsm_create_hash_entry(uint64 bucket_id, uint64 queryid, PlanInfo *plan_info) +{ + pgsmEntry *entry; + int sec_ctx; + bool found_client_addr = false; + char app_name[APPLICATIONNAME_LEN] = ""; + char *app_name_ptr = app_name; + int app_name_len = 0; + MemoryContext oldctx; + char *datname = NULL; + char *username = NULL; + + /* Create an entry in the pgsm memory context */ + oldctx = MemoryContextSwitchTo(pgsm_get_ss()->pgsm_mem_cxt); + entry = palloc0(sizeof(pgsmEntry)); + + /* + * Get the user ID. Let's use this instead of GetUserID as this + * won't throw an assertion in case of an error. + */ + GetUserIdAndSecContext((Oid *) &entry->key.userid, &sec_ctx); + + /* Get the application name and set appid */ + app_name_len = pg_get_application_name(app_name, APPLICATIONNAME_LEN); + entry->key.appid = pgsm_hash_string((const char *)app_name_ptr, app_name_len); + + /* client address */ + entry->key.ip = pg_get_client_addr(&found_client_addr); + + /* PlanID, if there is one */ + entry->key.planid = plan_info ? plan_info->planid : 0; + + /* Set remaining data */ + entry->key.dbid = MyDatabaseId; + entry->key.queryid = queryid; + entry->key.bucket_id = bucket_id; + +#if PG_VERSION_NUM < 140000 + entry->key.toplevel = 1; +#else + entry->key.toplevel = ((exec_nested_level + plan_nested_level) == 0); +#endif + + if (IsTransactionState()) + { + datname = get_database_name(entry->key.dbid); + username = GetUserNameFromId(entry->key.userid, true); + } + + if (!datname) + datname = pnstrdup("", sizeof(entry->datname) - 1); + + if (!username) + username = pnstrdup("", sizeof(entry->username) - 1); + + snprintf(entry->datname, sizeof(entry->datname), "%s", datname); + snprintf(entry->username, sizeof(entry->username), "%s", username); + + pfree(datname); + pfree(username); + + MemoryContextSwitchTo(oldctx); + + return entry; +} + + /* * Store some statistics for a statement. * @@ -1472,179 +1716,50 @@ pgss_store_error(uint64 queryid, * query string. total_time, rows, bufusage are ignored in this case. */ static void -pgss_store(uint64 queryid, - const char *query, - int query_location, - int query_len, - PlanInfo * plan_info, - CmdType cmd_type, - SysInfo * sys_info, - ErrorInfo * error_info, - double total_time, - uint64 rows, - BufferUsage *bufusage, - WalUsage *walusage, - const struct JitInstrumentation *jitusage, - JumbleState *jstate, - pgssStoreKind kind) +pgsm_store(pgsmEntry *entry) { - pgssHashKey key; - pgssEntry *entry; - pgssSharedState *pgss; - char *app_name_ptr; - char app_name[APPLICATIONNAME_LEN] = ""; - int app_name_len = 0; - bool reset = false; - uint64 pgsm_query_id = 0; + pgsmEntry *shared_hash_entry; + pgsmSharedState *pgsm; + bool found; uint64 bucketid; uint64 prev_bucket_id; - uint64 userid; - uint64 planid; - uint64 appid = 0; - int norm_query_len = 0; - char *norm_query = NULL; - char comments[512] = ""; - bool found_app_name = false; - bool found_client_addr = false; - uint client_addr = 0; - bool found; + bool reset = false; /* Only used in update function - HAMID */ + char *query; + int query_len; + BufferUsage bufusage; + WalUsage walusage; + JitInstrumentation jitusage; /* Safety check... */ if (!IsSystemInitialized()) return; - pgss = pgsm_get_ss(); + pgsm = pgsm_get_ss(); -#if PG_VERSION_NUM >= 140000 - - /* - * Nothing to do if compute_query_id isn't enabled and no other module - * computed a query identifier. - */ - if (queryid == UINT64CONST(0)) - return; -#endif - - query = CleanQuerytext(query, &query_location, &query_len); - -#if PG_VERSION_NUM < 140000 - - /* - * For utility statements, we just hash the query string to get an ID. - */ - if (queryid == UINT64CONST(0)) - { - queryid = pgss_hash_string(query, query_len); - - /* - * If we are unlucky enough to get a hash of zero(invalid), use - * queryID as 2 instead, queryID 1 is already in use for normal - * statements. - */ - if (queryid == UINT64CONST(0)) - queryid = UINT64CONST(2); - } -#endif - - Assert(query != NULL); - if (kind == PGSS_ERROR) - { - int sec_ctx; - - GetUserIdAndSecContext((Oid *) &userid, &sec_ctx); - } - else - userid = GetUserId(); - - /* Try to read application name from GUC directly */ - if (application_name && *application_name) - { - app_name_ptr = application_name; - appid = djb2_hash_str((unsigned char *) application_name, &app_name_len); - } - else - { - app_name_len = pg_get_application_name(app_name, &found_app_name); - if (found_app_name) - appid = djb2_hash((unsigned char *) app_name, app_name_len); - app_name_ptr = app_name; - } - - if (!found_client_addr) - client_addr = pg_get_client_addr(&found_client_addr); - - planid = plan_info ? plan_info->planid : 0; - - /* Extract comments if enabled. */ - if (PGSM_EXTRACT_COMMENTS) - extract_query_comments(query, comments, sizeof(comments)); - - prev_bucket_id = pg_atomic_read_u64(&pgss->current_wbucket); - bucketid = get_next_wbucket(pgss); + /* We should lock the hash table here what if the bucket is removed; e.g. reset is called - HAMID */ + prev_bucket_id = pg_atomic_read_u64(&pgsm->current_wbucket); + bucketid = get_next_wbucket(pgsm); if (bucketid != prev_bucket_id) reset = true; - key.bucket_id = bucketid; - key.userid = userid; - key.dbid = MyDatabaseId; - key.queryid = queryid; - key.ip = client_addr; - key.planid = planid; - key.appid = appid; -#if PG_VERSION_NUM < 140000 - key.toplevel = 1; -#else - key.toplevel = ((exec_nested_level + plan_nested_level) == 0); -#endif + entry->key.bucket_id = bucketid; + query = entry->query_text.query_pointer; + query_len = strlen(query); - LWLockAcquire(pgss->lock, LW_SHARED); + /* + * Acquire a share lock to start with. We'd have to acquire exclusive + * if we need ot create the entry. + */ + LWLockAcquire(pgsm->lock, LW_SHARED); + shared_hash_entry = (pgsmEntry *) pgsm_hash_find(get_pgsmHash(), &entry->key, &found); - entry = (pgssEntry *) pgsm_hash_find(get_pgssHash(), &key, &found); - if (!entry) + if (!shared_hash_entry) { dsa_pointer dsa_query_pointer; dsa_area *query_dsa_area; char *query_buff; - /* - * Create a new, normalized query string if caller asked. We don't - * need to hold the lock while doing this work. (Note: in any case, - * it's possible that someone else creates a duplicate hashtable entry - * in the interval where we don't hold the lock below. That case is - * handled by entry_alloc. - */ - if (jstate) - { - norm_query_len = query_len; - - LWLockRelease(pgss->lock); - norm_query = generate_normalized_query(jstate, query, - query_location, - &norm_query_len, - GetDatabaseEncoding()); - LWLockAcquire(pgss->lock, LW_SHARED); - - pgsm_query_id = pgss_hash_string(norm_query, norm_query_len); - - /* Free up norm_query if we don't intend to show normalized version in the view */ - if (PGSM_NORMALIZED_QUERY) - { - query_len = norm_query_len; - } - else - { - if (norm_query) - pfree(norm_query); - - norm_query = NULL; - } - } - else - { - pgsm_query_id = pgss_hash_string(query, query_len); - } - /* New query, truncate length if necessary. */ if (query_len > PGSM_QUERY_MAX_LEN) query_len = PGSM_QUERY_MAX_LEN; @@ -1654,27 +1769,28 @@ pgss_store(uint64 queryid, dsa_query_pointer = dsa_allocate_extended(query_dsa_area, query_len+1, DSA_ALLOC_NO_OOM | DSA_ALLOC_ZERO); if (!DsaPointerIsValid(dsa_query_pointer)) { - LWLockRelease(pgss->lock); - if (norm_query) - pfree(norm_query); + LWLockRelease(pgsm->lock); return; } + /* Get the memory address from DSA pointer and copy the query text in local variable */ query_buff = dsa_get_address(query_dsa_area, dsa_query_pointer); - memcpy(query_buff, norm_query ? norm_query : query, query_len); - /* OK to create a new hashtable entry */ + memcpy(query_buff, query, query_len); + LWLockRelease(pgsm->lock); + LWLockAcquire(pgsm->lock, LW_EXCLUSIVE); + + /* OK to create a new hashtable entry */ PGSM_DISABLE_ERROR_CAPUTRE(); { PG_TRY(); { - entry = hash_entry_alloc(pgss, &key, GetDatabaseEncoding()); + shared_hash_entry = hash_entry_alloc(pgsm, &entry->key, GetDatabaseEncoding()); } PG_CATCH(); { - LWLockRelease(pgss->lock); - if (norm_query) - pfree(norm_query); + LWLockRelease(pgsm->lock); + if (DsaPointerIsValid(dsa_query_pointer)) dsa_free(query_dsa_area, dsa_query_pointer); PG_RE_THROW(); @@ -1682,50 +1798,91 @@ pgss_store(uint64 queryid, PG_END_TRY(); }PGSM_END_DISABLE_ERROR_CAPTURE(); - if (entry == NULL) + if (shared_hash_entry == NULL) { - LWLockRelease(pgss->lock); - if (norm_query) - pfree(norm_query); + /* Out of memory; report only if the state has changed now. Otherwise we risk filling up the log file with these message. */ + if (!IsSystemOOM()) + { + pgsm->pgsm_oom = true; + + ereport(WARNING, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("[pg_stat_monitor] pgsm_store: Hash table is out of memory and can no longer store queries!"), + errdetail("You may reset the view or when the buckets are deallocated, pg_stat_monitor will resume saving " \ + "queries. Alternatively, try increasing the value of pg_stat_monitor.pgsm_max."))); + } + + LWLockRelease(pgsm->lock); + if (DsaPointerIsValid(dsa_query_pointer)) dsa_free(query_dsa_area, dsa_query_pointer); + return; } - entry->query_pos = dsa_query_pointer; - entry->pgsm_query_id = pgsm_query_id; - } - else - { - #if USE_DYNAMIC_HASH - if(entry) - dshash_release_lock(get_pgssHash(), entry); - #endif + /* If we already have the pointer set, free this one */ + if (DsaPointerIsValid(shared_hash_entry->query_text.query_pos)) + dsa_free(query_dsa_area, dsa_query_pointer); + else + shared_hash_entry->query_text.query_pos = dsa_query_pointer; + + shared_hash_entry->pgsm_query_id = entry->pgsm_query_id; + shared_hash_entry->encoding = entry->encoding; + shared_hash_entry->counters.info.cmd_type = entry->counters.info.cmd_type; + + snprintf(shared_hash_entry->datname, sizeof(shared_hash_entry->datname), "%s", entry->datname); + snprintf(shared_hash_entry->username, sizeof(shared_hash_entry->username), "%s", entry->username); } - if (jstate == NULL) - pgss_update_entry(entry, /* entry */ - bucketid, /* bucketid */ - queryid, /* queryid */ - query, /* query */ - comments, /* comments */ - plan_info, /* PlanInfo */ - cmd_type, /* CmdType */ - sys_info, /* SysInfo */ - error_info, /* ErrorInfo */ - total_time, /* total_time */ - rows, /* rows */ - bufusage, /* bufusage */ - walusage, /* walusage */ - jitusage, - reset, /* reset */ - kind, /* kind */ - app_name_ptr, - app_name_len); + /* bufusage */ + bufusage.shared_blks_hit = entry->counters.blocks.shared_blks_hit; + bufusage.shared_blks_read = entry->counters.blocks.shared_blks_read; + bufusage.shared_blks_dirtied = entry->counters.blocks.shared_blks_dirtied; + bufusage.shared_blks_written = entry->counters.blocks.shared_blks_written; + bufusage.local_blks_hit = entry->counters.blocks.local_blks_hit; + bufusage.local_blks_read = entry->counters.blocks.local_blks_read; + bufusage.local_blks_dirtied = entry->counters.blocks.local_blks_dirtied; + bufusage.local_blks_written = entry->counters.blocks.local_blks_written; + bufusage.temp_blks_read = entry->counters.blocks.temp_blks_read; + bufusage.temp_blks_written = entry->counters.blocks.temp_blks_written; - LWLockRelease(pgss->lock); - if (norm_query) - pfree(norm_query); + memcpy(&bufusage.blk_read_time, &entry->counters.blocks.instr_blk_read_time, sizeof(instr_time)); + memcpy(&bufusage.blk_write_time, &entry->counters.blocks.instr_blk_write_time, sizeof(instr_time)); + + #if PG_VERSION_NUM >= 150000 + memcpy(&bufusage.temp_blk_read_time, &entry->counters.blocks.instr_temp_blk_read_time, sizeof(instr_time)); + memcpy(&bufusage.temp_blk_write_time, &entry->counters.blocks.instr_temp_blk_write_time, sizeof(instr_time)); + #endif + + /* walusage */ + walusage.wal_records = entry->counters.walusage.wal_records; + walusage.wal_fpi = entry->counters.walusage.wal_fpi; + walusage.wal_bytes = entry->counters.walusage.wal_bytes; + + /* jit */ + jitusage.created_functions = entry->counters.jitinfo.jit_functions; + memcpy(&jitusage.generation_counter, &entry->counters.jitinfo.instr_generation_counter, sizeof(instr_time)); + memcpy(&jitusage.inlining_counter, &entry->counters.jitinfo.instr_inlining_counter, sizeof(instr_time)); + memcpy(&jitusage.optimization_counter, &entry->counters.jitinfo.instr_optimization_counter, sizeof(instr_time)); + memcpy(&jitusage.emission_counter, &entry->counters.jitinfo.instr_emission_counter, sizeof(instr_time)); + + + pgsm_update_entry(shared_hash_entry, /* entry */ + query, /* query */ + &entry->counters.planinfo, /* PlanInfo */ + &entry->counters.sysinfo, /* SysInfo */ + &entry->counters.error, /* ErrorInfo */ + entry->counters.plantime.total_time, /* plan_total_time */ + entry->counters.time.total_time, /* exec_total_time */ + entry->counters.calls.rows, /* rows */ + &bufusage, /* bufusage */ + &walusage, /* walusage */ + &jitusage, /* jitusage */ + reset, /* reset */ + PGSM_STORE); + + memset(&entry->counters, 0, sizeof(entry->counters)); + LWLockRelease(pgsm->lock); } /* @@ -1734,7 +1891,7 @@ pgss_store(uint64 queryid, Datum pg_stat_monitor_reset(PG_FUNCTION_ARGS) { - pgssSharedState *pgss; + pgsmSharedState *pgsm; /* Safety check... */ if (!IsSystemInitialized()) @@ -1742,11 +1899,11 @@ pg_stat_monitor_reset(PG_FUNCTION_ARGS) (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("pg_stat_monitor: must be loaded via shared_preload_libraries"))); - pgss = pgsm_get_ss(); - LWLockAcquire(pgss->lock, LW_EXCLUSIVE); + pgsm = pgsm_get_ss(); + LWLockAcquire(pgsm->lock, LW_EXCLUSIVE); hash_entry_dealloc(-1, -1, NULL); - LWLockRelease(pgss->lock); + LWLockRelease(pgsm->lock); PG_RETURN_VOID(); } @@ -1780,9 +1937,9 @@ IsBucketValid(uint64 bucketid) long secs; int microsecs; TimestampTz current_tz = GetCurrentTimestamp(); - pgssSharedState *pgss = pgsm_get_ss(); + pgsmSharedState *pgsm = pgsm_get_ss(); - TimestampDifference(pgss->bucket_start_time[bucketid], current_tz,&secs, µsecs); + TimestampDifference(pgsm->bucket_start_time[bucketid], current_tz,&secs, µsecs); if (secs > (PGSM_BUCKET_TIME * PGSM_MAX_BUCKETS)) return false; @@ -1801,36 +1958,40 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, MemoryContext per_query_ctx; MemoryContext oldcontext; PGSM_HASH_SEQ_STATUS hstat; - pgssEntry *entry; - pgssSharedState *pgss; - char *query_txt = NULL; - char *parent_query_txt = NULL; + pgsmEntry *entry; + pgsmSharedState *pgsm; int expected_columns = (api_version >= PGSM_V2_0)?PG_STAT_MONITOR_COLS_V2_0:PG_STAT_MONITOR_COLS_V1_0; /* Disallow old api usage */ if (api_version < PGSM_V2_0) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("pg_stat_monitor: API version not supported"), - errhint("upgrade pg_stat_monitor extension"))); + errmsg("[pg_stat_monitor] pg_stat_monitor_internal: API version not supported."), + errhint("Upgrade pg_stat_monitor extension"))); /* Safety check... */ if (!IsSystemInitialized()) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("pg_stat_monitor: must be loaded via shared_preload_libraries"))); + errmsg("[pg_stat_monitor] pg_stat_monitor_internal: Must be loaded via shared_preload_libraries."))); + + /* Out of memory? */ + if (IsSystemOOM()) + ereport(WARNING, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("[pg_stat_monitor] pg_stat_monitor_internal: Hash table is out of memory and can no longer store queries!"), + errdetail("You may reset the view or when the buckets are deallocated, pg_stat_monitor will resume saving " \ + "queries. Alternatively, try increasing the value of pg_stat_monitor.pgsm_max."))); /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("pg_stat_monitor: set-valued function called in context that cannot accept a set"))); + errmsg("[pg_stat_monitor] pg_stat_monitor_internal: Set-valued function called in context that cannot accept a set."))); if (!(rsinfo->allowedModes & SFRM_Materialize)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("pg_stat_monitor: materialize mode required, but it is not " \ - "allowed in this context"))); - - pgss = pgsm_get_ss(); + errmsg("[pg_stat_monitor] pg_stat_monitor_internal: Materialize mode required, but it is not " \ + "allowed in this context."))); /* Switch into long-lived context to construct returned data structures */ per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; @@ -1838,10 +1999,10 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, /* Build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "pg_stat_monitor: return type must be a row type"); + elog(ERROR, "[pg_stat_monitor] pg_stat_monitor_internal: Return type must be a row type."); if (tupdesc->natts != expected_columns) - elog(ERROR, "pg_stat_monitor: incorrect number of output arguments, required %d", tupdesc->natts); + elog(ERROR, "[pg_stat_monitor] pg_stat_monitor_internal: Incorrect number of output arguments, received %d, required %d.", tupdesc->natts, expected_columns); tupstore = tuplestore_begin_heap(true, false, work_mem); rsinfo->returnMode = SFRM_Materialize; @@ -1850,9 +2011,9 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, MemoryContextSwitchTo(oldcontext); - LWLockAcquire(pgss->lock, LW_SHARED); - - pgsm_hash_seq_init(&hstat, get_pgssHash(), false); + pgsm = pgsm_get_ss(); + LWLockAcquire(pgsm->lock, LW_SHARED); + pgsm_hash_seq_init(&hstat, get_pgsmHash(), false); while ((entry = pgsm_hash_seq_next(&hstat)) != NULL) { @@ -1863,25 +2024,27 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, double stddev; uint64 queryid = entry->key.queryid; int64 bucketid = entry->key.bucket_id; - uint64 dbid = entry->key.dbid; - uint64 userid = entry->key.userid; - int64 ip = entry->key.ip; + Oid dbid = entry->key.dbid; + Oid userid = entry->key.userid; + uint32 ip = entry->key.ip; uint64 planid = entry->key.planid; uint64 pgsm_query_id = entry->pgsm_query_id; dsa_area *query_dsa_area; char *query_ptr; + char *query_txt = NULL; + char *parent_query_txt = NULL; + + bool toplevel = entry->key.toplevel; #if PG_VERSION_NUM < 140000 - bool toplevel = 1; bool is_allowed_role = is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_ALL_STATS); #else bool is_allowed_role = is_member_of_role(GetUserId(), ROLE_PG_READ_ALL_STATS); - bool toplevel = entry->key.toplevel; #endif /* Load the query text from dsa area */ - if (DsaPointerIsValid(entry->query_pos)) + if (DsaPointerIsValid(entry->query_text.query_pos)) { query_dsa_area = get_dsa_area_for_query_text(); - query_ptr = dsa_get_address(query_dsa_area, entry->query_pos); + query_ptr = dsa_get_address(query_dsa_area, entry->query_text.query_pos); query_txt = pstrdup(query_ptr); } else @@ -1889,7 +2052,7 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, /* copy counters to a local variable to keep locking time short */ { - volatile pgssEntry *e = (volatile pgssEntry *) entry; + volatile pgsmEntry *e = (volatile pgsmEntry *) entry; SpinLockAcquire(&e->mutex); tmp = e->counters; @@ -1905,13 +2068,8 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, if (!IsBucketValid(bucketid)) { - if (tmp.state == PGSS_FINISHED) - continue; - } - - /* Skip queries such as, $1, $2 := $3, etc. */ - if (tmp.state == PGSS_PARSE || tmp.state == PGSS_PLAN) continue; + } /* read the parent query text if any */ if (tmp.info.parentid != UINT64CONST(0)) @@ -1931,15 +2089,21 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, /* userid at column number 1 */ values[i++] = ObjectIdGetDatum(userid); + /* userid at column number 1 */ + values[i++] = CStringGetTextDatum(entry->username); + /* dbid at column number 2 */ values[i++] = ObjectIdGetDatum(dbid); + /* userid at column number 1 */ + values[i++] = CStringGetTextDatum(entry->datname); + /* * ip address at column number 3, Superusers or members of * pg_read_all_stats members are allowed */ if (is_allowed_role || userid == GetUserId()) - values[i++] = Int64GetDatumFast(ip); + values[i++] = UInt32GetDatum(ip); else nulls[i++] = true; @@ -1989,10 +2153,6 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, values[i++] = UInt64GetDatum(pgsm_query_id); - /* state at column number 8 for V1.0 API*/ - if (api_version <= PGSM_V1_0) - values[i++] = Int64GetDatumFast(tmp.state); - /* parentid at column number 9 */ if (tmp.info.parentid != UINT64CONST(0)) { @@ -2015,8 +2175,8 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, if (tmp.info.num_relations > 0) { int j; - char *text_str = palloc0(1024); - char *tmp_str = palloc0(1024); + char *text_str = palloc0(TOTAL_RELS_LENGTH); + char *tmp_str = palloc0(TOTAL_RELS_LENGTH); bool first = true; /* @@ -2062,7 +2222,7 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, values[i++] = CStringGetTextDatum(tmp.error.message); /* bucket_start_time at column number 15 */ - values[i++] = TimestampTzGetDatum(pgss->bucket_start_time[entry->key.bucket_id]); + values[i++] = TimestampTzGetDatum(pgsm->bucket_start_time[entry->key.bucket_id]); if (tmp.calls.calls == 0) { @@ -2138,12 +2298,8 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, values[i++] = Int64GetDatumFast(tmp.blocks.temp_blks_written); values[i++] = Float8GetDatumFast(tmp.blocks.blk_read_time); values[i++] = Float8GetDatumFast(tmp.blocks.blk_write_time); - - if (api_version >= PGSM_V2_0) - { - values[i++] = Float8GetDatumFast(tmp.blocks.temp_blk_read_time); - values[i++] = Float8GetDatumFast(tmp.blocks.temp_blk_write_time); - } + values[i++] = Float8GetDatumFast(tmp.blocks.temp_blk_read_time); + values[i++] = Float8GetDatumFast(tmp.blocks.temp_blk_write_time); /* resp_calls at column number 41 */ values[i++] = IntArrayGetTextDatum(tmp.resp_calls, hist_bucket_count_total); @@ -2179,39 +2335,36 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, else nulls[i++] = true; - if (api_version >= PGSM_V2_0) - { - values[i++] = Int64GetDatumFast(tmp.jitinfo.jit_functions); - values[i++] = Float8GetDatumFast(tmp.jitinfo.jit_generation_time); - values[i++] = Int64GetDatumFast(tmp.jitinfo.jit_inlining_count); - values[i++] = Float8GetDatumFast(tmp.jitinfo.jit_inlining_time); - values[i++] = Int64GetDatumFast(tmp.jitinfo.jit_optimization_count); - values[i++] = Float8GetDatumFast(tmp.jitinfo.jit_optimization_time); - values[i++] = Int64GetDatumFast(tmp.jitinfo.jit_emission_count); - values[i++] = Float8GetDatumFast(tmp.jitinfo.jit_emission_time); - } - + values[i++] = Int64GetDatumFast(tmp.jitinfo.jit_functions); + values[i++] = Float8GetDatumFast(tmp.jitinfo.jit_generation_time); + values[i++] = Int64GetDatumFast(tmp.jitinfo.jit_inlining_count); + values[i++] = Float8GetDatumFast(tmp.jitinfo.jit_inlining_time); + values[i++] = Int64GetDatumFast(tmp.jitinfo.jit_optimization_count); + values[i++] = Float8GetDatumFast(tmp.jitinfo.jit_optimization_time); + values[i++] = Int64GetDatumFast(tmp.jitinfo.jit_emission_count); + values[i++] = Float8GetDatumFast(tmp.jitinfo.jit_emission_time); } values[i++] = BoolGetDatum(toplevel); - values[i++] = BoolGetDatum(pg_atomic_read_u64(&pgss->current_wbucket) != bucketid); + values[i++] = BoolGetDatum(pg_atomic_read_u64(&pgsm->current_wbucket) != bucketid); /* clean up and return the tuplestore */ tuplestore_putvalues(tupstore, tupdesc, values, nulls); + + if(query_txt) + pfree(query_txt); + if(parent_query_txt) + pfree(parent_query_txt); } /* clean up and return the tuplestore */ pgsm_hash_seq_term(&hstat); - LWLockRelease(pgss->lock); + LWLockRelease(pgsm->lock); - if(query_txt) - pfree(query_txt); - if(parent_query_txt) - pfree(parent_query_txt); tuplestore_donestoring(tupstore); } static uint64 -get_next_wbucket(pgssSharedState *pgss) +get_next_wbucket(pgsmSharedState *pgsm) { struct timeval tv; uint64 current_bucket_sec; @@ -2221,7 +2374,7 @@ get_next_wbucket(pgssSharedState *pgss) bool update_bucket = false; gettimeofday(&tv, NULL); - current_bucket_sec = pg_atomic_read_u64(&pgss->prev_bucket_sec); + current_bucket_sec = pg_atomic_read_u64(&pgsm->prev_bucket_sec); /* * If current bucket expired we loop attempting to update prev_bucket_sec. @@ -2240,13 +2393,13 @@ get_next_wbucket(pgssSharedState *pgss) */ while ((tv.tv_sec - (uint)current_bucket_sec) >= ((uint)PGSM_BUCKET_TIME)) { - if (pg_atomic_compare_exchange_u64(&pgss->prev_bucket_sec, ¤t_bucket_sec, (uint64)tv.tv_sec)) + if (pg_atomic_compare_exchange_u64(&pgsm->prev_bucket_sec, ¤t_bucket_sec, (uint64)tv.tv_sec)) { update_bucket = true; break; } - current_bucket_sec = pg_atomic_read_u64(&pgss->prev_bucket_sec); + current_bucket_sec = pg_atomic_read_u64(&pgsm->prev_bucket_sec); } if (update_bucket) @@ -2255,25 +2408,99 @@ get_next_wbucket(pgssSharedState *pgss) new_bucket_id = (tv.tv_sec / PGSM_BUCKET_TIME) % PGSM_MAX_BUCKETS; /* Update bucket id and retrieve the previous one. */ - prev_bucket_id = pg_atomic_exchange_u64(&pgss->current_wbucket, new_bucket_id); + prev_bucket_id = pg_atomic_exchange_u64(&pgsm->current_wbucket, new_bucket_id); - LWLockAcquire(pgss->lock, LW_EXCLUSIVE); + LWLockAcquire(pgsm->lock, LW_EXCLUSIVE); hash_entry_dealloc(new_bucket_id, prev_bucket_id, NULL); - LWLockRelease(pgss->lock); + LWLockRelease(pgsm->lock); /* Allign the value in prev_bucket_sec to the bucket start time */ tv.tv_sec = (tv.tv_sec) - (tv.tv_sec % PGSM_BUCKET_TIME); - pg_atomic_exchange_u64(&pgss->prev_bucket_sec, (uint64)tv.tv_sec); + pg_atomic_exchange_u64(&pgsm->prev_bucket_sec, (uint64)tv.tv_sec); - pgss->bucket_start_time[new_bucket_id] = (TimestampTz) tv.tv_sec - + pgsm->bucket_start_time[new_bucket_id] = (TimestampTz) tv.tv_sec - ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY); - pgss->bucket_start_time[new_bucket_id] = pgss->bucket_start_time[new_bucket_id] * USECS_PER_SEC; + pgsm->bucket_start_time[new_bucket_id] = pgsm->bucket_start_time[new_bucket_id] * USECS_PER_SEC; return new_bucket_id; } - return pg_atomic_read_u64(&pgss->current_wbucket); + return pg_atomic_read_u64(&pgsm->current_wbucket); +} + +/* + * This function expects a NORMALIZED query as the input. + * It iterates oer the normalized query skipping comments and + * multiple spaces. All spaces are converted to ' ' so that we + * the calculation is independent of the space type whether + * newline, tab, or any other type. Trailing and leading spaces + * are also removed before calculating the hash. + */ +uint64 +get_pgsm_query_id_hash(const char *norm_query, int norm_len) +{ + char *query = palloc(norm_len + 1); + char *q_iter = query; + char *norm_q_iter = (char *)norm_query; + uint64 pgsm_query_id = 0; + + while (norm_q_iter && *norm_q_iter && norm_q_iter < (norm_query + norm_len)) + { + /* Skip multiline comments, + 1 is safe even if we've reach end of string */ + if (*norm_q_iter == '/' && *(norm_q_iter + 1) == '*') + { + while (*norm_q_iter && *norm_q_iter != '*' && *(norm_q_iter + 1) != '/') + norm_q_iter++; + + /* Skip the '/' if the current character is valid. */ + if (*norm_q_iter) + norm_q_iter++; + } + + /* Skip single line comments, + 1 is safe even if we've reach end of string */ + if (*norm_q_iter == '-' && *(norm_q_iter + 1) == '-') + { + while (*norm_q_iter && *norm_q_iter != '\n') + norm_q_iter++; + } + + /* Skip white spaces */ + if (scanner_isspace(*norm_q_iter)) + { + while (scanner_isspace(*++norm_q_iter)); + + /* + * Let's replace it with a simple space. -1 is safe as we + * are making sure we are not at the start of the string. + */ + if (q_iter != query && !scanner_isspace(*(q_iter - 1))) + *q_iter++ = ' '; + + continue; + } + + *q_iter++ = *norm_q_iter++; + } + + /* Ensure we have a terminating zero at the end */ + *q_iter = '\0'; + + /* Get rid of trailing spaces */ + while (q_iter > query && *q_iter == '\0') + { + q_iter--; + + /* Continue reducing the string size if space is found. */ + if (scanner_isspace(*q_iter)) + *q_iter = '\0'; + } + + /* Calcuate the hash. */ + pgsm_query_id = pgsm_hash_string(query, strlen(query)); + + pfree(query); + return pgsm_query_id; } #if PG_VERSION_NUM < 140000 @@ -2300,7 +2527,7 @@ AppendJumble(JumbleState *jstate, const unsigned char *item, Size size) { uint64 start_hash; - start_hash = pgss_hash_string((char *)jumble, JUMBLE_SIZE); + start_hash = pgsm_hash_string((char *)jumble, JUMBLE_SIZE); memcpy(jumble, &start_hash, sizeof(start_hash)); jumble_len = sizeof(start_hash); } @@ -2408,7 +2635,7 @@ JumbleRangeTable(JumbleState *jstate, List *rtable, CmdType cmd_type) APP_JUMB_STRING(rte->enrname); break; default: - elog(ERROR, "unrecognized RTE kind: %d", (int) rte->rtekind); + elog(ERROR, "[pg_stat_monitor] JumbleRangeTable: unrecognized RTE kind: %d.", (int) rte->rtekind); break; } } @@ -2909,7 +3136,7 @@ JumbleExpr(JumbleState *jstate, Node *node) break; default: /* Only a warning, since we can stumble along anyway */ - elog(INFO, "unrecognized node type: %d", + elog(INFO, "[pg_stat_monitor] JumbleExpr: unrecognized node type: %d.", (int) nodeTag(node)); break; } @@ -3273,16 +3500,10 @@ pgsm_emit_log_hook(ErrorData *edata) if (MyProc == NULL) goto exit; - if (PGSM_ERROR_CAPTURE_ENABLED && - (edata->elevel == ERROR || edata->elevel == WARNING || edata->elevel == INFO || edata->elevel == DEBUG1)) + /* Do not store */ + if (PGSM_ERROR_CAPTURE_ENABLED && edata->elevel >= WARNING && IsSystemOOM() == false) { - uint64 queryid = 0; - - if (debug_query_string) - queryid = pgss_hash_string(debug_query_string, strlen(debug_query_string)); - - pgss_store_error(queryid, - debug_query_string ? debug_query_string : "", + pgsm_store_error(debug_query_string ? debug_query_string : "", edata); } exit: @@ -3296,7 +3517,6 @@ IsSystemInitialized(void) return (system_init && IsHashInitialize()); } - static double time_diff(struct timeval end, struct timeval start) { @@ -3488,38 +3708,8 @@ get_query_id(JumbleState *jstate, Query *query) /* Compute query ID and mark the Query node with it */ JumbleQuery(jstate, query); - queryid = pgss_hash_string((const char *)jstate->jumble, jstate->jumble_len); + queryid = pgsm_hash_string((const char *)jstate->jumble, jstate->jumble_len); return queryid; } #endif -static uint64 -djb2_hash(unsigned char *str, size_t len) -{ - uint64 hash = 5381LLU; - - while (len--) - hash = ((hash << 5) + hash) ^ *str++; - /* hash(i - 1) * 33 ^ str[i] */ - - return hash; -} - -static uint64 -djb2_hash_str(unsigned char *str, int *out_len) -{ - uint64 hash = 5381LLU; - unsigned char *start = str; - unsigned char c; - - while ((c = *str) != '\0') - { - hash = ((hash << 5) + hash) ^ c; - /* hash(i - 1) * 33 ^ str[i] */ - ++str; - } - - *out_len = str - start; - - return hash; -} diff --git a/pg_stat_monitor.h b/pg_stat_monitor.h index 6bcf089..a65af74 100644 --- a/pg_stat_monitor.h +++ b/pg_stat_monitor.h @@ -57,6 +57,9 @@ #include "utils/lsyscache.h" #include "utils/guc.h" #include "utils/guc_tables.h" +#include "utils/memutils.h" +#include "utils/palloc.h" + #define MAX_BACKEND_PROCESES (MaxBackends + NUM_AUXILIARY_PROCS + max_prepared_xacts) #define IntArrayGetTextDatum(x,y) intarray_get_datum(x,y) @@ -64,7 +67,6 @@ /* XXX: Should USAGE_EXEC reflect execution time and/or buffer usage? */ #define USAGE_EXEC(duration) (1.0) #define USAGE_INIT (1.0) /* including initial planning */ -#define ASSUMED_MEDIAN_INIT (10.0) /* initial assumed median usage */ #define ASSUMED_LENGTH_INIT 1024 /* initial assumed mean query length */ #define USAGE_DECREASE_FACTOR (0.99) /* decreased every entry_dealloc */ #define STICKY_DECREASE_FACTOR (0.50) /* factor for sticky entries */ @@ -74,16 +76,16 @@ #define MAX_RESPONSE_BUCKET 50 #define INVALID_BUCKET_ID -1 -#define MAX_REL_LEN 255 #define MAX_BUCKETS 10 #define TEXT_LEN 255 #define ERROR_MESSAGE_LEN 100 +#define REL_TYPENAME_LEN 64 #define REL_LST 10 -#define REL_LEN 1000 +#define REL_LEN 132 /* REL_TYPENAME_LEN * 2 (relname + schema) + 1 (for view indication) + 1 and dot and string terminator */ #define CMD_LST 10 #define CMD_LEN 20 -#define APPLICATIONNAME_LEN 100 -#define COMMENTS_LEN 512 +#define APPLICATIONNAME_LEN NAMEDATALEN +#define COMMENTS_LEN 256 #define PGSM_OVER_FLOW_MAX 10 #define PLAN_TEXT_LEN 1024 /* the assumption of query max nested level */ @@ -91,12 +93,13 @@ #define MAX_QUERY_BUF (PGSM_QUERY_SHARED_BUFFER * 1024 * 1024) #define MAX_BUCKETS_MEM (PGSM_MAX * 1024 * 1024) -#define BUCKETS_MEM_OVERFLOW() ((hash_get_num_entries(pgss_hash) * sizeof(pgssEntry)) >= MAX_BUCKETS_MEM) -#define MAX_BUCKET_ENTRIES (MAX_BUCKETS_MEM / sizeof(pgssEntry)) +#define BUCKETS_MEM_OVERFLOW() ((hash_get_num_entries(pgsm_hash) * sizeof(pgsmEntry)) >= MAX_BUCKETS_MEM) +#define MAX_BUCKET_ENTRIES (MAX_BUCKETS_MEM / sizeof(pgsmEntry)) #define QUERY_BUFFER_OVERFLOW(x,y) ((x + y + sizeof(uint64) + sizeof(uint64)) > MAX_QUERY_BUF) #define QUERY_MARGIN 100 #define MIN_QUERY_LEN 10 #define SQLCODE_LEN 20 +#define TOTAL_RELS_LENGTH (REL_LST * REL_LEN) #if PG_VERSION_NUM >= 130000 #define MAX_SETTINGS 15 @@ -196,23 +199,23 @@ typedef enum OVERFLOW_TARGET OVERFLOW_TARGET_DISK } OVERFLOW_TARGET; -typedef enum pgssStoreKind +typedef enum pgsmStoreKind { - PGSS_INVALID = -1, + PGSM_INVALID = -1, /* - * PGSS_PLAN and PGSS_EXEC must be respectively 0 and 1 as they're used to + * PGSM_PLAN and PGSM_EXEC must be respectively 0 and 1 as they're used to * reference the underlying values in the arrays in the Counters struct, - * and this order is required in pg_stat_statements_internal(). + * and this order is required in pg_stat_monitor_internal(). */ - PGSS_PARSE = 0, - PGSS_PLAN, - PGSS_EXEC, - PGSS_FINISHED, - PGSS_ERROR, + PGSM_PARSE = 0, + PGSM_PLAN, + PGSM_EXEC, + PGSM_STORE, + PGSM_ERROR, - PGSS_NUMKIND /* Must be last value of this enum */ -} pgssStoreKind; + PGSM_NUMKIND /* Must be last value of this enum */ +} pgsmStoreKind; /* the assumption of query max nested level */ #define DEFAULT_MAX_NESTED_LEVEL 10 @@ -247,17 +250,17 @@ typedef struct PlanInfo size_t plan_len; /* strlen(plan_text) */ } PlanInfo; -typedef struct pgssHashKey +typedef struct pgsmHashKey { uint64 bucket_id; /* bucket number */ uint64 queryid; /* query identifier */ - uint64 userid; /* user OID */ - uint64 dbid; /* database OID */ - uint64 ip; /* client ip address */ uint64 planid; /* plan identifier */ uint64 appid; /* hash of application name */ - uint64 toplevel; /* query executed at top level */ -} pgssHashKey; + Oid userid; /* user OID */ + Oid dbid; /* database OID */ + uint32 ip; /* client ip address */ + bool toplevel; /* query executed at top level */ +} pgsmHashKey; typedef struct QueryInfo { @@ -307,6 +310,15 @@ typedef struct Blocks double temp_blk_read_time; /* time spent reading temp blocks, in msec */ double temp_blk_write_time; /* time spent writing temp blocks, in * msec */ + + /* + * Variables for local entry. The values to be passed to pgsm_update_entry + * from pgsm_store. + */ + instr_time instr_blk_read_time; /* time spent reading blocks */ + instr_time instr_blk_write_time; /* time spent writing blocks */ + instr_time instr_temp_blk_read_time; /* time spent reading temp blocks */ + instr_time instr_temp_blk_write_time; /* time spent writing temp blocks */ } Blocks; typedef struct JitInfo @@ -322,12 +334,21 @@ typedef struct JitInfo int64 jit_emission_count; /* number of times emission time has been * > 0 */ double jit_emission_time; /* total time to emit jit code */ + + /* + * Variables for local entry. The values to be passed to pgsm_update_entry + * from pgsm_store. + */ + instr_time instr_generation_counter; /* generation counter */ + instr_time instr_inlining_counter; /* inlining counter */ + instr_time instr_optimization_counter; /* optimization counter */ + instr_time instr_emission_counter; /* emission counter */ } JitInfo; typedef struct SysInfo { - float utime; /* user cpu time */ - float stime; /* system cpu time */ + double utime; /* user cpu time */ + double stime; /* system cpu time */ } SysInfo; typedef struct Wal_Usage @@ -339,7 +360,6 @@ typedef struct Wal_Usage typedef struct Counters { - uint64 bucket_id; /* bucket id */ Calls calls; QueryInfo info; CallTime time; @@ -355,7 +375,6 @@ typedef struct Counters Wal_Usage walusage; int resp_calls[MAX_RESPONSE_BUCKET]; /* execution time's in * msec */ - int64 state; /* query state */ } Counters; /* Some global structure to get the cpu usage, really don't like the idea of global variable */ @@ -363,32 +382,33 @@ typedef struct Counters /* * Statistics per statement */ -typedef struct pgssEntry +typedef struct pgsmEntry { - pgssHashKey key; /* hash key of entry - MUST BE FIRST */ - uint64 pgsm_query_id; /* pgsm generate normalized query hash */ - Counters counters; /* the statistics for this query */ - int encoding; /* query text encoding */ - slock_t mutex; /* protects the counters only */ - dsa_pointer query_pos; /* query location within query buffer */ -} pgssEntry; + pgsmHashKey key; /* hash key of entry - MUST BE FIRST */ + uint64 pgsm_query_id; /* pgsm generate normalized query hash */ + char datname[NAMEDATALEN]; /* database name */ + char username[NAMEDATALEN]; /* user name */ + Counters counters; /* the statistics for this query */ + int encoding; /* query text encoding */ + slock_t mutex; /* protects the counters only */ + union + { + dsa_pointer query_pos; /* query location within query buffer */ + char* query_pointer; + }query_text; +} pgsmEntry; /* * Global shared state */ -typedef struct pgssSharedState +typedef struct pgsmSharedState { LWLock *lock; /* protects hashtable search/modification */ - double cur_median_usage; /* current median usage in hashtable */ slock_t mutex; /* protects following fields only: */ - Size extent; /* current extent of query file */ - int64 n_writers; /* number of active writers to query file */ pg_atomic_uint64 current_wbucket; pg_atomic_uint64 prev_bucket_sec; uint64 bucket_entry[MAX_BUCKETS]; TimestampTz bucket_start_time[MAX_BUCKETS]; /* start time of the bucket */ - LWLock *errors_lock; /* protects errors hashtable - * search/modification */ int hash_tranche_id; void *raw_dsa_area; /* DSA area pointer to store query texts. * dshash also lives in this memory when @@ -398,28 +418,22 @@ typedef struct pgssSharedState * classic shared memory hash or dshash * (if we are using USE_DYNAMIC_HASH) */ -} pgssSharedState; + MemoryContext pgsm_mem_cxt; + /* context to store stats in local + * memory until they are pushed to shared hash + */ + bool pgsm_oom; +} pgsmSharedState; typedef struct pgsmLocalState { - pgssSharedState *shared_pgssState; + pgsmSharedState *shared_pgsmState; dsa_area *dsa; /* local dsa area for backend attached to the * dsa area created by postmaster at startup. */ PGSM_HASH_TABLE *shared_hash; }pgsmLocalState; -#define ResetSharedState(x) \ -do { \ - x->cur_median_usage = ASSUMED_MEDIAN_INIT; \ - x->cur_median_usage = ASSUMED_MEDIAN_INIT; \ - x->n_writers = 0; \ - pg_atomic_init_u64(&x->current_wbucket, 0); \ - pg_atomic_init_u64(&x->prev_bucket_sec, 0); \ - memset(&x->bucket_entry, 0, MAX_BUCKETS * sizeof(uint64)); \ -} while(0) - - #if PG_VERSION_NUM < 140000 /* * Struct for tracking locations/lengths of constants during normalization @@ -456,40 +470,30 @@ typedef struct JumbleState } JumbleState; #endif -/* Links to shared memory state */ - -bool SaveQueryText(uint64 bucketid, - uint64 queryid, - unsigned char *buf, - const char *query, - uint64 query_len, - size_t *query_pos); - /* guc.c */ void init_guc(void); GucVariable *get_conf(int i); /* hash_create.c */ dsa_area *get_dsa_area_for_query_text(void); -PGSM_HASH_TABLE *get_pgssHash(void); +PGSM_HASH_TABLE *get_pgsmHash(void); void pgsm_attach_shmem(void); bool IsHashInitialize(void); -void pgss_shmem_startup(void); -void pgss_shmem_shutdown(int code, Datum arg); +bool IsSystemOOM(void); +void pgsm_shmem_startup(void); +void pgsm_shmem_shutdown(int code, Datum arg); int pgsm_get_bucket_size(void); -pgssSharedState *pgsm_get_ss(void); -void hash_entry_reset(void); -void hash_query_entryies_reset(void); +pgsmSharedState *pgsm_get_ss(void); void hash_query_entries(); void hash_query_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_buffer[]); void hash_entry_dealloc(int new_bucket_id, int old_bucket_id, unsigned char *query_buffer); -pgssEntry *hash_entry_alloc(pgssSharedState *pgss, pgssHashKey *key, int encoding); +pgsmEntry *hash_entry_alloc(pgsmSharedState *pgsm, pgsmHashKey *key, int encoding); Size pgsm_ShmemSize(void); -void pgss_startup(void); +void pgsm_startup(void); /* hash_query.c */ -void pgss_startup(void); +void pgsm_startup(void); /*---- GUC variables ----*/ typedef enum @@ -528,8 +532,8 @@ static const struct config_enum_entry track_options[] = #define HOOK_STATS_SIZE 0 #endif -void *pgsm_hash_find_or_insert(PGSM_HASH_TABLE *shared_hash, pgssHashKey *key, bool* found); -void *pgsm_hash_find(PGSM_HASH_TABLE *shared_hash, pgssHashKey *key, bool* found); +void *pgsm_hash_find_or_insert(PGSM_HASH_TABLE *shared_hash, pgsmHashKey *key, bool* found); +void *pgsm_hash_find(PGSM_HASH_TABLE *shared_hash, pgsmHashKey *key, bool* found); void pgsm_hash_seq_init(PGSM_HASH_SEQ_STATUS *hstat, PGSM_HASH_TABLE *shared_hash, bool lock); void *pgsm_hash_seq_next(PGSM_HASH_SEQ_STATUS *hstat); void pgsm_hash_seq_term(PGSM_HASH_SEQ_STATUS *hstat); diff --git a/regression/expected/cmd_type.out b/regression/expected/cmd_type.out index 2981c8f..b006af1 100644 --- a/regression/expected/cmd_type.out +++ b/regression/expected/cmd_type.out @@ -23,6 +23,7 @@ SELECT b FROM t2 FOR UPDATE; TRUNCATE t1; DROP TABLE t1; +DROP TABLE t2; SELECT query, cmd_type, cmd_type_text FROM pg_stat_monitor ORDER BY query COLLATE "C"; query | cmd_type | cmd_type_text --------------------------------+----------+--------------- @@ -30,13 +31,14 @@ SELECT query, cmd_type, cmd_type_text FROM pg_stat_monitor ORDER BY query COLLA CREATE TABLE t2 (b INTEGER) | 0 | DELETE FROM t1 | 4 | DELETE DROP TABLE t1 | 0 | + DROP TABLE t2 | 0 | INSERT INTO t1 VALUES(1) | 3 | INSERT SELECT a FROM t1 | 1 | SELECT SELECT b FROM t2 FOR UPDATE | 1 | SELECT SELECT pg_stat_monitor_reset() | 1 | SELECT TRUNCATE t1 | 0 | UPDATE t1 SET a = 2 | 2 | UPDATE -(10 rows) +(11 rows) SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset diff --git a/regression/expected/database.out b/regression/expected/database.out index 7062272..bd377f6 100644 --- a/regression/expected/database.out +++ b/regression/expected/database.out @@ -27,13 +27,15 @@ SELECT * FROM t3,t4 WHERE t3.c = t4.d; (0 rows) \c contrib_regression +DROP DATABASE db2; SELECT datname, query FROM pg_stat_monitor ORDER BY query COLLATE "C"; datname | query --------------------+--------------------------------------- + contrib_regression | DROP DATABASE db2 db1 | SELECT * FROM t1,t2 WHERE t1.a = t2.b db2 | SELECT * FROM t3,t4 WHERE t3.c = t4.d contrib_regression | SELECT pg_stat_monitor_reset() -(3 rows) +(4 rows) SELECT pg_stat_monitor_reset(); pg_stat_monitor_reset @@ -44,10 +46,6 @@ SELECT pg_stat_monitor_reset(); \c db1 DROP TABLE t1; DROP TABLE t2; -\c db2 -DROP TABLE t3; -DROP TABLE t4; \c contrib_regression DROP DATABASE db1; -DROP DATABASE db2; DROP EXTENSION pg_stat_monitor; diff --git a/regression/expected/error_2.out b/regression/expected/error_2.out new file mode 100644 index 0000000..c89920b --- /dev/null +++ b/regression/expected/error_2.out @@ -0,0 +1,42 @@ +CREATE EXTENSION pg_stat_monitor; +SELECT pg_stat_monitor_reset(); + pg_stat_monitor_reset +----------------------- + +(1 row) + +SELECT 1/0; -- divide by zero +ERROR: division by zero +SELECT * FROM unknown; -- unknown table +ERROR: relation "unknown" does not exist +LINE 1: SELECT * FROM unknown; + ^ +ELECET * FROM unknown; -- syntax error +ERROR: syntax error at or near "ELECET" +LINE 1: ELECET * FROM unknown; + ^ +do $$ +BEGIN +RAISE WARNING 'warning message'; +END $$; +WARNING: warning message +SELECT query, elevel, sqlcode, message FROM pg_stat_monitor ORDER BY query COLLATE "C",elevel; + query | elevel | sqlcode | message +----------------------------------+--------+---------+----------------------------------- + ELECET * FROM unknown; | 20 | 42601 | syntax error at or near "ELECET" + SELECT * FROM unknown; | 20 | 42P01 | relation "unknown" does not exist + SELECT 1/0; | 20 | 22012 | division by zero + SELECT pg_stat_monitor_reset() | 0 | | + do $$ +| 0 | 01000 | warning message + BEGIN +| | | + RAISE WARNING 'warning message';+| | | + END $$; | | | +(5 rows) + +SELECT pg_stat_monitor_reset(); + pg_stat_monitor_reset +----------------------- + +(1 row) + +DROP EXTENSION pg_stat_monitor; diff --git a/regression/expected/functions.out b/regression/expected/functions.out index aa22a3a..b924c8d 100644 --- a/regression/expected/functions.out +++ b/regression/expected/functions.out @@ -1,4 +1,5 @@ DROP ROLE IF EXISTS su; +NOTICE: role "su" does not exist, skipping CREATE USER su WITH SUPERUSER; SET ROLE su; CREATE EXTENSION pg_stat_monitor; @@ -29,13 +30,19 @@ SELECT routine_schema, routine_name, routine_type, data_type FROM information_sc SET ROLE u1; SELECT routine_schema, routine_name, routine_type, data_type FROM information_schema.routines WHERE routine_schema = 'public' ORDER BY routine_name COLLATE "C"; - routine_schema | routine_name | routine_type | data_type -----------------+-------------------------+--------------+----------- - public | histogram | FUNCTION | record - public | pg_stat_monitor_reset | FUNCTION | void - public | pg_stat_monitor_version | FUNCTION | text -(3 rows) + routine_schema | routine_name | routine_type | data_type +----------------+--------------------------+--------------+----------- + public | decode_error_level | FUNCTION | text + public | get_cmd_type | FUNCTION | text + public | get_histogram_timings | FUNCTION | text + public | histogram | FUNCTION | record + public | pg_stat_monitor_internal | FUNCTION | record + public | pg_stat_monitor_version | FUNCTION | text + public | range | FUNCTION | ARRAY +(7 rows) SET ROLE su; DROP USER u1; DROP EXTENSION pg_stat_monitor; +DROP USER su; +ERROR: current user cannot be dropped diff --git a/regression/expected/guc.out b/regression/expected/guc.out index c0702a9..e62dc2d 100644 --- a/regression/expected/guc.out +++ b/regression/expected/guc.out @@ -24,8 +24,8 @@ COLLATE "C"; pg_stat_monitor.pgsm_histogram_buckets | 20 | | postmaster | integer | default | 2 | 50 | | 20 | 20 | f pg_stat_monitor.pgsm_histogram_max | 100000 | | postmaster | integer | default | 10 | 2147483647 | | 100000 | 100000 | f pg_stat_monitor.pgsm_histogram_min | 1 | | postmaster | integer | default | 0 | 2147483647 | | 1 | 1 | f - pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | default | 1 | 1000 | | 100 | 100 | f - pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 10 | | 10 | 10 | f + pg_stat_monitor.pgsm_max | 256 | MB | postmaster | integer | default | 10 | 10240 | | 256 | 256 | f + pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 20000 | | 10 | 10 | f pg_stat_monitor.pgsm_normalized_query | off | | user | bool | default | | | | off | off | f pg_stat_monitor.pgsm_overflow_target | 1 | | postmaster | integer | default | 0 | 1 | | 1 | 1 | f pg_stat_monitor.pgsm_query_max_len | 2048 | | postmaster | integer | default | 1024 | 2147483647 | | 2048 | 2048 | f diff --git a/regression/expected/guc_1.out b/regression/expected/guc_1.out index e83f603..087cf5d 100644 --- a/regression/expected/guc_1.out +++ b/regression/expected/guc_1.out @@ -24,8 +24,8 @@ COLLATE "C"; pg_stat_monitor.pgsm_histogram_buckets | 20 | | postmaster | integer | default | 2 | 50 | | 20 | 20 | f pg_stat_monitor.pgsm_histogram_max | 100000 | | postmaster | integer | default | 10 | 2147483647 | | 100000 | 100000 | f pg_stat_monitor.pgsm_histogram_min | 1 | | postmaster | integer | default | 0 | 2147483647 | | 1 | 1 | f - pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | default | 1 | 1000 | | 100 | 100 | f - pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 10 | | 10 | 10 | f + pg_stat_monitor.pgsm_max | 256 | MB | postmaster | integer | default | 10 | 10240 | | 256 | 256 | f + pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 20000 | | 10 | 10 | f pg_stat_monitor.pgsm_normalized_query | off | | user | bool | default | | | | off | off | f pg_stat_monitor.pgsm_overflow_target | 1 | | postmaster | integer | default | 0 | 1 | | 1 | 1 | f pg_stat_monitor.pgsm_query_max_len | 2048 | | postmaster | integer | default | 1024 | 2147483647 | | 2048 | 2048 | f diff --git a/regression/expected/pgsm_query_id.out b/regression/expected/pgsm_query_id.out index 09897f7..be1009f 100644 --- a/regression/expected/pgsm_query_id.out +++ b/regression/expected/pgsm_query_id.out @@ -40,6 +40,22 @@ SELECT * FROM t2; --- (0 rows) +-- Check that spaces and comments do not generate a different pgsm_query_id +SELECT * FROM t2 --WHATEVER; +; + b +--- +(0 rows) + +SELECT * FROM t2 /* ... +... +More comments to check for spaces. +*/ + ; + b +--- +(0 rows) + \c db2 SELECT * FROM t1; a @@ -57,18 +73,18 @@ SELECT * FROM t3; (0 rows) \c contrib_regression -SELECT datname, pgsm_query_id, query FROM pg_stat_monitor ORDER BY pgsm_query_id, query, datname; - datname | pgsm_query_id | query ---------------------+----------------------+-------------------------------- - db2 | -5029137034974447432 | SELECT * FROM t3 - contrib_regression | 689150021118383254 | SELECT pg_stat_monitor_reset() - db1 | 1897482803466821995 | SELECT * FROM t2 - db1 | 1988437669671417938 | SELECT * FROM t1 - db2 | 1988437669671417938 | SELECT * FROM t1 - db1 | 2864453209316739369 | select $1 + $2 - db2 | 2864453209316739369 | select $1 + $2 - db1 | 8140395000078788481 | SELECT *, ADD(1, 2) FROM t1 - db2 | 8140395000078788481 | SELECT *, ADD(1, 2) FROM t1 +SELECT datname, pgsm_query_id, query, calls FROM pg_stat_monitor ORDER BY pgsm_query_id, query, datname; + datname | pgsm_query_id | query | calls +--------------------+----------------------+--------------------------------+------- + db2 | -5029137034974447432 | SELECT * FROM t3 | 1 + contrib_regression | 689150021118383254 | SELECT pg_stat_monitor_reset() | 1 + db1 | 1897482803466821995 | SELECT * FROM t2 | 3 + db1 | 1988437669671417938 | SELECT * FROM t1 | 1 + db2 | 1988437669671417938 | SELECT * FROM t1 | 1 + db1 | 2864453209316739369 | select $1 + $2 | 1 + db2 | 2864453209316739369 | select $1 + $2 | 1 + db1 | 8140395000078788481 | SELECT *, ADD(1, 2) FROM t1 | 1 + db2 | 8140395000078788481 | SELECT *, ADD(1, 2) FROM t1 | 1 (9 rows) SELECT pg_stat_monitor_reset(); diff --git a/regression/expected/rows.out b/regression/expected/rows.out index e06cd5d..6350ecd 100644 --- a/regression/expected/rows.out +++ b/regression/expected/rows.out @@ -1,7 +1,6 @@ CREATE EXTENSION pg_stat_monitor; CREATE TABLE t1(a int); CREATE TABLE t2(b int); -ERROR: relation "t2" already exists INSERT INTO t1 VALUES(generate_series(1,1000)); INSERT INTO t2 VALUES(generate_series(1,5000)); SELECT pg_stat_monitor_reset(); @@ -8557,4 +8556,5 @@ SELECT pg_stat_monitor_reset(); (1 row) DROP TABLE t1; +DROP TABLE t2; DROP EXTENSION pg_stat_monitor; diff --git a/regression/expected/user.out b/regression/expected/user.out new file mode 100644 index 0000000..a3eb20f --- /dev/null +++ b/regression/expected/user.out @@ -0,0 +1,65 @@ +CREATE USER su WITH SUPERUSER; +ERROR: role "su" already exists +SET ROLE su; +CREATE EXTENSION pg_stat_monitor; +CREATE USER u1; +CREATE USER u2; +GRANT ALL ON SCHEMA public TO u1; +GRANT ALL ON SCHEMA public TO u2; +SET ROLE su; +SELECT pg_stat_monitor_reset(); + pg_stat_monitor_reset +----------------------- + +(1 row) + +SET ROLE u1; +SELECT pg_stat_monitor_reset(); +ERROR: permission denied for function pg_stat_monitor_reset +CREATE TABLE t1 (a int); +SELECT * FROM t1; + a +--- +(0 rows) + +SET ROLE u2; +CREATE TABLE t2 (a int); +SELECT * FROM t2; + a +--- +(0 rows) + +DROP TABLE t2; +SET ROLE su; +DROP OWNED BY u2; +DROP USER u2; +SELECT username, query FROM pg_stat_monitor ORDER BY username, query COLLATE "C"; + username | query +----------+--------------------------------- + su | DROP OWNED BY u2 + su | DROP USER u2 + su | SELECT pg_stat_monitor_reset() + su | SET ROLE su + u1 | CREATE TABLE t1 (a int) + u1 | SELECT * FROM t1 + u1 | SELECT pg_stat_monitor_reset(); + u1 | SET ROLE u1 + u2 | CREATE TABLE t2 (a int) + u2 | DROP TABLE t2 + u2 | SELECT * FROM t2 + u2 | SET ROLE u2 +(12 rows) + +SELECT pg_stat_monitor_reset(); + pg_stat_monitor_reset +----------------------- + +(1 row) + +DROP TABLE t1; +DROP OWNED BY u1; +DROP USER u1; +DROP EXTENSION pg_stat_monitor; +SET ROLE NONE; +DROP OWNED BY su; +DROP USER su; diff --git a/regression/sql/cmd_type.sql b/regression/sql/cmd_type.sql index 5f1fa1b..9fb2fa6 100644 --- a/regression/sql/cmd_type.sql +++ b/regression/sql/cmd_type.sql @@ -10,6 +10,7 @@ DELETE FROM t1; SELECT b FROM t2 FOR UPDATE; TRUNCATE t1; DROP TABLE t1; +DROP TABLE t2; SELECT query, cmd_type, cmd_type_text FROM pg_stat_monitor ORDER BY query COLLATE "C"; SELECT pg_stat_monitor_reset(); diff --git a/regression/sql/database.sql b/regression/sql/database.sql index 0431004..0e38754 100644 --- a/regression/sql/database.sql +++ b/regression/sql/database.sql @@ -20,6 +20,8 @@ SELECT * FROM t1,t2 WHERE t1.a = t2.b; SELECT * FROM t3,t4 WHERE t3.c = t4.d; \c contrib_regression +DROP DATABASE db2; + SELECT datname, query FROM pg_stat_monitor ORDER BY query COLLATE "C"; SELECT pg_stat_monitor_reset(); @@ -27,11 +29,6 @@ SELECT pg_stat_monitor_reset(); DROP TABLE t1; DROP TABLE t2; -\c db2 -DROP TABLE t3; -DROP TABLE t4; - \c contrib_regression DROP DATABASE db1; -DROP DATABASE db2; DROP EXTENSION pg_stat_monitor; diff --git a/regression/sql/functions.sql b/regression/sql/functions.sql index d16453b..75f0ad9 100644 --- a/regression/sql/functions.sql +++ b/regression/sql/functions.sql @@ -14,3 +14,4 @@ SELECT routine_schema, routine_name, routine_type, data_type FROM information_sc SET ROLE su; DROP USER u1; DROP EXTENSION pg_stat_monitor; +DROP USER su; diff --git a/regression/sql/pgsm_query_id.sql b/regression/sql/pgsm_query_id.sql index c26296e..eb5bebc 100644 --- a/regression/sql/pgsm_query_id.sql +++ b/regression/sql/pgsm_query_id.sql @@ -29,6 +29,14 @@ SELECT pg_stat_monitor_reset(); SELECT * FROM t1; SELECT *, ADD(1, 2) FROM t1; SELECT * FROM t2; +-- Check that spaces and comments do not generate a different pgsm_query_id +SELECT * FROM t2 --WHATEVER; +; +SELECT * FROM t2 /* ... +... +More comments to check for spaces. +*/ + ; \c db2 SELECT * FROM t1; @@ -36,7 +44,7 @@ SELECT *, ADD(1, 2) FROM t1; SELECT * FROM t3; \c contrib_regression -SELECT datname, pgsm_query_id, query FROM pg_stat_monitor ORDER BY pgsm_query_id, query, datname; +SELECT datname, pgsm_query_id, query, calls FROM pg_stat_monitor ORDER BY pgsm_query_id, query, datname; SELECT pg_stat_monitor_reset(); \c db1 diff --git a/regression/sql/rows.sql b/regression/sql/rows.sql index 02f0f70..7a09398 100644 --- a/regression/sql/rows.sql +++ b/regression/sql/rows.sql @@ -16,4 +16,5 @@ SELECT query, rows FROM pg_stat_monitor ORDER BY query COLLATE "C"; SELECT pg_stat_monitor_reset(); DROP TABLE t1; +DROP TABLE t2; DROP EXTENSION pg_stat_monitor; diff --git a/regression/sql/user.sql b/regression/sql/user.sql index 03237d2..1e3086d 100644 --- a/regression/sql/user.sql +++ b/regression/sql/user.sql @@ -6,25 +6,37 @@ CREATE EXTENSION pg_stat_monitor; CREATE USER u1; CREATE USER u2; -SET ROLE su; +GRANT ALL ON SCHEMA public TO u1; +GRANT ALL ON SCHEMA public TO u2; +SET ROLE su; SELECT pg_stat_monitor_reset(); + SET ROLE u1; +SELECT pg_stat_monitor_reset(); CREATE TABLE t1 (a int); SELECT * FROM t1; SET ROLE u2; CREATE TABLE t2 (a int); SELECT * FROM t2; +DROP TABLE t2; SET ROLE su; -SELECT userid, query FROM pg_stat_monitor ORDER BY query COLLATE "C"; + +DROP OWNED BY u2; +DROP USER u2; + +SELECT username, query FROM pg_stat_monitor ORDER BY username, query COLLATE "C"; SELECT pg_stat_monitor_reset(); DROP TABLE t1; -DROP TABLE t2; - +DROP OWNED BY u1; DROP USER u1; -DROP USER u2; DROP EXTENSION pg_stat_monitor; + +SET ROLE NONE; + +DROP OWNED BY su; +DROP USER su; diff --git a/t/018_column_names.pl b/t/018_column_names.pl index 584b9d8..a43780c 100644 --- a/t/018_column_names.pl +++ b/t/018_column_names.pl @@ -36,7 +36,7 @@ my %pg_versions_pgsm_columns = ( 15 => "application_name,blk_read_time," . "shared_blks_written,sqlcode,stddev_exec_time,stddev_plan_time," . "temp_blk_read_time,temp_blk_write_time,temp_blks_read,temp_blks_written," . "top_query,top_queryid,toplevel,total_exec_time,total_plan_time," . - "user,userid,wal_bytes,wal_fpi,wal_records", + "userid,username,wal_bytes,wal_fpi,wal_records", 14 => "application_name,blk_read_time," . "blk_write_time,bucket,bucket_done,bucket_start_time,calls," . "client_ip,cmd_type,cmd_type_text,comments,cpu_sys_time,cpu_user_time," . @@ -47,7 +47,7 @@ my %pg_versions_pgsm_columns = ( 15 => "application_name,blk_read_time," . "rows,shared_blks_dirtied,shared_blks_hit,shared_blks_read," . "shared_blks_written,sqlcode,stddev_exec_time,stddev_plan_time," . "temp_blks_read,temp_blks_written,top_query,top_queryid,toplevel," . - "total_exec_time,total_plan_time,user,userid,wal_bytes,wal_fpi,wal_records", + "total_exec_time,total_plan_time,userid,username,wal_bytes,wal_fpi,wal_records", 13 => "application_name,blk_read_time," . "blk_write_time,bucket,bucket_done,bucket_start_time,calls," . "client_ip,cmd_type,cmd_type_text,comments,cpu_sys_time,cpu_user_time," . @@ -58,7 +58,7 @@ my %pg_versions_pgsm_columns = ( 15 => "application_name,blk_read_time," . "rows,shared_blks_dirtied,shared_blks_hit,shared_blks_read," . "shared_blks_written,sqlcode,stddev_exec_time,stddev_plan_time," . "temp_blks_read,temp_blks_written,top_query,top_queryid,toplevel," . - "total_exec_time,total_plan_time,user,userid,wal_bytes,wal_fpi,wal_records", + "total_exec_time,total_plan_time,userid,username,wal_bytes,wal_fpi,wal_records", 12 => "application_name,blk_read_time,blk_write_time,bucket,bucket_done," . "bucket_start_time,calls,client_ip,cmd_type,cmd_type_text,comments," . "cpu_sys_time,cpu_user_time,datname,dbid,elevel,local_blks_dirtied," . @@ -66,7 +66,7 @@ my %pg_versions_pgsm_columns = ( 15 => "application_name,blk_read_time," . "message,min_time,pgsm_query_id,planid,query,query_plan,queryid,relations,resp_calls," . "rows,shared_blks_dirtied,shared_blks_hit,shared_blks_read," . "shared_blks_written,sqlcode,stddev_time,temp_blks_read,temp_blks_written," . - "top_query,top_queryid,total_time,user,userid" + "top_query,top_queryid,total_time,userid,username" ); # Start server diff --git a/t/expected/001_settings_default.out b/t/expected/001_settings_default.out index 498bb39..57ac4c4 100644 --- a/t/expected/001_settings_default.out +++ b/t/expected/001_settings_default.out @@ -14,8 +14,8 @@ SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals pg_stat_monitor.pgsm_histogram_buckets | 20 | | postmaster | integer | default | 2 | 50 | | 20 | 20 | f pg_stat_monitor.pgsm_histogram_max | 100000 | | postmaster | integer | default | 10 | 2147483647 | | 100000 | 100000 | f pg_stat_monitor.pgsm_histogram_min | 1 | | postmaster | integer | default | 0 | 2147483647 | | 1 | 1 | f - pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | default | 1 | 1000 | | 100 | 100 | f - pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 10 | | 10 | 10 | f + pg_stat_monitor.pgsm_max | 256 | MB | postmaster | integer | default | 10 | 10240 | | 256 | 256 | f + pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 20000 | | 10 | 10 | f pg_stat_monitor.pgsm_normalized_query | off | | user | bool | default | | | | off | off | f pg_stat_monitor.pgsm_overflow_target | 1 | | postmaster | integer | default | 0 | 1 | | 1 | 1 | f pg_stat_monitor.pgsm_query_max_len | 2048 | | postmaster | integer | default | 1024 | 2147483647 | | 2048 | 2048 | f @@ -41,8 +41,8 @@ SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals pg_stat_monitor.pgsm_histogram_buckets | 20 | | postmaster | integer | default | 2 | 50 | | 20 | 20 | f pg_stat_monitor.pgsm_histogram_max | 100000 | | postmaster | integer | default | 10 | 2147483647 | | 100000 | 100000 | f pg_stat_monitor.pgsm_histogram_min | 1 | | postmaster | integer | default | 0 | 2147483647 | | 1 | 1 | f - pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | default | 1 | 1000 | | 100 | 100 | f - pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 10 | | 10 | 10 | f + pg_stat_monitor.pgsm_max | 256 | MB | postmaster | integer | default | 10 | 10240 | | 256 | 256 | f + pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 20000 | | 10 | 10 | f pg_stat_monitor.pgsm_normalized_query | off | | user | bool | default | | | | off | off | f pg_stat_monitor.pgsm_overflow_target | 1 | | postmaster | integer | default | 0 | 1 | | 1 | 1 | f pg_stat_monitor.pgsm_query_max_len | 2048 | | postmaster | integer | default | 1024 | 2147483647 | | 2048 | 2048 | f diff --git a/t/expected/001_settings_default.out.12 b/t/expected/001_settings_default.out.12 old mode 100755 new mode 100644 index d77c7e4..157d687 --- a/t/expected/001_settings_default.out.12 +++ b/t/expected/001_settings_default.out.12 @@ -14,8 +14,8 @@ SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals pg_stat_monitor.pgsm_histogram_buckets | 20 | | postmaster | integer | default | 2 | 50 | | 20 | 20 | f pg_stat_monitor.pgsm_histogram_max | 100000 | | postmaster | integer | default | 10 | 2147483647 | | 100000 | 100000 | f pg_stat_monitor.pgsm_histogram_min | 1 | | postmaster | integer | default | 0 | 2147483647 | | 1 | 1 | f - pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | default | 1 | 1000 | | 100 | 100 | f - pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 10 | | 10 | 10 | f + pg_stat_monitor.pgsm_max | 256 | MB | postmaster | integer | default | 10 | 10240 | | 256 | 256 | f + pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 20000 | | 10 | 10 | f pg_stat_monitor.pgsm_normalized_query | off | | user | bool | default | | | | off | off | f pg_stat_monitor.pgsm_overflow_target | 1 | | postmaster | integer | default | 0 | 1 | | 1 | 1 | f pg_stat_monitor.pgsm_query_max_len | 2048 | | postmaster | integer | default | 1024 | 2147483647 | | 2048 | 2048 | f @@ -40,8 +40,8 @@ SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals pg_stat_monitor.pgsm_histogram_buckets | 20 | | postmaster | integer | default | 2 | 50 | | 20 | 20 | f pg_stat_monitor.pgsm_histogram_max | 100000 | | postmaster | integer | default | 10 | 2147483647 | | 100000 | 100000 | f pg_stat_monitor.pgsm_histogram_min | 1 | | postmaster | integer | default | 0 | 2147483647 | | 1 | 1 | f - pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | default | 1 | 1000 | | 100 | 100 | f - pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 10 | | 10 | 10 | f + pg_stat_monitor.pgsm_max | 256 | MB | postmaster | integer | default | 10 | 10240 | | 256 | 256 | f + pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 20000 | | 10 | 10 | f pg_stat_monitor.pgsm_normalized_query | off | | user | bool | default | | | | off | off | f pg_stat_monitor.pgsm_overflow_target | 1 | | postmaster | integer | default | 0 | 1 | | 1 | 1 | f pg_stat_monitor.pgsm_query_max_len | 2048 | | postmaster | integer | default | 1024 | 2147483647 | | 2048 | 2048 | f diff --git a/t/expected/012_settings_pgsm_max_buckets.out b/t/expected/012_settings_pgsm_max_buckets.out index 3942aae..2560281 100644 --- a/t/expected/012_settings_pgsm_max_buckets.out +++ b/t/expected/012_settings_pgsm_max_buckets.out @@ -8,7 +8,7 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart ----------------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max_buckets | 1 | | postmaster | integer | configuration file | 1 | 10 | | 10 | 1 | f + pg_stat_monitor.pgsm_max_buckets | 1 | | postmaster | integer | configuration file | 1 | 20000 | | 10 | 1 | f (1 row) SELECT pg_stat_monitor_reset(); @@ -20,7 +20,7 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart ----------------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max_buckets | 2 | | postmaster | integer | configuration file | 1 | 10 | | 10 | 2 | f + pg_stat_monitor.pgsm_max_buckets | 2 | | postmaster | integer | configuration file | 1 | 20000 | | 10 | 2 | f (1 row) SELECT pg_stat_monitor_reset(); @@ -32,7 +32,7 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart ----------------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max_buckets | 5 | | postmaster | integer | configuration file | 1 | 10 | | 10 | 5 | f + pg_stat_monitor.pgsm_max_buckets | 5 | | postmaster | integer | configuration file | 1 | 20000 | | 10 | 5 | f (1 row) SELECT pg_stat_monitor_reset(); @@ -44,7 +44,19 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart ----------------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | configuration file | 1 | 10 | | 10 | 10 | f + pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | configuration file | 1 | 20000 | | 10 | 10 | f +(1 row) + +SELECT pg_stat_monitor_reset(); + pg_stat_monitor_reset +----------------------- + +(1 row) + +SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets'; + name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart +----------------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+----------------- + pg_stat_monitor.pgsm_max_buckets | 11 | | postmaster | integer | configuration file | 1 | 20000 | | 10 | 11 | f (1 row) SELECT pg_stat_monitor_reset(); @@ -56,19 +68,7 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart ----------------------------------+---------+------+------------+---------+---------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 10 | | 10 | 10 | f -(1 row) - -SELECT pg_stat_monitor_reset(); - pg_stat_monitor_reset ------------------------ - -(1 row) - -SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max_buckets'; - name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart -----------------------------------+---------+------+------------+---------+---------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 10 | | 10 | 10 | f + pg_stat_monitor.pgsm_max_buckets | 10 | | postmaster | integer | default | 1 | 20000 | | 10 | 10 | f (1 row) DROP EXTENSION pg_stat_monitor; diff --git a/t/expected/016_settings_pgsm_max.out b/t/expected/016_settings_pgsm_max.out index 4f7e090..111ad5f 100644 --- a/t/expected/016_settings_pgsm_max.out +++ b/t/expected/016_settings_pgsm_max.out @@ -8,7 +8,7 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart --------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max | 1000 | MB | postmaster | integer | configuration file | 1 | 1000 | | 100 | 1000 | f + pg_stat_monitor.pgsm_max | 1000 | MB | postmaster | integer | configuration file | 10 | 10240 | | 256 | 1000 | f (1 row) SELECT pg_stat_monitor_reset(); @@ -20,7 +20,7 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart --------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max | 500 | MB | postmaster | integer | configuration file | 1 | 1000 | | 100 | 500 | f + pg_stat_monitor.pgsm_max | 500 | MB | postmaster | integer | configuration file | 10 | 10240 | | 256 | 500 | f (1 row) SELECT pg_stat_monitor_reset(); @@ -32,7 +32,7 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart --------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | configuration file | 1 | 1000 | | 100 | 100 | f + pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | configuration file | 10 | 10240 | | 256 | 100 | f (1 row) SELECT pg_stat_monitor_reset(); @@ -44,7 +44,7 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart --------------------------+---------+------+------------+---------+--------------------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max | 10 | MB | postmaster | integer | configuration file | 1 | 1000 | | 100 | 10 | f + pg_stat_monitor.pgsm_max | 10 | MB | postmaster | integer | configuration file | 10 | 10240 | | 256 | 10 | f (1 row) SELECT pg_stat_monitor_reset(); @@ -56,7 +56,7 @@ SELECT pg_stat_monitor_reset(); SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_max'; name | setting | unit | context | vartype | source | min_val | max_val | enumvals | boot_val | reset_val | pending_restart --------------------------+---------+------+------------+---------+---------+---------+---------+----------+----------+-----------+----------------- - pg_stat_monitor.pgsm_max | 100 | MB | postmaster | integer | default | 1 | 1000 | | 100 | 100 | f + pg_stat_monitor.pgsm_max | 256 | MB | postmaster | integer | default | 10 | 10240 | | 256 | 256 | f (1 row) DROP EXTENSION pg_stat_monitor;