From 12ba1e39d154445512531ac3bb48fced966e6974 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Tue, 2 Feb 2021 15:30:28 +0000 Subject: [PATCH] PG-172: Exponential histogram for time buckets. --- guc.c | 56 ++++++++++++++++++--------- pg_stat_monitor--1.0.sql | 10 +++++ pg_stat_monitor.c | 83 ++++++++++++++++++++++++++++++++-------- pg_stat_monitor.h | 11 +++--- 4 files changed, 121 insertions(+), 39 deletions(-) diff --git a/guc.c b/guc.c index f51a7a5..6acacd1 100644 --- a/guc.c +++ b/guc.c @@ -18,7 +18,7 @@ #include "pg_stat_monitor.h" -GucVariable conf[12]; +GucVariable conf[13]; /* * Define (or redefine) custom GUC variables. @@ -87,19 +87,27 @@ init_guc(void) }; conf[i++] = (GucVariable) { - .guc_name = "pg_stat_monitor.pgsm_respose_time_lower_bound", + .guc_name = "pg_stat_monitor.pgsm_histogram_min", .guc_desc = "Sets the time in millisecond.", - .guc_default = 1, - .guc_min = 1, + .guc_default = 0, + .guc_min = 0, + .guc_max = INT_MAX, + .guc_restart = true + }; + conf[i++] = (GucVariable) { + .guc_name = "pg_stat_monitor.pgsm_histogram_max", + .guc_desc = "Sets the time in millisecond.", + .guc_default = 10, + .guc_min = 10, .guc_max = INT_MAX, .guc_restart = true }; conf[i++] = (GucVariable) { - .guc_name = "pg_stat_monitor.pgsm_respose_time_step", - .guc_desc = "Sets the response time steps in millisecond.", - .guc_default = 1, - .guc_min = 1, + .guc_name = "pg_stat_monitor.pgsm_histogram_buckets", + .guc_desc = "Sets the maximum number of histogram buckets", + .guc_default = 10, + .guc_min = 2, .guc_max = INT_MAX, .guc_restart = true }; @@ -208,13 +216,25 @@ init_guc(void) NULL, NULL); - - DefineCustomIntVariable("pg_stat_monitor.pgsm_respose_time_lower_bound", + DefineCustomIntVariable("pg_stat_monitor.pgsm_histogram_min", "Sets the time in millisecond.", NULL, - &PGSM_RESPOSE_TIME_LOWER_BOUND, - 1, - 1, + &PGSM_HISTOGRAM_MIN, + 0, + 0, + INT_MAX, + PGC_POSTMASTER, + 0, + NULL, + NULL, + NULL); + + DefineCustomIntVariable("pg_stat_monitor.pgsm_histogram_max", + "Sets the time in millisecond.", + NULL, + &PGSM_HISTOGRAM_MAX, + 10, + 10, INT_MAX, PGC_POSTMASTER, 0, @@ -222,12 +242,12 @@ init_guc(void) NULL, NULL); - DefineCustomIntVariable("pg_stat_monitor.pgsm_respose_time_step", - "Sets the response time steps in millisecond.", + DefineCustomIntVariable("pg_stat_monitor.pgsm_histogram_buckets", + "Sets the total number of histogram buckets", NULL, - &PGSM_RESPOSE_TIME_STEP, - 1, - 1, + &PGSM_HISTOGRAM_BUCKETS, + 10, + 2, INT_MAX, PGC_POSTMASTER, 0, diff --git a/pg_stat_monitor--1.0.sql b/pg_stat_monitor--1.0.sql index 35c9f76..0d58aac 100644 --- a/pg_stat_monitor--1.0.sql +++ b/pg_stat_monitor--1.0.sql @@ -14,6 +14,16 @@ RETURNS text AS 'MODULE_PATHNAME' LANGUAGE C PARALLEL SAFE; +CREATE FUNCTION get_histogram_timings() +RETURNS text +AS 'MODULE_PATHNAME' +LANGUAGE C PARALLEL SAFE; + +CREATE FUNCTION range() +RETURNS text[] AS $$ +SELECT string_to_array(get_histogram_timings(), ','); +$$ LANGUAGE SQL; + CREATE FUNCTION pg_stat_monitor(IN showtext boolean, OUT bucket int, OUT userid oid, diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 4310953..e674526 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -54,6 +54,7 @@ static struct rusage rusage_start; static struct rusage rusage_end; static unsigned char *pgss_qbuf[MAX_BUCKETS]; +static int get_histogram_bucket(double q_time); static bool IsSystemInitialized(void); static void dump_queries_buffer(int bucket_id, unsigned char *buf, int buf_len); static double time_diff(struct timeval end, struct timeval start); @@ -77,11 +78,12 @@ PG_FUNCTION_INFO_V1(pg_stat_monitor_1_2); PG_FUNCTION_INFO_V1(pg_stat_monitor_1_3); PG_FUNCTION_INFO_V1(pg_stat_monitor); PG_FUNCTION_INFO_V1(pg_stat_monitor_settings); +PG_FUNCTION_INFO_V1(get_histogram_timings); static uint pg_get_client_addr(void); static int pg_get_application_name(char* application_name); static PgBackendStatus *pg_get_backend_status(void); -static Datum textarray_get_datum(char arr[][CMD_LEN], int len); +static Datum textarray_get_datum(char arr[][1024], int len, int str_len); static Datum intarray_get_datum(int32 arr[], int len); #if PG_VERSION_NUM >= 130000 @@ -949,16 +951,8 @@ static void pgss_store(uint64 queryId, /* increment only in case of PGSS_EXEC */ if (kind == PGSS_EXEC) { - for (i = 0; i < MAX_RESPONSE_BUCKET; i++) - { - if (total_time < PGSM_RESPOSE_TIME_LOWER_BOUND + (PGSM_RESPOSE_TIME_STEP * i)) - { - e->counters.resp_calls[i]++; - break; - } - } - if (total_time > PGSM_RESPOSE_TIME_LOWER_BOUND + (PGSM_RESPOSE_TIME_STEP * MAX_RESPONSE_BUCKET)) - e->counters.resp_calls[MAX_RESPONSE_BUCKET - 1]++; + int index = get_histogram_bucket(total_time); + e->counters.resp_calls[index]++; } _snprintf(e->counters.info.application_name, application_name, application_name_len, APPLICATIONNAME_LEN); @@ -1209,7 +1203,7 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, else values[i++] = IntArrayGetTextDatum(tmp.info.relations, len); - values[i++] = TextArrayGetTextDatum(tmp.info.cmd_type, CMD_LST); + values[i++] = TextArrayGetTextDatum((char (*)[1024])tmp.info.cmd_type, CMD_LST, CMD_LEN); values[i++] = Int64GetDatumFast(tmp.error.elevel); if (strlen(tmp.error.sqlcode) <= 0) values[i++] = CStringGetTextDatum("0"); @@ -2213,9 +2207,10 @@ comp_location(const void *a, const void *b) else return 0; } + /* Convert array into Text dataum */ static Datum -textarray_get_datum(char arr[][CMD_LEN], int len) +textarray_get_datum(char arr[][1024], int len, int str_len) { int j; char str[1024]; @@ -2226,15 +2221,15 @@ textarray_get_datum(char arr[][CMD_LEN], int len) /* Need to calculate the actual size, and avoid unnessary memory usage */ for (j = 0; j < len; j++) { - if (strlen(arr[j]) <= 0) + if (arr[j] == NULL || strlen(arr[j]) <= 0) continue; if (first) { - snprintf(str, CMD_LEN, "%s", arr[j]); + snprintf(str, str_len, "%s", arr[j]); first = false; continue; } - snprintf(str, CMD_LEN, "%s,%s", str, arr[j]); + snprintf(str, str_len, "%s,%s", str, arr[j]); } return CStringGetTextDatum(str); @@ -2665,3 +2660,59 @@ unpack_sql_state(int sql_state) return buf; } +static int +get_histogram_bucket(double q_time) +{ + double q_min = PGSM_HISTOGRAM_MIN; + double q_max = PGSM_HISTOGRAM_MAX; + int b_count = PGSM_HISTOGRAM_BUCKETS; + int index = 0; + double b_max; + double b_min; + double bucket_size; + + q_time -= q_min; + + b_max = log(q_max - q_min); + b_min = 0; + + bucket_size = (b_max - b_min) / (double)b_count; + + for(index = 1; index <= b_count; index++) + { + double b_start = (index == 1)? 0 : exp(bucket_size * (index - 1)); + double b_end = exp(bucket_size * index); + if( (index == 1 && q_time < b_start) + || (q_time >= b_start && q_time <= b_end) + || (index == b_count && q_time > b_end) ) + { + return index - 1; + } + } + return 0; +} + +Datum +get_histogram_timings(PG_FUNCTION_ARGS) +{ + double q_min = PGSM_HISTOGRAM_MIN; + double q_max = PGSM_HISTOGRAM_MAX; + int b_count = PGSM_HISTOGRAM_BUCKETS; + int index = 0; + double b_max; + double b_min; + double bucket_size; + char range[50][1024] = {0}; + + b_max = log(q_max - q_min); + b_min = 0; + bucket_size = (b_max - b_min) / (double)b_count; + for(index = 1; index <= b_count; index++) + { + int64 b_start = (index == 1)? 0 : exp(bucket_size * (index - 1)); + int64 b_end = exp(bucket_size * index); + sprintf(range[index-1], "(%d - %d)}", b_start, b_end); + } + return TextArrayGetTextDatum(range, b_count, 1024); +} + diff --git a/pg_stat_monitor.h b/pg_stat_monitor.h index 28a54fa..e76b68a 100644 --- a/pg_stat_monitor.h +++ b/pg_stat_monitor.h @@ -54,7 +54,7 @@ #include "utils/guc.h" #define MAX_BACKEND_PROCESES (MaxBackends + NUM_AUXILIARY_PROCS + max_prepared_xacts) -#define TextArrayGetTextDatum(x,y) textarray_get_datum(x,y) +#define TextArrayGetTextDatum(x,y,z) textarray_get_datum(x,y,z) #define IntArrayGetTextDatum(x,y) intarray_get_datum(x,y) /* XXX: Should USAGE_EXEC reflect execution time and/or buffer usage? */ @@ -341,9 +341,10 @@ void pgss_startup(void); #define PGSM_NORMALIZED_QUERY get_conf(4)->guc_variable #define PGSM_MAX_BUCKETS get_conf(5)->guc_variable #define PGSM_BUCKET_TIME get_conf(6)->guc_variable -#define PGSM_RESPOSE_TIME_LOWER_BOUND get_conf(7)->guc_variable -#define PGSM_RESPOSE_TIME_STEP get_conf(8)->guc_variable -#define PGSM_QUERY_BUF_SIZE get_conf(9)->guc_variable -#define PGSM_TRACK_PLANNING get_conf(10)->guc_variable +#define PGSM_HISTOGRAM_MIN get_conf(7)->guc_variable +#define PGSM_HISTOGRAM_MAX get_conf(8)->guc_variable +#define PGSM_HISTOGRAM_BUCKETS get_conf(9)->guc_variable +#define PGSM_QUERY_BUF_SIZE get_conf(10)->guc_variable +#define PGSM_TRACK_PLANNING get_conf(11)->guc_variable #endif