Move stat view implementations into a submodule (#7975)

Also move serialize_distributed_ddls into commands submodule, seems like
an oversight from last year (by me).

src/backend/distributed/Makefile

@@ -18,7 +18,7 @@ generated_downgrade_sql_files += $(patsubst %,$(citus_abs_srcdir)/build/sql/%,$(
 DATA_built = $(generated_sql_files)
 
 # directories with source files
-SUBDIRS = . commands connection ddl deparser executor metadata operations planner progress relay safeclib shardsplit test transaction utils worker clock
+SUBDIRS = . commands connection ddl deparser executor metadata operations planner progress relay safeclib shardsplit stats test transaction utils worker clock
 # enterprise modules
 SUBDIRS += replication
 

src/backend/distributed/README.md

@@ -2718,7 +2718,7 @@ And beside these, Citus itself also provides some additional statistic views to
 ### Citus stat counters
 Citus keeps track of several stat counters and exposes them via the `citus_stat_counters` view. The counters are tracked once `citus.enable_stat_counters` is set to true. Also, `citus_stat_counters_reset()` can be used to reset the counters for the given database if a database id different than 0 (default, InvalidOid) is provided, otherwise, it resets the counters for the current database.
 
-Details about the implementation and its caveats can be found in the header comment of [stat_counters.c](/src/backend/distributed/stat_counters.c). However, at the high level;
+Details about the implementation and its caveats can be found in the header comment of [stat_counters.c](/src/backend/distributed/stats/stat_counters.c). However, at the high level;
 1. We allocate a shared memory array of length `MaxBackends` so that each backend has its own counter slot to reduce the contention while incrementing the counters at the runtime.
 2. We also allocate a shared hash, whose entries correspond to individual databases. Then, when a backend exits, it first aggregates its counters to the relevant entry in the shared hash, and then it resets its own counters because the same counter slot might be reused by another backend later.
 

src/backend/distributed/commands/database.c

@@ -35,6 +35,7 @@
 
 #include "distributed/adaptive_executor.h"
 #include "distributed/commands.h"
+#include "distributed/commands/serialize_distributed_ddls.h"
 #include "distributed/commands/utility_hook.h"
 #include "distributed/comment.h"
 #include "distributed/deparse_shard_query.h"
@@ -46,7 +47,6 @@
 #include "distributed/metadata_utility.h"
 #include "distributed/multi_executor.h"
 #include "distributed/relation_access_tracking.h"
-#include "distributed/serialize_distributed_ddls.h"
 #include "distributed/shard_cleaner.h"
 #include "distributed/worker_protocol.h"
 #include "distributed/worker_transaction.h"

src/backend/distributed/commands/multi_copy.c

@@ -106,7 +106,7 @@
 #include "distributed/resource_lock.h"
 #include "distributed/shard_pruning.h"
 #include "distributed/shared_connection_stats.h"
-#include "distributed/stat_counters.h"
+#include "distributed/stats/stat_counters.h"
 #include "distributed/transmit.h"
 #include "distributed/version_compat.h"
 #include "distributed/worker_protocol.h"

src/backend/distributed/commands/serialize_distributed_ddls.c

@@ -26,9 +26,9 @@
 
 #include "distributed/adaptive_executor.h"
 #include "distributed/argutils.h"
+#include "distributed/commands/serialize_distributed_ddls.h"
 #include "distributed/deparse_shard_query.h"
 #include "distributed/resource_lock.h"
-#include "distributed/serialize_distributed_ddls.h"
 
 
 PG_FUNCTION_INFO_V1(citus_internal_acquire_citus_advisory_object_class_lock);

src/backend/distributed/connection/connection_management.c

@@ -39,7 +39,7 @@
 #include "distributed/remote_commands.h"
 #include "distributed/run_from_same_connection.h"
 #include "distributed/shared_connection_stats.h"
-#include "distributed/stat_counters.h"
+#include "distributed/stats/stat_counters.h"
 #include "distributed/time_constants.h"
 #include "distributed/version_compat.h"
 #include "distributed/worker_log_messages.h"

src/backend/distributed/executor/adaptive_executor.c

@@ -171,7 +171,7 @@
 #include "distributed/repartition_join_execution.h"
 #include "distributed/resource_lock.h"
 #include "distributed/shared_connection_stats.h"
-#include "distributed/stat_counters.h"
+#include "distributed/stats/stat_counters.h"
 #include "distributed/subplan_execution.h"
 #include "distributed/transaction_identifier.h"
 #include "distributed/transaction_management.h"

src/backend/distributed/executor/citus_custom_scan.c

@@ -43,9 +43,9 @@
 #include "distributed/multi_executor.h"
 #include "distributed/multi_router_planner.h"
 #include "distributed/multi_server_executor.h"
-#include "distributed/query_stats.h"
 #include "distributed/shard_utils.h"
-#include "distributed/stat_counters.h"
+#include "distributed/stats/query_stats.h"
+#include "distributed/stats/stat_counters.h"
 #include "distributed/subplan_execution.h"
 #include "distributed/worker_log_messages.h"
 #include "distributed/worker_protocol.h"

src/backend/distributed/executor/insert_select_executor.c

@@ -50,7 +50,7 @@
 #include "distributed/repartition_executor.h"
 #include "distributed/resource_lock.h"
 #include "distributed/shardinterval_utils.h"
-#include "distributed/stat_counters.h"
+#include "distributed/stats/stat_counters.h"
 #include "distributed/subplan_execution.h"
 #include "distributed/transaction_management.h"
 #include "distributed/version_compat.h"

src/backend/distributed/executor/local_executor.c

@@ -104,8 +104,8 @@
 #include "distributed/query_utils.h"
 #include "distributed/relation_access_tracking.h"
 #include "distributed/remote_commands.h" /* to access LogRemoteCommands */
+#include "distributed/stats/stat_tenants.h"
 #include "distributed/transaction_management.h"
-#include "distributed/utils/citus_stat_tenants.h"
 #include "distributed/version_compat.h"
 #include "distributed/worker_protocol.h"
 

src/backend/distributed/executor/merge_executor.c

@@ -26,7 +26,7 @@
 #include "distributed/multi_partitioning_utils.h"
 #include "distributed/multi_router_planner.h"
 #include "distributed/repartition_executor.h"
-#include "distributed/stat_counters.h"
+#include "distributed/stats/stat_counters.h"
 #include "distributed/subplan_execution.h"
 
 static void ExecuteSourceAtWorkerAndRepartition(CitusScanState *scanState);

src/backend/distributed/planner/deparse_shard_query.c

@@ -39,7 +39,7 @@
 #include "distributed/multi_physical_planner.h"
 #include "distributed/multi_router_planner.h"
 #include "distributed/shard_utils.h"
-#include "distributed/utils/citus_stat_tenants.h"
+#include "distributed/stats/stat_tenants.h"
 #include "distributed/version_compat.h"
 
 

src/backend/distributed/planner/distributed_planner.c

@@ -66,7 +66,7 @@
 #include "distributed/recursive_planning.h"
 #include "distributed/shard_utils.h"
 #include "distributed/shardinterval_utils.h"
-#include "distributed/utils/citus_stat_tenants.h"
+#include "distributed/stats/stat_tenants.h"
 #include "distributed/version_compat.h"
 #include "distributed/worker_shard_visibility.h"
 

src/backend/distributed/shared_library_init.c

@@ -89,7 +89,6 @@
 #include "distributed/placement_connection.h"
 #include "distributed/priority.h"
 #include "distributed/query_pushdown_planning.h"
-#include "distributed/query_stats.h"
 #include "distributed/recursive_planning.h"
 #include "distributed/reference_table_utils.h"
 #include "distributed/relation_access_tracking.h"
@@ -105,13 +104,14 @@
 #include "distributed/shardsplit_shared_memory.h"
 #include "distributed/shared_connection_stats.h"
 #include "distributed/shared_library_init.h"
-#include "distributed/stat_counters.h"
 #include "distributed/statistics_collection.h"
+#include "distributed/stats/query_stats.h"
+#include "distributed/stats/stat_counters.h"
+#include "distributed/stats/stat_tenants.h"
 #include "distributed/subplan_execution.h"
 #include "distributed/time_constants.h"
 #include "distributed/transaction_management.h"
 #include "distributed/transaction_recovery.h"
-#include "distributed/utils/citus_stat_tenants.h"
 #include "distributed/utils/directory.h"
 #include "distributed/worker_log_messages.h"
 #include "distributed/worker_manager.h"

src/backend/distributed/sql/udfs/citus_stat_counters/13.1-1.sql

@@ -1,14 +1,14 @@
 -- See the comments for the function in
--- src/backend/distributed/stat_counters.c for more details.
+-- src/backend/distributed/stats/stat_counters.c for more details.
 CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_counters(
 	database_id oid DEFAULT 0,
 
 	-- must always be the first column or you should accordingly update
-	-- StoreDatabaseStatsIntoTupStore() function in src/backend/distributed/stat_counters.c
+	-- StoreDatabaseStatsIntoTupStore() function in src/backend/distributed/stats/stat_counters.c
 	OUT database_id oid,
 
 	-- Following stat counter columns must be in the same order as the
-	-- StatType enum defined in src/include/distributed/stat_counters.h
+	-- StatType enum defined in src/include/distributed/stats/stat_counters.h
 	OUT connection_establishment_succeeded bigint,
 	OUT connection_establishment_failed bigint,
 	OUT connection_reused bigint,
@@ -16,7 +16,7 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_counters(
 	OUT query_execution_multi_shard bigint,
 
 	-- must always be the last column or you should accordingly update
-	-- StoreDatabaseStatsIntoTupStore() function in src/backend/distributed/stat_counters.c
+	-- StoreDatabaseStatsIntoTupStore() function in src/backend/distributed/stats/stat_counters.c
 	OUT stats_reset timestamp with time zone
 )
 RETURNS SETOF RECORD

src/backend/distributed/sql/udfs/citus_stat_counters/latest.sql

@@ -1,14 +1,14 @@
 -- See the comments for the function in
--- src/backend/distributed/stat_counters.c for more details.
+-- src/backend/distributed/stats/stat_counters.c for more details.
 CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_counters(
 	database_id oid DEFAULT 0,
 
 	-- must always be the first column or you should accordingly update
-	-- StoreDatabaseStatsIntoTupStore() function in src/backend/distributed/stat_counters.c
+	-- StoreDatabaseStatsIntoTupStore() function in src/backend/distributed/stats/stat_counters.c
 	OUT database_id oid,
 
 	-- Following stat counter columns must be in the same order as the
-	-- StatType enum defined in src/include/distributed/stat_counters.h
+	-- StatType enum defined in src/include/distributed/stats/stat_counters.h
 	OUT connection_establishment_succeeded bigint,
 	OUT connection_establishment_failed bigint,
 	OUT connection_reused bigint,
@@ -16,7 +16,7 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_counters(
 	OUT query_execution_multi_shard bigint,
 
 	-- must always be the last column or you should accordingly update
-	-- StoreDatabaseStatsIntoTupStore() function in src/backend/distributed/stat_counters.c
+	-- StoreDatabaseStatsIntoTupStore() function in src/backend/distributed/stats/stat_counters.c
 	OUT stats_reset timestamp with time zone
 )
 RETURNS SETOF RECORD

src/backend/distributed/stats/query_stats.c

@@ -32,7 +32,7 @@
 #include "distributed/hash_helpers.h"
 #include "distributed/multi_executor.h"
 #include "distributed/multi_server_executor.h"
-#include "distributed/query_stats.h"
+#include "distributed/stats/query_stats.h"
 #include "distributed/tuplestore.h"
 #include "distributed/version_compat.h"
 

src/backend/distributed/stats/stat_counters.c

@@ -76,7 +76,7 @@
 
 #include "distributed/argutils.h"
 #include "distributed/metadata_cache.h"
-#include "distributed/stat_counters.h"
+#include "distributed/stats/stat_counters.h"
 #include "distributed/tuplestore.h"
 
 

src/backend/distributed/stats/stat_tenants.c

@@ -1,6 +1,6 @@
 /*-------------------------------------------------------------------------
  *
- * citus_stat_tenants.c
+ * stat_tenants.c
  *	  Routines related to the multi tenant monitor.
  *
  * Copyright (c) Citus Data, Inc.
@@ -35,9 +35,9 @@
 #include "distributed/log_utils.h"
 #include "distributed/metadata_cache.h"
 #include "distributed/multi_executor.h"
+#include "distributed/stats/stat_tenants.h"
 #include "distributed/tenant_schema_metadata.h"
 #include "distributed/tuplestore.h"
-#include "distributed/utils/citus_stat_tenants.h"
 
 static void AttributeMetricsIfApplicable(void);
 

src/backend/distributed/test/citus_stat_tenants.c

@@ -15,7 +15,7 @@
 
 #include "sys/time.h"
 
-#include "distributed/utils/citus_stat_tenants.h"
+#include "distributed/stats/stat_tenants.h"
 
 PG_FUNCTION_INFO_V1(sleep_until_next_period);
 

src/backend/distributed/utils/maintenanced.c

@@ -54,10 +54,10 @@
 #include "distributed/maintenanced.h"
 #include "distributed/metadata_cache.h"
 #include "distributed/metadata_sync.h"
-#include "distributed/query_stats.h"
 #include "distributed/resource_lock.h"
 #include "distributed/shard_cleaner.h"
 #include "distributed/statistics_collection.h"
+#include "distributed/stats/query_stats.h"
 #include "distributed/transaction_recovery.h"
 #include "distributed/version_compat.h"
 

src/include/distributed/stats/stat_tenants.h

@@ -1,6 +1,6 @@
 /*-------------------------------------------------------------------------
  *
- * citus_stat_tenants.h
+ * stat_tenants.h
  *	  Routines related to the multi tenant monitor.
  *
  * Copyright (c) Citus Data, Inc.