diff --git a/src/backend/columnar/cstore_tableam.c b/src/backend/columnar/cstore_tableam.c index 2109627dd..2b18eb712 100644 --- a/src/backend/columnar/cstore_tableam.c +++ b/src/backend/columnar/cstore_tableam.c @@ -44,12 +44,16 @@ #include "utils/memutils.h" #include "utils/pg_rusage.h" #include "utils/rel.h" +#include "utils/lsyscache.h" #include "utils/syscache.h" #include "columnar/cstore.h" #include "columnar/cstore_customscan.h" #include "columnar/cstore_tableam.h" #include "columnar/cstore_version_compat.h" +#include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/metadata_cache.h" #define CSTORE_TABLEAM_NAME "cstore_tableam" @@ -1286,6 +1290,131 @@ columnar_handler(PG_FUNCTION_ARGS) } +/* + * CitusCreateAlterColumnarTableSet generates a portable + */ +static char * +CitusCreateAlterColumnarTableSet(char *qualifiedRelationName, + const ColumnarOptions *options) +{ + StringInfoData buf = { 0 }; + initStringInfo(&buf); + + appendStringInfo(&buf, + "SELECT alter_columnar_table_set(%s, " + "block_row_count => %d, " + "stripe_row_count => %lu, " + "compression => %s);", + quote_literal_cstr(qualifiedRelationName), + options->blockRowCount, + options->stripeRowCount, + quote_literal_cstr(CompressionTypeStr(options->compressionType))); + + return buf.data; +} + + +/* + * ColumnarTableDDLContext holds the instance variable for the TableDDLCommandFunction + * instance described below. + */ +typedef struct ColumnarTableDDLContext +{ + char *schemaName; + char *relationName; + ColumnarOptions options; +} ColumnarTableDDLContext; + + +/* + * GetTableDDLCommandColumnar is an internal function used to turn a + * ColumnarTableDDLContext stored on the context of a TableDDLCommandFunction into a sql + * command that will be executed against a table. The resulting command will set the + * options of the table to the same options as the relation on the coordinator. + */ +static char * +GetTableDDLCommandColumnar(void *context) +{ + ColumnarTableDDLContext *tableDDLContext = (ColumnarTableDDLContext *) context; + + char *qualifiedShardName = quote_qualified_identifier(tableDDLContext->schemaName, + tableDDLContext->relationName); + + return CitusCreateAlterColumnarTableSet(qualifiedShardName, + &tableDDLContext->options); +} + + +/* + * GetShardedTableDDLCommandColumnar is an internal function used to turn a + * ColumnarTableDDLContext stored on the context of a TableDDLCommandFunction into a sql + * command that will be executed against a shard. The resulting command will set the + * options of the shard to the same options as the relation the shard is based on. + */ +static char * +GetShardedTableDDLCommandColumnar(uint64 shardId, void *context) +{ + ColumnarTableDDLContext *tableDDLContext = (ColumnarTableDDLContext *) context; + + /* + * AppendShardId is destructive of the original cahr *, given we want to serialize + * more than once we copy it before appending the shard id. + */ + char *relationName = pstrdup(tableDDLContext->relationName); + AppendShardIdToName(&relationName, shardId); + + char *qualifiedShardName = quote_qualified_identifier(tableDDLContext->schemaName, + relationName); + + return CitusCreateAlterColumnarTableSet(qualifiedShardName, + &tableDDLContext->options); +} + + +/* + * ColumnarGetCustomTableOptionsDDL returns a TableDDLCommand representing a command that + * will apply the passed columnar options to the relation identified by relationId on a + * new table or shard. + */ +static TableDDLCommand * +ColumnarGetCustomTableOptionsDDL(char *schemaName, char *relationName, + ColumnarOptions options) +{ + ColumnarTableDDLContext *context = (ColumnarTableDDLContext *) palloc0( + sizeof(ColumnarTableDDLContext)); + + /* build the context */ + context->schemaName = schemaName; + context->relationName = relationName; + context->options = options; + + /* create TableDDLCommand based on the context build above */ + return makeTableDDLCommandFunction( + GetTableDDLCommandColumnar, + GetShardedTableDDLCommandColumnar, + context); +} + + +/* + * ColumnarGetTableOptionsDDL returns a TableDDLCommand representing a command that will + * apply the columnar options currently applicable to the relation identified by + * relationId on a new table or shard. + */ +TableDDLCommand * +ColumnarGetTableOptionsDDL(Oid relationId) +{ + Oid namespaceId = get_rel_namespace(relationId); + char *schemaName = get_namespace_name(namespaceId); + char *relationName = get_rel_name(relationId); + + ColumnarOptions options = { 0 }; + ReadColumnarOptions(relationId, &options); + + return ColumnarGetCustomTableOptionsDDL(schemaName, relationName, options); +} + + /* * alter_columnar_table_set is a UDF exposed in postgres to change settings on a columnar * table. Calling this function on a non-columnar table gives an error. @@ -1356,6 +1485,20 @@ alter_columnar_table_set(PG_FUNCTION_ARGS) CompressionTypeStr(options.compressionType)))); } + if (EnableDDLPropagation && IsCitusTable(relationId)) + { + /* when a columnar table is distributed update all settings on the shards */ + Oid namespaceId = get_rel_namespace(relationId); + char *schemaName = get_namespace_name(namespaceId); + char *relationName = get_rel_name(relationId); + TableDDLCommand *command = ColumnarGetCustomTableOptionsDDL(schemaName, + relationName, + options); + DDLJob *ddljob = CreateCustomDDLTaskList(relationId, command); + + ExecuteDistributedDDLJob(ddljob); + } + SetColumnarOptions(relationId, &options); table_close(rel, NoLock); @@ -1364,6 +1507,24 @@ alter_columnar_table_set(PG_FUNCTION_ARGS) } +/* + * alter_columnar_table_reset is a UDF exposed in postgres to reset the settings on a + * columnar table. Calling this function on a non-columnar table gives an error. + * + * sql syntax: + * pg_catalog.alter_columnar_table_re + * teset( + * table_name regclass, + * block_row_count bool DEFAULT FALSE, + * stripe_row_count bool DEFAULT FALSE, + * compression bool DEFAULT FALSE) + * + * All arguments except the table name are optional. The UDF is supposed to be called + * like: + * SELECT alter_columnar_table_set('table', compression => true); + * + * All options set to true will be reset to the default system value. + */ PG_FUNCTION_INFO_V1(alter_columnar_table_reset); Datum alter_columnar_table_reset(PG_FUNCTION_ARGS) @@ -1408,6 +1569,20 @@ alter_columnar_table_reset(PG_FUNCTION_ARGS) CompressionTypeStr(options.compressionType)))); } + if (EnableDDLPropagation && IsCitusTable(relationId)) + { + /* when a columnar table is distributed update all settings on the shards */ + Oid namespaceId = get_rel_namespace(relationId); + char *schemaName = get_namespace_name(namespaceId); + char *relationName = get_rel_name(relationId); + TableDDLCommand *command = ColumnarGetCustomTableOptionsDDL(schemaName, + relationName, + options); + DDLJob *ddljob = CreateCustomDDLTaskList(relationId, command); + + ExecuteDistributedDDLJob(ddljob); + } + SetColumnarOptions(relationId, &options); table_close(rel, NoLock); diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index 17253854d..13be409a4 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -75,7 +75,6 @@ static int activeDropSchemaOrDBs = 0; /* Local functions forward declarations for helper functions */ -static void ExecuteDistributedDDLJob(DDLJob *ddlJob); static char * SetSearchPathToCurrentSearchPathCommand(void); static char * CurrentSearchPath(void); static void IncrementUtilityHookCountersIfNecessary(Node *parsetree); @@ -645,7 +644,7 @@ IsDropSchemaOrDB(Node *parsetree) * a partitioned table which has replication factor > 1. * */ -static void +void ExecuteDistributedDDLJob(DDLJob *ddlJob) { bool shouldSyncMetadata = false; @@ -744,6 +743,54 @@ ExecuteDistributedDDLJob(DDLJob *ddlJob) } +/* + * CreateCustomDDLTaskList creates a DDLJob which will apply a command to all placements + * of shards of a distributed table. The command to be applied is generated by the + * TableDDLCommand structure passed in. + */ +DDLJob * +CreateCustomDDLTaskList(Oid relationId, TableDDLCommand *command) +{ + List *taskList = NIL; + List *shardIntervalList = LoadShardIntervalList(relationId); + uint64 jobId = INVALID_JOB_ID; + Oid namespace = get_rel_namespace(relationId); + char *namespaceName = get_namespace_name(namespace); + int taskId = 1; + + /* lock metadata before getting placement lists */ + LockShardListMetadata(shardIntervalList, ShareLock); + + ShardInterval *shardInterval = NULL; + foreach_ptr(shardInterval, shardIntervalList) + { + uint64 shardId = shardInterval->shardId; + + char *commandStr = GetShardedTableDDLCommand(command, shardId, namespaceName); + + Task *task = CitusMakeNode(Task); + task->jobId = jobId; + task->taskId = taskId++; + task->taskType = DDL_TASK; + SetTaskQueryString(task, commandStr); + task->replicationModel = REPLICATION_MODEL_INVALID; + task->dependentTaskList = NULL; + task->anchorShardId = shardId; + task->taskPlacementList = ActiveShardPlacementList(shardId); + + taskList = lappend(taskList, task); + } + + DDLJob *ddlJob = palloc0(sizeof(DDLJob)); + ddlJob->targetRelationId = relationId; + ddlJob->concurrentIndexCmd = false; + ddlJob->commandString = GetTableDDLCommand(command); + ddlJob->taskList = taskList; + + return ddlJob; +} + + /* * SetSearchPathToCurrentSearchPathCommand generates a command which can * set the search path to the exact same search path that the issueing node diff --git a/src/backend/distributed/operations/node_protocol.c b/src/backend/distributed/operations/node_protocol.c index 17c385f62..2b65fd7c9 100644 --- a/src/backend/distributed/operations/node_protocol.c +++ b/src/backend/distributed/operations/node_protocol.c @@ -64,6 +64,7 @@ #include "utils/ruleutils.h" #include "utils/varlena.h" +#include "columnar/cstore_tableam.h" /* Shard related configuration */ int ShardCount = 32; @@ -646,6 +647,19 @@ GetPreLoadTableCreationCommands(Oid relationId, bool includeSequenceDefaults) tableColumnOptionsDef)); } +#if PG_VERSION_NUM >= 120000 + + /* add columnar options for cstore tables */ + if (IsCStoreTableAmTable(relationId)) + { + TableDDLCommand *cstoreOptionsDDL = ColumnarGetTableOptionsDDL(relationId); + if (cstoreOptionsDDL != NULL) + { + tableDDLEventList = lappend(tableDDLEventList, cstoreOptionsDDL); + } + } +#endif + char *tableOwnerDef = TableOwnerResetCommand(relationId); if (tableOwnerDef != NULL) { @@ -871,6 +885,33 @@ makeTableDDLCommandString(char *commandStr) } +/* + * makeTableDDLCommandString creates an implementation of TableDDLCommand that creates the + * final sql command based on function pointers being passed. + */ +TableDDLCommand * +makeTableDDLCommandFunction(TableDDLFunction function, + TableDDLShardedFunction shardedFunction, + void *context) +{ + TableDDLCommand *command = CitusMakeNode(TableDDLCommand); + + /* + * Function pointers are called later without verifying them not being NULL. Guard + * developers from making a mistake with them directly when they could be made. + */ + Assert(function != NULL); + Assert(shardedFunction != NULL); + + command->type = TABLE_DDL_COMMAND_FUNCTION; + command->function.function = function; + command->function.shardedFunction = shardedFunction; + command->function.context = context; + + return command; +} + + /* * GetShardedTableDDLCommandString is the internal function for TableDDLCommand objects * created with makeTableDDLCommandString. @@ -918,6 +959,9 @@ GetTableDDLCommandString(TableDDLCommand *command) * GetShardedTableDDLCommand returns the ddl command expressed by this TableDDLCommand * where all applicable names are transformed into the names for a shard identified by * shardId + * + * schemaName is deprecated but used for TableDDLCommandString. All other implementations + * will need to rely solely on the shardId. */ char * GetShardedTableDDLCommand(TableDDLCommand *command, uint64 shardId, char *schemaName) @@ -928,6 +972,11 @@ GetShardedTableDDLCommand(TableDDLCommand *command, uint64 shardId, char *schema { return GetShardedTableDDLCommandString(command, shardId, schemaName); } + + case TABLE_DDL_COMMAND_FUNCTION: + { + return command->function.shardedFunction(shardId, command->function.context); + } } /* unreachable: compiler should warn/error when not all cases are covered above */ @@ -948,6 +997,11 @@ GetTableDDLCommand(TableDDLCommand *command) { return GetTableDDLCommandString(command); } + + case TABLE_DDL_COMMAND_FUNCTION: + { + return command->function.function(command->function.context); + } } /* unreachable: compiler should warn/error when not all cases are covered above */ diff --git a/src/backend/distributed/utils/citus_outfuncs.c b/src/backend/distributed/utils/citus_outfuncs.c index b02c7daf7..9adbca31a 100644 --- a/src/backend/distributed/utils/citus_outfuncs.c +++ b/src/backend/distributed/utils/citus_outfuncs.c @@ -115,6 +115,10 @@ (appendStringInfo(str, " :" CppAsString(fldname) " "), \ _outBitmapset(str, node->fldname)) +#define WRITE_CUSTOM_FIELD(fldname, fldvalue) \ + (appendStringInfo(str, " :" CppAsString(fldname) " "), \ + appendStringInfoString(str, (fldvalue))) + /* Write an integer array (anything written as ":fldname (%d, %d") */ #define WRITE_INT_ARRAY(fldname, count) \ @@ -580,5 +584,12 @@ OutTableDDLCommand(OUTFUNC_ARGS) WRITE_STRING_FIELD(commandStr); break; } + + case TABLE_DDL_COMMAND_FUNCTION: + { + char *example = node->function.function(node->function.context); + WRITE_CUSTOM_FIELD(function, example); + break; + } } } diff --git a/src/include/columnar/cstore.h b/src/include/columnar/cstore.h index 2e5c743cb..71be3a2c6 100644 --- a/src/include/columnar/cstore.h +++ b/src/include/columnar/cstore.h @@ -95,7 +95,6 @@ typedef struct StripeMetadata typedef struct DataFileMetadata { List *stripeMetadataList; - ColumnarOptions options; } DataFileMetadata; diff --git a/src/include/columnar/cstore_tableam.h b/src/include/columnar/cstore_tableam.h index b137de899..aa320a864 100644 --- a/src/include/columnar/cstore_tableam.h +++ b/src/include/columnar/cstore_tableam.h @@ -7,6 +7,8 @@ #include "access/skey.h" #include "nodes/bitmapset.h" +#include "distributed/coordinator_protocol.h" + const TableAmRoutine * GetColumnarTableAmRoutine(void); extern void cstore_tableam_init(void); extern void cstore_tableam_finish(void); @@ -16,4 +18,7 @@ extern TableScanDesc cstore_beginscan_extended(Relation relation, Snapshot snaps ParallelTableScanDesc parallel_scan, uint32 flags, Bitmapset *attr_needed, List *scanQual); + +extern bool IsCStoreTableAmTable(Oid relationId); +extern TableDDLCommand * ColumnarGetTableOptionsDDL(Oid relationId); #endif diff --git a/src/include/distributed/commands/utility_hook.h b/src/include/distributed/commands/utility_hook.h index c7814f658..dc3cb3d71 100644 --- a/src/include/distributed/commands/utility_hook.h +++ b/src/include/distributed/commands/utility_hook.h @@ -17,6 +17,7 @@ #include "utils/relcache.h" #include "tcop/utility.h" +#include "distributed/coordinator_protocol.h" #include "distributed/version_compat.h" #include "distributed/worker_transaction.h" @@ -75,5 +76,9 @@ extern List * DDLTaskList(Oid relationId, const char *commandString); extern List * NodeDDLTaskList(TargetWorkerSet targets, List *commands); extern bool AlterTableInProgress(void); extern bool DropSchemaOrDBInProgress(void); +extern void ExecuteDistributedDDLJob(DDLJob *ddlJob); + +/* forward declarations for sending custom commands to a distributed table */ +extern DDLJob * CreateCustomDDLTaskList(Oid relationId, TableDDLCommand *command); #endif /* MULTI_UTILITY_H */ diff --git a/src/include/distributed/coordinator_protocol.h b/src/include/distributed/coordinator_protocol.h index 9bc819265..f871db5b2 100644 --- a/src/include/distributed/coordinator_protocol.h +++ b/src/include/distributed/coordinator_protocol.h @@ -90,9 +90,15 @@ typedef enum typedef enum TableDDLCommandType { TABLE_DDL_COMMAND_STRING, + TABLE_DDL_COMMAND_FUNCTION, } TableDDLCommandType; +struct TableDDLCommand; +typedef struct TableDDLCommand TableDDLCommand; +typedef char *(*TableDDLFunction)(void *context); +typedef char *(*TableDDLShardedFunction)(uint64 shardId, void *context); + /* * TableDDLCommand holds the definition of a command to be executed to bring the table and * or shard into a certain state. The command needs to be able to serialized into two @@ -107,7 +113,7 @@ typedef enum TableDDLCommandType * command that will be wrapped in `worker_apply_shard_ddl_command` when applied * against a shard. */ -typedef struct TableDDLCommand +struct TableDDLCommand { CitusNode node; @@ -126,11 +132,28 @@ typedef struct TableDDLCommand * sql ddl command string representing the ddl command. */ char *commandStr; + + /* + * function is used when type is set to TABLE_DDL_COMMAND_FUNCTION. It contains + * function pointers and a context to be passed to the functions to be able to + * construct the sql commands for sharded and non-sharded tables. + */ + struct + { + TableDDLFunction function; + TableDDLShardedFunction shardedFunction; + void *context; + } + function; }; -} TableDDLCommand; +}; /* make functions for TableDDLCommand */ extern TableDDLCommand * makeTableDDLCommandString(char *commandStr); +extern TableDDLCommand * makeTableDDLCommandFunction(TableDDLFunction function, + TableDDLShardedFunction + shardedFunction, + void *context); extern char * GetShardedTableDDLCommand(TableDDLCommand *command, uint64 shardId, char *schemaName); diff --git a/src/test/regress/columnar_am_schedule b/src/test/regress/columnar_am_schedule index 39fcd78bb..b17b796e6 100644 --- a/src/test/regress/columnar_am_schedule +++ b/src/test/regress/columnar_am_schedule @@ -1,3 +1,7 @@ +test: multi_cluster_management +test: multi_test_helpers multi_test_helpers_superuser +test: multi_test_catalog_views + test: am_create test: am_load test: am_query @@ -20,3 +24,4 @@ test: am_recursive test: am_transactions test: am_matview test: am_memory +test: columnar_citus_integration diff --git a/src/test/regress/expected/columnar_citus_integration.out b/src/test/regress/expected/columnar_citus_integration.out new file mode 100644 index 000000000..0bccaa750 --- /dev/null +++ b/src/test/regress/expected/columnar_citus_integration.out @@ -0,0 +1,603 @@ +CREATE SCHEMA columnar_citus_integration; +SET search_path TO columnar_citus_integration; +SET citus.next_shard_id TO 20090000; +SET citus.shard_replication_factor TO 1; +SET citus.shard_count TO 4; +-- table options integration testing +-- distributed table 1 placement +CREATE TABLE table_option (a int, b text) USING columnar; +SELECT create_distributed_table('table_option', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- setting: compression +-- get baseline for setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090000,t,none) + (localhost,57638,20090001,t,none) + (localhost,57637,20090002,t,none) + (localhost,57638,20090003,t,none) +(4 rows) + +-- change setting +SELECT alter_columnar_table_set('table_option', compression => 'pglz'); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090000,t,pglz) + (localhost,57638,20090001,t,pglz) + (localhost,57637,20090002,t,pglz) + (localhost,57638,20090003,t,pglz) +(4 rows) + +-- reset setting +SELECT alter_columnar_table_reset('table_option', compression => true); + alter_columnar_table_reset +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090000,t,none) + (localhost,57638,20090001,t,none) + (localhost,57637,20090002,t,none) + (localhost,57638,20090003,t,none) +(4 rows) + +-- setting: block_row_count +-- get baseline for setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090000,t,10000) + (localhost,57638,20090001,t,10000) + (localhost,57637,20090002,t,10000) + (localhost,57638,20090003,t,10000) +(4 rows) + +-- change setting +SELECT alter_columnar_table_set('table_option', block_row_count => 100); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090000,t,100) + (localhost,57638,20090001,t,100) + (localhost,57637,20090002,t,100) + (localhost,57638,20090003,t,100) +(4 rows) + +-- reset setting +SELECT alter_columnar_table_reset('table_option', block_row_count => true); + alter_columnar_table_reset +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090000,t,10000) + (localhost,57638,20090001,t,10000) + (localhost,57637,20090002,t,10000) + (localhost,57638,20090003,t,10000) +(4 rows) + +-- setting: stripe_row_count +-- get baseline for setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT stripe_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090000,t,150000) + (localhost,57638,20090001,t,150000) + (localhost,57637,20090002,t,150000) + (localhost,57638,20090003,t,150000) +(4 rows) + +-- change setting +SELECT alter_columnar_table_set('table_option', stripe_row_count => 100); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT stripe_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090000,t,100) + (localhost,57638,20090001,t,100) + (localhost,57637,20090002,t,100) + (localhost,57638,20090003,t,100) +(4 rows) + +-- reset setting +SELECT alter_columnar_table_reset('table_option', stripe_row_count => true); + alter_columnar_table_reset +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT stripe_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090000,t,150000) + (localhost,57638,20090001,t,150000) + (localhost,57637,20090002,t,150000) + (localhost,57638,20090003,t,150000) +(4 rows) + +-- verify settings are propagated when creating a table +CREATE TABLE table_option_2 (a int, b text) USING columnar; +SELECT alter_columnar_table_set('table_option_2', + block_row_count => 100, + stripe_row_count => 1000, + compression => 'pglz'); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('table_option_2', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- verify settings on placements +SELECT run_command_on_placements('table_option_2',$cmd$ + SELECT ROW(block_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090004,t,"(100,1000,pglz)") + (localhost,57638,20090005,t,"(100,1000,pglz)") + (localhost,57637,20090006,t,"(100,1000,pglz)") + (localhost,57638,20090007,t,"(100,1000,pglz)") +(4 rows) + +DROP TABLE table_option, table_option_2; +-- verify settings get to all placements when there are multiple replica's +SET citus.shard_replication_factor TO 2; +-- table options integration testing +CREATE TABLE table_option (a int, b text) USING columnar; +SELECT create_distributed_table('table_option', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- setting: compression +-- get baseline for setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090008,t,none) + (localhost,57638,20090008,t,none) + (localhost,57637,20090009,t,none) + (localhost,57638,20090009,t,none) + (localhost,57637,20090010,t,none) + (localhost,57638,20090010,t,none) + (localhost,57637,20090011,t,none) + (localhost,57638,20090011,t,none) +(8 rows) + +-- change setting +SELECT alter_columnar_table_set('table_option', compression => 'pglz'); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090008,t,pglz) + (localhost,57638,20090008,t,pglz) + (localhost,57637,20090009,t,pglz) + (localhost,57638,20090009,t,pglz) + (localhost,57637,20090010,t,pglz) + (localhost,57638,20090010,t,pglz) + (localhost,57637,20090011,t,pglz) + (localhost,57638,20090011,t,pglz) +(8 rows) + +-- reset setting +SELECT alter_columnar_table_reset('table_option', compression => true); + alter_columnar_table_reset +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090008,t,none) + (localhost,57638,20090008,t,none) + (localhost,57637,20090009,t,none) + (localhost,57638,20090009,t,none) + (localhost,57637,20090010,t,none) + (localhost,57638,20090010,t,none) + (localhost,57637,20090011,t,none) + (localhost,57638,20090011,t,none) +(8 rows) + +-- setting: block_row_count +-- get baseline for setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090008,t,10000) + (localhost,57638,20090008,t,10000) + (localhost,57637,20090009,t,10000) + (localhost,57638,20090009,t,10000) + (localhost,57637,20090010,t,10000) + (localhost,57638,20090010,t,10000) + (localhost,57637,20090011,t,10000) + (localhost,57638,20090011,t,10000) +(8 rows) + +-- change setting +SELECT alter_columnar_table_set('table_option', block_row_count => 100); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090008,t,100) + (localhost,57638,20090008,t,100) + (localhost,57637,20090009,t,100) + (localhost,57638,20090009,t,100) + (localhost,57637,20090010,t,100) + (localhost,57638,20090010,t,100) + (localhost,57637,20090011,t,100) + (localhost,57638,20090011,t,100) +(8 rows) + +-- reset setting +SELECT alter_columnar_table_reset('table_option', block_row_count => true); + alter_columnar_table_reset +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090008,t,10000) + (localhost,57638,20090008,t,10000) + (localhost,57637,20090009,t,10000) + (localhost,57638,20090009,t,10000) + (localhost,57637,20090010,t,10000) + (localhost,57638,20090010,t,10000) + (localhost,57637,20090011,t,10000) + (localhost,57638,20090011,t,10000) +(8 rows) + +-- setting: stripe_row_count +-- get baseline for setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT stripe_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090008,t,150000) + (localhost,57638,20090008,t,150000) + (localhost,57637,20090009,t,150000) + (localhost,57638,20090009,t,150000) + (localhost,57637,20090010,t,150000) + (localhost,57638,20090010,t,150000) + (localhost,57637,20090011,t,150000) + (localhost,57638,20090011,t,150000) +(8 rows) + +-- change setting +SELECT alter_columnar_table_set('table_option', stripe_row_count => 100); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT stripe_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090008,t,100) + (localhost,57638,20090008,t,100) + (localhost,57637,20090009,t,100) + (localhost,57638,20090009,t,100) + (localhost,57637,20090010,t,100) + (localhost,57638,20090010,t,100) + (localhost,57637,20090011,t,100) + (localhost,57638,20090011,t,100) +(8 rows) + +-- reset setting +SELECT alter_columnar_table_reset('table_option', stripe_row_count => true); + alter_columnar_table_reset +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT stripe_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090008,t,150000) + (localhost,57638,20090008,t,150000) + (localhost,57637,20090009,t,150000) + (localhost,57638,20090009,t,150000) + (localhost,57637,20090010,t,150000) + (localhost,57638,20090010,t,150000) + (localhost,57637,20090011,t,150000) + (localhost,57638,20090011,t,150000) +(8 rows) + +-- verify settings are propagated when creating a table +CREATE TABLE table_option_2 (a int, b text) USING columnar; +SELECT alter_columnar_table_set('table_option_2', + block_row_count => 100, + stripe_row_count => 1000, + compression => 'pglz'); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('table_option_2', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- verify settings on placements +SELECT run_command_on_placements('table_option_2',$cmd$ + SELECT ROW(block_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090012,t,"(100,1000,pglz)") + (localhost,57638,20090012,t,"(100,1000,pglz)") + (localhost,57637,20090013,t,"(100,1000,pglz)") + (localhost,57638,20090013,t,"(100,1000,pglz)") + (localhost,57637,20090014,t,"(100,1000,pglz)") + (localhost,57638,20090014,t,"(100,1000,pglz)") + (localhost,57637,20090015,t,"(100,1000,pglz)") + (localhost,57638,20090015,t,"(100,1000,pglz)") +(8 rows) + +DROP TABLE table_option, table_option_2; +-- test options on a reference table +CREATE TABLE table_option_reference (a int, b text) USING columnar; +SELECT create_reference_table('table_option_reference'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +-- setting: compression +-- get baseline for setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090016,t,none) + (localhost,57638,20090016,t,none) +(2 rows) + +-- change setting +SELECT alter_columnar_table_set('table_option_reference', compression => 'pglz'); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090016,t,pglz) + (localhost,57638,20090016,t,pglz) +(2 rows) + +-- reset setting +SELECT alter_columnar_table_reset('table_option_reference', compression => true); + alter_columnar_table_reset +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090016,t,none) + (localhost,57638,20090016,t,none) +(2 rows) + +-- setting: block_row_count +-- get baseline for setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090016,t,10000) + (localhost,57638,20090016,t,10000) +(2 rows) + +-- change setting +SELECT alter_columnar_table_set('table_option_reference', block_row_count => 100); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090016,t,100) + (localhost,57638,20090016,t,100) +(2 rows) + +-- reset setting +SELECT alter_columnar_table_reset('table_option_reference', block_row_count => true); + alter_columnar_table_reset +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090016,t,10000) + (localhost,57638,20090016,t,10000) +(2 rows) + +-- setting: stripe_row_count +-- get baseline for setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT stripe_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090016,t,150000) + (localhost,57638,20090016,t,150000) +(2 rows) + +-- change setting +SELECT alter_columnar_table_set('table_option_reference', stripe_row_count => 100); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT stripe_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090016,t,100) + (localhost,57638,20090016,t,100) +(2 rows) + +-- reset setting +SELECT alter_columnar_table_reset('table_option_reference', stripe_row_count => true); + alter_columnar_table_reset +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT stripe_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090016,t,150000) + (localhost,57638,20090016,t,150000) +(2 rows) + +-- verify settings are propagated when creating a table +CREATE TABLE table_option_reference_2 (a int, b text) USING columnar; +SELECT alter_columnar_table_set('table_option_reference_2', + block_row_count => 100, + stripe_row_count => 1000, + compression => 'pglz'); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +SELECT create_reference_table('table_option_reference_2'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +-- verify settings on placements +SELECT run_command_on_placements('table_option_reference_2',$cmd$ + SELECT ROW(block_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090017,t,"(100,1000,pglz)") + (localhost,57638,20090017,t,"(100,1000,pglz)") +(2 rows) + +DROP TABLE table_option_reference, table_option_reference_2; +SET client_min_messages TO WARNING; +DROP SCHEMA columnar_citus_integration CASCADE; diff --git a/src/test/regress/sql/columnar_citus_integration.sql b/src/test/regress/sql/columnar_citus_integration.sql new file mode 100644 index 000000000..7ee5c2e14 --- /dev/null +++ b/src/test/regress/sql/columnar_citus_integration.sql @@ -0,0 +1,232 @@ +CREATE SCHEMA columnar_citus_integration; +SET search_path TO columnar_citus_integration; +SET citus.next_shard_id TO 20090000; + +SET citus.shard_replication_factor TO 1; +SET citus.shard_count TO 4; + +-- table options integration testing +-- distributed table 1 placement +CREATE TABLE table_option (a int, b text) USING columnar; +SELECT create_distributed_table('table_option', 'a'); + +-- setting: compression +-- get baseline for setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); +-- change setting +SELECT alter_columnar_table_set('table_option', compression => 'pglz'); +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); +-- reset setting +SELECT alter_columnar_table_reset('table_option', compression => true); +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + +-- setting: block_row_count +-- get baseline for setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); +-- change setting +SELECT alter_columnar_table_set('table_option', block_row_count => 100); +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); +-- reset setting +SELECT alter_columnar_table_reset('table_option', block_row_count => true); +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + +-- setting: stripe_row_count +-- get baseline for setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT stripe_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); +-- change setting +SELECT alter_columnar_table_set('table_option', stripe_row_count => 100); +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT stripe_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); +-- reset setting +SELECT alter_columnar_table_reset('table_option', stripe_row_count => true); +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT stripe_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + +-- verify settings are propagated when creating a table +CREATE TABLE table_option_2 (a int, b text) USING columnar; +SELECT alter_columnar_table_set('table_option_2', + block_row_count => 100, + stripe_row_count => 1000, + compression => 'pglz'); +SELECT create_distributed_table('table_option_2', 'a'); + +-- verify settings on placements +SELECT run_command_on_placements('table_option_2',$cmd$ + SELECT ROW(block_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + +DROP TABLE table_option, table_option_2; + +-- verify settings get to all placements when there are multiple replica's +SET citus.shard_replication_factor TO 2; + +-- table options integration testing +CREATE TABLE table_option (a int, b text) USING columnar; +SELECT create_distributed_table('table_option', 'a'); + +-- setting: compression +-- get baseline for setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); +-- change setting +SELECT alter_columnar_table_set('table_option', compression => 'pglz'); +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); +-- reset setting +SELECT alter_columnar_table_reset('table_option', compression => true); +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + +-- setting: block_row_count +-- get baseline for setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); +-- change setting +SELECT alter_columnar_table_set('table_option', block_row_count => 100); +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); +-- reset setting +SELECT alter_columnar_table_reset('table_option', block_row_count => true); +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + +-- setting: stripe_row_count +-- get baseline for setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT stripe_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); +-- change setting +SELECT alter_columnar_table_set('table_option', stripe_row_count => 100); +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT stripe_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); +-- reset setting +SELECT alter_columnar_table_reset('table_option', stripe_row_count => true); +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT stripe_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + +-- verify settings are propagated when creating a table +CREATE TABLE table_option_2 (a int, b text) USING columnar; +SELECT alter_columnar_table_set('table_option_2', + block_row_count => 100, + stripe_row_count => 1000, + compression => 'pglz'); +SELECT create_distributed_table('table_option_2', 'a'); + +-- verify settings on placements +SELECT run_command_on_placements('table_option_2',$cmd$ + SELECT ROW(block_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + +DROP TABLE table_option, table_option_2; + +-- test options on a reference table +CREATE TABLE table_option_reference (a int, b text) USING columnar; +SELECT create_reference_table('table_option_reference'); + +-- setting: compression +-- get baseline for setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); +-- change setting +SELECT alter_columnar_table_set('table_option_reference', compression => 'pglz'); +-- verify setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); +-- reset setting +SELECT alter_columnar_table_reset('table_option_reference', compression => true); +-- verify setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + +-- setting: block_row_count +-- get baseline for setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); +-- change setting +SELECT alter_columnar_table_set('table_option_reference', block_row_count => 100); +-- verify setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); +-- reset setting +SELECT alter_columnar_table_reset('table_option_reference', block_row_count => true); +-- verify setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + +-- setting: stripe_row_count +-- get baseline for setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT stripe_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); +-- change setting +SELECT alter_columnar_table_set('table_option_reference', stripe_row_count => 100); +-- verify setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT stripe_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); +-- reset setting +SELECT alter_columnar_table_reset('table_option_reference', stripe_row_count => true); +-- verify setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT stripe_row_count FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + +-- verify settings are propagated when creating a table +CREATE TABLE table_option_reference_2 (a int, b text) USING columnar; +SELECT alter_columnar_table_set('table_option_reference_2', + block_row_count => 100, + stripe_row_count => 1000, + compression => 'pglz'); +SELECT create_reference_table('table_option_reference_2'); + +-- verify settings on placements +SELECT run_command_on_placements('table_option_reference_2',$cmd$ + SELECT ROW(block_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass; +$cmd$); + +DROP TABLE table_option_reference, table_option_reference_2; + +SET client_min_messages TO WARNING; +DROP SCHEMA columnar_citus_integration CASCADE;