external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix matviews for citus_add_local_table_to_metadata (#6023)

pull/6033/head
Ahmet Gedemenli 2022-07-04 17:00:07 +03:00 committed by GitHub
parent f60809a6c1
commit c8e1e243b8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 331 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
src/backend/distributed/commands/alter_table.c
View File

@ -207,6 +207,7 @@ static char * CreateWorkerChangeSequenceDependencyCommand(char *sequenceSchemaNa
char *sourceName,
char *targetSchemaName,
char *targetName);
static void ErrorIfMatViewSizeExceedsTheLimit(Oid matViewOid);
static char * CreateMaterializedViewDDLCommand(Oid matViewOid);
static char * GetAccessMethodForMatViewIfExists(Oid viewOid);
static bool WillRecreateForeignKeyToReferenceTable(Oid relationId,
@ -223,6 +224,8 @@ PG_FUNCTION_INFO_V1(worker_change_sequence_dependency);
/* global variable keeping track of whether we are in a table type conversion function */
bool InTableTypeConversionFunctionCall = false;
/* controlled by GUC, in MB */
int MaxMatViewSizeToAutoRecreate = 1024;
/*
* undistribute_table gets a distributed table name and
@ -1357,6 +1360,7 @@ List *
GetViewCreationCommandsOfTable(Oid relationId)
{
List *views = GetDependingViews(relationId);
List *commands = NIL;
Oid viewOid = InvalidOid;
@ -1367,6 +1371,8 @@ GetViewCreationCommandsOfTable(Oid relationId)
/* See comments on CreateMaterializedViewDDLCommand for its limitations */
if (get_rel_relkind(viewOid) == RELKIND_MATVIEW)
{
ErrorIfMatViewSizeExceedsTheLimit(viewOid);
char *matViewCreateCommands = CreateMaterializedViewDDLCommand(viewOid);
appendStringInfoString(query, matViewCreateCommands);
}
@ -1406,6 +1412,42 @@ GetViewCreationTableDDLCommandsOfTable(Oid relationId)
}
/*
* ErrorIfMatViewSizeExceedsTheLimit takes the oid of a materialized view and errors
* out if the size of the matview exceeds the limit set by the GUC
* citus.max_matview_size_to_auto_recreate.
*/
static void
ErrorIfMatViewSizeExceedsTheLimit(Oid matViewOid)
{
if (MaxMatViewSizeToAutoRecreate >= 0)
{
/* if it's below 0, it means the user has removed the limit */
Datum relSizeDatum = DirectFunctionCall1(pg_total_relation_size,
ObjectIdGetDatum(matViewOid));
uint64 matViewSize = DatumGetInt64(relSizeDatum);
/* convert from MB to bytes */
uint64 limitSizeInBytes = MaxMatViewSizeToAutoRecreate * 1024L * 1024L;
if (matViewSize > limitSizeInBytes)
{
ereport(ERROR, (errmsg("size of the materialized view %s exceeds "
"citus.max_matview_size_to_auto_recreate "
"(currently %d MB)", get_rel_name(matViewOid),
MaxMatViewSizeToAutoRecreate),
errdetail("Citus restricts automatically recreating "
"materialized views that are larger than the "
"limit, because it could take too long."),
errhint(
"Consider increasing the size limit by setting "
"citus.max_matview_size_to_auto_recreate; "
"or you can remove the limit by setting it to -1")));
}
}
}
/*
* CreateMaterializedViewDDLCommand creates the command to create materialized view.
* Note that this function doesn't support
@ -1967,6 +2009,19 @@ ExecuteQueryViaSPI(char *query, int SPIOK)
}
/*
* ExecuteAndLogQueryViaSPI is a wrapper around ExecuteQueryViaSPI, that logs
* the query to be executed, with the given log level.
*/
void
ExecuteAndLogQueryViaSPI(char *query, int SPIOK, int logLevel)
{
ereport(logLevel, (errmsg("executing \"%s\"", query)));
ExecuteQueryViaSPI(query, SPIOK);
}
/*
* SwitchToSequentialAndLocalExecutionIfRelationNameTooLong generates the longest shard name
* on the shards of a distributed table, and if exceeds the limit switches to sequential and

19
src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c
View File

@ -26,6 +26,7 @@
#include "distributed/reference_table_utils.h"
#include "distributed/relation_access_tracking.h"
#include "distributed/worker_protocol.h"
#include "executor/spi.h"
#include "miscadmin.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
@ -513,12 +514,12 @@ ExecuteCascadeOperationForRelationIdList(List *relationIdList,
/*
* ExecuteAndLogUtilityCommandListInTableTypeConversion is a wrapper function
* around ExecuteAndLogUtilityCommandList, that makes it execute with the flag
* InTableTypeConversionFunctionCall is set to true.
* ExecuteAndLogUtilityCommandListInTableTypeConversionViaSPI is a wrapper function
* around ExecuteAndLogQueryViaSPI, that executes view creation commands
* with the flag InTableTypeConversionFunctionCall set to true.
*/
void
ExecuteAndLogUtilityCommandListInTableTypeConversion(List *utilityCommandList)
ExecuteAndLogUtilityCommandListInTableTypeConversionViaSPI(List *utilityCommandList)
{
bool oldValue = InTableTypeConversionFunctionCall;
InTableTypeConversionFunctionCall = true;
@ -526,7 +527,15 @@ ExecuteAndLogUtilityCommandListInTableTypeConversion(List *utilityCommandList)
MemoryContext savedMemoryContext = CurrentMemoryContext;
PG_TRY();
{
ExecuteAndLogUtilityCommandList(utilityCommandList);
char *utilityCommand = NULL;
foreach_ptr(utilityCommand, utilityCommandList)
{
/*
* CREATE MATERIALIZED VIEW commands need to be parsed/transformed,
* which SPI does for us.
*/
ExecuteAndLogQueryViaSPI(utilityCommand, SPI_OK_UTILITY, DEBUG1);
}
}
PG_CATCH();
{

6
src/backend/distributed/commands/citus_add_local_table_to_metadata.c
View File

@ -349,7 +349,7 @@ CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys, bool autoConve
* Execute the view creation commands with the shell table.
* Views will be distributed via FinalizeCitusLocalTableCreation below.
*/
ExecuteAndLogUtilityCommandListInTableTypeConversion(tableViewCreationCommands);
ExecuteAndLogUtilityCommandListInTableTypeConversionViaSPI(tableViewCreationCommands);
/*
* Set shellRelationId as the relation with relationId now points
@ -1053,7 +1053,9 @@ DropViewsOnTable(Oid relationId)
char *qualifiedViewName = quote_qualified_identifier(schemaName, viewName);
StringInfo dropCommand = makeStringInfo();
appendStringInfo(dropCommand, "DROP VIEW IF EXISTS %s",
appendStringInfo(dropCommand, "DROP %sVIEW IF EXISTS %s",
get_rel_relkind(viewId) == RELKIND_MATVIEW ? "MATERIALIZED " :
"",
qualifiedViewName);
ExecuteAndLogUtilityCommand(dropCommand->data);

11
src/backend/distributed/shared_library_init.c
View File

@ -1460,6 +1460,17 @@ RegisterCitusConfigVariables(void)
GUC_UNIT_KB | GUC_STANDARD,
NULL, NULL, NULL);
DefineCustomIntVariable(
"citus.max_matview_size_to_auto_recreate",
gettext_noop("Sets the maximum size of materialized views in MB to "
"automatically distribute them."),
NULL,
&MaxMatViewSizeToAutoRecreate,
1024, -1, INT_MAX,
PGC_USERSET,
GUC_UNIT_MB | GUC_STANDARD,
NULL, NULL, NULL);
DefineCustomIntVariable(
"citus.max_rebalancer_logged_ignored_moves",
gettext_noop("Sets the maximum number of ignored moves the rebalance logs"),

3
src/include/distributed/commands.h
View File

@ -29,6 +29,7 @@ extern bool EnableLocalReferenceForeignKeys;
extern bool EnableUnsafeTriggers;
extern int MaxMatViewSizeToAutoRecreate;
extern void SwitchToSequentialAndLocalExecutionIfRelationNameTooLong(Oid relationId,
char *
@ -642,7 +643,7 @@ extern bool RelationIdListHasReferenceTable(List *relationIdList);
extern List * GetFKeyCreationCommandsForRelationIdList(List *relationIdList);
extern void DropRelationForeignKeys(Oid relationId, int flags);
extern void SetLocalEnableLocalReferenceForeignKeys(bool state);
extern void ExecuteAndLogUtilityCommandListInTableTypeConversion(
extern void ExecuteAndLogUtilityCommandListInTableTypeConversionViaSPI(
List *utilityCommandList);
extern void ExecuteAndLogUtilityCommandList(List *ddlCommandList);
extern void ExecuteAndLogUtilityCommand(const char *commandString);

1
src/include/distributed/metadata_utility.h
View File

@ -305,6 +305,7 @@ extern bool GetNodeDiskSpaceStatsForConnection(MultiConnection *connection,
uint64 *availableBytes,
uint64 *totalBytes);
extern void ExecuteQueryViaSPI(char *query, int SPIOK);
extern void ExecuteAndLogQueryViaSPI(char *query, int SPIOK, int logLevel);
extern void EnsureSequenceTypeSupported(Oid seqOid, Oid attributeTypeId, Oid
ownerRelationId);
extern void AlterSequenceType(Oid seqOid, Oid typeOid);

168
src/test/regress/expected/citus_local_tables_mx.out
View File

@ -876,7 +876,29 @@ CREATE TABLE loc_tb (a int );
CREATE VIEW v100 AS SELECT * FROM loc_tb;
CREATE VIEW v101 AS SELECT * FROM loc_tb JOIN ref_tb USING (a);
CREATE VIEW v102 AS SELECT * FROM v101;
-- a regular matview that depends on local table
CREATE MATERIALIZED VIEW matview_101 AS SELECT * from loc_tb;
-- a matview and a view that depend on the local table + each other
CREATE VIEW v103 AS SELECT * from loc_tb;
CREATE MATERIALIZED VIEW matview_102 AS SELECT * from loc_tb JOIN v103 USING (a);
CREATE OR REPLACE VIEW v103 AS SELECT * from loc_tb JOIN matview_102 USING (a);
SET client_min_messages TO DEBUG1;
-- auto undistribute
ALTER TABLE loc_tb ADD CONSTRAINT fkey FOREIGN KEY (a) references ref_tb(a);
DEBUG: executing "CREATE OR REPLACE VIEW citus_local_tables_mx.v100 (a) AS SELECT loc_tb.a
FROM citus_local_tables_mx.loc_tb; ALTER VIEW citus_local_tables_mx.v100 OWNER TO postgres"
DEBUG: "view v100" has dependency to "table loc_tb" that is not in Citus' metadata
DEBUG: executing "CREATE OR REPLACE VIEW citus_local_tables_mx.v101 (a) AS SELECT loc_tb.a
FROM (citus_local_tables_mx.loc_tb
JOIN citus_local_tables_mx.ref_tb USING (a)); ALTER VIEW citus_local_tables_mx.v101 OWNER TO postgres"
DEBUG: "view v101" has dependency to "table loc_tb" that is not in Citus' metadata
DEBUG: executing "CREATE MATERIALIZED VIEW citus_local_tables_mx.matview_101 USING heap AS SELECT loc_tb.a
FROM citus_local_tables_mx.loc_tb;ALTER MATERIALIZED VIEW citus_local_tables_mx.matview_101 OWNER TO postgres"
DEBUG: executing "CREATE OR REPLACE VIEW citus_local_tables_mx.v102 (a) AS SELECT v101.a
FROM citus_local_tables_mx.v101; ALTER VIEW citus_local_tables_mx.v102 OWNER TO postgres"
DEBUG: "view v102" has dependency to "table loc_tb" that is not in Citus' metadata
DEBUG: validating foreign key constraint "fkey_xxxxxxx"
SET client_min_messages TO WARNING;
-- works fine
select run_command_on_workers($$SELECT count(*) from citus_local_tables_mx.v100, citus_local_tables_mx.v101, citus_local_tables_mx.v102$$);
run_command_on_workers
@ -908,6 +930,152 @@ select run_command_on_workers($$SELECT count(*) from citus_local_tables_mx.v100$
(localhost,57638,f,"ERROR: relation ""citus_local_tables_mx.v102"" does not exist")
(2 rows)
INSERT INTO loc_tb VALUES (1), (2);
-- test a matview with columnar
CREATE MATERIALIZED VIEW matview_columnar USING COLUMNAR AS SELECT * FROM loc_tb WITH DATA;
-- cant recreate matviews, because the size limit is set to zero, by the GUC
SET citus.max_matview_size_to_auto_recreate TO 0;
SELECT citus_add_local_table_to_metadata('loc_tb', true);
ERROR: size of the materialized view matview_columnar exceeds citus.max_matview_size_to_auto_recreate (currently 0 MB)
-- remove the limit
SET citus.max_matview_size_to_auto_recreate TO -1;
SELECT citus_add_local_table_to_metadata('loc_tb', true);
citus_add_local_table_to_metadata
---------------------------------------------------------------------
(1 row)
-- test REFRESH MAT VIEW
SELECT * FROM matview_101 ORDER BY a;
a
---------------------------------------------------------------------
(0 rows)
REFRESH MATERIALIZED VIEW matview_101;
SELECT * FROM matview_101 ORDER BY a;
a
---------------------------------------------------------------------
1
2
(2 rows)
-- verify columnar matview works on a table added to metadata
SELECT * FROM matview_columnar;
a
---------------------------------------------------------------------
(0 rows)
REFRESH MATERIALIZED VIEW matview_columnar;
SELECT * FROM matview_columnar ORDER BY a;
a
---------------------------------------------------------------------
1
2
(2 rows)
-- test with partitioned tables
SET citus.use_citus_managed_tables TO ON;
CREATE TABLE parent_1 (a INT UNIQUE) PARTITION BY RANGE(a);
SET citus.use_citus_managed_tables TO OFF;
CREATE MATERIALIZED VIEW part_matview1 as SELECT count(*) FROM parent_1 JOIN parent_1 p2 ON (true);
CREATE MATERIALIZED VIEW part_matview2 as SELECT count(*) FROM parent_1 JOIN part_matview1 on (true);
SELECT count(*) FROM citus_local_tables_mx.part_matview1 JOIN citus_local_tables_mx.part_matview2 ON (true);
count
---------------------------------------------------------------------
1
(1 row)
CREATE TABLE parent_1_child_1 (a int);
CREATE TABLE parent_1_child_2 (a int);
-- create matviews on partition tables
CREATE MATERIALIZED VIEW mv1 AS SELECT * FROM parent_1_child_1;
CREATE MATERIALIZED VIEW mv2 AS SELECT * FROM parent_1_child_2;
CREATE MATERIALIZED VIEW mv3 AS SELECT parent_1_child_2.* FROM parent_1_child_2 JOIN parent_1_child_1 USING(a);
CREATE MATERIALIZED VIEW mv4 AS SELECT * FROM mv3;
alter table parent_1 attach partition parent_1_child_1 FOR VALUES FROM (0) TO (10) ;
-- all matviews work
SELECT count(*) FROM citus_local_tables_mx.mv1;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM citus_local_tables_mx.mv2;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM citus_local_tables_mx.mv3;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM citus_local_tables_mx.mv4;
count
---------------------------------------------------------------------
0
(1 row)
-- recreate matviews and verify they still work
alter table parent_1 attach partition parent_1_child_2 FOR VALUES FROM (10) TO (20);
SELECT count(*) FROM citus_local_tables_mx.mv1;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM citus_local_tables_mx.mv2;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM citus_local_tables_mx.mv3;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM citus_local_tables_mx.mv4;
count
---------------------------------------------------------------------
0
(1 row)
-- verify matviews work after undistributing
SELECT undistribute_table('parent_1');
undistribute_table
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM citus_local_tables_mx.mv1;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM citus_local_tables_mx.mv2;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM citus_local_tables_mx.mv3;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM citus_local_tables_mx.mv4;
count
---------------------------------------------------------------------
0
(1 row)
-- todo: add more matview tests once 5968 and 6028 are fixed
-- cleanup at exit
set client_min_messages to error;
DROP SCHEMA citus_local_tables_mx CASCADE;

75
src/test/regress/sql/citus_local_tables_mx.sql
View File

@ -453,7 +453,17 @@ CREATE VIEW v100 AS SELECT * FROM loc_tb;
CREATE VIEW v101 AS SELECT * FROM loc_tb JOIN ref_tb USING (a);
CREATE VIEW v102 AS SELECT * FROM v101;
-- a regular matview that depends on local table
CREATE MATERIALIZED VIEW matview_101 AS SELECT * from loc_tb;
-- a matview and a view that depend on the local table + each other
CREATE VIEW v103 AS SELECT * from loc_tb;
CREATE MATERIALIZED VIEW matview_102 AS SELECT * from loc_tb JOIN v103 USING (a);
CREATE OR REPLACE VIEW v103 AS SELECT * from loc_tb JOIN matview_102 USING (a);
SET client_min_messages TO DEBUG1;
-- auto undistribute
ALTER TABLE loc_tb ADD CONSTRAINT fkey FOREIGN KEY (a) references ref_tb(a);
SET client_min_messages TO WARNING;
-- works fine
select run_command_on_workers($$SELECT count(*) from citus_local_tables_mx.v100, citus_local_tables_mx.v101, citus_local_tables_mx.v102$$);
@ -464,6 +474,71 @@ select run_command_on_workers($$SELECT count(*) from citus_local_tables_mx.v100$
select run_command_on_workers($$SELECT count(*) from citus_local_tables_mx.v101$$);
select run_command_on_workers($$SELECT count(*) from citus_local_tables_mx.v102$$);
INSERT INTO loc_tb VALUES (1), (2);
-- test a matview with columnar
CREATE MATERIALIZED VIEW matview_columnar USING COLUMNAR AS SELECT * FROM loc_tb WITH DATA;
-- cant recreate matviews, because the size limit is set to zero, by the GUC
SET citus.max_matview_size_to_auto_recreate TO 0;
SELECT citus_add_local_table_to_metadata('loc_tb', true);
-- remove the limit
SET citus.max_matview_size_to_auto_recreate TO -1;
SELECT citus_add_local_table_to_metadata('loc_tb', true);
-- test REFRESH MAT VIEW
SELECT * FROM matview_101 ORDER BY a;
REFRESH MATERIALIZED VIEW matview_101;
SELECT * FROM matview_101 ORDER BY a;
-- verify columnar matview works on a table added to metadata
SELECT * FROM matview_columnar;
REFRESH MATERIALIZED VIEW matview_columnar;
SELECT * FROM matview_columnar ORDER BY a;
-- test with partitioned tables
SET citus.use_citus_managed_tables TO ON;
CREATE TABLE parent_1 (a INT UNIQUE) PARTITION BY RANGE(a);
SET citus.use_citus_managed_tables TO OFF;
CREATE MATERIALIZED VIEW part_matview1 as SELECT count(*) FROM parent_1 JOIN parent_1 p2 ON (true);
CREATE MATERIALIZED VIEW part_matview2 as SELECT count(*) FROM parent_1 JOIN part_matview1 on (true);
SELECT count(*) FROM citus_local_tables_mx.part_matview1 JOIN citus_local_tables_mx.part_matview2 ON (true);
CREATE TABLE parent_1_child_1 (a int);
CREATE TABLE parent_1_child_2 (a int);
-- create matviews on partition tables
CREATE MATERIALIZED VIEW mv1 AS SELECT * FROM parent_1_child_1;
CREATE MATERIALIZED VIEW mv2 AS SELECT * FROM parent_1_child_2;
CREATE MATERIALIZED VIEW mv3 AS SELECT parent_1_child_2.* FROM parent_1_child_2 JOIN parent_1_child_1 USING(a);
CREATE MATERIALIZED VIEW mv4 AS SELECT * FROM mv3;
alter table parent_1 attach partition parent_1_child_1 FOR VALUES FROM (0) TO (10) ;
-- all matviews work
SELECT count(*) FROM citus_local_tables_mx.mv1;
SELECT count(*) FROM citus_local_tables_mx.mv2;
SELECT count(*) FROM citus_local_tables_mx.mv3;
SELECT count(*) FROM citus_local_tables_mx.mv4;
-- recreate matviews and verify they still work
alter table parent_1 attach partition parent_1_child_2 FOR VALUES FROM (10) TO (20);
SELECT count(*) FROM citus_local_tables_mx.mv1;
SELECT count(*) FROM citus_local_tables_mx.mv2;
SELECT count(*) FROM citus_local_tables_mx.mv3;
SELECT count(*) FROM citus_local_tables_mx.mv4;
-- verify matviews work after undistributing
SELECT undistribute_table('parent_1');
SELECT count(*) FROM citus_local_tables_mx.mv1;
SELECT count(*) FROM citus_local_tables_mx.mv2;
SELECT count(*) FROM citus_local_tables_mx.mv3;
SELECT count(*) FROM citus_local_tables_mx.mv4;
-- todo: add more matview tests once 5968 and 6028 are fixed
-- cleanup at exit
set client_min_messages to error;
DROP SCHEMA citus_local_tables_mx CASCADE;