mirror of https://github.com/citusdata/citus.git
Fix multi-row & router INSERT crash with local exec. when def. cols not specified (#4197)
Multi-row & router INSERT's were crashing with local execution if at least one of the DEFAULT columns were not specified in VALUES list. This was because, the changes we make on query->values_lists and query->targetList was sufficient for deparsing given INSERT for remote execution but not sufficient for local execution. With this commit, DEFAULT value normalization for multi-row & router INSERT's is fixed by adding dummy column references for unspecified DEFAULT columns.pull/4214/head
Onur Tirtir
GitHub
parent
ba88ed3f0b
commit
2cd0a69dfb
|
|
@ -159,6 +159,8 @@ static Job * RouterJob(Query *originalQuery,
|
|||
DeferredErrorMessage **planningError);
|
||||
static bool RelationPrunesToMultipleShards(List *relationShardList);
|
||||
static void NormalizeMultiRowInsertTargetList(Query *query);
|
||||
static void AppendNextDummyColReference(Alias *expendedReferenceNames);
|
||||
static Value * MakeDummyColumnString(int dummyColumnId);
|
||||
static List * BuildRoutesForInsert(Query *query, DeferredErrorMessage **planningError);
|
||||
static List * GroupInsertValuesByShardId(List *insertValuesList);
|
||||
static List * ExtractInsertValuesList(Query *query, Var *partitionColumn);
|
||||
|
|
@ -3096,10 +3098,48 @@ NormalizeMultiRowInsertTargetList(Query *query)
|
|||
Var *syntheticVar = makeVar(valuesVarno, targetEntryNo, targetType, targetTypmod,
|
||||
targetColl, 0);
|
||||
targetEntry->expr = (Expr *) syntheticVar;
|
||||
|
||||
/*
|
||||
* Postgres appends a dummy column reference into valuesRTE->eref->colnames
|
||||
* list in addRangeTableEntryForValues for each column specified in VALUES
|
||||
* clause. Now that we replaced DEFAULT column with a synthetic Var, we also
|
||||
* need to add a dummy column reference for that column.
|
||||
*/
|
||||
AppendNextDummyColReference(valuesRTE->eref);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* AppendNextDummyColReference appends a new dummy column reference to colnames
|
||||
* list of given Alias object.
|
||||
*/
|
||||
static void
|
||||
AppendNextDummyColReference(Alias *expendedReferenceNames)
|
||||
{
|
||||
int existingColReferences = list_length(expendedReferenceNames->colnames);
|
||||
int nextColReferenceId = existingColReferences + 1;
|
||||
Value *missingColumnString = MakeDummyColumnString(nextColReferenceId);
|
||||
expendedReferenceNames->colnames = lappend(expendedReferenceNames->colnames,
|
||||
missingColumnString);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* MakeDummyColumnString returns a String (Value) object by appending given
|
||||
* integer to end of the "column" string.
|
||||
*/
|
||||
static Value *
|
||||
MakeDummyColumnString(int dummyColumnId)
|
||||
{
|
||||
StringInfo dummyColumnStringInfo = makeStringInfo();
|
||||
appendStringInfo(dummyColumnStringInfo, "column%d", dummyColumnId);
|
||||
Value *dummyColumnString = makeString(dummyColumnStringInfo->data);
|
||||
|
||||
return dummyColumnString;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* IntersectPlacementList performs placement pruning based on matching on
|
||||
* nodeName:nodePort fields of shard placement data. We start pruning from all
|
||||
|
|
|
|||
|
|
@ -0,0 +1,105 @@
|
|||
\set VERBOSITY terse
|
||||
SET citus.next_shard_id TO 1511000;
|
||||
SET citus.shard_replication_factor TO 1;
|
||||
SET citus.enable_local_execution TO ON;
|
||||
SET citus.log_local_commands TO ON;
|
||||
CREATE SCHEMA multi_row_router_insert;
|
||||
SET search_path TO multi_row_router_insert;
|
||||
SET client_min_messages to ERROR;
|
||||
SELECT 1 FROM master_add_node('localhost', :master_port, groupId => 0);
|
||||
?column?
|
||||
---------------------------------------------------------------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
RESET client_min_messages;
|
||||
-- when using local execution, multi-row & router inserts works fine
|
||||
-- even when not specifying some default columns
|
||||
CREATE TABLE reference_table(column1 INT DEFAULT 1111, column2 INT DEFAULT 2222);
|
||||
SELECT create_reference_table('reference_table');
|
||||
create_reference_table
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
INSERT INTO reference_table VALUES (5), (6);
|
||||
NOTICE: executing the command locally: INSERT INTO multi_row_router_insert.reference_table_1511000 AS citus_table_alias (column1, column2) VALUES (5,2222), (6,2222)
|
||||
-- note that first column is specified in below INSERT
|
||||
INSERT INTO reference_table VALUES (DEFAULT), (7);
|
||||
NOTICE: executing the command locally: INSERT INTO multi_row_router_insert.reference_table_1511000 AS citus_table_alias (column1, column2) VALUES (1111,2222), (7,2222)
|
||||
INSERT INTO reference_table (column2) VALUES (8), (9);
|
||||
NOTICE: executing the command locally: INSERT INTO multi_row_router_insert.reference_table_1511000 AS citus_table_alias (column1, column2) VALUES (1111,8), (1111,9)
|
||||
PREPARE prepared_statement(int) AS INSERT INTO reference_table (column2) VALUES ($1), ($1 * 500);
|
||||
EXECUTE prepared_statement(1);
|
||||
NOTICE: executing the command locally: INSERT INTO multi_row_router_insert.reference_table_1511000 AS citus_table_alias (column1, column2) VALUES (1111,1), (1111,500)
|
||||
EXECUTE prepared_statement(1);
|
||||
NOTICE: executing the command locally: INSERT INTO multi_row_router_insert.reference_table_1511000 AS citus_table_alias (column1, column2) VALUES (1111,1), (1111,500)
|
||||
EXECUTE prepared_statement(1);
|
||||
NOTICE: executing the command locally: INSERT INTO multi_row_router_insert.reference_table_1511000 AS citus_table_alias (column1, column2) VALUES (1111,1), (1111,500)
|
||||
EXECUTE prepared_statement(2);
|
||||
NOTICE: executing the command locally: INSERT INTO multi_row_router_insert.reference_table_1511000 AS citus_table_alias (column1, column2) VALUES (1111,2), (1111,1000)
|
||||
EXECUTE prepared_statement(3);
|
||||
NOTICE: executing the command locally: INSERT INTO multi_row_router_insert.reference_table_1511000 AS citus_table_alias (column1, column2) VALUES (1111,3), (1111,1500)
|
||||
EXECUTE prepared_statement(3);
|
||||
NOTICE: executing the command locally: INSERT INTO multi_row_router_insert.reference_table_1511000 AS citus_table_alias (column1, column2) VALUES (1111,3), (1111,1500)
|
||||
SELECT * FROM reference_table ORDER BY 1,2;
|
||||
NOTICE: executing the command locally: SELECT column1, column2 FROM multi_row_router_insert.reference_table_1511000 reference_table ORDER BY column1, column2
|
||||
column1 | column2
|
||||
---------------------------------------------------------------------
|
||||
5 | 2222
|
||||
6 | 2222
|
||||
7 | 2222
|
||||
1111 | 1
|
||||
1111 | 1
|
||||
1111 | 1
|
||||
1111 | 2
|
||||
1111 | 3
|
||||
1111 | 3
|
||||
1111 | 8
|
||||
1111 | 9
|
||||
1111 | 500
|
||||
1111 | 500
|
||||
1111 | 500
|
||||
1111 | 1000
|
||||
1111 | 1500
|
||||
1111 | 1500
|
||||
1111 | 2222
|
||||
(18 rows)
|
||||
|
||||
CREATE OR REPLACE FUNCTION square(a INT) RETURNS INT AS $$
|
||||
BEGIN
|
||||
RETURN a*a;
|
||||
END; $$ LANGUAGE PLPGSQL STABLE;
|
||||
CREATE TABLE citus_local_table(a int, b int DEFAULT square(10));
|
||||
SELECT create_citus_local_table('citus_local_table');
|
||||
create_citus_local_table
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
INSERT INTO citus_local_table VALUES (10), (11);
|
||||
NOTICE: executing the command locally: INSERT INTO multi_row_router_insert.citus_local_table_1511001 AS citus_table_alias (a, b) VALUES (10,100), (11,100)
|
||||
INSERT INTO citus_local_table (a) VALUES (12), (13);
|
||||
NOTICE: executing the command locally: INSERT INTO multi_row_router_insert.citus_local_table_1511001 AS citus_table_alias (a, b) VALUES (12,100), (13,100)
|
||||
ALTER TABLE citus_local_table ADD COLUMN c INT DEFAULT to_number('5', '91');
|
||||
NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1511001, 'multi_row_router_insert', 'ALTER TABLE citus_local_table ADD COLUMN c INT DEFAULT to_number(''5'', ''91'');')
|
||||
ALTER TABLE citus_local_table ADD COLUMN d INT;
|
||||
NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1511001, 'multi_row_router_insert', 'ALTER TABLE citus_local_table ADD COLUMN d INT;')
|
||||
INSERT INTO citus_local_table (d, a, b) VALUES (13, 14, 15), (16, 17, 18), (19, 20, 21);
|
||||
NOTICE: executing the command locally: INSERT INTO multi_row_router_insert.citus_local_table_1511001 AS citus_table_alias (a, b, c, d) VALUES (14,15,5,13), (17,18,5,16), (20,21,5,19)
|
||||
SELECT * FROM citus_local_table ORDER BY 1,2,3,4;
|
||||
NOTICE: executing the command locally: SELECT a, b, c, d FROM multi_row_router_insert.citus_local_table_1511001 citus_local_table ORDER BY a, b, c, d
|
||||
a | b | c | d
|
||||
---------------------------------------------------------------------
|
||||
10 | 100 | 5 |
|
||||
11 | 100 | 5 |
|
||||
12 | 100 | 5 |
|
||||
13 | 100 | 5 |
|
||||
14 | 15 | 5 | 13
|
||||
17 | 18 | 5 | 16
|
||||
20 | 21 | 5 | 19
|
||||
(7 rows)
|
||||
|
||||
-- cleanup at exit
|
||||
DROP SCHEMA multi_row_router_insert CASCADE;
|
||||
NOTICE: drop cascades to 5 other objects
|
||||
|
|
@ -306,6 +306,7 @@ test: replicate_reference_tables_to_coordinator
|
|||
test: coordinator_shouldhaveshards
|
||||
test: local_shard_utility_command_execution
|
||||
test: citus_local_tables
|
||||
test: multi_row_router_insert
|
||||
|
||||
test: remove_coordinator
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,55 @@
|
|||
\set VERBOSITY terse
|
||||
|
||||
SET citus.next_shard_id TO 1511000;
|
||||
SET citus.shard_replication_factor TO 1;
|
||||
SET citus.enable_local_execution TO ON;
|
||||
SET citus.log_local_commands TO ON;
|
||||
|
||||
CREATE SCHEMA multi_row_router_insert;
|
||||
SET search_path TO multi_row_router_insert;
|
||||
|
||||
SET client_min_messages to ERROR;
|
||||
SELECT 1 FROM master_add_node('localhost', :master_port, groupId => 0);
|
||||
RESET client_min_messages;
|
||||
|
||||
-- when using local execution, multi-row & router inserts works fine
|
||||
-- even when not specifying some default columns
|
||||
|
||||
CREATE TABLE reference_table(column1 INT DEFAULT 1111, column2 INT DEFAULT 2222);
|
||||
SELECT create_reference_table('reference_table');
|
||||
|
||||
INSERT INTO reference_table VALUES (5), (6);
|
||||
-- note that first column is specified in below INSERT
|
||||
INSERT INTO reference_table VALUES (DEFAULT), (7);
|
||||
INSERT INTO reference_table (column2) VALUES (8), (9);
|
||||
|
||||
PREPARE prepared_statement(int) AS INSERT INTO reference_table (column2) VALUES ($1), ($1 * 500);
|
||||
EXECUTE prepared_statement(1);
|
||||
EXECUTE prepared_statement(1);
|
||||
EXECUTE prepared_statement(1);
|
||||
EXECUTE prepared_statement(2);
|
||||
EXECUTE prepared_statement(3);
|
||||
EXECUTE prepared_statement(3);
|
||||
|
||||
SELECT * FROM reference_table ORDER BY 1,2;
|
||||
|
||||
CREATE OR REPLACE FUNCTION square(a INT) RETURNS INT AS $$
|
||||
BEGIN
|
||||
RETURN a*a;
|
||||
END; $$ LANGUAGE PLPGSQL STABLE;
|
||||
|
||||
CREATE TABLE citus_local_table(a int, b int DEFAULT square(10));
|
||||
SELECT create_citus_local_table('citus_local_table');
|
||||
|
||||
INSERT INTO citus_local_table VALUES (10), (11);
|
||||
INSERT INTO citus_local_table (a) VALUES (12), (13);
|
||||
|
||||
ALTER TABLE citus_local_table ADD COLUMN c INT DEFAULT to_number('5', '91');
|
||||
ALTER TABLE citus_local_table ADD COLUMN d INT;
|
||||
|
||||
INSERT INTO citus_local_table (d, a, b) VALUES (13, 14, 15), (16, 17, 18), (19, 20, 21);
|
||||
|
||||
SELECT * FROM citus_local_table ORDER BY 1,2,3,4;
|
||||
|
||||
-- cleanup at exit
|
||||
DROP SCHEMA multi_row_router_insert CASCADE;
|
||||
Loading…
Reference in New Issue