mirror of https://github.com/citusdata/citus.git
Fix crash with single node dummy placement (#3993)
Static analysis found an issue where we could dereference `NULL`, because `CreateDummyPlacement` could return `NULL` when there were no workers. This PR changes it so that it never returns `NULL`, which was intended by @marcocitus when doing this change: https://github.com/citusdata/citus/pull/3887/files#r438136433 While adding tests for citus on a single node I also added some more basic tests and it turns out we error out on repartition joins. This has been present since `shouldhaveshards` was introduced and is not trivial to fix. So I created a separate issue for this: https://github.com/citusdata/citus/issues/3996pull/3973/head^2
Jelte Fennema
GitHub
parent
f6e2f1b1cb
commit
ab01571c9e
|
|
@ -156,6 +156,7 @@ static DeferredErrorMessage * MultiRouterPlannableQuery(Query *query);
|
|||
static DeferredErrorMessage * ErrorIfQueryHasUnroutableModifyingCTE(Query *queryTree);
|
||||
static bool SelectsFromDistributedTable(List *rangeTableList, Query *query);
|
||||
static ShardPlacement * CreateDummyPlacement(bool hasLocalRelation);
|
||||
static ShardPlacement * CreateLocalDummyPlacement();
|
||||
static List * get_all_actual_clauses(List *restrictinfo_list);
|
||||
static int CompareInsertValuesByShardId(const void *leftElement,
|
||||
const void *rightElement);
|
||||
|
|
@ -2233,6 +2234,25 @@ CreateTaskPlacementListForShardIntervals(List *shardIntervalListList, bool shard
|
|||
}
|
||||
|
||||
|
||||
/*
|
||||
* CreateLocalDummyPlacement creates a dummy placement for the local node that
|
||||
* can be used for queries that don't involve any shards. The typical examples
|
||||
* are:
|
||||
* (a) queries that consist of only intermediate results
|
||||
* (b) queries that hit zero shards (... WHERE false;)
|
||||
*/
|
||||
static ShardPlacement *
|
||||
CreateLocalDummyPlacement()
|
||||
{
|
||||
ShardPlacement *dummyPlacement = CitusMakeNode(ShardPlacement);
|
||||
dummyPlacement->nodeId = LOCAL_NODE_ID;
|
||||
dummyPlacement->nodeName = LOCAL_HOST_NAME;
|
||||
dummyPlacement->nodePort = PostPortNumber;
|
||||
dummyPlacement->groupId = GetLocalGroupId();
|
||||
return dummyPlacement;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* CreateDummyPlacement creates a dummy placement that can be used for queries
|
||||
* that don't involve any shards. The typical examples are:
|
||||
|
|
@ -2249,31 +2269,32 @@ static ShardPlacement *
|
|||
CreateDummyPlacement(bool hasLocalRelation)
|
||||
{
|
||||
static uint32 zeroShardQueryRoundRobin = 0;
|
||||
|
||||
if (TaskAssignmentPolicy != TASK_ASSIGNMENT_ROUND_ROBIN || hasLocalRelation)
|
||||
{
|
||||
return CreateLocalDummyPlacement();
|
||||
}
|
||||
|
||||
List *workerNodeList = ActiveReadableWorkerNodeList();
|
||||
if (workerNodeList == NIL)
|
||||
{
|
||||
/*
|
||||
* We want to round-robin over the workers, but there are no workers.
|
||||
* To make sure the query can still succeed we fall back to returning
|
||||
* a local dummy placement.
|
||||
*/
|
||||
return CreateLocalDummyPlacement();
|
||||
}
|
||||
|
||||
int workerNodeCount = list_length(workerNodeList);
|
||||
int workerNodeIndex = zeroShardQueryRoundRobin % workerNodeCount;
|
||||
WorkerNode *workerNode = (WorkerNode *) list_nth(workerNodeList,
|
||||
workerNodeIndex);
|
||||
|
||||
ShardPlacement *dummyPlacement = CitusMakeNode(ShardPlacement);
|
||||
SetPlacementNodeMetadata(dummyPlacement, workerNode);
|
||||
|
||||
if (TaskAssignmentPolicy == TASK_ASSIGNMENT_ROUND_ROBIN && !hasLocalRelation)
|
||||
{
|
||||
List *workerNodeList = ActiveReadableWorkerNodeList();
|
||||
if (workerNodeList == NIL)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int workerNodeCount = list_length(workerNodeList);
|
||||
int workerNodeIndex = zeroShardQueryRoundRobin % workerNodeCount;
|
||||
WorkerNode *workerNode = (WorkerNode *) list_nth(workerNodeList,
|
||||
workerNodeIndex);
|
||||
SetPlacementNodeMetadata(dummyPlacement, workerNode);
|
||||
|
||||
zeroShardQueryRoundRobin++;
|
||||
}
|
||||
else
|
||||
{
|
||||
dummyPlacement->nodeId = LOCAL_NODE_ID;
|
||||
dummyPlacement->nodeName = LOCAL_HOST_NAME;
|
||||
dummyPlacement->nodePort = PostPortNumber;
|
||||
dummyPlacement->groupId = GetLocalGroupId();
|
||||
}
|
||||
zeroShardQueryRoundRobin++;
|
||||
|
||||
return dummyPlacement;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -198,6 +198,13 @@ NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinato
|
|||
(1 row)
|
||||
|
||||
ROLLBACK;
|
||||
-- Commented out since it currently does not clean up task files on the
|
||||
-- coordinator
|
||||
-- See this issue for details: https://github.com/citusdata/citus/issues/3996
|
||||
-- BEGIN;
|
||||
-- SET citus.enable_repartition_joins TO ON;
|
||||
-- SELECT count(*) FROM test t1, test t2 WHERE t1.x = t2.y;
|
||||
-- ROLLBACK;
|
||||
BEGIN;
|
||||
SET citus.enable_repartition_joins TO ON;
|
||||
-- trigger local execution
|
||||
|
|
|
|||
|
|
@ -0,0 +1,282 @@
|
|||
\c - - - :master_port
|
||||
CREATE SCHEMA single_node;
|
||||
SET search_path TO single_node;
|
||||
SET citus.shard_count TO 4;
|
||||
SET citus.shard_replication_factor TO 1;
|
||||
SET citus.next_shard_id TO 93630500;
|
||||
SELECT 1 FROM master_add_node('localhost', :master_port, groupid => 0);
|
||||
?column?
|
||||
---------------------------------------------------------------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT 1 FROM master_set_node_property('localhost', :master_port, 'shouldhaveshards', true);
|
||||
?column?
|
||||
---------------------------------------------------------------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
CREATE TABLE test(x int, y int);
|
||||
SELECT create_distributed_table('test','x');
|
||||
create_distributed_table
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
CREATE TABLE ref(a int, b int);
|
||||
SELECT create_reference_table('ref');
|
||||
create_reference_table
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
CREATE TABLE local(c int, d int);
|
||||
INSERT INTO test VALUES (1, 2), (3, 4), (5, 6), (2, 7), (4, 5);
|
||||
INSERT INTO ref VALUES (1, 2), (5, 6), (7, 8);
|
||||
INSERT INTO local VALUES (1, 2), (3, 4), (7, 8);
|
||||
-- connect to the follower and check that a simple select query works, the follower
|
||||
-- is still in the default cluster and will send queries to the primary nodes
|
||||
\c - - - :follower_master_port
|
||||
SET search_path TO single_node;
|
||||
SELECT * FROM test WHERE x = 1;
|
||||
x | y
|
||||
---------------------------------------------------------------------
|
||||
1 | 2
|
||||
(1 row)
|
||||
|
||||
SELECT count(*) FROM test;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
5
|
||||
(1 row)
|
||||
|
||||
SELECT * FROM test ORDER BY x;
|
||||
x | y
|
||||
---------------------------------------------------------------------
|
||||
1 | 2
|
||||
2 | 7
|
||||
3 | 4
|
||||
4 | 5
|
||||
5 | 6
|
||||
(5 rows)
|
||||
|
||||
SELECT count(*) FROM ref;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
3
|
||||
(1 row)
|
||||
|
||||
SELECT * FROM ref ORDER BY a;
|
||||
a | b
|
||||
---------------------------------------------------------------------
|
||||
1 | 2
|
||||
5 | 6
|
||||
7 | 8
|
||||
(3 rows)
|
||||
|
||||
SELECT * FROM test, ref WHERE x = a ORDER BY x;
|
||||
x | y | a | b
|
||||
---------------------------------------------------------------------
|
||||
1 | 2 | 1 | 2
|
||||
5 | 6 | 5 | 6
|
||||
(2 rows)
|
||||
|
||||
SELECT count(*) FROM local;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
3
|
||||
(1 row)
|
||||
|
||||
SELECT * FROM local ORDER BY c;
|
||||
c | d
|
||||
---------------------------------------------------------------------
|
||||
1 | 2
|
||||
3 | 4
|
||||
7 | 8
|
||||
(3 rows)
|
||||
|
||||
SELECT * FROM ref, local WHERE a = c ORDER BY a;
|
||||
a | b | c | d
|
||||
---------------------------------------------------------------------
|
||||
1 | 2 | 1 | 2
|
||||
7 | 8 | 7 | 8
|
||||
(2 rows)
|
||||
|
||||
-- Check repartion joins are support
|
||||
SELECT * FROM test t1, test t2 WHERE t1.x = t2.y ORDER BY t1.x;
|
||||
ERROR: floating-point exception
|
||||
DETAIL: An invalid floating-point operation was signaled. This probably means an out-of-range result or an invalid operation, such as division by zero.
|
||||
SET citus.enable_repartition_joins TO ON;
|
||||
SELECT * FROM test t1, test t2 WHERE t1.x = t2.y ORDER BY t1.x;
|
||||
ERROR: floating-point exception
|
||||
DETAIL: An invalid floating-point operation was signaled. This probably means an out-of-range result or an invalid operation, such as division by zero.
|
||||
RESET citus.enable_repartition_joins;
|
||||
-- Confirm that dummy placements work
|
||||
SELECT count(*) FROM test WHERE false;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT count(*) FROM test WHERE false GROUP BY GROUPING SETS (x,y);
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
(0 rows)
|
||||
|
||||
-- Confirm that they work with round-robin task assignment policy
|
||||
SET citus.task_assignment_policy TO 'round-robin';
|
||||
SELECT count(*) FROM test WHERE false;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT count(*) FROM test WHERE false GROUP BY GROUPING SETS (x,y);
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
(0 rows)
|
||||
|
||||
RESET citus.task_assignment_policy;
|
||||
-- now, connect to the follower but tell it to use secondary nodes. There are no
|
||||
-- secondary nodes so this should fail.
|
||||
-- (this is :follower_master_port but substitution doesn't work here)
|
||||
\c "port=9070 dbname=regression options='-c\ citus.use_secondary_nodes=always'"
|
||||
SET search_path TO single_node;
|
||||
SELECT * FROM test WHERE x = 1;
|
||||
ERROR: node group 0 does not have a secondary node
|
||||
-- add the the follower as secondary nodes and try again, the SELECT statement
|
||||
-- should work this time
|
||||
\c - - - :master_port
|
||||
SET search_path TO single_node;
|
||||
SELECT 1 FROM master_add_node('localhost', :follower_master_port, groupid => 0, noderole => 'secondary');
|
||||
?column?
|
||||
---------------------------------------------------------------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT 1 FROM master_set_node_property('localhost', :master_port, 'shouldhaveshards', true);
|
||||
?column?
|
||||
---------------------------------------------------------------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
\c "port=9070 dbname=regression options='-c\ citus.use_secondary_nodes=always'"
|
||||
SET search_path TO single_node;
|
||||
SELECT * FROM test WHERE x = 1;
|
||||
x | y
|
||||
---------------------------------------------------------------------
|
||||
1 | 2
|
||||
(1 row)
|
||||
|
||||
SELECT count(*) FROM test;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
5
|
||||
(1 row)
|
||||
|
||||
SELECT * FROM test ORDER BY x;
|
||||
x | y
|
||||
---------------------------------------------------------------------
|
||||
1 | 2
|
||||
2 | 7
|
||||
3 | 4
|
||||
4 | 5
|
||||
5 | 6
|
||||
(5 rows)
|
||||
|
||||
SELECT count(*) FROM ref;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
3
|
||||
(1 row)
|
||||
|
||||
SELECT * FROM ref ORDER BY a;
|
||||
a | b
|
||||
---------------------------------------------------------------------
|
||||
1 | 2
|
||||
5 | 6
|
||||
7 | 8
|
||||
(3 rows)
|
||||
|
||||
SELECT * FROM test, ref WHERE x = a ORDER BY x;
|
||||
x | y | a | b
|
||||
---------------------------------------------------------------------
|
||||
1 | 2 | 1 | 2
|
||||
5 | 6 | 5 | 6
|
||||
(2 rows)
|
||||
|
||||
SELECT count(*) FROM local;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
3
|
||||
(1 row)
|
||||
|
||||
SELECT * FROM local ORDER BY c;
|
||||
c | d
|
||||
---------------------------------------------------------------------
|
||||
1 | 2
|
||||
3 | 4
|
||||
7 | 8
|
||||
(3 rows)
|
||||
|
||||
SELECT * FROM ref, local WHERE a = c ORDER BY a;
|
||||
a | b | c | d
|
||||
---------------------------------------------------------------------
|
||||
1 | 2 | 1 | 2
|
||||
7 | 8 | 7 | 8
|
||||
(2 rows)
|
||||
|
||||
-- Check repartion joins are support
|
||||
SELECT * FROM test t1, test t2 WHERE t1.x = t2.y ORDER BY t1.x;
|
||||
ERROR: floating-point exception
|
||||
DETAIL: An invalid floating-point operation was signaled. This probably means an out-of-range result or an invalid operation, such as division by zero.
|
||||
SET citus.enable_repartition_joins TO ON;
|
||||
SELECT * FROM test t1, test t2 WHERE t1.x = t2.y ORDER BY t1.x;
|
||||
ERROR: floating-point exception
|
||||
DETAIL: An invalid floating-point operation was signaled. This probably means an out-of-range result or an invalid operation, such as division by zero.
|
||||
RESET citus.enable_repartition_joins;
|
||||
-- Confirm that dummy placements work
|
||||
SELECT count(*) FROM test WHERE false;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT count(*) FROM test WHERE false GROUP BY GROUPING SETS (x,y);
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
(0 rows)
|
||||
|
||||
-- Confirm that they work with round-robin task assignment policy
|
||||
SET citus.task_assignment_policy TO 'round-robin';
|
||||
SELECT count(*) FROM test WHERE false;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT count(*) FROM test WHERE false GROUP BY GROUPING SETS (x,y);
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
(0 rows)
|
||||
|
||||
RESET citus.task_assignment_policy;
|
||||
-- Cleanup
|
||||
\c - - - :master_port
|
||||
SET search_path TO single_node;
|
||||
SET client_min_messages TO WARNING;
|
||||
DROP SCHEMA single_node CASCADE;
|
||||
-- Remove the coordinator again
|
||||
SELECT 1 FROM master_remove_node('localhost', :master_port);
|
||||
?column?
|
||||
---------------------------------------------------------------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
-- Remove the secondary coordinator again
|
||||
SELECT 1 FROM master_remove_node('localhost', :follower_master_port);
|
||||
?column?
|
||||
---------------------------------------------------------------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
|
|
@ -0,0 +1,241 @@
|
|||
CREATE SCHEMA single_node;
|
||||
SET search_path TO single_node;
|
||||
SET citus.shard_count TO 4;
|
||||
SET citus.shard_replication_factor TO 1;
|
||||
SET citus.next_shard_id TO 90630500;
|
||||
-- idempotently add node to allow this test to run without add_coordinator
|
||||
SET client_min_messages TO WARNING;
|
||||
SELECT 1 FROM master_add_node('localhost', :master_port, groupid => 0);
|
||||
?column?
|
||||
---------------------------------------------------------------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
RESET client_min_messages;
|
||||
SELECT 1 FROM master_set_node_property('localhost', :master_port, 'shouldhaveshards', true);
|
||||
?column?
|
||||
---------------------------------------------------------------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
CREATE TABLE test(x int, y int);
|
||||
SELECT create_distributed_table('test','x');
|
||||
create_distributed_table
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
CREATE TABLE ref(a int, b int);
|
||||
SELECT create_reference_table('ref');
|
||||
create_reference_table
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
CREATE TABLE local(c int, d int);
|
||||
-- Confirm the basics work
|
||||
INSERT INTO test VALUES (1, 2), (3, 4), (5, 6), (2, 7), (4, 5);
|
||||
SELECT * FROM test WHERE x = 1;
|
||||
x | y
|
||||
---------------------------------------------------------------------
|
||||
1 | 2
|
||||
(1 row)
|
||||
|
||||
SELECT count(*) FROM test;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
5
|
||||
(1 row)
|
||||
|
||||
SELECT * FROM test ORDER BY x;
|
||||
x | y
|
||||
---------------------------------------------------------------------
|
||||
1 | 2
|
||||
2 | 7
|
||||
3 | 4
|
||||
4 | 5
|
||||
5 | 6
|
||||
(5 rows)
|
||||
|
||||
INSERT INTO ref VALUES (1, 2), (5, 6), (7, 8);
|
||||
SELECT count(*) FROM ref;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
3
|
||||
(1 row)
|
||||
|
||||
SELECT * FROM ref ORDER BY a;
|
||||
a | b
|
||||
---------------------------------------------------------------------
|
||||
1 | 2
|
||||
5 | 6
|
||||
7 | 8
|
||||
(3 rows)
|
||||
|
||||
SELECT * FROM test, ref WHERE x = a ORDER BY x;
|
||||
x | y | a | b
|
||||
---------------------------------------------------------------------
|
||||
1 | 2 | 1 | 2
|
||||
5 | 6 | 5 | 6
|
||||
(2 rows)
|
||||
|
||||
INSERT INTO local VALUES (1, 2), (3, 4), (7, 8);
|
||||
SELECT count(*) FROM local;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
3
|
||||
(1 row)
|
||||
|
||||
SELECT * FROM local ORDER BY c;
|
||||
c | d
|
||||
---------------------------------------------------------------------
|
||||
1 | 2
|
||||
3 | 4
|
||||
7 | 8
|
||||
(3 rows)
|
||||
|
||||
SELECT * FROM ref, local WHERE a = c ORDER BY a;
|
||||
a | b | c | d
|
||||
---------------------------------------------------------------------
|
||||
1 | 2 | 1 | 2
|
||||
7 | 8 | 7 | 8
|
||||
(2 rows)
|
||||
|
||||
-- Check repartion joins are support
|
||||
SELECT * FROM test t1, test t2 WHERE t1.x = t2.y ORDER BY t1.x;
|
||||
ERROR: floating-point exception
|
||||
DETAIL: An invalid floating-point operation was signaled. This probably means an out-of-range result or an invalid operation, such as division by zero.
|
||||
SET citus.enable_repartition_joins TO ON;
|
||||
SELECT * FROM test t1, test t2 WHERE t1.x = t2.y ORDER BY t1.x;
|
||||
ERROR: floating-point exception
|
||||
DETAIL: An invalid floating-point operation was signaled. This probably means an out-of-range result or an invalid operation, such as division by zero.
|
||||
RESET citus.enable_repartition_joins;
|
||||
-- INSERT SELECT router
|
||||
BEGIN;
|
||||
INSERT INTO test(x, y) SELECT x, y FROM test WHERE x = 1;
|
||||
SELECT count(*) from test;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
6
|
||||
(1 row)
|
||||
|
||||
ROLLBACK;
|
||||
-- INSERT SELECT analytical query
|
||||
BEGIN;
|
||||
INSERT INTO test(x, y) SELECT count(x), max(y) FROM test;
|
||||
SELECT count(*) from test;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
6
|
||||
(1 row)
|
||||
|
||||
ROLLBACK;
|
||||
-- INSERT SELECT repartition
|
||||
BEGIN;
|
||||
INSERT INTO test(x, y) SELECT y, x FROM test;
|
||||
SELECT count(*) from test;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
10
|
||||
(1 row)
|
||||
|
||||
ROLLBACK;
|
||||
-- INSERT SELECT from reference table into distributed
|
||||
BEGIN;
|
||||
INSERT INTO test(x, y) SELECT a, b FROM ref;
|
||||
SELECT count(*) from test;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
8
|
||||
(1 row)
|
||||
|
||||
ROLLBACK;
|
||||
-- INSERT SELECT from local table into distributed
|
||||
BEGIN;
|
||||
INSERT INTO test(x, y) SELECT c, d FROM local;
|
||||
SELECT count(*) from test;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
8
|
||||
(1 row)
|
||||
|
||||
ROLLBACK;
|
||||
-- INSERT SELECT from distributed table to local table
|
||||
BEGIN;
|
||||
INSERT INTO ref(a, b) SELECT x, y FROM test;
|
||||
SELECT count(*) from ref;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
8
|
||||
(1 row)
|
||||
|
||||
ROLLBACK;
|
||||
-- INSERT SELECT from distributed table to local table
|
||||
BEGIN;
|
||||
INSERT INTO ref(a, b) SELECT c, d FROM local;
|
||||
SELECT count(*) from ref;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
6
|
||||
(1 row)
|
||||
|
||||
ROLLBACK;
|
||||
-- INSERT SELECT from distributed table to local table
|
||||
BEGIN;
|
||||
INSERT INTO local(c, d) SELECT x, y FROM test;
|
||||
SELECT count(*) from local;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
8
|
||||
(1 row)
|
||||
|
||||
ROLLBACK;
|
||||
-- INSERT SELECT from distributed table to local table
|
||||
BEGIN;
|
||||
INSERT INTO local(c, d) SELECT a, b FROM ref;
|
||||
SELECT count(*) from local;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
6
|
||||
(1 row)
|
||||
|
||||
ROLLBACK;
|
||||
-- Confirm that dummy placements work
|
||||
SELECT count(*) FROM test WHERE false;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT count(*) FROM test WHERE false GROUP BY GROUPING SETS (x,y);
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
(0 rows)
|
||||
|
||||
-- Confirm that they work with round-robin task assignment policy
|
||||
SET citus.task_assignment_policy TO 'round-robin';
|
||||
SELECT count(*) FROM test WHERE false;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT count(*) FROM test WHERE false GROUP BY GROUPING SETS (x,y);
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
(0 rows)
|
||||
|
||||
RESET citus.task_assignment_policy;
|
||||
-- Cleanup
|
||||
SET client_min_messages TO WARNING;
|
||||
DROP SCHEMA single_node CASCADE;
|
||||
-- Remove the coordinator again
|
||||
SELECT 1 FROM master_remove_node('localhost', :master_port);
|
||||
?column?
|
||||
---------------------------------------------------------------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
-- restart nodeid sequence so that multi_cluster_management still has the same
|
||||
-- nodeids
|
||||
ALTER SEQUENCE pg_dist_node_nodeid_seq RESTART 1;
|
||||
|
|
@ -19,5 +19,7 @@ SET citus.next_shard_id TO 280000;
|
|||
\copy part_append FROM '@abs_srcdir@/data/part.more.data' with delimiter '|'
|
||||
|
||||
-- Exchange partition files in binary format in remaining tests
|
||||
ALTER SYSTEM SET citus.binary_worker_copy_format TO on;
|
||||
SELECT pg_reload_conf();
|
||||
SELECT success FROM run_command_on_workers('ALTER SYSTEM SET citus.binary_worker_copy_format TO on');
|
||||
SELECT success FROM run_command_on_workers('SELECT pg_reload_conf()');
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
test: multi_follower_sanity_check
|
||||
test: follower_single_node
|
||||
test: multi_follower_select_statements
|
||||
test: multi_follower_dml
|
||||
test: multi_follower_configure_followers
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@
|
|||
# ---
|
||||
test: multi_extension
|
||||
test: multi_703_upgrade
|
||||
test: single_node
|
||||
test: multi_cluster_management
|
||||
test: alter_role_propagation
|
||||
test: propagate_extension_commands
|
||||
|
|
|
|||
|
|
@ -12,6 +12,13 @@ SET citus.next_shard_id TO 280000;
|
|||
\copy customer_append FROM '@abs_srcdir@/data/customer.3.data' with delimiter '|'
|
||||
\copy part_append FROM '@abs_srcdir@/data/part.more.data' with delimiter '|'
|
||||
-- Exchange partition files in binary format in remaining tests
|
||||
ALTER SYSTEM SET citus.binary_worker_copy_format TO on;
|
||||
SELECT pg_reload_conf();
|
||||
pg_reload_conf
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
SELECT success FROM run_command_on_workers('ALTER SYSTEM SET citus.binary_worker_copy_format TO on');
|
||||
success
|
||||
---------
|
||||
|
|
|
|||
|
|
@ -87,6 +87,14 @@ SELECT create_distributed_table('dist_table', 'a', colocate_with := 'none');
|
|||
SELECT count(*) FROM dist_table;
|
||||
ROLLBACK;
|
||||
|
||||
-- Commented out since it currently does not clean up task files on the
|
||||
-- coordinator
|
||||
-- See this issue for details: https://github.com/citusdata/citus/issues/3996
|
||||
-- BEGIN;
|
||||
-- SET citus.enable_repartition_joins TO ON;
|
||||
-- SELECT count(*) FROM test t1, test t2 WHERE t1.x = t2.y;
|
||||
-- ROLLBACK;
|
||||
|
||||
BEGIN;
|
||||
SET citus.enable_repartition_joins TO ON;
|
||||
-- trigger local execution
|
||||
|
|
|
|||
|
|
@ -0,0 +1,108 @@
|
|||
\c - - - :master_port
|
||||
CREATE SCHEMA single_node;
|
||||
SET search_path TO single_node;
|
||||
SET citus.shard_count TO 4;
|
||||
SET citus.shard_replication_factor TO 1;
|
||||
SET citus.next_shard_id TO 93630500;
|
||||
|
||||
SELECT 1 FROM master_add_node('localhost', :master_port, groupid => 0);
|
||||
SELECT 1 FROM master_set_node_property('localhost', :master_port, 'shouldhaveshards', true);
|
||||
|
||||
CREATE TABLE test(x int, y int);
|
||||
SELECT create_distributed_table('test','x');
|
||||
CREATE TABLE ref(a int, b int);
|
||||
SELECT create_reference_table('ref');
|
||||
CREATE TABLE local(c int, d int);
|
||||
|
||||
INSERT INTO test VALUES (1, 2), (3, 4), (5, 6), (2, 7), (4, 5);
|
||||
INSERT INTO ref VALUES (1, 2), (5, 6), (7, 8);
|
||||
INSERT INTO local VALUES (1, 2), (3, 4), (7, 8);
|
||||
|
||||
-- connect to the follower and check that a simple select query works, the follower
|
||||
-- is still in the default cluster and will send queries to the primary nodes
|
||||
\c - - - :follower_master_port
|
||||
SET search_path TO single_node;
|
||||
SELECT * FROM test WHERE x = 1;
|
||||
SELECT count(*) FROM test;
|
||||
SELECT * FROM test ORDER BY x;
|
||||
|
||||
SELECT count(*) FROM ref;
|
||||
SELECT * FROM ref ORDER BY a;
|
||||
SELECT * FROM test, ref WHERE x = a ORDER BY x;
|
||||
|
||||
SELECT count(*) FROM local;
|
||||
SELECT * FROM local ORDER BY c;
|
||||
SELECT * FROM ref, local WHERE a = c ORDER BY a;
|
||||
|
||||
-- Check repartion joins are support
|
||||
SELECT * FROM test t1, test t2 WHERE t1.x = t2.y ORDER BY t1.x;
|
||||
SET citus.enable_repartition_joins TO ON;
|
||||
SELECT * FROM test t1, test t2 WHERE t1.x = t2.y ORDER BY t1.x;
|
||||
RESET citus.enable_repartition_joins;
|
||||
|
||||
-- Confirm that dummy placements work
|
||||
SELECT count(*) FROM test WHERE false;
|
||||
SELECT count(*) FROM test WHERE false GROUP BY GROUPING SETS (x,y);
|
||||
-- Confirm that they work with round-robin task assignment policy
|
||||
SET citus.task_assignment_policy TO 'round-robin';
|
||||
SELECT count(*) FROM test WHERE false;
|
||||
SELECT count(*) FROM test WHERE false GROUP BY GROUPING SETS (x,y);
|
||||
RESET citus.task_assignment_policy;
|
||||
|
||||
|
||||
-- now, connect to the follower but tell it to use secondary nodes. There are no
|
||||
-- secondary nodes so this should fail.
|
||||
|
||||
-- (this is :follower_master_port but substitution doesn't work here)
|
||||
\c "port=9070 dbname=regression options='-c\ citus.use_secondary_nodes=always'"
|
||||
SET search_path TO single_node;
|
||||
|
||||
SELECT * FROM test WHERE x = 1;
|
||||
|
||||
-- add the the follower as secondary nodes and try again, the SELECT statement
|
||||
-- should work this time
|
||||
\c - - - :master_port
|
||||
SET search_path TO single_node;
|
||||
|
||||
SELECT 1 FROM master_add_node('localhost', :follower_master_port, groupid => 0, noderole => 'secondary');
|
||||
SELECT 1 FROM master_set_node_property('localhost', :master_port, 'shouldhaveshards', true);
|
||||
|
||||
\c "port=9070 dbname=regression options='-c\ citus.use_secondary_nodes=always'"
|
||||
SET search_path TO single_node;
|
||||
|
||||
SELECT * FROM test WHERE x = 1;
|
||||
SELECT count(*) FROM test;
|
||||
SELECT * FROM test ORDER BY x;
|
||||
|
||||
SELECT count(*) FROM ref;
|
||||
SELECT * FROM ref ORDER BY a;
|
||||
SELECT * FROM test, ref WHERE x = a ORDER BY x;
|
||||
|
||||
SELECT count(*) FROM local;
|
||||
SELECT * FROM local ORDER BY c;
|
||||
SELECT * FROM ref, local WHERE a = c ORDER BY a;
|
||||
|
||||
-- Check repartion joins are support
|
||||
SELECT * FROM test t1, test t2 WHERE t1.x = t2.y ORDER BY t1.x;
|
||||
SET citus.enable_repartition_joins TO ON;
|
||||
SELECT * FROM test t1, test t2 WHERE t1.x = t2.y ORDER BY t1.x;
|
||||
RESET citus.enable_repartition_joins;
|
||||
|
||||
-- Confirm that dummy placements work
|
||||
SELECT count(*) FROM test WHERE false;
|
||||
SELECT count(*) FROM test WHERE false GROUP BY GROUPING SETS (x,y);
|
||||
-- Confirm that they work with round-robin task assignment policy
|
||||
SET citus.task_assignment_policy TO 'round-robin';
|
||||
SELECT count(*) FROM test WHERE false;
|
||||
SELECT count(*) FROM test WHERE false GROUP BY GROUPING SETS (x,y);
|
||||
RESET citus.task_assignment_policy;
|
||||
|
||||
-- Cleanup
|
||||
\c - - - :master_port
|
||||
SET search_path TO single_node;
|
||||
SET client_min_messages TO WARNING;
|
||||
DROP SCHEMA single_node CASCADE;
|
||||
-- Remove the coordinator again
|
||||
SELECT 1 FROM master_remove_node('localhost', :master_port);
|
||||
-- Remove the secondary coordinator again
|
||||
SELECT 1 FROM master_remove_node('localhost', :follower_master_port);
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
CREATE SCHEMA single_node;
|
||||
SET search_path TO single_node;
|
||||
SET citus.shard_count TO 4;
|
||||
SET citus.shard_replication_factor TO 1;
|
||||
SET citus.next_shard_id TO 90630500;
|
||||
|
||||
-- idempotently add node to allow this test to run without add_coordinator
|
||||
SET client_min_messages TO WARNING;
|
||||
SELECT 1 FROM master_add_node('localhost', :master_port, groupid => 0);
|
||||
RESET client_min_messages;
|
||||
|
||||
SELECT 1 FROM master_set_node_property('localhost', :master_port, 'shouldhaveshards', true);
|
||||
|
||||
CREATE TABLE test(x int, y int);
|
||||
SELECT create_distributed_table('test','x');
|
||||
CREATE TABLE ref(a int, b int);
|
||||
SELECT create_reference_table('ref');
|
||||
CREATE TABLE local(c int, d int);
|
||||
|
||||
-- Confirm the basics work
|
||||
INSERT INTO test VALUES (1, 2), (3, 4), (5, 6), (2, 7), (4, 5);
|
||||
SELECT * FROM test WHERE x = 1;
|
||||
SELECT count(*) FROM test;
|
||||
SELECT * FROM test ORDER BY x;
|
||||
|
||||
INSERT INTO ref VALUES (1, 2), (5, 6), (7, 8);
|
||||
SELECT count(*) FROM ref;
|
||||
SELECT * FROM ref ORDER BY a;
|
||||
SELECT * FROM test, ref WHERE x = a ORDER BY x;
|
||||
|
||||
INSERT INTO local VALUES (1, 2), (3, 4), (7, 8);
|
||||
SELECT count(*) FROM local;
|
||||
SELECT * FROM local ORDER BY c;
|
||||
SELECT * FROM ref, local WHERE a = c ORDER BY a;
|
||||
|
||||
-- Check repartion joins are support
|
||||
SELECT * FROM test t1, test t2 WHERE t1.x = t2.y ORDER BY t1.x;
|
||||
SET citus.enable_repartition_joins TO ON;
|
||||
SELECT * FROM test t1, test t2 WHERE t1.x = t2.y ORDER BY t1.x;
|
||||
RESET citus.enable_repartition_joins;
|
||||
|
||||
-- INSERT SELECT router
|
||||
BEGIN;
|
||||
INSERT INTO test(x, y) SELECT x, y FROM test WHERE x = 1;
|
||||
SELECT count(*) from test;
|
||||
ROLLBACK;
|
||||
|
||||
-- INSERT SELECT analytical query
|
||||
BEGIN;
|
||||
INSERT INTO test(x, y) SELECT count(x), max(y) FROM test;
|
||||
SELECT count(*) from test;
|
||||
ROLLBACK;
|
||||
|
||||
-- INSERT SELECT repartition
|
||||
BEGIN;
|
||||
INSERT INTO test(x, y) SELECT y, x FROM test;
|
||||
SELECT count(*) from test;
|
||||
ROLLBACK;
|
||||
|
||||
-- INSERT SELECT from reference table into distributed
|
||||
BEGIN;
|
||||
INSERT INTO test(x, y) SELECT a, b FROM ref;
|
||||
SELECT count(*) from test;
|
||||
ROLLBACK;
|
||||
|
||||
-- INSERT SELECT from local table into distributed
|
||||
BEGIN;
|
||||
INSERT INTO test(x, y) SELECT c, d FROM local;
|
||||
SELECT count(*) from test;
|
||||
ROLLBACK;
|
||||
|
||||
-- INSERT SELECT from distributed table to local table
|
||||
BEGIN;
|
||||
INSERT INTO ref(a, b) SELECT x, y FROM test;
|
||||
SELECT count(*) from ref;
|
||||
ROLLBACK;
|
||||
|
||||
-- INSERT SELECT from distributed table to local table
|
||||
BEGIN;
|
||||
INSERT INTO ref(a, b) SELECT c, d FROM local;
|
||||
SELECT count(*) from ref;
|
||||
ROLLBACK;
|
||||
|
||||
-- INSERT SELECT from distributed table to local table
|
||||
BEGIN;
|
||||
INSERT INTO local(c, d) SELECT x, y FROM test;
|
||||
SELECT count(*) from local;
|
||||
ROLLBACK;
|
||||
|
||||
-- INSERT SELECT from distributed table to local table
|
||||
BEGIN;
|
||||
INSERT INTO local(c, d) SELECT a, b FROM ref;
|
||||
SELECT count(*) from local;
|
||||
ROLLBACK;
|
||||
|
||||
-- Confirm that dummy placements work
|
||||
SELECT count(*) FROM test WHERE false;
|
||||
SELECT count(*) FROM test WHERE false GROUP BY GROUPING SETS (x,y);
|
||||
-- Confirm that they work with round-robin task assignment policy
|
||||
SET citus.task_assignment_policy TO 'round-robin';
|
||||
SELECT count(*) FROM test WHERE false;
|
||||
SELECT count(*) FROM test WHERE false GROUP BY GROUPING SETS (x,y);
|
||||
RESET citus.task_assignment_policy;
|
||||
|
||||
|
||||
-- Cleanup
|
||||
SET client_min_messages TO WARNING;
|
||||
DROP SCHEMA single_node CASCADE;
|
||||
-- Remove the coordinator again
|
||||
SELECT 1 FROM master_remove_node('localhost', :master_port);
|
||||
-- restart nodeid sequence so that multi_cluster_management still has the same
|
||||
-- nodeids
|
||||
ALTER SEQUENCE pg_dist_node_nodeid_seq RESTART 1;
|
||||
Loading…
Reference in New Issue