external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix Test Failure in multi-mx in PG17 (#7722) 

This PR addresses a regression test failure in the multi-mx feature of
Citus with the new PostgreSQL 17 version. The regression was identified
during the execution of multi-node tests, specifically targeting
compatibility issues introduced with PostgreSQL 17.

---------

Co-authored-by: Mehmet YILMAZ <mehmet.yilmaz@microsoft.com>
naisila/pg17_support_backup
Mehmet YILMAZ 2024-11-08 14:49:50 +03:00 committed by GitHub
parent 6c12b10c3a
commit 70cf729ba3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 4047 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/test/regress/bin/normalize.sed
View File

@ -309,3 +309,8 @@ s/permission denied to terminate process/must be a superuser to terminate superu
s/permission denied to cancel query/must be a superuser to cancel superuser query/g
#endif /* PG_VERSION_NUM < PG_VERSION_16 */
# pg17 changes
# multi_mx_router_planner normalize public.pg_temp_xxxxx
s/DEBUG: drop auto-cascades to type public\.pg_temp_[0-9]+/DEBUG: drop auto-cascades to type public.pg_temp_xxxxx/g

5
src/test/regress/expected/ch_bench_having_mx.out
View File

@ -1,3 +1,8 @@
-- Two alternative test outputs:
-- ch_bench_having_mx.out for PG16 and before
-- ch_bench_having_mx_0.out for PG17
-- related commit
-- https://github.com/postgres/postgres/commit/fd0398fc
ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 1640000;
SET citus.shard_replication_factor to 1;
SET citus.shard_count to 4;

383
src/test/regress/expected/ch_bench_having_mx_0.out Normal file
View File

@ -0,0 +1,383 @@
-- Two alternative test outputs:
-- ch_bench_having_mx.out for PG16 and before
-- ch_bench_having_mx_0.out for PG17
-- related commit
-- https://github.com/postgres/postgres/commit/fd0398fc
ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 1640000;
SET citus.shard_replication_factor to 1;
SET citus.shard_count to 4;
CREATE SCHEMA ch_bench_having;
SET search_path = ch_bench_having;
CREATE TABLE stock (
s_w_id int NOT NULL,
s_i_id int NOT NULL,
s_order_cnt int NOT NULL
);
SELECT create_distributed_table('stock','s_w_id');
create_distributed_table
---------------------------------------------------------------------
(1 row)
\c - - - :worker_1_port
SET search_path = ch_bench_having;
explain (costs false, summary false, timing false)
select s_i_id, sum(s_order_cnt) as ordercount
from stock
where s_order_cnt > (select sum(s_order_cnt) * .005 as where_query from stock)
group by s_i_id
having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from stock)
order by s_i_id;
QUERY PLAN
---------------------------------------------------------------------
Sort
Sort Key: remote_scan.s_i_id
InitPlan 1
-> Function Scan on read_intermediate_result intermediate_result
-> HashAggregate
Group Key: remote_scan.s_i_id
Filter: ((pg_catalog.sum(remote_scan.worker_column_3))::bigint > (InitPlan 1).col1)
-> Custom Scan (Citus Adaptive)
-> Distributed Subplan XXX_1
-> Aggregate
-> Custom Scan (Citus Adaptive)
Task Count: 4
Tasks Shown: One of 4
-> Task
Node: host=localhost port=xxxxx dbname=regression
-> Aggregate
-> Seq Scan on stock_1640000 stock
-> Distributed Subplan XXX_2
-> Aggregate
-> Custom Scan (Citus Adaptive)
Task Count: 4
Tasks Shown: One of 4
-> Task
Node: host=localhost port=xxxxx dbname=regression
-> Aggregate
-> Seq Scan on stock_1640000 stock
Task Count: 4
Tasks Shown: One of 4
-> Task
Node: host=localhost port=xxxxx dbname=regression
-> HashAggregate
Group Key: stock.s_i_id
InitPlan 1
-> Function Scan on read_intermediate_result intermediate_result
-> Seq Scan on stock_1640000 stock
Filter: ((s_order_cnt)::numeric > (InitPlan 1).col1)
(36 rows)
explain (costs false, summary false, timing false)
select s_i_id, sum(s_order_cnt) as ordercount
from stock
group by s_i_id
having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from stock)
order by s_i_id;
QUERY PLAN
---------------------------------------------------------------------
Sort
Sort Key: remote_scan.s_i_id
InitPlan 1
-> Function Scan on read_intermediate_result intermediate_result
-> HashAggregate
Group Key: remote_scan.s_i_id
Filter: ((pg_catalog.sum(remote_scan.worker_column_3))::bigint > (InitPlan 1).col1)
-> Custom Scan (Citus Adaptive)
-> Distributed Subplan XXX_1
-> Aggregate
-> Custom Scan (Citus Adaptive)
Task Count: 4
Tasks Shown: One of 4
-> Task
Node: host=localhost port=xxxxx dbname=regression
-> Aggregate
-> Seq Scan on stock_1640000 stock
Task Count: 4
Tasks Shown: One of 4
-> Task
Node: host=localhost port=xxxxx dbname=regression
-> HashAggregate
Group Key: stock.s_i_id
-> Seq Scan on stock_1640000 stock
(24 rows)
explain (costs false, summary false, timing false)
select s_i_id, sum(s_order_cnt) as ordercount
from stock
group by s_i_id
having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from stock);
QUERY PLAN
---------------------------------------------------------------------
HashAggregate
Group Key: remote_scan.s_i_id
Filter: ((pg_catalog.sum(remote_scan.worker_column_3))::bigint > (InitPlan 1).col1)
InitPlan 1
-> Function Scan on read_intermediate_result intermediate_result
-> Custom Scan (Citus Adaptive)
-> Distributed Subplan XXX_1
-> Aggregate
-> Custom Scan (Citus Adaptive)
Task Count: 4
Tasks Shown: One of 4
-> Task
Node: host=localhost port=xxxxx dbname=regression
-> Aggregate
-> Seq Scan on stock_1640000 stock
Task Count: 4
Tasks Shown: One of 4
-> Task
Node: host=localhost port=xxxxx dbname=regression
-> HashAggregate
Group Key: stock.s_i_id
-> Seq Scan on stock_1640000 stock
(22 rows)
explain (costs false)
select s_i_id, sum(s_order_cnt) as ordercount
from stock s
group by s_i_id
having (select true)
order by s_i_id;
QUERY PLAN
---------------------------------------------------------------------
Sort
Sort Key: remote_scan.s_i_id
InitPlan 1
-> Result
-> HashAggregate
Group Key: remote_scan.s_i_id
-> Result
One-Time Filter: (InitPlan 1).col1
-> Custom Scan (Citus Adaptive)
Filter: (InitPlan 1).col1
Task Count: 4
Tasks Shown: One of 4
-> Task
Node: host=localhost port=xxxxx dbname=regression
-> HashAggregate
Group Key: s.s_i_id
-> Seq Scan on stock_1640000 s
(17 rows)
explain select s_i_id, sum(s_order_cnt) as ordercount
from stock s
group by s_i_id
having (select true);
QUERY PLAN
---------------------------------------------------------------------
HashAggregate (cost=500.01..503.01 rows=200 width=12)
Group Key: remote_scan.s_i_id
InitPlan 1
-> Result (cost=0.00..0.01 rows=1 width=1)
-> Result (cost=0.00..0.00 rows=100000 width=12)
One-Time Filter: (InitPlan 1).col1
-> Custom Scan (Citus Adaptive) (cost=0.00..0.00 rows=100000 width=12)
Filter: (InitPlan 1).col1
Task Count: 4
Tasks Shown: One of 4
-> Task
Node: host=localhost port=xxxxx dbname=regression
-> HashAggregate (cost=40.60..42.60 rows=200 width=12)
Group Key: s.s_i_id
-> Seq Scan on stock_1640000 s (cost=0.00..30.40 rows=2040 width=8)
(15 rows)
select s_i_id, sum(s_order_cnt) as ordercount
from stock
where s_order_cnt > (select sum(s_order_cnt) * .005 as where_query from stock)
group by s_i_id
having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from stock)
order by s_i_id;
s_i_id | ordercount
---------------------------------------------------------------------
(0 rows)
INSERT INTO stock SELECT c, c, c FROM generate_series(1, 5) as c;
select s_i_id, sum(s_order_cnt) as ordercount
from stock
where s_order_cnt > (select sum(s_order_cnt) * .005 as where_query from stock)
group by s_i_id
having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from stock)
order by s_i_id;
s_i_id | ordercount
---------------------------------------------------------------------
3 | 3
4 | 4
5 | 5
(3 rows)
select s_i_id, sum(s_order_cnt) as ordercount
from stock
group by s_i_id
having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from stock)
order by s_i_id;
s_i_id | ordercount
---------------------------------------------------------------------
3 | 3
4 | 4
5 | 5
(3 rows)
select s_i_id, sum(s_order_cnt) as ordercount
from stock s
where s_order_cnt > (select sum(s_order_cnt) * .005 as where_query from stock)
group by s_i_id
having (select true)
order by s_i_id;
s_i_id | ordercount
---------------------------------------------------------------------
1 | 1
2 | 2
3 | 3
4 | 4
5 | 5
(5 rows)
select s_i_id, sum(s_order_cnt) as ordercount
from stock s
where s_order_cnt > (select sum(s_order_cnt) * .005 as where_query from stock)
group by s_i_id
having (select false)
order by s_i_id;
s_i_id | ordercount
---------------------------------------------------------------------
(0 rows)
select s_i_id, sum(s_order_cnt) as ordercount
from stock s
group by s_i_id
having (select true)
order by s_i_id;
s_i_id | ordercount
---------------------------------------------------------------------
1 | 1
2 | 2
3 | 3
4 | 4
5 | 5
(5 rows)
select s_i_id, sum(s_order_cnt) as ordercount
from stock s
group by s_i_id
having (select false)
order by s_i_id;
s_i_id | ordercount
---------------------------------------------------------------------
(0 rows)
select s_i_id, sum(s_order_cnt) as ordercount
from stock s
group by s_i_id
having (select true)
order by s_i_id;
s_i_id | ordercount
---------------------------------------------------------------------
1 | 1
2 | 2
3 | 3
4 | 4
5 | 5
(5 rows)
-- We don't support correlated subqueries in having
select s_i_id, sum(s_order_cnt) as ordercount
from stock s
where s_order_cnt > (select sum(s_order_cnt) * .005 as where_query from stock)
group by s_i_id
having (select max(s_order_cnt) > 2 as having_query from stock where s_i_id = s.s_i_id)
order by s_i_id;
ERROR: Subqueries in HAVING cannot refer to outer query
-- We don't support correlated subqueries in having
select s_i_id, sum(s_order_cnt) as ordercount
from stock s
group by s_i_id
having (select max(s_order_cnt) > 2 as having_query from stock where s_i_id = s.s_i_id)
order by s_i_id;
ERROR: Subqueries in HAVING cannot refer to outer query
\c - - - :master_port
SET citus.shard_replication_factor to 1;
SET citus.shard_count to 4;
SET search_path = ch_bench_having, public;
DROP TABLE stock;
CREATE TABLE stock (
s_w_id int NOT NULL,
s_i_id int NOT NULL,
s_quantity decimal(4,0) NOT NULL,
s_ytd decimal(8,2) NOT NULL,
s_order_cnt int NOT NULL,
s_remote_cnt int NOT NULL,
s_data varchar(50) NOT NULL,
s_dist_01 char(24) NOT NULL,
s_dist_02 char(24) NOT NULL,
s_dist_03 char(24) NOT NULL,
s_dist_04 char(24) NOT NULL,
s_dist_05 char(24) NOT NULL,
s_dist_06 char(24) NOT NULL,
s_dist_07 char(24) NOT NULL,
s_dist_08 char(24) NOT NULL,
s_dist_09 char(24) NOT NULL,
s_dist_10 char(24) NOT NULL,
PRIMARY KEY (s_w_id,s_i_id)
);
insert into stock VALUES
(1, 33, 1, 1, 1, 1, '', '','','','','','','','','',''),
(33, 1, 1, 1, 1, 1, '', '','','','','','','','','',''),
(32, 1, 1, 1, 1, 1, '', '','','','','','','','','','');
SELECT create_distributed_table('stock','s_w_id');
NOTICE: Copying data from local table...
NOTICE: copying the data has completed
DETAIL: The local data in the table is no longer visible, but is still on disk.
HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$ch_bench_having.stock$$)
create_distributed_table
---------------------------------------------------------------------
(1 row)
\c - - - :worker_1_port
SET search_path = ch_bench_having, public;
select s_i_id, sum(s_order_cnt) as ordercount
from stock, supplier_mx, nation_mx
where mod((s_w_id * s_i_id),10000) = s_suppkey
and s_nationkey = n_nationkey
and n_name = 'GERMANY'
group by s_i_id
having sum(s_order_cnt) >
(select sum(s_order_cnt) * .005
from stock, supplier_mx, nation_mx
where mod((s_w_id * s_i_id),10000) = s_suppkey
and s_nationkey = n_nationkey
and n_name = 'GERMANY')
order by s_i_id, ordercount desc;
s_i_id | ordercount
---------------------------------------------------------------------
1 | 1
33 | 1
(2 rows)
insert into stock VALUES
(10033, 1, 1, 1, 100000, 1, '', '','','','','','','','','','');
select s_i_id, sum(s_order_cnt) as ordercount
from stock, supplier_mx, nation_mx
where mod((s_w_id * s_i_id),10000) = s_suppkey
and s_nationkey = n_nationkey
and n_name = 'GERMANY'
group by s_i_id
having sum(s_order_cnt) >
(select sum(s_order_cnt) * .005
from stock, supplier_mx, nation_mx
where mod((s_w_id * s_i_id),10000) = s_suppkey
and s_nationkey = n_nationkey
and n_name = 'GERMANY')
order by s_i_id, ordercount desc;
s_i_id | ordercount
---------------------------------------------------------------------
1 | 100001
(1 row)
\c - - - :master_port
BEGIN;
SET LOCAL client_min_messages TO WARNING;
DROP SCHEMA ch_bench_having CASCADE;
COMMIT;

6
src/test/regress/expected/multi_extension.out
View File

@ -7,6 +7,12 @@
-- not done yet.
--
-- Upgrade tests verify the objects are added in citus_finish_pg_upgrade()
--
-- Two alternative test outputs:
-- multi_extension.out for PG16 and before
-- multi_extension_0.out for PG17
-- related commit
-- https://git.postgresql.org/gitweb/?p=postgresql.git;a=commit;h=e5bc9454e527b1cba97553531d8d4992892fdeef
SET citus.next_shard_id TO 580000;
CREATE SCHEMA multi_extension;
SELECT $definition$

2084
src/test/regress/expected/multi_extension_0.out Normal file
View File

File diff suppressed because it is too large Load Diff

26
src/test/regress/expected/multi_mx_hide_shard_names.out
View File

@ -473,19 +473,25 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name
-- PG16 added one more backend type B_STANDALONE_BACKEND
-- and also alphabetized the backend types, hence the orders changed
-- Relevant PG commit:
-- Relevant PG16 commit:
-- https://github.com/postgres/postgres/commit/0c679464a837079acc75ff1d45eaa83f79e05690
-- Relevant Pg17 commit:
-- https://github.com/postgres/postgres/commit/067701f57758f9baed5bd9d868539738d77bfa92#diff-afc0ebd67534b71b5b94b29a1387aa6eedffe342a5539f52d686428be323e802
SHOW server_version \gset
SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16
\gset
\if :server_version_ge_16
SELECT 4 AS client_backend \gset
SELECT 5 AS bgworker \gset
SELECT 12 AS walsender \gset
SELECT substring(:'server_version', '\d+')::int >= 17 AS server_version_ge_17 \gset
SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 \gset
\if :server_version_ge_17
SELECT 1 AS client_backend \gset
SELECT 4 AS bgworker \gset
SELECT 5 AS walsender \gset
\elif :server_version_ge_16
SELECT 4 AS client_backend \gset
SELECT 5 AS bgworker \gset
SELECT 12 AS walsender \gset
\else
SELECT 3 AS client_backend \gset
SELECT 4 AS bgworker \gset
SELECT 9 AS walsender \gset
SELECT 3 AS client_backend \gset
SELECT 4 AS bgworker \gset
SELECT 9 AS walsender \gset
\endif
-- say, we set it to bgworker
-- the shards and indexes do not show up

3
src/test/regress/expected/multi_mx_router_planner.out
View File

@ -1,3 +1,6 @@
-- Two alternative test outputs:
-- multi_mx_router_planner.out for PG16 and before
-- multi_mx_router_planner_0.out for PG17
-- ===================================================================
-- test router planner functionality for single shard select queries
-- ===================================================================

1514
src/test/regress/expected/multi_mx_router_planner_0.out Normal file
View File

File diff suppressed because it is too large Load Diff

6
src/test/regress/sql/ch_bench_having_mx.sql
View File

@ -1,3 +1,9 @@
-- Two alternative test outputs:
-- ch_bench_having_mx.out for PG16 and before
-- ch_bench_having_mx_0.out for PG17
-- related commit
-- https://github.com/postgres/postgres/commit/fd0398fc
ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 1640000;
SET citus.shard_replication_factor to 1;
SET citus.shard_count to 4;

6
src/test/regress/sql/multi_extension.sql
View File

@ -7,6 +7,12 @@
-- not done yet.
--
-- Upgrade tests verify the objects are added in citus_finish_pg_upgrade()
--
-- Two alternative test outputs:
-- multi_extension.out for PG16 and before
-- multi_extension_0.out for PG17
-- related commit
-- https://git.postgresql.org/gitweb/?p=postgresql.git;a=commit;h=e5bc9454e527b1cba97553531d8d4992892fdeef
SET citus.next_shard_id TO 580000;
CREATE SCHEMA multi_extension;

27
src/test/regress/sql/multi_mx_hide_shard_names.sql
View File

@ -246,20 +246,25 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name
-- PG16 added one more backend type B_STANDALONE_BACKEND
-- and also alphabetized the backend types, hence the orders changed
-- Relevant PG commit:
-- Relevant PG16 commit:
-- https://github.com/postgres/postgres/commit/0c679464a837079acc75ff1d45eaa83f79e05690
-- Relevant Pg17 commit:
-- https://github.com/postgres/postgres/commit/067701f57758f9baed5bd9d868539738d77bfa92#diff-afc0ebd67534b71b5b94b29a1387aa6eedffe342a5539f52d686428be323e802
SHOW server_version \gset
SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16
\gset
\if :server_version_ge_16
SELECT 4 AS client_backend \gset
SELECT 5 AS bgworker \gset
SELECT 12 AS walsender \gset
SELECT substring(:'server_version', '\d+')::int >= 17 AS server_version_ge_17 \gset
SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 \gset
\if :server_version_ge_17
SELECT 1 AS client_backend \gset
SELECT 4 AS bgworker \gset
SELECT 5 AS walsender \gset
\elif :server_version_ge_16
SELECT 4 AS client_backend \gset
SELECT 5 AS bgworker \gset
SELECT 12 AS walsender \gset
\else
SELECT 3 AS client_backend \gset
SELECT 4 AS bgworker \gset
SELECT 9 AS walsender \gset
SELECT 3 AS client_backend \gset
SELECT 4 AS bgworker \gset
SELECT 9 AS walsender \gset
\endif
-- say, we set it to bgworker

3
src/test/regress/sql/multi_mx_router_planner.sql
View File

@ -1,4 +1,7 @@
-- Two alternative test outputs:
-- multi_mx_router_planner.out for PG16 and before
-- multi_mx_router_planner_0.out for PG17
-- ===================================================================
-- test router planner functionality for single shard select queries