diff --git a/src/test/regress/expected/multi_explain.out b/src/test/regress/expected/multi_explain.out index 54b220627..7e12de7d8 100644 --- a/src/test/regress/expected/multi_explain.out +++ b/src/test/regress/expected/multi_explain.out @@ -688,16 +688,16 @@ Master Query -> Seq Scan on pg_merge_job_570038 SET citus.task_executor_type TO 'real-time'; PREPARE router_executor_query AS SELECT l_quantity FROM lineitem WHERE l_orderkey = 5; -EXPLAIN EXECUTE router_executor_query; +EXPLAIN (COSTS FALSE) EXECUTE router_executor_query; Distributed Query into pg_merge_job_570039 Executor: Router Task Count: 1 Tasks Shown: All -> Task Node: host=localhost port=57637 dbname=regression - -> Bitmap Heap Scan on lineitem_290000 lineitem (cost=4.30..13.44 rows=3 width=18) + -> Bitmap Heap Scan on lineitem_290000 lineitem Recheck Cond: (l_orderkey = 5) - -> Bitmap Index Scan on lineitem_pkey_290000 (cost=0.00..4.30 rows=3 width=0) + -> Bitmap Index Scan on lineitem_pkey_290000 Index Cond: (l_orderkey = 5) PREPARE real_time_executor_query AS SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030; diff --git a/src/test/regress/expected/multi_explain_0.out b/src/test/regress/expected/multi_explain_0.out index 68a35bf69..eb7078c79 100644 --- a/src/test/regress/expected/multi_explain_0.out +++ b/src/test/regress/expected/multi_explain_0.out @@ -659,16 +659,16 @@ Master Query -> Seq Scan on pg_merge_job_570038 SET citus.task_executor_type TO 'real-time'; PREPARE router_executor_query AS SELECT l_quantity FROM lineitem WHERE l_orderkey = 5; -EXPLAIN EXECUTE router_executor_query; +EXPLAIN (COSTS FALSE) EXECUTE router_executor_query; Distributed Query into pg_merge_job_570039 Executor: Router Task Count: 1 Tasks Shown: All -> Task Node: host=localhost port=57637 dbname=regression - -> Bitmap Heap Scan on lineitem_290000 lineitem (cost=4.30..13.44 rows=3 width=18) + -> Bitmap Heap Scan on lineitem_290000 lineitem Recheck Cond: (l_orderkey = 5) - -> Bitmap Index Scan on lineitem_pkey_290000 (cost=0.00..4.30 rows=3 width=0) + -> Bitmap Index Scan on lineitem_pkey_290000 Index Cond: (l_orderkey = 5) PREPARE real_time_executor_query AS SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030; diff --git a/src/test/regress/expected/multi_generate_ddl_commands.out b/src/test/regress/expected/multi_generate_ddl_commands.out index 1594b6361..e10c7e62b 100644 --- a/src/test/regress/expected/multi_generate_ddl_commands.out +++ b/src/test/regress/expected/multi_generate_ddl_commands.out @@ -108,9 +108,9 @@ CREATE TABLE clustered_table ( CREATE INDEX clustered_time_idx ON clustered_table (received_at); CLUSTER clustered_table USING clustered_time_idx; SELECT table_ddl_command_array('clustered_table'); - table_ddl_command_array -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - {"CREATE TABLE public.clustered_table (data json NOT NULL, received_at timestamp without time zone NOT NULL)","CREATE INDEX clustered_time_idx ON public.clustered_table USING btree (received_at)","ALTER TABLE public.clustered_table CLUSTER ON clustered_time_idx"} + table_ddl_command_array +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {"CREATE TABLE public.clustered_table (data json NOT NULL, received_at timestamp without time zone NOT NULL)","CREATE INDEX clustered_time_idx ON public.clustered_table USING btree (received_at) TABLESPACE pg_default","ALTER TABLE public.clustered_table CLUSTER ON clustered_time_idx"} (1 row) -- fiddly things like storage type and statistics also work diff --git a/src/test/regress/expected/multi_generate_ddl_commands_0.out b/src/test/regress/expected/multi_generate_ddl_commands_0.out new file mode 100644 index 000000000..1594b6361 --- /dev/null +++ b/src/test/regress/expected/multi_generate_ddl_commands_0.out @@ -0,0 +1,156 @@ +ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 610000; +ALTER SEQUENCE pg_catalog.pg_dist_jobid_seq RESTART 610000; +-- =================================================================== +-- create test functions +-- =================================================================== +CREATE FUNCTION table_ddl_command_array(regclass) + RETURNS text[] + AS 'citus' + LANGUAGE C STRICT; +-- =================================================================== +-- test ddl command generation functionality +-- =================================================================== +-- first make sure a simple table works +CREATE TABLE simple_table ( + first_name text, + last_name text, + id bigint +); +SELECT table_ddl_command_array('simple_table'); + table_ddl_command_array +----------------------------------------------------------------------------------- + {"CREATE TABLE public.simple_table (first_name text, last_name text, id bigint)"} +(1 row) + +-- ensure not-null constraints are propagated +CREATE TABLE not_null_table ( + city text, + id bigint not null +); +SELECT table_ddl_command_array('not_null_table'); + table_ddl_command_array +------------------------------------------------------------------------ + {"CREATE TABLE public.not_null_table (city text, id bigint NOT NULL)"} +(1 row) + +-- ensure tables not in search path are schema-prefixed +CREATE SCHEMA not_in_path CREATE TABLE simple_table (id bigint); +SELECT table_ddl_command_array('not_in_path.simple_table'); + table_ddl_command_array +------------------------------------------------------------------------------------------------- + {"CREATE SCHEMA IF NOT EXISTS not_in_path","CREATE TABLE not_in_path.simple_table (id bigint)"} +(1 row) + +-- even more complex constraints should be preserved... +CREATE TABLE column_constraint_table ( + first_name text, + last_name text, + age int CONSTRAINT non_negative_age CHECK (age >= 0) +); +SELECT table_ddl_command_array('column_constraint_table'); + table_ddl_command_array +---------------------------------------------------------------------------------------------------------------------------------------------- + {"CREATE TABLE public.column_constraint_table (first_name text, last_name text, age integer, CONSTRAINT non_negative_age CHECK (age >= 0))"} +(1 row) + +-- including table constraints +CREATE TABLE table_constraint_table ( + bid_item_id bigint, + min_bid decimal not null, + max_bid decimal not null, + CONSTRAINT bids_ordered CHECK (min_bid > max_bid) +); +SELECT table_ddl_command_array('table_constraint_table'); + table_ddl_command_array +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {"CREATE TABLE public.table_constraint_table (bid_item_id bigint, min_bid numeric NOT NULL, max_bid numeric NOT NULL, CONSTRAINT bids_ordered CHECK (min_bid > max_bid))"} +(1 row) + +-- default values are supported +CREATE TABLE default_value_table ( + name text, + price decimal default 0.00 +); +SELECT table_ddl_command_array('default_value_table'); + table_ddl_command_array +------------------------------------------------------------------------------------- + {"CREATE TABLE public.default_value_table (name text, price numeric DEFAULT 0.00)"} +(1 row) + +-- of course primary keys work... +CREATE TABLE pkey_table ( + first_name text, + last_name text, + id bigint PRIMARY KEY +); +SELECT table_ddl_command_array('pkey_table'); + table_ddl_command_array +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {"CREATE TABLE public.pkey_table (first_name text, last_name text, id bigint NOT NULL)","ALTER TABLE public.pkey_table ADD CONSTRAINT pkey_table_pkey PRIMARY KEY (id)"} +(1 row) + +-- as do unique indexes... +CREATE TABLE unique_table ( + user_id bigint not null, + username text UNIQUE not null +); +SELECT table_ddl_command_array('unique_table'); + table_ddl_command_array +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {"CREATE TABLE public.unique_table (user_id bigint NOT NULL, username text NOT NULL)","ALTER TABLE public.unique_table ADD CONSTRAINT unique_table_username_key UNIQUE (username)"} +(1 row) + +-- and indexes used for clustering +CREATE TABLE clustered_table ( + data json not null, + received_at timestamp not null +); +CREATE INDEX clustered_time_idx ON clustered_table (received_at); +CLUSTER clustered_table USING clustered_time_idx; +SELECT table_ddl_command_array('clustered_table'); + table_ddl_command_array +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {"CREATE TABLE public.clustered_table (data json NOT NULL, received_at timestamp without time zone NOT NULL)","CREATE INDEX clustered_time_idx ON public.clustered_table USING btree (received_at)","ALTER TABLE public.clustered_table CLUSTER ON clustered_time_idx"} +(1 row) + +-- fiddly things like storage type and statistics also work +CREATE TABLE fiddly_table ( + hostname char(255) not null, + os char(255) not null, + ip_addr inet not null, + traceroute text not null +); +ALTER TABLE fiddly_table + ALTER hostname SET STORAGE PLAIN, + ALTER os SET STORAGE MAIN, + ALTER ip_addr SET STORAGE EXTENDED, + ALTER traceroute SET STORAGE EXTERNAL, + ALTER ip_addr SET STATISTICS 500; +SELECT table_ddl_command_array('fiddly_table'); + table_ddl_command_array +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {"CREATE TABLE public.fiddly_table (hostname character(255) NOT NULL, os character(255) NOT NULL, ip_addr inet NOT NULL, traceroute text NOT NULL)","ALTER TABLE ONLY public.fiddly_table ALTER COLUMN hostname SET STORAGE PLAIN, ALTER COLUMN os SET STORAGE MAIN, ALTER COLUMN ip_addr SET STORAGE EXTENDED, ALTER COLUMN ip_addr SET STATISTICS 500, ALTER COLUMN traceroute SET STORAGE EXTERNAL"} +(1 row) + +-- test foreign tables using fake FDW +CREATE FOREIGN TABLE foreign_table ( + id bigint not null, + full_name text not null default '' +) SERVER fake_fdw_server OPTIONS (encoding 'utf-8', compression 'true'); +SELECT table_ddl_command_array('foreign_table'); +NOTICE: foreign-data wrapper "fake_fdw" does not have an extension defined + table_ddl_command_array +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {"CREATE SERVER fake_fdw_server FOREIGN DATA WRAPPER fake_fdw","CREATE FOREIGN TABLE public.foreign_table (id bigint NOT NULL, full_name text DEFAULT ''::text NOT NULL) SERVER fake_fdw_server OPTIONS (encoding 'utf-8', compression 'true')"} +(1 row) + +-- propagating views is not supported +CREATE VIEW local_view AS SELECT * FROM simple_table; +SELECT table_ddl_command_array('local_view'); +ERROR: public.local_view is not a regular or foreign table +-- clean up +DROP VIEW IF EXISTS local_view; +DROP FOREIGN TABLE IF EXISTS foreign_table; +DROP TABLE IF EXISTS simple_table, not_null_table, column_constraint_table, + table_constraint_table, default_value_table, pkey_table, + unique_table, clustered_table, fiddly_table; diff --git a/src/test/regress/expected/multi_master_protocol.out b/src/test/regress/expected/multi_master_protocol.out index 465897787..b02eba0e8 100644 --- a/src/test/regress/expected/multi_master_protocol.out +++ b/src/test/regress/expected/multi_master_protocol.out @@ -15,7 +15,7 @@ SELECT * FROM master_get_table_ddl_events('lineitem'); master_get_table_ddl_events ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- CREATE TABLE public.lineitem (l_orderkey bigint NOT NULL, l_partkey integer NOT NULL, l_suppkey integer NOT NULL, l_linenumber integer NOT NULL, l_quantity numeric(15,2) NOT NULL, l_extendedprice numeric(15,2) NOT NULL, l_discount numeric(15,2) NOT NULL, l_tax numeric(15,2) NOT NULL, l_returnflag character(1) NOT NULL, l_linestatus character(1) NOT NULL, l_shipdate date NOT NULL, l_commitdate date NOT NULL, l_receiptdate date NOT NULL, l_shipinstruct character(25) NOT NULL, l_shipmode character(10) NOT NULL, l_comment character varying(44) NOT NULL) - CREATE INDEX lineitem_time_index ON public.lineitem USING btree (l_shipdate) + CREATE INDEX lineitem_time_index ON public.lineitem USING btree (l_shipdate) TABLESPACE pg_default ALTER TABLE public.lineitem ADD CONSTRAINT lineitem_pkey PRIMARY KEY (l_orderkey, l_linenumber) (3 rows) diff --git a/src/test/regress/expected/multi_master_protocol_0.out b/src/test/regress/expected/multi_master_protocol_0.out new file mode 100644 index 000000000..465897787 --- /dev/null +++ b/src/test/regress/expected/multi_master_protocol_0.out @@ -0,0 +1,48 @@ +-- +-- MULTI_MASTER_PROTOCOL +-- +-- Tests that check the metadata returned by the master node. +ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 740000; +ALTER SEQUENCE pg_catalog.pg_dist_jobid_seq RESTART 740000; +SELECT part_storage_type, part_key, part_replica_count, part_max_size, + part_placement_policy FROM master_get_table_metadata('lineitem'); + part_storage_type | part_key | part_replica_count | part_max_size | part_placement_policy +-------------------+------------+--------------------+---------------+----------------------- + t | l_orderkey | 2 | 307200 | 2 +(1 row) + +SELECT * FROM master_get_table_ddl_events('lineitem'); + master_get_table_ddl_events +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + CREATE TABLE public.lineitem (l_orderkey bigint NOT NULL, l_partkey integer NOT NULL, l_suppkey integer NOT NULL, l_linenumber integer NOT NULL, l_quantity numeric(15,2) NOT NULL, l_extendedprice numeric(15,2) NOT NULL, l_discount numeric(15,2) NOT NULL, l_tax numeric(15,2) NOT NULL, l_returnflag character(1) NOT NULL, l_linestatus character(1) NOT NULL, l_shipdate date NOT NULL, l_commitdate date NOT NULL, l_receiptdate date NOT NULL, l_shipinstruct character(25) NOT NULL, l_shipmode character(10) NOT NULL, l_comment character varying(44) NOT NULL) + CREATE INDEX lineitem_time_index ON public.lineitem USING btree (l_shipdate) + ALTER TABLE public.lineitem ADD CONSTRAINT lineitem_pkey PRIMARY KEY (l_orderkey, l_linenumber) +(3 rows) + +SELECT * FROM master_get_new_shardid(); + master_get_new_shardid +------------------------ + 740000 +(1 row) + +SELECT * FROM master_get_round_robin_candidate_nodes(1); + node_name | node_port +-----------+----------- + localhost | 57638 + localhost | 57637 +(2 rows) + +SELECT * FROM master_get_round_robin_candidate_nodes(2); + node_name | node_port +-----------+----------- + localhost | 57637 + localhost | 57638 +(2 rows) + +SELECT * FROM master_get_active_worker_nodes(); + node_name | node_port +-----------+----------- + localhost | 57638 + localhost | 57637 +(2 rows) + diff --git a/src/test/regress/expected/multi_metadata_snapshot.out b/src/test/regress/expected/multi_metadata_snapshot.out index c33daeadf..4d09c3317 100644 --- a/src/test/regress/expected/multi_metadata_snapshot.out +++ b/src/test/regress/expected/multi_metadata_snapshot.out @@ -77,7 +77,7 @@ SELECT unnest(master_metadata_snapshot()); INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata) VALUES (2, 2, 'localhost', 57638, 'default', FALSE),(1, 1, 'localhost', 57637, 'default', FALSE) CREATE SEQUENCE IF NOT EXISTS public.mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE CREATE TABLE public.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 integer DEFAULT nextval('public.mx_test_table_col_3_seq'::regclass) NOT NULL) - CREATE INDEX mx_index ON public.mx_test_table USING btree (col_2) + CREATE INDEX mx_index ON public.mx_test_table USING btree (col_2) TABLESPACE pg_default ALTER TABLE public.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) ALTER TABLE public.mx_test_table OWNER TO postgres INSERT INTO pg_dist_partition (logicalrelid, partmethod, partkey, colocationid, repmodel) VALUES ('public.mx_test_table'::regclass, 'h', column_name_to_column('public.mx_test_table','col_1'), 0, 's') @@ -99,7 +99,7 @@ SELECT unnest(master_metadata_snapshot()); CREATE SCHEMA IF NOT EXISTS mx_testing_schema CREATE SEQUENCE IF NOT EXISTS mx_testing_schema.mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE CREATE TABLE mx_testing_schema.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 integer DEFAULT nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) NOT NULL) - CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) + CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) TABLESPACE pg_default ALTER TABLE mx_testing_schema.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres INSERT INTO pg_dist_partition (logicalrelid, partmethod, partkey, colocationid, repmodel) VALUES ('mx_testing_schema.mx_test_table'::regclass, 'h', column_name_to_column('mx_testing_schema.mx_test_table','col_1'), 0, 's') @@ -125,7 +125,7 @@ SELECT unnest(master_metadata_snapshot()); CREATE SCHEMA IF NOT EXISTS mx_testing_schema CREATE SEQUENCE IF NOT EXISTS mx_testing_schema.mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE CREATE TABLE mx_testing_schema.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 integer DEFAULT nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) NOT NULL) - CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) + CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) TABLESPACE pg_default ALTER TABLE mx_testing_schema.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres INSERT INTO pg_dist_partition (logicalrelid, partmethod, partkey, colocationid, repmodel) VALUES ('mx_testing_schema.mx_test_table'::regclass, 'h', column_name_to_column('mx_testing_schema.mx_test_table','col_1'), 0, 's') @@ -144,7 +144,7 @@ SELECT unnest(master_metadata_snapshot()); CREATE SCHEMA IF NOT EXISTS mx_testing_schema CREATE SEQUENCE IF NOT EXISTS mx_testing_schema.mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE CREATE TABLE mx_testing_schema.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 integer DEFAULT nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) NOT NULL) - CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) + CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) TABLESPACE pg_default ALTER TABLE mx_testing_schema.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres INSERT INTO pg_dist_partition (logicalrelid, partmethod, partkey, colocationid, repmodel) VALUES ('mx_testing_schema.mx_test_table'::regclass, 'h', column_name_to_column('mx_testing_schema.mx_test_table','col_1'), 0, 's') diff --git a/src/test/regress/expected/multi_metadata_snapshot_0.out b/src/test/regress/expected/multi_metadata_snapshot_0.out new file mode 100644 index 000000000..c33daeadf --- /dev/null +++ b/src/test/regress/expected/multi_metadata_snapshot_0.out @@ -0,0 +1,155 @@ +-- +-- MULTI_METADATA_SNAPSHOT +-- +-- Tests for metadata snapshot functions. +ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 1310000; +ALTER SEQUENCE pg_catalog.pg_dist_jobid_seq RESTART 1310000; +SELECT nextval('pg_catalog.pg_dist_shard_placement_placementid_seq') AS last_placement_id +\gset +ALTER SEQUENCE pg_catalog.pg_dist_shard_placement_placementid_seq RESTART 100000; +-- Create the necessary test utility function +CREATE FUNCTION master_metadata_snapshot() + RETURNS text[] + LANGUAGE C STRICT + AS 'citus'; + +COMMENT ON FUNCTION master_metadata_snapshot() + IS 'commands to create the metadata snapshot'; + +-- Show that none of the existing tables are qualified to be MX tables +SELECT * FROM pg_dist_partition WHERE partmethod='h' AND repmodel='s'; + logicalrelid | partmethod | partkey | colocationid | repmodel +--------------+------------+---------+--------------+---------- +(0 rows) + +-- Show that, with no MX tables, metadata snapshot contains only the delete commands and +-- pg_dist_node entries +SELECT unnest(master_metadata_snapshot()); + unnest +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + TRUNCATE pg_dist_node + SELECT worker_drop_distributed_table(logicalrelid) FROM pg_dist_partition + INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata) VALUES (2, 2, 'localhost', 57638, 'default', FALSE),(1, 1, 'localhost', 57637, 'default', FALSE) +(3 rows) + +-- Create a test table with constraints and SERIAL +CREATE TABLE mx_test_table (col_1 int UNIQUE, col_2 text NOT NULL, col_3 SERIAL); +SELECT master_create_distributed_table('mx_test_table', 'col_1', 'hash'); + master_create_distributed_table +--------------------------------- + +(1 row) + +SELECT master_create_worker_shards('mx_test_table', 8, 1); + master_create_worker_shards +----------------------------- + +(1 row) + +-- Set the replication model of the test table to streaming replication so that it is +-- considered as an MX table +UPDATE pg_dist_partition SET repmodel='s' WHERE logicalrelid='mx_test_table'::regclass; +-- Show that the created MX table is included in the metadata snapshot +SELECT unnest(master_metadata_snapshot()); + unnest +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + TRUNCATE pg_dist_node + SELECT worker_drop_distributed_table(logicalrelid) FROM pg_dist_partition + INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata) VALUES (2, 2, 'localhost', 57638, 'default', FALSE),(1, 1, 'localhost', 57637, 'default', FALSE) + CREATE SEQUENCE IF NOT EXISTS public.mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE + CREATE TABLE public.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 integer DEFAULT nextval('public.mx_test_table_col_3_seq'::regclass) NOT NULL) + ALTER TABLE public.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) + ALTER TABLE public.mx_test_table OWNER TO postgres + INSERT INTO pg_dist_partition (logicalrelid, partmethod, partkey, colocationid, repmodel) VALUES ('public.mx_test_table'::regclass, 'h', column_name_to_column('public.mx_test_table','col_1'), 0, 's') + INSERT INTO pg_dist_shard_placement (shardid, shardstate, shardlength, nodename, nodeport, placementid) VALUES (1310000, 1, 0, 'localhost', 57637, 100000),(1310001, 1, 0, 'localhost', 57638, 100001),(1310002, 1, 0, 'localhost', 57637, 100002),(1310003, 1, 0, 'localhost', 57638, 100003),(1310004, 1, 0, 'localhost', 57637, 100004),(1310005, 1, 0, 'localhost', 57638, 100005),(1310006, 1, 0, 'localhost', 57637, 100006),(1310007, 1, 0, 'localhost', 57638, 100007) + INSERT INTO pg_dist_shard (logicalrelid, shardid, shardstorage, shardminvalue, shardmaxvalue) VALUES ('public.mx_test_table'::regclass, 1310000, 't', '-2147483648', '-1610612737'),('public.mx_test_table'::regclass, 1310001, 't', '-1610612736', '-1073741825'),('public.mx_test_table'::regclass, 1310002, 't', '-1073741824', '-536870913'),('public.mx_test_table'::regclass, 1310003, 't', '-536870912', '-1'),('public.mx_test_table'::regclass, 1310004, 't', '0', '536870911'),('public.mx_test_table'::regclass, 1310005, 't', '536870912', '1073741823'),('public.mx_test_table'::regclass, 1310006, 't', '1073741824', '1610612735'),('public.mx_test_table'::regclass, 1310007, 't', '1610612736', '2147483647') +(10 rows) + +-- Show that CREATE INDEX commands are included in the metadata snapshot +CREATE INDEX mx_index ON mx_test_table(col_2); +NOTICE: using one-phase commit for distributed DDL commands +HINT: You can enable two-phase commit for extra safety with: SET citus.multi_shard_commit_protocol TO '2pc' +SELECT unnest(master_metadata_snapshot()); + unnest +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + TRUNCATE pg_dist_node + SELECT worker_drop_distributed_table(logicalrelid) FROM pg_dist_partition + INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata) VALUES (2, 2, 'localhost', 57638, 'default', FALSE),(1, 1, 'localhost', 57637, 'default', FALSE) + CREATE SEQUENCE IF NOT EXISTS public.mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE + CREATE TABLE public.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 integer DEFAULT nextval('public.mx_test_table_col_3_seq'::regclass) NOT NULL) + CREATE INDEX mx_index ON public.mx_test_table USING btree (col_2) + ALTER TABLE public.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) + ALTER TABLE public.mx_test_table OWNER TO postgres + INSERT INTO pg_dist_partition (logicalrelid, partmethod, partkey, colocationid, repmodel) VALUES ('public.mx_test_table'::regclass, 'h', column_name_to_column('public.mx_test_table','col_1'), 0, 's') + INSERT INTO pg_dist_shard_placement (shardid, shardstate, shardlength, nodename, nodeport, placementid) VALUES (1310000, 1, 0, 'localhost', 57637, 100000),(1310001, 1, 0, 'localhost', 57638, 100001),(1310002, 1, 0, 'localhost', 57637, 100002),(1310003, 1, 0, 'localhost', 57638, 100003),(1310004, 1, 0, 'localhost', 57637, 100004),(1310005, 1, 0, 'localhost', 57638, 100005),(1310006, 1, 0, 'localhost', 57637, 100006),(1310007, 1, 0, 'localhost', 57638, 100007) + INSERT INTO pg_dist_shard (logicalrelid, shardid, shardstorage, shardminvalue, shardmaxvalue) VALUES ('public.mx_test_table'::regclass, 1310000, 't', '-2147483648', '-1610612737'),('public.mx_test_table'::regclass, 1310001, 't', '-1610612736', '-1073741825'),('public.mx_test_table'::regclass, 1310002, 't', '-1073741824', '-536870913'),('public.mx_test_table'::regclass, 1310003, 't', '-536870912', '-1'),('public.mx_test_table'::regclass, 1310004, 't', '0', '536870911'),('public.mx_test_table'::regclass, 1310005, 't', '536870912', '1073741823'),('public.mx_test_table'::regclass, 1310006, 't', '1073741824', '1610612735'),('public.mx_test_table'::regclass, 1310007, 't', '1610612736', '2147483647') +(11 rows) + +-- Show that schema changes are included in the metadata snapshot +CREATE SCHEMA mx_testing_schema; +ALTER TABLE mx_test_table SET SCHEMA mx_testing_schema; +WARNING: not propagating ALTER ... SET SCHEMA commands to worker nodes +HINT: Connect to worker nodes directly to manually change schemas of affected objects. +SELECT unnest(master_metadata_snapshot()); + unnest +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + TRUNCATE pg_dist_node + SELECT worker_drop_distributed_table(logicalrelid) FROM pg_dist_partition + INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata) VALUES (2, 2, 'localhost', 57638, 'default', FALSE),(1, 1, 'localhost', 57637, 'default', FALSE) + CREATE SCHEMA IF NOT EXISTS mx_testing_schema + CREATE SEQUENCE IF NOT EXISTS mx_testing_schema.mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE + CREATE TABLE mx_testing_schema.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 integer DEFAULT nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) NOT NULL) + CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) + ALTER TABLE mx_testing_schema.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) + ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres + INSERT INTO pg_dist_partition (logicalrelid, partmethod, partkey, colocationid, repmodel) VALUES ('mx_testing_schema.mx_test_table'::regclass, 'h', column_name_to_column('mx_testing_schema.mx_test_table','col_1'), 0, 's') + INSERT INTO pg_dist_shard_placement (shardid, shardstate, shardlength, nodename, nodeport, placementid) VALUES (1310000, 1, 0, 'localhost', 57637, 100000),(1310001, 1, 0, 'localhost', 57638, 100001),(1310002, 1, 0, 'localhost', 57637, 100002),(1310003, 1, 0, 'localhost', 57638, 100003),(1310004, 1, 0, 'localhost', 57637, 100004),(1310005, 1, 0, 'localhost', 57638, 100005),(1310006, 1, 0, 'localhost', 57637, 100006),(1310007, 1, 0, 'localhost', 57638, 100007) + INSERT INTO pg_dist_shard (logicalrelid, shardid, shardstorage, shardminvalue, shardmaxvalue) VALUES ('mx_testing_schema.mx_test_table'::regclass, 1310000, 't', '-2147483648', '-1610612737'),('mx_testing_schema.mx_test_table'::regclass, 1310001, 't', '-1610612736', '-1073741825'),('mx_testing_schema.mx_test_table'::regclass, 1310002, 't', '-1073741824', '-536870913'),('mx_testing_schema.mx_test_table'::regclass, 1310003, 't', '-536870912', '-1'),('mx_testing_schema.mx_test_table'::regclass, 1310004, 't', '0', '536870911'),('mx_testing_schema.mx_test_table'::regclass, 1310005, 't', '536870912', '1073741823'),('mx_testing_schema.mx_test_table'::regclass, 1310006, 't', '1073741824', '1610612735'),('mx_testing_schema.mx_test_table'::regclass, 1310007, 't', '1610612736', '2147483647') +(12 rows) + +-- Show that append distributed tables are not included in the metadata snapshot +CREATE TABLE non_mx_test_table (col_1 int, col_2 text); +SELECT master_create_distributed_table('non_mx_test_table', 'col_1', 'append'); + master_create_distributed_table +--------------------------------- + +(1 row) + +UPDATE pg_dist_partition SET repmodel='s' WHERE logicalrelid='non_mx_test_table'::regclass; +SELECT unnest(master_metadata_snapshot()); + unnest +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + TRUNCATE pg_dist_node + SELECT worker_drop_distributed_table(logicalrelid) FROM pg_dist_partition + INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata) VALUES (2, 2, 'localhost', 57638, 'default', FALSE),(1, 1, 'localhost', 57637, 'default', FALSE) + CREATE SCHEMA IF NOT EXISTS mx_testing_schema + CREATE SEQUENCE IF NOT EXISTS mx_testing_schema.mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE + CREATE TABLE mx_testing_schema.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 integer DEFAULT nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) NOT NULL) + CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) + ALTER TABLE mx_testing_schema.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) + ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres + INSERT INTO pg_dist_partition (logicalrelid, partmethod, partkey, colocationid, repmodel) VALUES ('mx_testing_schema.mx_test_table'::regclass, 'h', column_name_to_column('mx_testing_schema.mx_test_table','col_1'), 0, 's') + INSERT INTO pg_dist_shard_placement (shardid, shardstate, shardlength, nodename, nodeport, placementid) VALUES (1310000, 1, 0, 'localhost', 57637, 100000),(1310001, 1, 0, 'localhost', 57638, 100001),(1310002, 1, 0, 'localhost', 57637, 100002),(1310003, 1, 0, 'localhost', 57638, 100003),(1310004, 1, 0, 'localhost', 57637, 100004),(1310005, 1, 0, 'localhost', 57638, 100005),(1310006, 1, 0, 'localhost', 57637, 100006),(1310007, 1, 0, 'localhost', 57638, 100007) + INSERT INTO pg_dist_shard (logicalrelid, shardid, shardstorage, shardminvalue, shardmaxvalue) VALUES ('mx_testing_schema.mx_test_table'::regclass, 1310000, 't', '-2147483648', '-1610612737'),('mx_testing_schema.mx_test_table'::regclass, 1310001, 't', '-1610612736', '-1073741825'),('mx_testing_schema.mx_test_table'::regclass, 1310002, 't', '-1073741824', '-536870913'),('mx_testing_schema.mx_test_table'::regclass, 1310003, 't', '-536870912', '-1'),('mx_testing_schema.mx_test_table'::regclass, 1310004, 't', '0', '536870911'),('mx_testing_schema.mx_test_table'::regclass, 1310005, 't', '536870912', '1073741823'),('mx_testing_schema.mx_test_table'::regclass, 1310006, 't', '1073741824', '1610612735'),('mx_testing_schema.mx_test_table'::regclass, 1310007, 't', '1610612736', '2147483647') +(12 rows) + +-- Show that range distributed tables are not included in the metadata snapshot +UPDATE pg_dist_partition SET partmethod='r' WHERE logicalrelid='non_mx_test_table'::regclass; +SELECT unnest(master_metadata_snapshot()); + unnest +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + TRUNCATE pg_dist_node + SELECT worker_drop_distributed_table(logicalrelid) FROM pg_dist_partition + INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata) VALUES (2, 2, 'localhost', 57638, 'default', FALSE),(1, 1, 'localhost', 57637, 'default', FALSE) + CREATE SCHEMA IF NOT EXISTS mx_testing_schema + CREATE SEQUENCE IF NOT EXISTS mx_testing_schema.mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE + CREATE TABLE mx_testing_schema.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 integer DEFAULT nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) NOT NULL) + CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) + ALTER TABLE mx_testing_schema.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) + ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres + INSERT INTO pg_dist_partition (logicalrelid, partmethod, partkey, colocationid, repmodel) VALUES ('mx_testing_schema.mx_test_table'::regclass, 'h', column_name_to_column('mx_testing_schema.mx_test_table','col_1'), 0, 's') + INSERT INTO pg_dist_shard_placement (shardid, shardstate, shardlength, nodename, nodeport, placementid) VALUES (1310000, 1, 0, 'localhost', 57637, 100000),(1310001, 1, 0, 'localhost', 57638, 100001),(1310002, 1, 0, 'localhost', 57637, 100002),(1310003, 1, 0, 'localhost', 57638, 100003),(1310004, 1, 0, 'localhost', 57637, 100004),(1310005, 1, 0, 'localhost', 57638, 100005),(1310006, 1, 0, 'localhost', 57637, 100006),(1310007, 1, 0, 'localhost', 57638, 100007) + INSERT INTO pg_dist_shard (logicalrelid, shardid, shardstorage, shardminvalue, shardmaxvalue) VALUES ('mx_testing_schema.mx_test_table'::regclass, 1310000, 't', '-2147483648', '-1610612737'),('mx_testing_schema.mx_test_table'::regclass, 1310001, 't', '-1610612736', '-1073741825'),('mx_testing_schema.mx_test_table'::regclass, 1310002, 't', '-1073741824', '-536870913'),('mx_testing_schema.mx_test_table'::regclass, 1310003, 't', '-536870912', '-1'),('mx_testing_schema.mx_test_table'::regclass, 1310004, 't', '0', '536870911'),('mx_testing_schema.mx_test_table'::regclass, 1310005, 't', '536870912', '1073741823'),('mx_testing_schema.mx_test_table'::regclass, 1310006, 't', '1073741824', '1610612735'),('mx_testing_schema.mx_test_table'::regclass, 1310007, 't', '1610612736', '2147483647') +(12 rows) + +ALTER SEQUENCE pg_catalog.pg_dist_shard_placement_placementid_seq RESTART :last_placement_id; diff --git a/src/test/regress/input/multi_subquery.source b/src/test/regress/input/multi_subquery.source index 48e497dc0..45c1597f2 100644 --- a/src/test/regress/input/multi_subquery.source +++ b/src/test/regress/input/multi_subquery.source @@ -702,7 +702,7 @@ LIMIT -- Same queries above with explain -- Simple join subquery pushdown -EXPLAIN SELECT +EXPLAIN (COSTS FALSE) SELECT avg(array_length(events, 1)) AS event_average FROM (SELECT @@ -729,7 +729,7 @@ FROM user_id) AS subquery; -- Union and left join subquery pushdown -EXPLAIN SELECT +EXPLAIN (COSTS FALSE) SELECT avg(array_length(events, 1)) AS event_average, hasdone FROM @@ -793,7 +793,7 @@ GROUP BY hasdone; -- Union, left join and having subquery pushdown -EXPLAIN SELECT +EXPLAIN (COSTS FALSE) SELECT avg(array_length(events, 1)) AS event_average, count_pay FROM ( @@ -865,7 +865,7 @@ ORDER BY count_pay; -- Lateral join subquery pushdown -EXPLAIN SELECT +EXPLAIN (COSTS FALSE) SELECT tenant_id, user_id, user_lastseen, diff --git a/src/test/regress/output/multi_subquery.source b/src/test/regress/output/multi_subquery.source index 65f8526d3..0efb3842a 100644 --- a/src/test/regress/output/multi_subquery.source +++ b/src/test/regress/output/multi_subquery.source @@ -740,7 +740,7 @@ LIMIT -- Same queries above with explain -- Simple join subquery pushdown -EXPLAIN SELECT +EXPLAIN (COSTS FALSE) SELECT avg(array_length(events, 1)) AS event_average FROM (SELECT @@ -773,26 +773,26 @@ FROM Tasks Shown: One of 2 -> Task Node: host=localhost port=57637 dbname=regression - -> Aggregate (cost=40.01..40.02 rows=1 width=16) - -> GroupAggregate (cost=39.89..39.99 rows=1 width=48) + -> Aggregate + -> GroupAggregate Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) - -> Merge Join (cost=39.89..39.97 rows=1 width=540) + -> Merge Join Merge Cond: ((((users.composite_id).tenant_id) = ((events.composite_id).tenant_id)) AND (((users.composite_id).user_id) = ((events.composite_id).user_id))) - -> Sort (cost=28.08..28.09 rows=6 width=32) + -> Sort Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) - -> Seq Scan on users_270013 users (cost=0.00..28.00 rows=6 width=32) + -> Seq Scan on users_270013 users Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) - -> Sort (cost=11.81..11.82 rows=3 width=556) + -> Sort Sort Key: ((events.composite_id).tenant_id), ((events.composite_id).user_id) - -> Seq Scan on events_270009 events (cost=0.00..11.79 rows=3 width=556) + -> Seq Scan on events_270009 events Filter: ((event_type)::text = ANY ('{click,submit,pay}'::text[])) Master Query - -> Aggregate (cost=0.00..0.00 rows=0 width=0) - -> Seq Scan on pg_merge_job_270014 (cost=0.00..0.00 rows=0 width=0) + -> Aggregate + -> Seq Scan on pg_merge_job_270014 (22 rows) -- Union and left join subquery pushdown -EXPLAIN SELECT +EXPLAIN (COSTS FALSE) SELECT avg(array_length(events, 1)) AS event_average, hasdone FROM @@ -862,47 +862,47 @@ GROUP BY Tasks Shown: One of 2 -> Task Node: host=localhost port=57637 dbname=regression - -> GroupAggregate (cost=91.93..91.98 rows=2 width=48) + -> GroupAggregate Group Key: subquery_top.hasdone - -> Sort (cost=91.93..91.93 rows=2 width=64) + -> Sort Sort Key: subquery_top.hasdone - -> Subquery Scan on subquery_top (cost=91.85..91.92 rows=2 width=64) - -> GroupAggregate (cost=91.85..91.90 rows=2 width=112) + -> Subquery Scan on subquery_top + -> GroupAggregate Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), ('Has done paying'::text) - -> Sort (cost=91.85..91.85 rows=2 width=88) + -> Sort Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), ('Has done paying'::text) - -> Merge Left Join (cost=91.75..91.84 rows=2 width=88) + -> Merge Left Join Merge Cond: ((((users.composite_id).tenant_id) = ((events_2.composite_id).tenant_id)) AND (((users.composite_id).user_id) = ((events_2.composite_id).user_id))) - -> Unique (cost=79.46..79.48 rows=2 width=56) - -> Sort (cost=79.46..79.47 rows=2 width=56) + -> Unique + -> Sort Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), ('action=>1'::text), events.event_time - -> Append (cost=0.00..79.45 rows=2 width=56) - -> Nested Loop (cost=0.00..39.72 rows=1 width=56) + -> Append + -> Nested Loop Join Filter: (((users.composite_id).tenant_id = (events.composite_id).tenant_id) AND ((users.composite_id).user_id = (events.composite_id).user_id)) - -> Seq Scan on events_270009 events (cost=0.00..11.62 rows=1 width=40) + -> Seq Scan on events_270009 events Filter: ((event_type)::text = 'click'::text) - -> Seq Scan on users_270013 users (cost=0.00..28.00 rows=6 width=32) + -> Seq Scan on users_270013 users Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) - -> Nested Loop (cost=0.00..39.72 rows=1 width=56) + -> Nested Loop Join Filter: (((users_1.composite_id).tenant_id = (events_1.composite_id).tenant_id) AND ((users_1.composite_id).user_id = (events_1.composite_id).user_id)) - -> Seq Scan on events_270009 events_1 (cost=0.00..11.62 rows=1 width=40) + -> Seq Scan on events_270009 events_1 Filter: ((event_type)::text = 'submit'::text) - -> Seq Scan on users_270013 users_1 (cost=0.00..28.00 rows=6 width=32) + -> Seq Scan on users_270013 users_1 Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) - -> Materialize (cost=12.29..12.31 rows=1 width=48) - -> Unique (cost=12.29..12.30 rows=1 width=80) - -> Sort (cost=12.29..12.29 rows=1 width=80) + -> Materialize + -> Unique + -> Sort Sort Key: ((events_2.composite_id).tenant_id), ((events_2.composite_id).user_id) - -> Seq Scan on events_270009 events_2 (cost=0.00..12.28 rows=1 width=80) + -> Seq Scan on events_270009 events_2 Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type) AND ((event_type)::text = 'pay'::text)) Master Query - -> HashAggregate (cost=0.00..0.00 rows=0 width=0) + -> HashAggregate Group Key: intermediate_column_270015_2 - -> Seq Scan on pg_merge_job_270015 (cost=0.00..0.00 rows=0 width=0) + -> Seq Scan on pg_merge_job_270015 (43 rows) -- Union, left join and having subquery pushdown -EXPLAIN SELECT +EXPLAIN (COSTS FALSE) SELECT avg(array_length(events, 1)) AS event_average, count_pay FROM ( @@ -974,7 +974,7 @@ ORDER BY count_pay; ERROR: bogus varattno for OUTER_VAR var: 3 -- Lateral join subquery pushdown -EXPLAIN SELECT +EXPLAIN (COSTS FALSE) SELECT tenant_id, user_id, user_lastseen, @@ -1031,29 +1031,29 @@ LIMIT Tasks Shown: One of 2 -> Task Node: host=localhost port=57637 dbname=regression - -> Limit (cost=100.43..100.44 rows=6 width=56) - -> Sort (cost=100.43..100.44 rows=6 width=56) + -> Limit + -> Sort Sort Key: (max(users.lastseen)) DESC - -> GroupAggregate (cost=100.14..100.29 rows=6 width=56) + -> GroupAggregate Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) - -> Sort (cost=100.14..100.16 rows=6 width=548) + -> Sort Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) - -> Nested Loop Left Join (cost=40.04..100.06 rows=6 width=548) - -> Limit (cost=28.08..28.09 rows=6 width=24) - -> Sort (cost=28.08..28.09 rows=6 width=24) + -> Nested Loop Left Join + -> Limit + -> Sort Sort Key: users.lastseen DESC - -> Seq Scan on users_270013 users (cost=0.00..28.00 rows=6 width=24) + -> Seq Scan on users_270013 users Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) - -> Limit (cost=11.96..11.96 rows=1 width=524) - -> Sort (cost=11.96..11.96 rows=1 width=524) + -> Limit + -> Sort Sort Key: events.event_time DESC - -> Seq Scan on events_270009 events (cost=0.00..11.95 rows=1 width=524) + -> Seq Scan on events_270009 events Filter: (((composite_id).tenant_id = ((users.composite_id).tenant_id)) AND ((composite_id).user_id = ((users.composite_id).user_id))) Master Query - -> Limit (cost=0.00..0.00 rows=0 width=0) - -> Sort (cost=0.00..0.00 rows=0 width=0) + -> Limit + -> Sort Sort Key: intermediate_column_270017_2 DESC - -> Seq Scan on pg_merge_job_270017 (cost=0.00..0.00 rows=0 width=0) + -> Seq Scan on pg_merge_job_270017 (29 rows) SET citusdb.task_executor_type TO 'real-time'; diff --git a/src/test/regress/output/multi_subquery_0.out b/src/test/regress/output/multi_subquery_0.out new file mode 100644 index 000000000..e3354ad1e --- /dev/null +++ b/src/test/regress/output/multi_subquery_0.out @@ -0,0 +1,1056 @@ +-- +-- MULTI_SUBQUERY +-- +ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 270000; +ALTER SEQUENCE pg_catalog.pg_dist_jobid_seq RESTART 270000; +-- print major version to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+\.\d+') AS major_version; + major_version +--------------- + 9.5 +(1 row) + +-- Create tables for subquery tests +CREATE TABLE lineitem_subquery ( + l_orderkey bigint not null, + l_partkey integer not null, + l_suppkey integer not null, + l_linenumber integer not null, + l_quantity decimal(15, 2) not null, + l_extendedprice decimal(15, 2) not null, + l_discount decimal(15, 2) not null, + l_tax decimal(15, 2) not null, + l_returnflag char(1) not null, + l_linestatus char(1) not null, + l_shipdate date not null, + l_commitdate date not null, + l_receiptdate date not null, + l_shipinstruct char(25) not null, + l_shipmode char(10) not null, + l_comment varchar(44) not null, + PRIMARY KEY(l_orderkey, l_linenumber) ); +SELECT master_create_distributed_table('lineitem_subquery', 'l_orderkey', 'range'); + master_create_distributed_table +--------------------------------- + +(1 row) + +CREATE TABLE orders_subquery ( + o_orderkey bigint not null, + o_custkey integer not null, + o_orderstatus char(1) not null, + o_totalprice decimal(15,2) not null, + o_orderdate date not null, + o_orderpriority char(15) not null, + o_clerk char(15) not null, + o_shippriority integer not null, + o_comment varchar(79) not null, + PRIMARY KEY(o_orderkey) ); +SELECT master_create_distributed_table('orders_subquery', 'o_orderkey', 'range'); + master_create_distributed_table +--------------------------------- + +(1 row) + +SET citus.task_executor_type TO 'task-tracker'; +-- Check that we don't allow subquery pushdown in default settings. +SELECT + avg(unit_price) +FROM + (SELECT + l_orderkey, + avg(o_totalprice) AS unit_price + FROM + lineitem_subquery, + orders_subquery + WHERE + l_orderkey = o_orderkey + GROUP BY + l_orderkey) AS unit_prices; +ERROR: cannot perform distributed planning on this query +DETAIL: Join in subqueries is not supported yet +SET citus.subquery_pushdown to TRUE; +-- Check that we don't crash if there are not any shards. +SELECT + avg(unit_price) +FROM + (SELECT + l_orderkey, + avg(o_totalprice) AS unit_price + FROM + lineitem_subquery, + orders_subquery + WHERE + l_orderkey = o_orderkey + GROUP BY + l_orderkey) AS unit_prices; + avg +----- + +(1 row) + +-- Load data into tables. +SELECT master_create_empty_shard('lineitem_subquery') AS new_shard_id +\gset +UPDATE pg_dist_shard SET shardminvalue = 1, shardmaxvalue = 5986 +WHERE shardid = :new_shard_id; +SELECT master_create_empty_shard('lineitem_subquery') AS new_shard_id +\gset +UPDATE pg_dist_shard SET shardminvalue = 8997, shardmaxvalue = 14947 +WHERE shardid = :new_shard_id; +SELECT master_create_empty_shard('orders_subquery') AS new_shard_id +\gset +UPDATE pg_dist_shard SET shardminvalue = 1, shardmaxvalue = 5986 +WHERE shardid = :new_shard_id; +SELECT master_create_empty_shard('orders_subquery') AS new_shard_id +\gset +UPDATE pg_dist_shard SET shardminvalue = 8997, shardmaxvalue = 14946 +WHERE shardid = :new_shard_id; +SET citus.shard_max_size TO "1MB"; +\copy lineitem_subquery FROM '@abs_srcdir@/data/lineitem.1.data' with delimiter '|' +\copy lineitem_subquery FROM '@abs_srcdir@/data/lineitem.2.data' with delimiter '|' +\copy orders_subquery FROM '@abs_srcdir@/data/orders.1.data' with delimiter '|' +\copy orders_subquery FROM '@abs_srcdir@/data/orders.2.data' with delimiter '|' +-- Check that we error out if shard min/max values are not exactly same. +SELECT + avg(unit_price) +FROM + (SELECT + l_orderkey, + avg(o_totalprice) AS unit_price + FROM + lineitem_subquery, + orders_subquery + WHERE + l_orderkey = o_orderkey + GROUP BY + l_orderkey) AS unit_prices; +ERROR: cannot push down this subquery +DETAIL: Shards of relations in subquery need to have 1-to-1 shard partitioning +-- Update metadata in order to make all shards equal. +UPDATE pg_dist_shard SET shardmaxvalue = '14947' WHERE shardid = 270003; +-- If group by is not on partition column then we error out. +SELECT + avg(order_count) +FROM + (SELECT + l_suppkey, + count(*) AS order_count + FROM + lineitem_subquery + GROUP BY + l_suppkey) AS order_counts; +ERROR: cannot push down this subquery +DETAIL: Group by list without partition column is currently unsupported +-- Check that we error out if join is not on partition columns. +SELECT + avg(unit_price) +FROM + (SELECT + l_orderkey, + avg(o_totalprice / l_quantity) AS unit_price + FROM + lineitem_subquery, + orders_subquery + GROUP BY + l_orderkey) AS unit_prices; +ERROR: cannot push down this subquery +DETAIL: Relations need to be joining on partition columns +SELECT + avg(unit_price) +FROM + (SELECT + l_orderkey, + avg(o_totalprice / l_quantity) AS unit_price + FROM + lineitem_subquery, + orders_subquery + WHERE + l_orderkey = o_custkey + GROUP BY + l_orderkey) AS unit_prices; +ERROR: cannot push down this subquery +DETAIL: Relations need to be joining on partition columns +-- Check that we error out if there is union all. +SELECT count(*) FROM +( + (SELECT l_orderkey FROM lineitem_subquery) UNION ALL + (SELECT 1::bigint) +) b; +ERROR: cannot perform distributed planning on this query +DETAIL: Complex table expressions are currently unsupported +-- Check that we error out if queries in union do not include partition columns. +SELECT count(*) FROM +( + (SELECT l_orderkey FROM lineitem_subquery) UNION + (SELECT l_partkey FROM lineitem_subquery) +) b; +ERROR: cannot push down this subquery +DETAIL: Union clauses need to select partition columns +-- Check that we run union queries if partition column is selected. +SELECT count(*) FROM +( + (SELECT l_orderkey FROM lineitem_subquery) UNION + (SELECT l_orderkey FROM lineitem_subquery) +) b; + count +------- + 2985 +(1 row) + +-- Check that we error out if the outermost query has subquery join. +SELECT + avg(o_totalprice/l_quantity) +FROM + (SELECT + l_orderkey, + l_quantity + FROM + lineitem_subquery + ORDER BY + l_quantity + LIMIT 10 + ) lineitem_quantities + JOIN LATERAL + (SELECT + o_totalprice + FROM + orders_subquery + WHERE + lineitem_quantities.l_orderkey = o_orderkey) orders_price ON true; +ERROR: cannot perform distributed planning on this query +DETAIL: Join in subqueries is not supported yet +-- Check that we error out if the outermost query is a distinct clause. +SELECT + count(DISTINCT a) +FROM ( + SELECT + count(*) a + FROM + lineitem_subquery +) z; +ERROR: cannot push down this subquery +DETAIL: distinct in the outermost query is unsupported +-- Check supported subquery types. +SELECT + o_custkey, + sum(order_count) as total_order_count +FROM + (SELECT + o_orderkey, + o_custkey, + count(*) AS order_count + FROM + orders_subquery + WHERE + o_orderkey > 0 AND + o_orderkey < 12000 + GROUP BY + o_orderkey, o_custkey) AS order_counts +GROUP BY + o_custkey +ORDER BY + total_order_count DESC, + o_custkey ASC +LIMIT 10; + o_custkey | total_order_count +-----------+------------------- + 1462 | 9 + 619 | 8 + 643 | 8 + 1030 | 8 + 1486 | 8 + 79 | 7 + 304 | 7 + 319 | 7 + 343 | 7 + 448 | 7 +(10 rows) + +SELECT + avg(unit_price) +FROM + (SELECT + l_orderkey, + avg(o_totalprice / l_quantity) AS unit_price + FROM + lineitem_subquery, + orders_subquery + WHERE + l_orderkey = o_orderkey + GROUP BY + l_orderkey) AS unit_prices +WHERE + unit_price > 1000 AND + unit_price < 10000; + avg +----------------------- + 4968.2889885208475549 +(1 row) + +-- Check that if subquery is pulled, we don't error and run query properly. +SELECT count(*) FROM +( + SELECT l_orderkey FROM ( + (SELECT l_orderkey FROM lineitem_subquery) UNION + (SELECT l_orderkey FROM lineitem_subquery) + ) a + WHERE l_orderkey = 1 +) b; + count +------- + 1 +(1 row) + +SELECT count(*) FROM +( + SELECT * FROM ( + (SELECT * FROM lineitem_subquery) UNION + (SELECT * FROM lineitem_subquery) + ) a + WHERE l_orderkey = 1 +) b; + count +------- + 6 +(1 row) + +SELECT max(l_orderkey) FROM +( + SELECT l_orderkey FROM ( + SELECT + l_orderkey + FROM + lineitem_subquery + WHERE + l_orderkey < 20000 + GROUP BY + l_orderkey + ) z +) y; + max +------- + 14947 +(1 row) + +-- Add one more shard to one relation, then test if we error out because of different +-- shard counts for joining relations. +SELECT master_create_empty_shard('orders_subquery') AS new_shard_id +\gset +UPDATE pg_dist_shard SET shardminvalue = 15000, shardmaxvalue = 20000 +WHERE shardid = :new_shard_id; +SELECT + avg(unit_price) +FROM + (SELECT + l_orderkey, + avg(o_totalprice / l_quantity) AS unit_price + FROM + lineitem_subquery, + orders_subquery + WHERE + l_orderkey = o_orderkey + GROUP BY + l_orderkey) AS unit_prices; +ERROR: cannot push down this subquery +DETAIL: Shards of relations in subquery need to have 1-to-1 shard partitioning +-- Check that we can prune shards in subqueries with VARCHAR partition columns +CREATE TABLE subquery_pruning_varchar_test_table +( + a varchar, + b int +); +SELECT master_create_distributed_table('subquery_pruning_varchar_test_table', 'a', 'hash'); + master_create_distributed_table +--------------------------------- + +(1 row) + +SELECT master_create_worker_shards('subquery_pruning_varchar_test_table', 4, 1); + master_create_worker_shards +----------------------------- + +(1 row) + +SET citus.subquery_pushdown TO TRUE; +SET client_min_messages TO DEBUG2; +SELECT * FROM + (SELECT count(*) FROM subquery_pruning_varchar_test_table WHERE a = 'onder' GROUP BY a) +AS foo; +DEBUG: predicate pruning for shardId 270005 +DEBUG: predicate pruning for shardId 270006 +DEBUG: predicate pruning for shardId 270008 + count +------- +(0 rows) + +SELECT * FROM + (SELECT count(*) FROM subquery_pruning_varchar_test_table WHERE 'eren' = a GROUP BY a) +AS foo; +DEBUG: predicate pruning for shardId 270005 +DEBUG: predicate pruning for shardId 270007 +DEBUG: predicate pruning for shardId 270008 + count +------- +(0 rows) + +SET client_min_messages TO NOTICE; +-- test subquery join on VARCHAR partition column +SELECT * FROM + (SELECT + a_inner AS a + FROM + (SELECT + subquery_pruning_varchar_test_table.a AS a_inner + FROM + subquery_pruning_varchar_test_table + GROUP BY + subquery_pruning_varchar_test_table.a + HAVING + count(subquery_pruning_varchar_test_table.a) < 3) + AS f1, + (SELECT + subquery_pruning_varchar_test_table.a + FROM + subquery_pruning_varchar_test_table + GROUP BY + subquery_pruning_varchar_test_table.a + HAVING + sum(coalesce(subquery_pruning_varchar_test_table.b,0)) > 20.0) + AS f2 + WHERE + f1.a_inner = f2.a + GROUP BY + a_inner) +AS foo; + a +--- +(0 rows) + +DROP TABLE subquery_pruning_varchar_test_table; +-- Create composite type to use in subquery pushdown +CREATE TYPE user_composite_type AS +( + tenant_id BIGINT, + user_id BIGINT +); +\c - - - :worker_1_port +CREATE TYPE user_composite_type AS +( + tenant_id BIGINT, + user_id BIGINT +); +\c - - - :worker_2_port +CREATE TYPE user_composite_type AS +( + tenant_id BIGINT, + user_id BIGINT +); +\c - - - :master_port +CREATE TABLE events ( + composite_id user_composite_type, + event_id bigint, + event_type character varying(255), + event_time bigint +); +SELECT master_create_distributed_table('events', 'composite_id', 'range'); + master_create_distributed_table +--------------------------------- + +(1 row) + +SELECT master_create_empty_shard('events') AS new_shard_id +\gset +UPDATE pg_dist_shard SET shardminvalue = '(1,1)', shardmaxvalue = '(1,2000000000)' +WHERE shardid = :new_shard_id; +SELECT master_create_empty_shard('events') AS new_shard_id +\gset +UPDATE pg_dist_shard SET shardminvalue = '(1,2000000001)', shardmaxvalue = '(1,4300000000)' +WHERE shardid = :new_shard_id; +SELECT master_create_empty_shard('events') AS new_shard_id +\gset +UPDATE pg_dist_shard SET shardminvalue = '(2,1)', shardmaxvalue = '(2,2000000000)' +WHERE shardid = :new_shard_id; +SELECT master_create_empty_shard('events') AS new_shard_id +\gset +UPDATE pg_dist_shard SET shardminvalue = '(2,2000000001)', shardmaxvalue = '(2,4300000000)' +WHERE shardid = :new_shard_id; +\COPY events FROM STDIN WITH CSV +CREATE TABLE users ( + composite_id user_composite_type, + lastseen bigint +); +SELECT master_create_distributed_table('users', 'composite_id', 'range'); + master_create_distributed_table +--------------------------------- + +(1 row) + +SELECT master_create_empty_shard('users') AS new_shard_id +\gset +UPDATE pg_dist_shard SET shardminvalue = '(1,1)', shardmaxvalue = '(1,2000000000)' +WHERE shardid = :new_shard_id; +SELECT master_create_empty_shard('users') AS new_shard_id +\gset +UPDATE pg_dist_shard SET shardminvalue = '(1,2000000001)', shardmaxvalue = '(1,4300000000)' +WHERE shardid = :new_shard_id; +SELECT master_create_empty_shard('users') AS new_shard_id +\gset +UPDATE pg_dist_shard SET shardminvalue = '(2,1)', shardmaxvalue = '(2,2000000000)' +WHERE shardid = :new_shard_id; +SELECT master_create_empty_shard('users') AS new_shard_id +\gset +UPDATE pg_dist_shard SET shardminvalue = '(2,2000000001)', shardmaxvalue = '(2,4300000000)' +WHERE shardid = :new_shard_id; +\COPY users FROM STDIN WITH CSV +SET citus.subquery_pushdown TO TRUE; +-- Simple join subquery pushdown +SELECT + avg(array_length(events, 1)) AS event_average +FROM + (SELECT + tenant_id, + user_id, + array_agg(event_type ORDER BY event_time) AS events + FROM + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + event_type, + events.event_time + FROM + users, + events + WHERE + (users.composite_id).tenant_id = (events.composite_id).tenant_id AND + (users.composite_id).user_id = (events.composite_id).user_id AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type IN ('click', 'submit', 'pay')) AS subquery + GROUP BY + tenant_id, + user_id) AS subquery; + event_average +-------------------- + 3.6666666666666667 +(1 row) + +-- Union and left join subquery pushdown +SELECT + avg(array_length(events, 1)) AS event_average, + hasdone +FROM + (SELECT + subquery_1.tenant_id, + subquery_1.user_id, + array_agg(event ORDER BY event_time) AS events, + COALESCE(hasdone, 'Has not done paying') AS hasdone + FROM + ( + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + 'action=>1'AS event, + events.event_time + FROM + users, + events + WHERE + (users.composite_id).tenant_id = (events.composite_id).tenant_id AND + (users.composite_id).user_id = (events.composite_id).user_id AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'click') + UNION + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + 'action=>2'AS event, + events.event_time + FROM + users, + events + WHERE + (users.composite_id).tenant_id = (events.composite_id).tenant_id AND + (users.composite_id).user_id = (events.composite_id).user_id AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'submit') + ) AS subquery_1 + LEFT JOIN + (SELECT + DISTINCT ON ((composite_id).tenant_id, (composite_id).user_id) composite_id, + (composite_id).tenant_id, + (composite_id).user_id, + 'Has done paying'::TEXT AS hasdone + FROM + events + WHERE + events.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + events.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'pay') AS subquery_2 + ON + subquery_1.tenant_id = subquery_2.tenant_id AND + subquery_1.user_id = subquery_2.user_id + GROUP BY + subquery_1.tenant_id, + subquery_1.user_id, + hasdone) AS subquery_top +GROUP BY + hasdone; + event_average | hasdone +--------------------+--------------------- + 4.0000000000000000 | Has not done paying + 2.5000000000000000 | Has done paying +(2 rows) + +-- Union, left join and having subquery pushdown +SELECT + avg(array_length(events, 1)) AS event_average, + count_pay + FROM ( + SELECT + subquery_1.tenant_id, + subquery_1.user_id, + array_agg(event ORDER BY event_time) AS events, + COALESCE(count_pay, 0) AS count_pay + FROM + ( + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + 'action=>1'AS event, + events.event_time + FROM + users, + events + WHERE + (users.composite_id).tenant_id = (events.composite_id).tenant_id AND + (users.composite_id).user_id = (events.composite_id).user_id AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'click') + UNION + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + 'action=>2'AS event, + events.event_time + FROM + users, + events + WHERE + (users.composite_id).tenant_id = (events.composite_id).tenant_id AND + (users.composite_id).user_id = (events.composite_id).user_id AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'submit') + ) AS subquery_1 + LEFT JOIN + (SELECT + (composite_id).tenant_id, + (composite_id).user_id, + COUNT(*) AS count_pay + FROM + events + WHERE + events.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + events.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'pay' + GROUP BY + tenant_id, + user_id + HAVING + COUNT(*) > 2) AS subquery_2 + ON + subquery_1.tenant_id = subquery_2.tenant_id AND + subquery_1.user_id = subquery_2.user_id + GROUP BY + subquery_1.tenant_id, + subquery_1.user_id, + count_pay) AS subquery_top +WHERE + array_ndims(events) > 0 +GROUP BY + count_pay +ORDER BY + count_pay; + event_average | count_pay +--------------------+----------- + 3.0000000000000000 | 0 +(1 row) + +-- Lateral join subquery pushdown +SELECT + tenant_id, + user_id, + user_lastseen, + event_array +FROM + (SELECT + tenant_id, + user_id, + max(lastseen) as user_lastseen, + array_agg(event_type ORDER BY event_time) AS event_array + FROM + (SELECT + (composite_id).tenant_id, + (composite_id).user_id, + lastseen + FROM + users + WHERE + composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + composite_id <= '(1, 9223372036854775807)'::user_composite_type + ORDER BY + lastseen DESC + LIMIT + 10 + ) AS subquery_top + LEFT JOIN LATERAL + (SELECT + event_type, + event_time + FROM + events + WHERE + (composite_id).tenant_id = subquery_top.tenant_id AND + (composite_id).user_id = subquery_top.user_id + ORDER BY + event_time DESC + LIMIT + 99) AS subquery_lateral + ON + true + GROUP BY + tenant_id, + user_id + ) AS shard_union +ORDER BY + user_lastseen DESC +LIMIT + 10; + tenant_id | user_id | user_lastseen | event_array +-----------+---------+---------------+---------------------------- + 1 | 1003 | 1472807315 | {click,click,click,submit} + 1 | 1002 | 1472807215 | {click,click,submit,pay} + 1 | 1001 | 1472807115 | {click,submit,pay} +(3 rows) + +-- Same queries above with explain +-- Simple join subquery pushdown +EXPLAIN (COSTS FALSE) SELECT + avg(array_length(events, 1)) AS event_average +FROM + (SELECT + tenant_id, + user_id, + array_agg(event_type ORDER BY event_time) AS events + FROM + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + event_type, + events.event_time + FROM + users, + events + WHERE + (users.composite_id).tenant_id = (events.composite_id).tenant_id AND + (users.composite_id).user_id = (events.composite_id).user_id AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type IN ('click', 'submit', 'pay')) AS subquery + GROUP BY + tenant_id, + user_id) AS subquery; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Distributed Query into pg_merge_job_270014 + Executor: Real-Time + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> GroupAggregate + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) + -> Merge Join + Merge Cond: ((((users.composite_id).tenant_id) = ((events.composite_id).tenant_id)) AND (((users.composite_id).user_id) = ((events.composite_id).user_id))) + -> Sort + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) + -> Seq Scan on users_270013 users + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) + -> Sort + Sort Key: ((events.composite_id).tenant_id), ((events.composite_id).user_id) + -> Seq Scan on events_270009 events + Filter: ((event_type)::text = ANY ('{click,submit,pay}'::text[])) + Master Query + -> Aggregate + -> Seq Scan on pg_merge_job_270014 +(22 rows) + +-- Union and left join subquery pushdown +EXPLAIN (COSTS FALSE) SELECT + avg(array_length(events, 1)) AS event_average, + hasdone +FROM + (SELECT + subquery_1.tenant_id, + subquery_1.user_id, + array_agg(event ORDER BY event_time) AS events, + COALESCE(hasdone, 'Has not done paying') AS hasdone + FROM + ( + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + 'action=>1'AS event, + events.event_time + FROM + users, + events + WHERE + (users.composite_id).tenant_id = (events.composite_id).tenant_id AND + (users.composite_id).user_id = (events.composite_id).user_id AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'click') + UNION + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + 'action=>2'AS event, + events.event_time + FROM + users, + events + WHERE + (users.composite_id).tenant_id = (events.composite_id).tenant_id AND + (users.composite_id).user_id = (events.composite_id).user_id AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'submit') + ) AS subquery_1 + LEFT JOIN + (SELECT + DISTINCT ON ((composite_id).tenant_id, (composite_id).user_id) composite_id, + (composite_id).tenant_id, + (composite_id).user_id, + 'Has done paying'::TEXT AS hasdone + FROM + events + WHERE + events.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + events.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'pay') AS subquery_2 + ON + subquery_1.tenant_id = subquery_2.tenant_id AND + subquery_1.user_id = subquery_2.user_id + GROUP BY + subquery_1.tenant_id, + subquery_1.user_id, + hasdone) AS subquery_top +GROUP BY + hasdone; + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Distributed Query into pg_merge_job_270015 + Executor: Real-Time + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=57637 dbname=regression + -> HashAggregate + Group Key: COALESCE(('Has done paying'::text), 'Has not done paying'::text) + -> GroupAggregate + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), ('Has done paying'::text) + -> Sort + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), ('Has done paying'::text) + -> Merge Left Join + Merge Cond: ((((users.composite_id).tenant_id) = ((events_2.composite_id).tenant_id)) AND (((users.composite_id).user_id) = ((events_2.composite_id).user_id))) + -> Unique + -> Sort + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), ('action=>1'::text), events.event_time + -> Append + -> Nested Loop + Join Filter: (((users.composite_id).tenant_id = (events.composite_id).tenant_id) AND ((users.composite_id).user_id = (events.composite_id).user_id)) + -> Seq Scan on events_270009 events + Filter: ((event_type)::text = 'click'::text) + -> Seq Scan on users_270013 users + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) + -> Nested Loop + Join Filter: (((users_1.composite_id).tenant_id = (events_1.composite_id).tenant_id) AND ((users_1.composite_id).user_id = (events_1.composite_id).user_id)) + -> Seq Scan on events_270009 events_1 + Filter: ((event_type)::text = 'submit'::text) + -> Seq Scan on users_270013 users_1 + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) + -> Materialize + -> Unique + -> Sort + Sort Key: ((events_2.composite_id).tenant_id), ((events_2.composite_id).user_id) + -> Seq Scan on events_270009 events_2 + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type) AND ((event_type)::text = 'pay'::text)) + Master Query + -> HashAggregate + Group Key: intermediate_column_270015_2 + -> Seq Scan on pg_merge_job_270015 +(40 rows) + +-- Union, left join and having subquery pushdown +EXPLAIN (COSTS FALSE) SELECT + avg(array_length(events, 1)) AS event_average, + count_pay + FROM ( + SELECT + subquery_1.tenant_id, + subquery_1.user_id, + array_agg(event ORDER BY event_time) AS events, + COALESCE(count_pay, 0) AS count_pay + FROM + ( + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + 'action=>1'AS event, + events.event_time + FROM + users, + events + WHERE + (users.composite_id).tenant_id = (events.composite_id).tenant_id AND + (users.composite_id).user_id = (events.composite_id).user_id AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'click') + UNION + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + 'action=>2'AS event, + events.event_time + FROM + users, + events + WHERE + (users.composite_id).tenant_id = (events.composite_id).tenant_id AND + (users.composite_id).user_id = (events.composite_id).user_id AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'submit') + ) AS subquery_1 + LEFT JOIN + (SELECT + (composite_id).tenant_id, + (composite_id).user_id, + COUNT(*) AS count_pay + FROM + events + WHERE + events.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + events.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'pay' + GROUP BY + tenant_id, + user_id + HAVING + COUNT(*) > 2) AS subquery_2 + ON + subquery_1.tenant_id = subquery_2.tenant_id AND + subquery_1.user_id = subquery_2.user_id + GROUP BY + subquery_1.tenant_id, + subquery_1.user_id, + count_pay) AS subquery_top +WHERE + array_ndims(events) > 0 +GROUP BY + count_pay +ORDER BY + count_pay; +ERROR: bogus varattno for OUTER_VAR var: 3 +-- Lateral join subquery pushdown +EXPLAIN (COSTS FALSE) SELECT + tenant_id, + user_id, + user_lastseen, + event_array +FROM + (SELECT + tenant_id, + user_id, + max(lastseen) as user_lastseen, + array_agg(event_type ORDER BY event_time) AS event_array + FROM + (SELECT + (composite_id).tenant_id, + (composite_id).user_id, + lastseen + FROM + users + WHERE + composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + composite_id <= '(1, 9223372036854775807)'::user_composite_type + ORDER BY + lastseen DESC + LIMIT + 10 + ) AS subquery_top + LEFT JOIN LATERAL + (SELECT + event_type, + event_time + FROM + events + WHERE + (composite_id).tenant_id = subquery_top.tenant_id AND + (composite_id).user_id = subquery_top.user_id + ORDER BY + event_time DESC + LIMIT + 99) AS subquery_lateral + ON + true + GROUP BY + tenant_id, + user_id + ) AS shard_union +ORDER BY + user_lastseen DESC +LIMIT + 10; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Distributed Query into pg_merge_job_270017 + Executor: Real-Time + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=57637 dbname=regression + -> Limit + -> Sort + Sort Key: (max(users.lastseen)) DESC + -> GroupAggregate + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) + -> Sort + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) + -> Nested Loop Left Join + -> Limit + -> Sort + Sort Key: users.lastseen DESC + -> Seq Scan on users_270013 users + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) + -> Limit + -> Sort + Sort Key: events.event_time DESC + -> Seq Scan on events_270009 events + Filter: (((composite_id).tenant_id = ((users.composite_id).tenant_id)) AND ((composite_id).user_id = ((users.composite_id).user_id))) + Master Query + -> Limit + -> Sort + Sort Key: intermediate_column_270017_2 DESC + -> Seq Scan on pg_merge_job_270017 +(29 rows) + +SET citusdb.task_executor_type TO 'real-time'; diff --git a/src/test/regress/output/multi_subquery_0.source b/src/test/regress/output/multi_subquery_0.source index 771942193..e3354ad1e 100644 --- a/src/test/regress/output/multi_subquery_0.source +++ b/src/test/regress/output/multi_subquery_0.source @@ -740,7 +740,7 @@ LIMIT -- Same queries above with explain -- Simple join subquery pushdown -EXPLAIN SELECT +EXPLAIN (COSTS FALSE) SELECT avg(array_length(events, 1)) AS event_average FROM (SELECT @@ -773,26 +773,26 @@ FROM Tasks Shown: One of 2 -> Task Node: host=localhost port=57637 dbname=regression - -> Aggregate (cost=40.01..40.02 rows=1 width=32) - -> GroupAggregate (cost=39.89..39.99 rows=1 width=556) + -> Aggregate + -> GroupAggregate Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) - -> Merge Join (cost=39.89..39.97 rows=1 width=556) + -> Merge Join Merge Cond: ((((users.composite_id).tenant_id) = ((events.composite_id).tenant_id)) AND (((users.composite_id).user_id) = ((events.composite_id).user_id))) - -> Sort (cost=28.08..28.09 rows=6 width=32) + -> Sort Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) - -> Seq Scan on users_270013 users (cost=0.00..28.00 rows=6 width=32) + -> Seq Scan on users_270013 users Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) - -> Sort (cost=11.81..11.82 rows=3 width=556) + -> Sort Sort Key: ((events.composite_id).tenant_id), ((events.composite_id).user_id) - -> Seq Scan on events_270009 events (cost=0.00..11.79 rows=3 width=556) + -> Seq Scan on events_270009 events Filter: ((event_type)::text = ANY ('{click,submit,pay}'::text[])) Master Query - -> Aggregate (cost=0.01..0.02 rows=1 width=0) - -> Seq Scan on pg_merge_job_270014 (cost=0.00..0.00 rows=0 width=0) + -> Aggregate + -> Seq Scan on pg_merge_job_270014 (22 rows) -- Union and left join subquery pushdown -EXPLAIN SELECT +EXPLAIN (COSTS FALSE) SELECT avg(array_length(events, 1)) AS event_average, hasdone FROM @@ -862,44 +862,44 @@ GROUP BY Tasks Shown: One of 2 -> Task Node: host=localhost port=57637 dbname=regression - -> HashAggregate (cost=91.94..91.96 rows=2 width=64) + -> HashAggregate Group Key: COALESCE(('Has done paying'::text), 'Has not done paying'::text) - -> GroupAggregate (cost=91.85..91.90 rows=2 width=88) + -> GroupAggregate Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), ('Has done paying'::text) - -> Sort (cost=91.85..91.85 rows=2 width=88) + -> Sort Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), ('Has done paying'::text) - -> Merge Left Join (cost=91.75..91.84 rows=2 width=88) + -> Merge Left Join Merge Cond: ((((users.composite_id).tenant_id) = ((events_2.composite_id).tenant_id)) AND (((users.composite_id).user_id) = ((events_2.composite_id).user_id))) - -> Unique (cost=79.46..79.48 rows=2 width=40) - -> Sort (cost=79.46..79.47 rows=2 width=40) + -> Unique + -> Sort Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), ('action=>1'::text), events.event_time - -> Append (cost=0.00..79.45 rows=2 width=40) - -> Nested Loop (cost=0.00..39.72 rows=1 width=40) + -> Append + -> Nested Loop Join Filter: (((users.composite_id).tenant_id = (events.composite_id).tenant_id) AND ((users.composite_id).user_id = (events.composite_id).user_id)) - -> Seq Scan on events_270009 events (cost=0.00..11.62 rows=1 width=40) + -> Seq Scan on events_270009 events Filter: ((event_type)::text = 'click'::text) - -> Seq Scan on users_270013 users (cost=0.00..28.00 rows=6 width=32) + -> Seq Scan on users_270013 users Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) - -> Nested Loop (cost=0.00..39.72 rows=1 width=40) + -> Nested Loop Join Filter: (((users_1.composite_id).tenant_id = (events_1.composite_id).tenant_id) AND ((users_1.composite_id).user_id = (events_1.composite_id).user_id)) - -> Seq Scan on events_270009 events_1 (cost=0.00..11.62 rows=1 width=40) + -> Seq Scan on events_270009 events_1 Filter: ((event_type)::text = 'submit'::text) - -> Seq Scan on users_270013 users_1 (cost=0.00..28.00 rows=6 width=32) + -> Seq Scan on users_270013 users_1 Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) - -> Materialize (cost=12.29..12.31 rows=1 width=48) - -> Unique (cost=12.29..12.30 rows=1 width=32) - -> Sort (cost=12.29..12.29 rows=1 width=32) + -> Materialize + -> Unique + -> Sort Sort Key: ((events_2.composite_id).tenant_id), ((events_2.composite_id).user_id) - -> Seq Scan on events_270009 events_2 (cost=0.00..12.28 rows=1 width=32) + -> Seq Scan on events_270009 events_2 Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type) AND ((event_type)::text = 'pay'::text)) Master Query - -> HashAggregate (cost=0.00..0.18 rows=10 width=0) + -> HashAggregate Group Key: intermediate_column_270015_2 - -> Seq Scan on pg_merge_job_270015 (cost=0.00..0.00 rows=0 width=0) + -> Seq Scan on pg_merge_job_270015 (40 rows) -- Union, left join and having subquery pushdown -EXPLAIN SELECT +EXPLAIN (COSTS FALSE) SELECT avg(array_length(events, 1)) AS event_average, count_pay FROM ( @@ -971,7 +971,7 @@ ORDER BY count_pay; ERROR: bogus varattno for OUTER_VAR var: 3 -- Lateral join subquery pushdown -EXPLAIN SELECT +EXPLAIN (COSTS FALSE) SELECT tenant_id, user_id, user_lastseen, @@ -1028,29 +1028,29 @@ LIMIT Tasks Shown: One of 2 -> Task Node: host=localhost port=57637 dbname=regression - -> Limit (cost=100.43..100.44 rows=6 width=56) - -> Sort (cost=100.43..100.44 rows=6 width=56) + -> Limit + -> Sort Sort Key: (max(users.lastseen)) DESC - -> GroupAggregate (cost=100.14..100.29 rows=6 width=548) + -> GroupAggregate Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) - -> Sort (cost=100.14..100.16 rows=6 width=548) + -> Sort Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) - -> Nested Loop Left Join (cost=40.04..100.06 rows=6 width=548) - -> Limit (cost=28.08..28.09 rows=6 width=40) - -> Sort (cost=28.08..28.09 rows=6 width=40) + -> Nested Loop Left Join + -> Limit + -> Sort Sort Key: users.lastseen DESC - -> Seq Scan on users_270013 users (cost=0.00..28.00 rows=6 width=40) + -> Seq Scan on users_270013 users Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) - -> Limit (cost=11.96..11.96 rows=1 width=524) - -> Sort (cost=11.96..11.96 rows=1 width=524) + -> Limit + -> Sort Sort Key: events.event_time DESC - -> Seq Scan on events_270009 events (cost=0.00..11.95 rows=1 width=524) + -> Seq Scan on events_270009 events Filter: (((composite_id).tenant_id = ((users.composite_id).tenant_id)) AND ((composite_id).user_id = ((users.composite_id).user_id))) Master Query - -> Limit (cost=0.01..0.02 rows=0 width=0) - -> Sort (cost=0.01..0.02 rows=0 width=0) + -> Limit + -> Sort Sort Key: intermediate_column_270017_2 DESC - -> Seq Scan on pg_merge_job_270017 (cost=0.00..0.00 rows=0 width=0) + -> Seq Scan on pg_merge_job_270017 (29 rows) SET citusdb.task_executor_type TO 'real-time'; diff --git a/src/test/regress/sql/multi_explain.sql b/src/test/regress/sql/multi_explain.sql index aa7eeb023..fc2cde54c 100644 --- a/src/test/regress/sql/multi_explain.sql +++ b/src/test/regress/sql/multi_explain.sql @@ -209,7 +209,7 @@ EXPLAIN (COSTS FALSE) EXECUTE task_tracker_query; SET citus.task_executor_type TO 'real-time'; PREPARE router_executor_query AS SELECT l_quantity FROM lineitem WHERE l_orderkey = 5; -EXPLAIN EXECUTE router_executor_query; +EXPLAIN (COSTS FALSE) EXECUTE router_executor_query; PREPARE real_time_executor_query AS SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030;