external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3111 from citusdata/refresh_materialized_view_with_subquery 

Refresh Materialized View with Subquery
pull/3115/head
Halil Ozan Akgül 2019-10-17 16:07:35 +03:00 committed by GitHub
parent e065933928 5f04ac774f
commit 210a6cc04b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 351 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

202
src/test/regress/expected/materialized_view.out Normal file
View File

@ -0,0 +1,202 @@
---
--- materialized_view
---
-- This file contains test cases for materialized view support.
-- materialized views work
-- insert into... select works with views
CREATE SCHEMA materialized_view;
SET search_path TO materialized_view, public;
CREATE VIEW air_shipped_lineitems AS SELECT * FROM lineitem_hash_part WHERE l_shipmode = 'AIR';
CREATE TABLE temp_lineitem(LIKE lineitem_hash_part);
SELECT create_distributed_table('temp_lineitem', 'l_orderkey', 'hash', 'lineitem_hash_part');
create_distributed_table
--------------------------
(1 row)
INSERT INTO temp_lineitem SELECT * FROM air_shipped_lineitems;
SELECT count(*) FROM temp_lineitem;
count
-------
1706
(1 row)
-- following is a where false query, should not be inserting anything
INSERT INTO temp_lineitem SELECT * FROM air_shipped_lineitems WHERE l_shipmode = 'MAIL';
SELECT count(*) FROM temp_lineitem;
count
-------
1706
(1 row)
-- can create and query materialized views
CREATE MATERIALIZED VIEW mode_counts
AS SELECT l_shipmode, count(*) FROM temp_lineitem GROUP BY l_shipmode;
SELECT * FROM mode_counts WHERE l_shipmode = 'AIR' ORDER BY 2 DESC, 1 LIMIT 10;
l_shipmode | count
------------+-------
AIR | 1706
(1 row)
-- materialized views are local, cannot join with distributed tables
SELECT count(*) FROM mode_counts JOIN temp_lineitem USING (l_shipmode);
ERROR: relation mode_counts is not distributed
-- new data is not immediately reflected in the view
INSERT INTO temp_lineitem SELECT * FROM air_shipped_lineitems;
SELECT * FROM mode_counts WHERE l_shipmode = 'AIR' ORDER BY 2 DESC, 1 LIMIT 10;
l_shipmode | count
------------+-------
AIR | 1706
(1 row)
-- refresh updates the materialised view with new data
REFRESH MATERIALIZED VIEW mode_counts;
SELECT * FROM mode_counts WHERE l_shipmode = 'AIR' ORDER BY 2 DESC, 1 LIMIT 10;
l_shipmode | count
------------+-------
AIR | 3412
(1 row)
DROP MATERIALIZED VIEW mode_counts;
DROP TABLE temp_lineitem CASCADE;
-- Refresh single-shard materialized view
CREATE MATERIALIZED VIEW materialized_view AS
SELECT orders_hash_part.o_orderdate, total_price.price_sum
FROM lineitem_hash_part, orders_hash_part, (SELECT SUM(l_extendedprice) AS price_sum FROM lineitem_hash_part where l_orderkey=3) AS total_price
WHERE lineitem_hash_part.l_orderkey=orders_hash_part.o_orderkey AND lineitem_hash_part.l_orderkey=3;
REFRESH MATERIALIZED VIEW materialized_view;
SELECT count(*) FROM materialized_view;
count
-------
6
(1 row)
DROP MATERIALIZED VIEW materialized_view;
-- Refresh multi-shard materialized view
CREATE MATERIALIZED VIEW materialized_view AS
SELECT orders_hash_part.o_orderdate, total_price.price_sum
FROM lineitem_hash_part, orders_hash_part, (SELECT SUM(l_extendedprice) AS price_sum FROM lineitem_hash_part) AS total_price
WHERE lineitem_hash_part.l_orderkey=orders_hash_part.o_orderkey;
REFRESH MATERIALIZED VIEW materialized_view;
SELECT count(*) FROM materialized_view;
count
-------
12000
(1 row)
DROP MATERIALIZED VIEW materialized_view;
-- Refresh materialized view with CTE
CREATE MATERIALIZED VIEW materialized_view AS
WITH total_price AS (SELECT SUM(l_extendedprice) AS price_sum FROM lineitem_hash_part)
SELECT orders_hash_part.o_orderdate, total_price.price_sum
FROM lineitem_hash_part, orders_hash_part, total_price
WHERE lineitem_hash_part.l_orderkey=orders_hash_part.o_orderkey;
REFRESH MATERIALIZED VIEW materialized_view;
SELECT count(*) FROM materialized_view;
count
-------
12000
(1 row)
DROP MATERIALIZED VIEW materialized_view;
-- Refresh materialized view with join
CREATE MATERIALIZED VIEW materialized_view AS
SELECT orders_hash_part.o_orderdate, quantity_sum
FROM orders_hash_part JOIN (
SELECT l_orderkey, SUM(l_quantity) AS quantity_sum
FROM lineitem_hash_part
GROUP BY l_orderkey
) AS total_quantity
ON total_quantity.l_orderkey=orders_hash_part.o_orderkey;
REFRESH MATERIALIZED VIEW materialized_view;
SELECT count(*) FROM materialized_view;
count
-------
2985
(1 row)
DROP MATERIALIZED VIEW materialized_view;
-- Refresh materialized view with reference tables
CREATE MATERIALIZED VIEW materialized_view AS
SELECT orders_reference.o_orderdate, total_price.price_sum
FROM lineitem_hash_part, orders_reference, (SELECT SUM(o_totalprice) AS price_sum FROM orders_reference) AS total_price
WHERE lineitem_hash_part.l_orderkey=orders_reference.o_orderkey;
REFRESH MATERIALIZED VIEW materialized_view;
SELECT count(*) FROM materialized_view;
count
-------
12000
(1 row)
DROP MATERIALIZED VIEW materialized_view;
-- Refresh materialized view with table distributed after creation of view
CREATE TABLE lineitem_local_to_hash_part AS SELECT * FROM lineitem_hash_part;
CREATE TABLE orders_local_to_hash_part AS SELECT * FROM orders_hash_part;
CREATE MATERIALIZED VIEW materialized_view AS
SELECT orders_local_to_hash_part.o_orderdate, total_price.price_sum
FROM lineitem_local_to_hash_part, orders_local_to_hash_part, (SELECT SUM(l_extendedprice) AS price_sum FROM lineitem_local_to_hash_part) AS total_price
WHERE lineitem_local_to_hash_part.l_orderkey=orders_local_to_hash_part.o_orderkey;
SELECT create_distributed_table('lineitem_local_to_hash_part', 'l_orderkey');
NOTICE: Copying data from local table...
create_distributed_table
--------------------------
(1 row)
SELECT create_distributed_table('orders_local_to_hash_part', 'o_orderkey');
NOTICE: Copying data from local table...
create_distributed_table
--------------------------
(1 row)
REFRESH MATERIALIZED VIEW materialized_view;
SELECT count(*) FROM materialized_view;
count
-------
12000
(1 row)
DROP MATERIALIZED VIEW materialized_view;
DROP TABLE lineitem_local_to_hash_part;
DROP TABLE orders_local_to_hash_part;
-- Refresh materialized view WITH DATA
CREATE MATERIALIZED VIEW materialized_view AS
SELECT orders_hash_part.o_orderdate, total_price.price_sum
FROM lineitem_hash_part, orders_hash_part, (SELECT SUM(l_extendedprice) AS price_sum FROM lineitem_hash_part) AS total_price
WHERE lineitem_hash_part.l_orderkey=orders_hash_part.o_orderkey;
REFRESH MATERIALIZED VIEW materialized_view WITH DATA;
SELECT count(*) FROM materialized_view;
count
-------
12000
(1 row)
DROP MATERIALIZED VIEW materialized_view;
-- Refresh materialized view WITH NO DATA
CREATE MATERIALIZED VIEW materialized_view AS
SELECT orders_hash_part.o_orderdate, total_price.price_sum
FROM lineitem_hash_part, orders_hash_part, (SELECT SUM(l_extendedprice) AS price_sum FROM lineitem_hash_part) AS total_price
WHERE lineitem_hash_part.l_orderkey=orders_hash_part.o_orderkey;
REFRESH MATERIALIZED VIEW materialized_view WITH NO DATA;
SELECT count(*) FROM materialized_view;
ERROR: materialized view "materialized_view" has not been populated
HINT: Use the REFRESH MATERIALIZED VIEW command.
DROP MATERIALIZED VIEW materialized_view;
-- Refresh materialized view CONCURRENTLY
CREATE MATERIALIZED VIEW materialized_view AS
SELECT orders_hash_part.o_orderdate, SUM(total_price.price_sum)
FROM lineitem_hash_part, orders_hash_part, (SELECT SUM(l_extendedprice) AS price_sum FROM lineitem_hash_part) AS total_price
WHERE lineitem_hash_part.l_orderkey=orders_hash_part.o_orderkey
GROUP BY orders_hash_part.o_orderdate;
CREATE UNIQUE INDEX materialized_view_index ON materialized_view (o_orderdate);
REFRESH MATERIALIZED VIEW CONCURRENTLY materialized_view;
SELECT count(*) FROM materialized_view;
count
-------
1699
(1 row)
DROP MATERIALIZED VIEW materialized_view;
DROP SCHEMA materialized_view CASCADE;
NOTICE: drop cascades to view air_shipped_lineitems

56
src/test/regress/expected/multi_view.out
View File

@ -198,59 +198,6 @@ SELECT count(*) FROM priority_orders JOIN air_shipped_lineitems ON (o_custkey =
192
(1 row)
-- materialized views work
-- insert into... select works with views
CREATE TABLE temp_lineitem(LIKE lineitem_hash_part);
SELECT create_distributed_table('temp_lineitem', 'l_orderkey', 'hash', 'lineitem_hash_part');
create_distributed_table
--------------------------
(1 row)
INSERT INTO temp_lineitem SELECT * FROM air_shipped_lineitems;
SELECT count(*) FROM temp_lineitem;
count
-------
1706
(1 row)
-- following is a where false query, should not be inserting anything
INSERT INTO temp_lineitem SELECT * FROM air_shipped_lineitems WHERE l_shipmode = 'MAIL';
SELECT count(*) FROM temp_lineitem;
count
-------
1706
(1 row)
-- can create and query materialized views
CREATE MATERIALIZED VIEW mode_counts
AS SELECT l_shipmode, count(*) FROM temp_lineitem GROUP BY l_shipmode;
SELECT * FROM mode_counts WHERE l_shipmode = 'AIR' ORDER BY 2 DESC, 1 LIMIT 10;
l_shipmode | count
------------+-------
AIR | 1706
(1 row)
-- materialized views are local, cannot join with distributed tables
SELECT count(*) FROM mode_counts JOIN temp_lineitem USING (l_shipmode);
ERROR: relation mode_counts is not distributed
-- new data is not immediately reflected in the view
INSERT INTO temp_lineitem SELECT * FROM air_shipped_lineitems;
SELECT * FROM mode_counts WHERE l_shipmode = 'AIR' ORDER BY 2 DESC, 1 LIMIT 10;
l_shipmode | count
------------+-------
AIR | 1706
(1 row)
-- refresh updates the materialised view with new data
REFRESH MATERIALIZED VIEW mode_counts;
SELECT * FROM mode_counts WHERE l_shipmode = 'AIR' ORDER BY 2 DESC, 1 LIMIT 10;
l_shipmode | count
------------+-------
AIR | 3412
(1 row)
DROP MATERIALIZED VIEW mode_counts;
SET citus.task_executor_type to "task-tracker";
-- single view repartition subqueries are not supported
SELECT l_suppkey, count(*) FROM
@ -343,7 +290,6 @@ SELECT * FROM lineitems_by_orderkey WHERE l_orderkey = 100;
100 | 5
(1 row)
DROP TABLE temp_lineitem CASCADE;
DROP VIEW supp_count_view;
DROP VIEW lineitems_by_orderkey;
DROP VIEW lineitems_by_shipping_method;
@ -867,7 +813,7 @@ EXPLAIN (COSTS FALSE) SELECT et.* FROM recent_10_users JOIN events_table et USIN
-> Sort
Sort Key: remote_scan."time" DESC
-> Custom Scan (Citus Adaptive)
-> Distributed Subplan 98_1
-> Distributed Subplan 90_1
-> Limit
-> Sort
Sort Key: (max(remote_scan.lastseen)) DESC

2
src/test/regress/multi_schedule
View File

@ -65,7 +65,7 @@ test: multi_deparse_shard_query multi_distributed_transaction_id multi_real_time
test: multi_explain hyperscale_tutorial
test: multi_basic_queries multi_complex_expressions multi_subquery multi_subquery_complex_queries multi_subquery_behavioral_analytics
test: multi_subquery_complex_reference_clause multi_subquery_window_functions multi_view multi_sql_function multi_prepare_sql
test: sql_procedure multi_function_in_join row_types
test: sql_procedure multi_function_in_join row_types materialized_view
test: multi_subquery_in_where_reference_clause full_join adaptive_executor propagate_set_commands
test: multi_subquery_union multi_subquery_in_where_clause multi_subquery_misc
test: multi_agg_distinct multi_agg_approximate_distinct multi_limit_clause_approximate multi_outer_join_reference multi_single_relation_subquery multi_prepare_plsql

147
src/test/regress/sql/materialized_view.sql Normal file
View File

@ -0,0 +1,147 @@
---
--- materialized_view
---
-- This file contains test cases for materialized view support.
-- materialized views work
-- insert into... select works with views
CREATE SCHEMA materialized_view;
SET search_path TO materialized_view, public;
CREATE VIEW air_shipped_lineitems AS SELECT * FROM lineitem_hash_part WHERE l_shipmode = 'AIR';
CREATE TABLE temp_lineitem(LIKE lineitem_hash_part);
SELECT create_distributed_table('temp_lineitem', 'l_orderkey', 'hash', 'lineitem_hash_part');
INSERT INTO temp_lineitem SELECT * FROM air_shipped_lineitems;
SELECT count(*) FROM temp_lineitem;
-- following is a where false query, should not be inserting anything
INSERT INTO temp_lineitem SELECT * FROM air_shipped_lineitems WHERE l_shipmode = 'MAIL';
SELECT count(*) FROM temp_lineitem;
-- can create and query materialized views
CREATE MATERIALIZED VIEW mode_counts
AS SELECT l_shipmode, count(*) FROM temp_lineitem GROUP BY l_shipmode;
SELECT * FROM mode_counts WHERE l_shipmode = 'AIR' ORDER BY 2 DESC, 1 LIMIT 10;
-- materialized views are local, cannot join with distributed tables
SELECT count(*) FROM mode_counts JOIN temp_lineitem USING (l_shipmode);
-- new data is not immediately reflected in the view
INSERT INTO temp_lineitem SELECT * FROM air_shipped_lineitems;
SELECT * FROM mode_counts WHERE l_shipmode = 'AIR' ORDER BY 2 DESC, 1 LIMIT 10;
-- refresh updates the materialised view with new data
REFRESH MATERIALIZED VIEW mode_counts;
SELECT * FROM mode_counts WHERE l_shipmode = 'AIR' ORDER BY 2 DESC, 1 LIMIT 10;
DROP MATERIALIZED VIEW mode_counts;
DROP TABLE temp_lineitem CASCADE;
-- Refresh single-shard materialized view
CREATE MATERIALIZED VIEW materialized_view AS
SELECT orders_hash_part.o_orderdate, total_price.price_sum
FROM lineitem_hash_part, orders_hash_part, (SELECT SUM(l_extendedprice) AS price_sum FROM lineitem_hash_part where l_orderkey=3) AS total_price
WHERE lineitem_hash_part.l_orderkey=orders_hash_part.o_orderkey AND lineitem_hash_part.l_orderkey=3;
REFRESH MATERIALIZED VIEW materialized_view;
SELECT count(*) FROM materialized_view;
DROP MATERIALIZED VIEW materialized_view;
-- Refresh multi-shard materialized view
CREATE MATERIALIZED VIEW materialized_view AS
SELECT orders_hash_part.o_orderdate, total_price.price_sum
FROM lineitem_hash_part, orders_hash_part, (SELECT SUM(l_extendedprice) AS price_sum FROM lineitem_hash_part) AS total_price
WHERE lineitem_hash_part.l_orderkey=orders_hash_part.o_orderkey;
REFRESH MATERIALIZED VIEW materialized_view;
SELECT count(*) FROM materialized_view;
DROP MATERIALIZED VIEW materialized_view;
-- Refresh materialized view with CTE
CREATE MATERIALIZED VIEW materialized_view AS
WITH total_price AS (SELECT SUM(l_extendedprice) AS price_sum FROM lineitem_hash_part)
SELECT orders_hash_part.o_orderdate, total_price.price_sum
FROM lineitem_hash_part, orders_hash_part, total_price
WHERE lineitem_hash_part.l_orderkey=orders_hash_part.o_orderkey;
REFRESH MATERIALIZED VIEW materialized_view;
SELECT count(*) FROM materialized_view;
DROP MATERIALIZED VIEW materialized_view;
-- Refresh materialized view with join
CREATE MATERIALIZED VIEW materialized_view AS
SELECT orders_hash_part.o_orderdate, quantity_sum
FROM orders_hash_part JOIN (
SELECT l_orderkey, SUM(l_quantity) AS quantity_sum
FROM lineitem_hash_part
GROUP BY l_orderkey
) AS total_quantity
ON total_quantity.l_orderkey=orders_hash_part.o_orderkey;
REFRESH MATERIALIZED VIEW materialized_view;
SELECT count(*) FROM materialized_view;
DROP MATERIALIZED VIEW materialized_view;
-- Refresh materialized view with reference tables
CREATE MATERIALIZED VIEW materialized_view AS
SELECT orders_reference.o_orderdate, total_price.price_sum
FROM lineitem_hash_part, orders_reference, (SELECT SUM(o_totalprice) AS price_sum FROM orders_reference) AS total_price
WHERE lineitem_hash_part.l_orderkey=orders_reference.o_orderkey;
REFRESH MATERIALIZED VIEW materialized_view;
SELECT count(*) FROM materialized_view;
DROP MATERIALIZED VIEW materialized_view;
-- Refresh materialized view with table distributed after creation of view
CREATE TABLE lineitem_local_to_hash_part AS SELECT * FROM lineitem_hash_part;
CREATE TABLE orders_local_to_hash_part AS SELECT * FROM orders_hash_part;
CREATE MATERIALIZED VIEW materialized_view AS
SELECT orders_local_to_hash_part.o_orderdate, total_price.price_sum
FROM lineitem_local_to_hash_part, orders_local_to_hash_part, (SELECT SUM(l_extendedprice) AS price_sum FROM lineitem_local_to_hash_part) AS total_price
WHERE lineitem_local_to_hash_part.l_orderkey=orders_local_to_hash_part.o_orderkey;
SELECT create_distributed_table('lineitem_local_to_hash_part', 'l_orderkey');
SELECT create_distributed_table('orders_local_to_hash_part', 'o_orderkey');
REFRESH MATERIALIZED VIEW materialized_view;
SELECT count(*) FROM materialized_view;
DROP MATERIALIZED VIEW materialized_view;
DROP TABLE lineitem_local_to_hash_part;
DROP TABLE orders_local_to_hash_part;
-- Refresh materialized view WITH DATA
CREATE MATERIALIZED VIEW materialized_view AS
SELECT orders_hash_part.o_orderdate, total_price.price_sum
FROM lineitem_hash_part, orders_hash_part, (SELECT SUM(l_extendedprice) AS price_sum FROM lineitem_hash_part) AS total_price
WHERE lineitem_hash_part.l_orderkey=orders_hash_part.o_orderkey;
REFRESH MATERIALIZED VIEW materialized_view WITH DATA;
SELECT count(*) FROM materialized_view;
DROP MATERIALIZED VIEW materialized_view;
-- Refresh materialized view WITH NO DATA
CREATE MATERIALIZED VIEW materialized_view AS
SELECT orders_hash_part.o_orderdate, total_price.price_sum
FROM lineitem_hash_part, orders_hash_part, (SELECT SUM(l_extendedprice) AS price_sum FROM lineitem_hash_part) AS total_price
WHERE lineitem_hash_part.l_orderkey=orders_hash_part.o_orderkey;
REFRESH MATERIALIZED VIEW materialized_view WITH NO DATA;
SELECT count(*) FROM materialized_view;
DROP MATERIALIZED VIEW materialized_view;
-- Refresh materialized view CONCURRENTLY
CREATE MATERIALIZED VIEW materialized_view AS
SELECT orders_hash_part.o_orderdate, SUM(total_price.price_sum)
FROM lineitem_hash_part, orders_hash_part, (SELECT SUM(l_extendedprice) AS price_sum FROM lineitem_hash_part) AS total_price
WHERE lineitem_hash_part.l_orderkey=orders_hash_part.o_orderkey
GROUP BY orders_hash_part.o_orderdate;
CREATE UNIQUE INDEX materialized_view_index ON materialized_view (o_orderdate);
REFRESH MATERIALIZED VIEW CONCURRENTLY materialized_view;
SELECT count(*) FROM materialized_view;
DROP MATERIALIZED VIEW materialized_view;
DROP SCHEMA materialized_view CASCADE;

30
src/test/regress/sql/multi_view.sql
View File

@ -85,35 +85,6 @@ RESET client_min_messages;
SELECT count(*) FROM priority_orders JOIN air_shipped_lineitems ON (o_custkey = l_suppkey);
-- materialized views work
-- insert into... select works with views
CREATE TABLE temp_lineitem(LIKE lineitem_hash_part);
SELECT create_distributed_table('temp_lineitem', 'l_orderkey', 'hash', 'lineitem_hash_part');
INSERT INTO temp_lineitem SELECT * FROM air_shipped_lineitems;
SELECT count(*) FROM temp_lineitem;
-- following is a where false query, should not be inserting anything
INSERT INTO temp_lineitem SELECT * FROM air_shipped_lineitems WHERE l_shipmode = 'MAIL';
SELECT count(*) FROM temp_lineitem;
-- can create and query materialized views
CREATE MATERIALIZED VIEW mode_counts
AS SELECT l_shipmode, count(*) FROM temp_lineitem GROUP BY l_shipmode;
SELECT * FROM mode_counts WHERE l_shipmode = 'AIR' ORDER BY 2 DESC, 1 LIMIT 10;
-- materialized views are local, cannot join with distributed tables
SELECT count(*) FROM mode_counts JOIN temp_lineitem USING (l_shipmode);
-- new data is not immediately reflected in the view
INSERT INTO temp_lineitem SELECT * FROM air_shipped_lineitems;
SELECT * FROM mode_counts WHERE l_shipmode = 'AIR' ORDER BY 2 DESC, 1 LIMIT 10;
-- refresh updates the materialised view with new data
REFRESH MATERIALIZED VIEW mode_counts;
SELECT * FROM mode_counts WHERE l_shipmode = 'AIR' ORDER BY 2 DESC, 1 LIMIT 10;
DROP MATERIALIZED VIEW mode_counts;
SET citus.task_executor_type to "task-tracker";
-- single view repartition subqueries are not supported
@ -162,7 +133,6 @@ SELECT * FROM lineitems_by_orderkey ORDER BY 2 DESC, 1 ASC LIMIT 10;
-- it would also work since it is made router plannable
SELECT * FROM lineitems_by_orderkey WHERE l_orderkey = 100;
DROP TABLE temp_lineitem CASCADE;
DROP VIEW supp_count_view;
DROP VIEW lineitems_by_orderkey;