diff --git a/src/test/regress/Makefile b/src/test/regress/Makefile index 398ed9cac..2beed2ed4 100644 --- a/src/test/regress/Makefile +++ b/src/test/regress/Makefile @@ -37,7 +37,7 @@ output_files := $(patsubst $(citus_abs_srcdir)/output/%.source,expected/%.out, $ # intermediate, for muscle memory backward compatibility. check: check-full # check-full triggers all tests that ought to be run routinely -check-full: check-multi check-multi-task-tracker check-multi-binary check-worker check-multi-fdw +check-full: check-multi check-multi-task-tracker-extra check-multi-binary check-worker check-multi-fdw # using pg_regress_multi_check unnecessarily starts up multiple nodes, which isn't needed # for check-worker. But that's harmless besides a few cycles. @@ -57,17 +57,17 @@ check-multi-hll: all tempinstall-main $(pg_regress_multi_check) --load-extension=citus --load-extension=hll -- \ $(MULTI_REGRESS_OPTS) $(EXTRA_TESTS) multi_create_table multi_master_protocol multi_stage_data multi_agg_approximate_distinct -check-multi-task-tracker: all tempinstall-main +check-multi-task-tracker-extra: all tempinstall-main $(pg_regress_multi_check) --load-extension=citus \ --server-option=citus.task_executor_type=task-tracker \ - --server-option=citus.task_tracker_delay=50ms \ --server-option=citus.large_table_shard_count=1 \ - -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule $(EXTRA_TESTS) + -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_task_tracker_extra_schedule $(EXTRA_TESTS) + check-multi-binary: all tempinstall-main $(pg_regress_multi_check) --load-extension=citus \ --server-option=citus.binary_worker_copy_format=on \ - -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule $(EXTRA_TESTS) + -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_binary_schedule $(EXTRA_TESTS) clean distclean maintainer-clean: rm -f $(output_files) $(input_files) diff --git a/src/test/regress/expected/multi_single_relation_subquery.out b/src/test/regress/expected/multi_single_relation_subquery.out index 7e3358819..c71cb8a05 100644 --- a/src/test/regress/expected/multi_single_relation_subquery.out +++ b/src/test/regress/expected/multi_single_relation_subquery.out @@ -198,7 +198,7 @@ SELECT max(l_suppkey) FROM l_suppkey, count(*) FROM - lineitem_subquery + lineitem WHERE l_orderkey < 20000 GROUP BY diff --git a/src/test/regress/multi_binary_schedule b/src/test/regress/multi_binary_schedule new file mode 100644 index 000000000..42bca0e45 --- /dev/null +++ b/src/test/regress/multi_binary_schedule @@ -0,0 +1,33 @@ +# ---------- +# $Id$ +# +# Regression tests that test binary mode data transfer between workers. +# No new tests are expected here unless they are specifically testing for changes +# in binary mode data transfer +# +# ---------- + +# --- +# Tests around schema changes, these are run first, so there's no preexisting objects. +# --- +test: multi_extension +test: multi_table_ddl + +# ---------- +# The following distributed tests depend on creating a partitioned table and +# uploading data to it. +# ---------- +test: multi_create_table +test: multi_stage_data + +test: multi_basic_queries multi_complex_expressions multi_verify_no_subquery +test: multi_single_relation_subquery +test: multi_binary_master_copy_format + +test: multi_simple_queries + +# --------- +# multi_copy creates hash and range-partitioned tables and performs COPY +# --------- +test: multi_copy + diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 0ecec80fb..5b3912c63 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -3,6 +3,10 @@ # # Regression tests that exercise distributed planning/execution functionality. # +# All new regression tests are expected to be run by this schedule. Tests that +# do not set specific task executor type should also be added to +# multi_task_tracker_extra_schedule. +# # Note that we use variant comparison files to test version dependent regression # test results. For more information: # http://www.postgresql.org/docs/9.5/static/regress-variant.html @@ -108,10 +112,11 @@ test: multi_outer_join # Tests covering mostly modification queries and required preliminary # functionality related to metadata, shard creation, shard pruning and # "hacky" copy script for hash partitioned tables. -# Note that the order of the following tests are important. +# Note that the order of the following tests are important. multi_complex_count_distinct +# is independed from the rest of the group, it is added to increase parallelism. # --- test: multi_create_fdw -test: multi_connection_cache +test: multi_connection_cache multi_complex_count_distinct test: multi_distribution_metadata test: multi_generate_ddl_commands test: multi_create_shards @@ -144,8 +149,3 @@ test: multi_large_shardid # multi_drop_extension makes sure we can safely drop and recreate the extension # ---------- test: multi_drop_extension - -# ---------- -# multi_complex_count_distinct creates table lineitem_hash, creates shards and load data -# ---------- -test: multi_complex_count_distinct diff --git a/src/test/regress/multi_task_tracker_extra_schedule b/src/test/regress/multi_task_tracker_extra_schedule new file mode 100644 index 000000000..9932b153c --- /dev/null +++ b/src/test/regress/multi_task_tracker_extra_schedule @@ -0,0 +1,111 @@ +# ---------- +# $Id$ +# +# Regression tests for task tracker executor. This schedule runs tests +# in task tracker executor. Any test that do not explicitly set the task executor +# are expected to be placed here in addition to multi_schedule. +# +# Note that we use variant comparison files to test version dependent regression +# test results. For more information: +# http://www.postgresql.org/docs/9.5/static/regress-variant.html +# ---------- + +# --- +# Tests around schema changes, these are run first, so there's no preexisting objects. +# --- +test: multi_extension +test: multi_table_ddl + +# ---------- +# The following distributed tests depend on creating a partitioned table and +# uploading data to it. +# ---------- +test: multi_create_table +test: multi_master_protocol +test: multi_stage_data + +# ---------- +# Miscellaneous tests to check our query planning behavior +# ---------- +test: multi_basic_queries multi_complex_expressions multi_verify_no_subquery +test: multi_agg_distinct multi_limit_clause multi_limit_clause_approximate +test: multi_average_expression multi_working_columns +test: multi_array_agg +test: multi_agg_type_conversion multi_count_type_conversion +test: multi_partition_pruning +test: multi_join_pruning multi_hash_pruning +test: multi_null_minmax_value_pruning +test: multi_query_directory_cleanup +test: multi_task_assignment_policy +test: multi_utility_statements +test: multi_dropped_column_aliases +test: multi_verify_no_join_with_alias + +# ---------- +# Parallel TPC-H tests to check our distributed execution behavior +# ---------- +test: multi_tpch_query1 multi_tpch_query3 multi_tpch_query6 multi_tpch_query10 +test: multi_tpch_query12 multi_tpch_query14 multi_tpch_query19 +test: multi_tpch_query7 multi_tpch_query7_nested + +# ---------- +# Parallel tests to check our join order planning logic. Note that we stage data +# below; and therefore these tests should come after the execution tests. +# ---------- +test: multi_join_order_tpch_small multi_join_order_additional +test: multi_stage_more_data +test: multi_join_order_tpch_large + +# ---------- +# Tests to check our large record staging and shard deletion behavior +# ---------- +test: multi_stage_large_records +test: multi_master_delete_protocol +test: multi_shard_modify + +# ---------- +# multi_create_schema tests creation, staging and querying of a table in a new +# schema (namespace). +# ---------- +test: multi_create_schema + +# --------- +# multi_outer_join stages shards to create different mappings for outer joins +# --------- +test: multi_outer_join + +# --- +# Tests covering mostly modification queries and required preliminary +# functionality related to metadata, shard creation, shard pruning and +# "hacky" copy script for hash partitioned tables. +# Note that the order of the following tests are important. +# --- +test: multi_create_fdw +test: multi_connection_cache +test: multi_distribution_metadata +test: multi_generate_ddl_commands +test: multi_create_shards +test: multi_prune_shard_list +test: multi_repair_shards +test: multi_modifications +test: multi_upsert +test: multi_simple_queries +test: multi_utilities +test: multi_create_insert_proxy +test: multi_data_types + +# --------- +# multi_copy creates hash and range-partitioned tables and performs COPY +# --------- +test: multi_copy + +# ---------- +# multi_large_shardid stages more shards into lineitem +# ---------- +test: multi_large_shardid + +# ---------- +# multi_drop_extension makes sure we can safely drop and recreate the extension +# ---------- +test: multi_drop_extension + diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index 09ef3085c..34c833c6e 100644 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -96,6 +96,8 @@ push(@pgOptions, '-c', "max_prepared_transactions=100"); push(@pgOptions, '-c', "citus.shard_max_size=300kB"); push(@pgOptions, '-c', "citus.max_running_tasks_per_node=4"); push(@pgOptions, '-c', "citus.expire_cached_shards=on"); +push(@pgOptions, '-c', "citus.task_tracker_delay=10ms"); +push(@pgOptions, '-c', "citus.remote_task_check_interval=1ms"); # Add externally added options last, so they overwrite the default ones above for my $option (@userPgOptions) @@ -301,8 +303,14 @@ for my $extension (@extensions) # Append remaining ARGV arguments to pg_regress arguments push(@arguments, @ARGV); +my $startTime = time(); + # Finally run the tests system("$pgxsdir/src/test/regress/pg_regress", @arguments) == 0 or die "Could not run regression tests"; +my $endTime = time(); + +print "Finished in ". ($endTime - $startTime)." seconds. \n"; + exit 0; diff --git a/src/test/regress/sql/multi_single_relation_subquery.sql b/src/test/regress/sql/multi_single_relation_subquery.sql index 688de921e..7e6f4f0fb 100644 --- a/src/test/regress/sql/multi_single_relation_subquery.sql +++ b/src/test/regress/sql/multi_single_relation_subquery.sql @@ -152,7 +152,7 @@ SELECT max(l_suppkey) FROM l_suppkey, count(*) FROM - lineitem_subquery + lineitem WHERE l_orderkey < 20000 GROUP BY