Merge pull request #5025 from citusdata/split_multi

Split multi schedule
pull/5049/head
SaitTalhaNisanci 2021-06-16 15:30:15 +03:00 committed by GitHub
commit 2511c4c045
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 375 additions and 313 deletions

View File

@ -451,6 +451,12 @@ workflows:
image_tag: '12.6'
make: check-multi
requires: [build-12]
- test-citus:
name: 'test-12_check-multi-1'
pg_major: 12
image_tag: '12.6'
make: check-multi-1
requires: [build-12]
- test-citus:
name: 'test-12_check-mx'
pg_major: 12
@ -519,6 +525,12 @@ workflows:
image_tag: '13.2'
make: check-multi
requires: [build-13]
- test-citus:
name: 'test-13_check-multi-1'
pg_major: 13
image_tag: '13.2'
make: check-multi-1
requires: [build-13]
- test-citus:
name: 'test-13_check-mx'
pg_major: 13

View File

@ -132,6 +132,10 @@ check-multi: all
$(pg_regress_multi_check) --load-extension=citus \
-- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule $(EXTRA_TESTS)
check-multi-1: all
$(pg_regress_multi_check) --load-extension=citus \
-- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_1_schedule $(EXTRA_TESTS)
check-multi-hyperscale: all
$(pg_regress_multi_check) --conninfo="$(conninfo)" --load-extension=citus \
-- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule_hyperscale $(EXTRA_TESTS)

View File

@ -1359,6 +1359,7 @@ SELECT create_reference_table('itemgroups');
(1 row)
DROP TABLE IF EXISTS users ;
CREATE TABLE users (
id int PRIMARY KEY,
name text,

View File

@ -0,0 +1,354 @@
# ----------
# $Id$
#
# Regression tests that exercise distributed planning/execution functionality.
#
# All new regression tests are expected to be run by this schedule. Tests that
# do not set specific task executor type should also be added to
# multi_task_tracker_extra_schedule.
#
# Note that we use variant comparison files to test version dependent regression
# test results. For more information:
# http://www.postgresql.org/docs/current/static/regress-variant.html
# ----------
# ---
# Tests around schema changes, these are run first, so there's no preexisting objects.
# ---
test: multi_extension
test: single_node
test: single_node_truncate
test: multi_cluster_management
# below tests are placed right after multi_cluster_management as we do
# remove/add node operations and we do not want any preexisting objects
test: alter_role_propagation
test: propagate_extension_commands
test: escape_extension_name
test: ref_citus_local_fkeys
test: alter_database_owner
test: multi_test_helpers multi_test_helpers_superuser
test: multi_test_catalog_views
test: multi_table_ddl
test: multi_sequence_default
test: multi_name_lengths
test: multi_name_resolution
test: multi_metadata_access
test: multi_metadata_attributes
test: multi_read_from_secondaries
# ----------
# The following distributed tests depend on creating a partitioned table and
# uploading data to it.
# ----------
test: multi_create_table
test: multi_create_table_superuser
test: multi_create_table_constraints multi_master_protocol multi_load_data multi_load_data_superuser multi_behavioral_analytics_create_table
test: multi_behavioral_analytics_basics multi_behavioral_analytics_single_shard_queries multi_insert_select_non_pushable_queries multi_insert_select multi_behavioral_analytics_create_table_superuser
test: multi_shard_update_delete recursive_dml_with_different_planners_executors
test: insert_select_repartition window_functions dml_recursive multi_insert_select_window
test: multi_insert_select_conflict citus_table_triggers
test: multi_row_insert insert_select_into_local_table multi_create_table_new_features alter_index
# following should not run in parallel because it relies on connection counts to workers
test: insert_select_connection_leak
# ---------
# at the end of the regression tests regarding recursively planned modifications
# ensure that we don't leak any intermediate results
# This test should not run in parallel with any other tests
# ---------
test: ensure_no_intermediate_data_leak
# ----------
# Tests for partitioning support
# ----------
test: multi_partitioning_utils multi_partitioning partitioning_issue_3970 replicated_partitioned_table
# ----------
# Tests for foreign data wrapper support
# ----------
test: multi_create_fdw
# ----------
# Tests for statistics propagation
# ----------
test: propagate_statistics
test: pg13_propagate_statistics
# ----------
# Test for updating table statistics
# ----------
test: citus_update_table_statistics
# ----------
# Parallel TPC-H tests to check our distributed execution behavior
# ----------
test: multi_tpch_query1 multi_tpch_query3 multi_tpch_query6 multi_tpch_query10
test: multi_tpch_query12 multi_tpch_query14 multi_tpch_query19
test: multi_tpch_query7 multi_tpch_query7_nested
# ----------
# Parallel tests to check our join order planning logic. Note that we load data
# below; and therefore these tests should come after the execution tests.
# ----------
test: multi_join_order_tpch_small multi_join_order_additional
test: multi_load_more_data
test: multi_join_order_tpch_repartition
# ----------
# Tests for repartition join planning and execution. Be careful when creating
# new shards before these tests, as they expect specific shard identifiers in
# the output.
# ----------
test: multi_repartition_join_planning multi_repartition_join_pruning multi_repartition_join_task_assignment multi_repartition_join_ref
test: adaptive_executor_repartition
# ---------
# Tests that modify data should run sequentially
# ---------
test: with_prepare
# ---------
# Tests for recursive planning.
# ---------
test: with_nested with_where with_basics with_set_operations
test: with_modifying cte_prepared_modify cte_nested_modification
test: ensure_no_intermediate_data_leak
test: with_executors with_join with_partitioning with_transactions with_dml
# ----------
# Tests to check our large record loading and shard deletion behavior
# ----------
test: multi_load_large_records
test: multi_master_delete_protocol
# ----------
# Tests around DDL statements run on distributed tables
# ----------
test: multi_index_statements
test: multi_alter_table_statements
test: multi_alter_table_add_constraints
# ----------
# multi_create_schema tests creation, loading, and querying of a table in a new
# schema (namespace).
# ----------
test: multi_create_schema
# ----------
# Tests to check if we inform the user about potential caveats of creating new
# databases, schemas, roles, and authentication information.
# ----------
test: multi_utility_warnings data_types
# ----------
# Tests to check the sequential and parallel executions of DDL and modification
# commands
# Should not be executed in parallel with other tests
# ----------
test: sequential_modifications
# ---------
# multi_append_table_to_shard loads data to create shards in a way that forces
# shard caching.
# ---------
test: multi_append_table_to_shard
# ---------
# multi_outer_join loads data to create shards to test outer join mappings
# ---------
test: multi_outer_join
# ---
# Tests covering mostly modification queries and required preliminary
# functionality related to metadata, shard creation, shard pruning and
# "hacky" copy script for hash partitioned tables.
# Note that the order of the following tests are important. multi_complex_count_distinct
# is independent from the rest of the group, it is added to increase parallelism.
# ---
test: multi_complex_count_distinct multi_select_distinct
test: multi_modifications
test: multi_distribution_metadata
test: multi_generate_ddl_commands multi_create_shards multi_prune_shard_list multi_repair_shards
test: multi_upsert multi_simple_queries multi_data_types
test: master_copy_shard_placement
# multi_utilities cannot be run in parallel with other tests because it checks
# global locks
test: multi_utilities
test: foreign_key_to_reference_table validate_constraint
test: multi_modifying_xacts
test: multi_repartition_udt multi_repartitioned_subquery_udf multi_subtransactions
test: multi_transaction_recovery
test: local_dist_join_modifications
test: local_table_join
test: local_dist_join_mixed
test: citus_local_dist_joins
# ---------
# multi_copy creates hash and range-partitioned tables and performs COPY
# multi_router_planner creates hash partitioned tables.
# ---------
test: multi_copy fast_path_router_modify pg_dump
test: multi_router_planner
# These 2 tests have prepared statements which sometimes get invalidated by concurrent tests,
# changing the debug output. We should not run them in parallel with others
test: null_parameters
test: multi_router_planner_fast_path
# ----------
# multi_large_shardid loads more lineitem data using high shard identifiers
# ----------
test: multi_large_shardid
# ----------
# multi_size_queries tests various size commands on distributed tables
# ----------
test: multi_size_queries
# ----------
# multi_drop_extension makes sure we can safely drop and recreate the extension
# ----------
test: multi_drop_extension
# ----------
# multi_metadata_sync tests the propagation of mx-related metadata changes to metadata workers
# multi_unsupported_worker_operations tests that unsupported operations error out on metadata workers
# ----------
test: multi_metadata_sync
test: multi_unsupported_worker_operations
# ----------
# grant_on_schema_propagation tests if the GRANT ... ON SCHEMA queries are propagated correctly
# multi_schema_support makes sure we can work with tables in schemas other than public with no problem
# ----------
test: grant_on_schema_propagation
test: multi_schema_support
# ----------
# multi_function_evaluation tests edge-cases in master-side function pre-evaluation
# ----------
test: multi_function_evaluation
# ----------
# multi_truncate tests truncate functionality for distributed tables
# ----------
test: multi_truncate
# ----------
# multi_colocation_utils tests utility functions written for co-location feature & internal API
# multi_colocated_shard_transfer tests master_copy_shard_placement with colocated tables.
# ----------
test: multi_colocation_utils
test: multi_colocated_shard_transfer
# ----------
# multi_citus_tools tests utility functions written for citus tools
# ----------
test: multi_citus_tools
# ----------
# node_conninfo_reload tests that node_conninfo changes take effect
# ----------
test: node_conninfo_reload
# ----------
# multi_foreign_key tests foreign key push down on distributed tables
# ----------
test: multi_foreign_key
test: multi_foreign_key_relation_graph
# ----------
# multi_replicate_reference_table tests replicating reference tables to new nodes after we add new nodes
# multi_remove_node_reference_table tests metadata changes after master_remove_node
# ----------
test: multi_replicate_reference_table
test: multi_remove_node_reference_table
# --------
# Replicating reference tables to coordinator. Add coordinator to pg_dist_node
# and rerun some of the tests.
# --------
test: add_coordinator
test: multi_reference_table citus_local_tables_queries
test: foreign_key_to_reference_table
test: citus_local_table_triggers
test: replicate_reference_tables_to_coordinator
test: coordinator_shouldhaveshards
test: local_shard_utility_command_execution
test: citus_local_tables
test: multi_row_router_insert mixed_relkind_tests create_ref_dist_from_citus_local
test: undistribute_table_cascade
test: create_citus_local_table_cascade
test: fkeys_between_local_ref
test: auto_undist_citus_local
test: remove_coordinator
# ----------
# multi_transactional_drop_shards tests for dropping shards using connection API
# ----------
test: multi_transactional_drop_shards
# ----------
# multi_multiuser tests simple combinations of permission access and queries
# ----------
test: multi_multiuser
# ---------
# multi_cache_invalidation tests for an obscure crash citus used to exhibit when shardids
# changed the table they belonged to during a session
# --------
test: multi_cache_invalidation
# ---------
# multi_task_string_size tests task string size checks
# ---------
test: multi_task_string_size
# ---------
# connection encryption tests
# ---------
test: ssl_by_default
# ---------
# object distribution tests
# ---------
test: distributed_types distributed_types_conflict disable_object_propagation distributed_types_xact_add_enum_value
test: distributed_functions distributed_functions_conflict
test: distributed_collations distributed_collations_conflict
test: distributed_procedure
# ---------
# deparsing logic tests
# ---------
test: multi_deparse_function multi_deparse_procedure
# --------
# cannot be run in parallel with any other tests as it checks
# statistics across sessions
# --------
test: shared_connection_stats
# ---------
# run queries generated by sql smith and sqlancer that caused issues in the past
# --------
test: sqlsmith_failures sqlancer_failures
# ---------
# test that no tests leaked intermediate results. This should always be last
# ---------
test: ensure_no_intermediate_data_leak
# ---------
# ensures that we never leak any connection counts
# in the shared memory
# --------
test: ensure_no_shared_connection_leak

View File

@ -1,43 +1,6 @@
# ----------
# $Id$
#
# Regression tests that exercise distributed planning/execution functionality.
#
# All new regression tests are expected to be run by this schedule. Tests that
# do not set specific task executor type should also be added to
# multi_task_tracker_extra_schedule.
#
# Note that we use variant comparison files to test version dependent regression
# test results. For more information:
# http://www.postgresql.org/docs/current/static/regress-variant.html
# ----------
# ---
# Tests around schema changes, these are run first, so there's no preexisting objects.
# ---
test: multi_extension
test: single_node
test: single_node_truncate
test: multi_cluster_management
# below tests are placed right after multi_cluster_management as we do
# remove/add node operations and we do not want any preexisting objects
test: alter_role_propagation
test: propagate_extension_commands
test: escape_extension_name
test: ref_citus_local_fkeys
test: alter_database_owner
test: multi_test_helpers multi_test_helpers_superuser
test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw
test: multi_test_catalog_views
test: multi_table_ddl
test: multi_sequence_default
test: multi_name_lengths
test: multi_name_resolution
test: multi_metadata_access
test: multi_metadata_attributes
test: multi_read_from_secondaries
# ----------
# The following distributed tests depend on creating a partitioned table and
@ -50,7 +13,7 @@ test: multi_behavioral_analytics_basics multi_behavioral_analytics_single_shard_
test: multi_shard_update_delete recursive_dml_with_different_planners_executors
test: insert_select_repartition window_functions dml_recursive multi_insert_select_window
test: multi_insert_select_conflict citus_table_triggers
test: multi_row_insert insert_select_into_local_table multi_create_table_new_features alter_index
test: multi_row_insert insert_select_into_local_table multi_create_table_new_features
# following should not run in parallel because it relies on connection counts to workers
test: insert_select_connection_leak
@ -65,12 +28,8 @@ test: ensure_no_intermediate_data_leak
# ----------
# Tests for partitioning support
# ----------
test: multi_partitioning_utils multi_partitioning partitioning_issue_3970 replicated_partitioned_table
test: multi_partitioning_utils multi_partitioning replicated_partitioned_table
# ----------
# Tests for foreign data wrapper support
# ----------
test: multi_create_fdw
# ----------
# Tests for recursive subquery planning
@ -94,17 +53,6 @@ test: cte_inline recursive_view_local_table values
test: pg13 pg12
test: tableam
# ----------
# Tests for statistics propagation
# ----------
test: propagate_statistics
test: pg13_propagate_statistics
# ----------
# Test for updating table statistics
# ----------
test: citus_update_table_statistics
# ----------
# Miscellaneous tests to check our query planning behavior
# ----------
@ -136,262 +84,6 @@ test: binary_protocol
test: alter_table_set_access_method
test: alter_distributed_table
# ----------
# Parallel TPC-H tests to check our distributed execution behavior
# ----------
test: multi_tpch_query1 multi_tpch_query3 multi_tpch_query6 multi_tpch_query10
test: multi_tpch_query12 multi_tpch_query14 multi_tpch_query19
test: multi_tpch_query7 multi_tpch_query7_nested
# ----------
# Parallel tests to check our join order planning logic. Note that we load data
# below; and therefore these tests should come after the execution tests.
# ----------
test: multi_join_order_tpch_small multi_join_order_additional
test: multi_load_more_data
test: multi_join_order_tpch_repartition
# ----------
# Tests for repartition join planning and execution. Be careful when creating
# new shards before these tests, as they expect specific shard identifiers in
# the output.
# ----------
test: multi_repartition_join_planning multi_repartition_join_pruning multi_repartition_join_task_assignment multi_repartition_join_ref
test: adaptive_executor_repartition
# ---------
# Tests that modify data should run sequentially
# ---------
test: with_prepare
# ---------
# Tests for recursive planning.
# ---------
test: with_nested with_where with_basics with_set_operations
test: with_modifying cte_prepared_modify cte_nested_modification
test: ensure_no_intermediate_data_leak
test: with_executors with_join with_partitioning with_transactions with_dml
# ----------
# Tests to check our large record loading and shard deletion behavior
# ----------
test: multi_load_large_records
test: multi_master_delete_protocol
# ----------
# Tests around DDL statements run on distributed tables
# ----------
test: multi_index_statements
test: multi_alter_table_statements
test: multi_alter_table_add_constraints
# ----------
# multi_create_schema tests creation, loading, and querying of a table in a new
# schema (namespace).
# ----------
test: multi_create_schema
# ----------
# Tests to check if we inform the user about potential caveats of creating new
# databases, schemas, roles, and authentication information.
# ----------
test: multi_utility_warnings data_types
# ----------
# Tests to check the sequential and parallel executions of DDL and modification
# commands
# Should not be executed in parallel with other tests
# ----------
test: sequential_modifications
# ---------
# multi_append_table_to_shard loads data to create shards in a way that forces
# shard caching.
# ---------
test: multi_append_table_to_shard
# ---------
# multi_outer_join loads data to create shards to test outer join mappings
# ---------
test: multi_outer_join
# ---
# Tests covering mostly modification queries and required preliminary
# functionality related to metadata, shard creation, shard pruning and
# "hacky" copy script for hash partitioned tables.
# Note that the order of the following tests are important. multi_complex_count_distinct
# is independent from the rest of the group, it is added to increase parallelism.
# ---
test: multi_complex_count_distinct multi_select_distinct
test: multi_modifications
test: multi_distribution_metadata
test: multi_generate_ddl_commands multi_create_shards multi_prune_shard_list multi_repair_shards
test: multi_upsert multi_simple_queries multi_data_types
test: master_copy_shard_placement
# multi_utilities cannot be run in parallel with other tests because it checks
# global locks
test: multi_utilities
test: foreign_key_to_reference_table validate_constraint
test: multi_modifying_xacts
test: multi_repartition_udt multi_repartitioned_subquery_udf multi_subtransactions
test: multi_transaction_recovery
test: local_dist_join_modifications
test: local_table_join
test: local_dist_join_mixed
test: citus_local_dist_joins
# ---------
# multi_copy creates hash and range-partitioned tables and performs COPY
# multi_router_planner creates hash partitioned tables.
# ---------
test: multi_copy fast_path_router_modify pg_dump
test: multi_router_planner
# These 2 tests have prepared statements which sometimes get invalidated by concurrent tests,
# changing the debug output. We should not run them in parallel with others
test: null_parameters
test: multi_router_planner_fast_path
# ----------
# multi_large_shardid loads more lineitem data using high shard identifiers
# ----------
test: multi_large_shardid
# ----------
# multi_size_queries tests various size commands on distributed tables
# ----------
test: multi_size_queries
# ----------
# multi_drop_extension makes sure we can safely drop and recreate the extension
# ----------
test: multi_drop_extension
# ----------
# multi_metadata_sync tests the propagation of mx-related metadata changes to metadata workers
# multi_unsupported_worker_operations tests that unsupported operations error out on metadata workers
# ----------
test: multi_metadata_sync
test: multi_unsupported_worker_operations
# ----------
# grant_on_schema_propagation tests if the GRANT ... ON SCHEMA queries are propagated correctly
# multi_schema_support makes sure we can work with tables in schemas other than public with no problem
# ----------
test: grant_on_schema_propagation
test: multi_schema_support
# ----------
# multi_function_evaluation tests edge-cases in master-side function pre-evaluation
# ----------
test: multi_function_evaluation
# ----------
# multi_truncate tests truncate functionality for distributed tables
# ----------
test: multi_truncate
# ----------
# multi_colocation_utils tests utility functions written for co-location feature & internal API
# multi_colocated_shard_transfer tests master_copy_shard_placement with colocated tables.
# ----------
test: multi_colocation_utils
test: multi_colocated_shard_transfer
# ----------
# multi_citus_tools tests utility functions written for citus tools
# ----------
test: multi_citus_tools
# ----------
# node_conninfo_reload tests that node_conninfo changes take effect
# ----------
test: node_conninfo_reload
# ----------
# multi_foreign_key tests foreign key push down on distributed tables
# ----------
test: multi_foreign_key
test: multi_foreign_key_relation_graph
# ----------
# multi_replicate_reference_table tests replicating reference tables to new nodes after we add new nodes
# multi_remove_node_reference_table tests metadata changes after master_remove_node
# ----------
test: multi_replicate_reference_table
test: multi_remove_node_reference_table
# --------
# Replicating reference tables to coordinator. Add coordinator to pg_dist_node
# and rerun some of the tests.
# --------
test: add_coordinator
test: multi_reference_table citus_local_tables_queries
test: foreign_key_to_reference_table
test: citus_local_table_triggers
test: replicate_reference_tables_to_coordinator
test: coordinator_shouldhaveshards
test: local_shard_utility_command_execution
test: citus_local_tables
test: multi_row_router_insert mixed_relkind_tests create_ref_dist_from_citus_local
test: undistribute_table_cascade
test: create_citus_local_table_cascade
test: fkeys_between_local_ref
test: auto_undist_citus_local
test: remove_coordinator
# ----------
# multi_transactional_drop_shards tests for dropping shards using connection API
# ----------
test: multi_transactional_drop_shards
# ----------
# multi_multiuser tests simple combinations of permission access and queries
# ----------
test: multi_multiuser
# ---------
# multi_cache_invalidation tests for an obscure crash citus used to exhibit when shardids
# changed the table they belonged to during a session
# --------
test: multi_cache_invalidation
# ---------
# multi_task_string_size tests task string size checks
# ---------
test: multi_task_string_size
# ---------
# connection encryption tests
# ---------
test: ssl_by_default
# ---------
# object distribution tests
# ---------
test: distributed_types distributed_types_conflict disable_object_propagation distributed_types_xact_add_enum_value
test: distributed_functions distributed_functions_conflict
test: distributed_collations distributed_collations_conflict
test: distributed_procedure
# ---------
# deparsing logic tests
# ---------
test: multi_deparse_function multi_deparse_procedure
# --------
# cannot be run in parallel with any other tests as it checks
# statistics across sessions
# --------
test: shared_connection_stats
# ---------
# run queries generated by sql smith and sqlancer that caused issues in the past
# --------
test: sqlsmith_failures sqlancer_failures
# ---------
# test that no tests leaked intermediate results. This should always be last
# ---------
@ -402,4 +94,3 @@ test: ensure_no_intermediate_data_leak
# in the shared memory
# --------
test: ensure_no_shared_connection_leak

View File

@ -1,4 +1,3 @@
SET citus.next_shard_id TO 1200000;
SET citus.next_placement_id TO 1200000;
@ -1022,6 +1021,7 @@ CREATE TABLE itemgroups (
);
SELECT create_reference_table('itemgroups');
DROP TABLE IF EXISTS users ;
CREATE TABLE users (
id int PRIMARY KEY,
name text,