From f45988962fb83ee8618c938611206c81ba1c18c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?mehmet=20furkan=20=C5=9Fahin?= <sahinffurkan@gmail.com>
Date: Tue, 14 Nov 2017 10:55:39 +0300
Subject: [PATCH 1/4] multi-shard update affecting the same/different rows

---
 .../isolation_multi_shard_modify_vs_all.out   | 122 ++++++++++++++++--
 .../isolation_multi_shard_modify_vs_all.spec  |  32 ++++-
 2 files changed, 133 insertions(+), 21 deletions(-)

diff --git a/src/test/regress/expected/isolation_multi_shard_modify_vs_all.out b/src/test/regress/expected/isolation_multi_shard_modify_vs_all.out
index fb0462e00..3c024c699 100644
--- a/src/test/regress/expected/isolation_multi_shard_modify_vs_all.out
+++ b/src/test/regress/expected/isolation_multi_shard_modify_vs_all.out
@@ -62,17 +62,17 @@ step s2-commit:
 	COMMIT;
 
 
-starting permutation: s1-begin s1-update_value_1_of_1_or_3 s2-begin s2-update_value_1_of_4_or_6 s1-commit s2-commit s2-select
+starting permutation: s1-begin s1-update_value_1_of_1_or_3_to_5 s2-begin s2-update_value_1_of_4_or_6_to_4 s1-commit s2-commit s2-select
 step s1-begin: 
     BEGIN;
 
-step s1-update_value_1_of_1_or_3: 
+step s1-update_value_1_of_1_or_3_to_5: 
 	UPDATE users_test_table SET value_1 = 5 WHERE user_id = 1 or user_id = 3;
 
 step s2-begin: 
 	BEGIN;
 
-step s2-update_value_1_of_4_or_6: 
+step s2-update_value_1_of_4_or_6_to_4: 
 	UPDATE users_test_table SET value_1 = 4 WHERE user_id = 4 or user_id = 6;
 
 step s1-commit: 
@@ -94,23 +94,23 @@ user_id        value_1        value_2        value_3
 6              4              11             25             
 7              27             12             18             
 
-starting permutation: s1-begin s1-update_value_1_of_1_or_3 s2-begin s2-update_value_1_of_1_or_3 s1-commit s2-commit s2-select
+starting permutation: s1-begin s1-update_value_1_of_1_or_3_to_5 s2-begin s2-update_value_1_of_1_or_3_to_8 s1-commit s2-commit s2-select
 step s1-begin: 
     BEGIN;
 
-step s1-update_value_1_of_1_or_3: 
+step s1-update_value_1_of_1_or_3_to_5: 
 	UPDATE users_test_table SET value_1 = 5 WHERE user_id = 1 or user_id = 3;
 
 step s2-begin: 
 	BEGIN;
 
-step s2-update_value_1_of_1_or_3: 
+step s2-update_value_1_of_1_or_3_to_8: 
 	UPDATE users_test_table SET value_1 = 8 WHERE user_id = 1 or user_id = 3;
  <waiting ...>
 step s1-commit: 
     COMMIT;
 
-step s2-update_value_1_of_1_or_3: <... completed>
+step s2-update_value_1_of_1_or_3_to_8: <... completed>
 step s2-commit: 
 	COMMIT;
 
@@ -200,6 +200,47 @@ user_id        value_1        value_2        value_3
 5              17             14             4              
 3              11             78             18             
 
+starting permutation: s1-begin s2-begin s1-update_value_1_of_1_or_3_to_5 s2-update_value_1_of_1_or_3_to_8 s1-commit s2-commit
+step s1-begin: 
+    BEGIN;
+
+step s2-begin: 
+	BEGIN;
+
+step s1-update_value_1_of_1_or_3_to_5: 
+	UPDATE users_test_table SET value_1 = 5 WHERE user_id = 1 or user_id = 3;
+
+step s2-update_value_1_of_1_or_3_to_8: 
+	UPDATE users_test_table SET value_1 = 8 WHERE user_id = 1 or user_id = 3;
+ <waiting ...>
+step s1-commit: 
+    COMMIT;
+
+step s2-update_value_1_of_1_or_3_to_8: <... completed>
+step s2-commit: 
+	COMMIT;
+
+
+starting permutation: s1-begin s2-begin s2-update_value_1_of_1_or_3_to_8 s1-update_value_1_of_2_or_4_to_5 s2-commit s1-commit
+step s1-begin: 
+    BEGIN;
+
+step s2-begin: 
+	BEGIN;
+
+step s2-update_value_1_of_1_or_3_to_8: 
+	UPDATE users_test_table SET value_1 = 8 WHERE user_id = 1 or user_id = 3;
+
+step s1-update_value_1_of_2_or_4_to_5: 
+	UPDATE users_test_table SET value_1 = 5 WHERE user_id = 2 or user_id = 4;
+
+step s2-commit: 
+	COMMIT;
+
+step s1-commit: 
+    COMMIT;
+
+
 starting permutation: s1-begin s1-change_connection_mode_to_sequential s1-update_all_value_1 s2-begin s2-change_connection_mode_to_sequential s2-update_all_value_1 s1-commit s2-commit s2-select
 step s1-begin: 
     BEGIN;
@@ -239,14 +280,14 @@ user_id        value_1        value_2        value_3
 6              6              11             25             
 7              6              12             18             
 
-starting permutation: s1-begin s1-change_connection_mode_to_sequential s1-update_value_1_of_1_or_3 s2-begin s2-change_connection_mode_to_sequential s2-update_value_1_of_1_or_3 s1-commit s2-commit s2-select
+starting permutation: s1-begin s1-change_connection_mode_to_sequential s1-update_value_1_of_1_or_3_to_5 s2-begin s2-change_connection_mode_to_sequential s2-update_value_1_of_1_or_3_to_8 s1-commit s2-commit s2-select
 step s1-begin: 
     BEGIN;
 
 step s1-change_connection_mode_to_sequential: 
     set citus.multi_shard_modify_mode to 'sequential';
 
-step s1-update_value_1_of_1_or_3: 
+step s1-update_value_1_of_1_or_3_to_5: 
 	UPDATE users_test_table SET value_1 = 5 WHERE user_id = 1 or user_id = 3;
 
 step s2-begin: 
@@ -255,13 +296,13 @@ step s2-begin:
 step s2-change_connection_mode_to_sequential: 
     set citus.multi_shard_modify_mode to 'sequential';
 
-step s2-update_value_1_of_1_or_3: 
+step s2-update_value_1_of_1_or_3_to_8: 
 	UPDATE users_test_table SET value_1 = 8 WHERE user_id = 1 or user_id = 3;
  <waiting ...>
 step s1-commit: 
     COMMIT;
 
-step s2-update_value_1_of_1_or_3: <... completed>
+step s2-update_value_1_of_1_or_3_to_8: <... completed>
 step s2-commit: 
 	COMMIT;
 
@@ -278,14 +319,14 @@ user_id        value_1        value_2        value_3
 6              21             11             25             
 7              27             12             18             
 
-starting permutation: s1-begin s1-change_connection_mode_to_sequential s1-update_value_1_of_1_or_3 s2-begin s2-change_connection_mode_to_sequential s2-update_value_1_of_4_or_6 s1-commit s2-commit s2-select
+starting permutation: s1-begin s1-change_connection_mode_to_sequential s1-update_value_1_of_1_or_3_to_5 s2-begin s2-change_connection_mode_to_sequential s2-update_value_1_of_4_or_6_to_4 s1-commit s2-commit s2-select
 step s1-begin: 
     BEGIN;
 
 step s1-change_connection_mode_to_sequential: 
     set citus.multi_shard_modify_mode to 'sequential';
 
-step s1-update_value_1_of_1_or_3: 
+step s1-update_value_1_of_1_or_3_to_5: 
 	UPDATE users_test_table SET value_1 = 5 WHERE user_id = 1 or user_id = 3;
 
 step s2-begin: 
@@ -294,7 +335,7 @@ step s2-begin:
 step s2-change_connection_mode_to_sequential: 
     set citus.multi_shard_modify_mode to 'sequential';
 
-step s2-update_value_1_of_4_or_6: 
+step s2-update_value_1_of_4_or_6_to_4: 
 	UPDATE users_test_table SET value_1 = 4 WHERE user_id = 4 or user_id = 6;
 
 step s1-commit: 
@@ -315,3 +356,56 @@ user_id        value_1        value_2        value_3
 5              35             10             17             
 6              4              11             25             
 7              27             12             18             
+
+starting permutation: s1-begin s2-begin s1-change_connection_mode_to_sequential s2-change_connection_mode_to_sequential s1-update_value_1_of_1_or_3_to_5 s2-update_value_1_of_1_or_3_to_8 s1-commit s2-commit
+step s1-begin: 
+    BEGIN;
+
+step s2-begin: 
+	BEGIN;
+
+step s1-change_connection_mode_to_sequential: 
+    set citus.multi_shard_modify_mode to 'sequential';
+
+step s2-change_connection_mode_to_sequential: 
+    set citus.multi_shard_modify_mode to 'sequential';
+
+step s1-update_value_1_of_1_or_3_to_5: 
+	UPDATE users_test_table SET value_1 = 5 WHERE user_id = 1 or user_id = 3;
+
+step s2-update_value_1_of_1_or_3_to_8: 
+	UPDATE users_test_table SET value_1 = 8 WHERE user_id = 1 or user_id = 3;
+ <waiting ...>
+step s1-commit: 
+    COMMIT;
+
+step s2-update_value_1_of_1_or_3_to_8: <... completed>
+step s2-commit: 
+	COMMIT;
+
+
+starting permutation: s1-begin s2-begin s1-change_connection_mode_to_sequential s2-change_connection_mode_to_sequential s2-update_value_1_of_1_or_3_to_8 s1-update_value_1_of_2_or_4_to_5 s1-commit s2-commit
+step s1-begin: 
+    BEGIN;
+
+step s2-begin: 
+	BEGIN;
+
+step s1-change_connection_mode_to_sequential: 
+    set citus.multi_shard_modify_mode to 'sequential';
+
+step s2-change_connection_mode_to_sequential: 
+    set citus.multi_shard_modify_mode to 'sequential';
+
+step s2-update_value_1_of_1_or_3_to_8: 
+	UPDATE users_test_table SET value_1 = 8 WHERE user_id = 1 or user_id = 3;
+
+step s1-update_value_1_of_2_or_4_to_5: 
+	UPDATE users_test_table SET value_1 = 5 WHERE user_id = 2 or user_id = 4;
+
+step s1-commit: 
+    COMMIT;
+
+step s2-commit: 
+	COMMIT;
+
diff --git a/src/test/regress/specs/isolation_multi_shard_modify_vs_all.spec b/src/test/regress/specs/isolation_multi_shard_modify_vs_all.spec
index b76220406..982776f97 100644
--- a/src/test/regress/specs/isolation_multi_shard_modify_vs_all.spec
+++ b/src/test/regress/specs/isolation_multi_shard_modify_vs_all.spec
@@ -55,11 +55,21 @@ step "s1-update_all_value_1"
 	UPDATE users_test_table SET value_1 = 3;
 }
 
-step "s1-update_value_1_of_1_or_3"
+step "s1-update_value_1_of_1_or_3_to_5"
 {
 	UPDATE users_test_table SET value_1 = 5 WHERE user_id = 1 or user_id = 3;
 }
 
+step "s1-update_value_1_of_1_or_3_to_7"
+{
+	UPDATE users_test_table SET value_1 = 7 WHERE user_id = 1 or user_id = 3;
+}
+
+step "s1-update_value_1_of_2_or_4_to_5"
+{
+	UPDATE users_test_table SET value_1 = 5 WHERE user_id = 2 or user_id = 4;
+}
+
 step "s1-commit"
 {
     COMMIT;
@@ -97,12 +107,12 @@ step "s2-update_all_value_1"
 	UPDATE users_test_table SET value_1 = 6;
 }
 
-step "s2-update_value_1_of_1_or_3"
+step "s2-update_value_1_of_1_or_3_to_8"
 {
 	UPDATE users_test_table SET value_1 = 8 WHERE user_id = 1 or user_id = 3;
 }
 
-step "s2-update_value_1_of_4_or_6"
+step "s2-update_value_1_of_4_or_6_to_4"
 {
 	UPDATE users_test_table SET value_1 = 4 WHERE user_id = 4 or user_id = 6;
 }
@@ -115,14 +125,22 @@ step "s2-commit"
 # test with parallel connections
 permutation "s1-begin" "s1-update_all_value_1" "s2-begin" "s2-select" "s1-commit" "s2-select" "s2-commit"
 permutation "s1-begin" "s1-update_all_value_1" "s2-begin" "s2-update_all_value_1" "s1-commit" "s2-commit"
-permutation "s1-begin" "s1-update_value_1_of_1_or_3" "s2-begin" "s2-update_value_1_of_4_or_6" "s1-commit" "s2-commit" "s2-select" 
-permutation "s1-begin" "s1-update_value_1_of_1_or_3" "s2-begin" "s2-update_value_1_of_1_or_3" "s1-commit" "s2-commit" "s2-select" 
+permutation "s1-begin" "s1-update_value_1_of_1_or_3_to_5" "s2-begin" "s2-update_value_1_of_4_or_6_to_4" "s1-commit" "s2-commit" "s2-select" 
+permutation "s1-begin" "s1-update_value_1_of_1_or_3_to_5" "s2-begin" "s2-update_value_1_of_1_or_3_to_8" "s1-commit" "s2-commit" "s2-select" 
 permutation "s1-begin" "s1-update_all_value_1" "s2-begin" "s2-insert-to-table" "s1-commit" "s2-commit" "s2-select" 
 permutation "s1-begin" "s1-update_all_value_1" "s2-begin" "s2-insert-into-select" "s1-commit" "s2-commit" "s2-select"
+# multi-shard update affecting the same rows
+permutation "s1-begin" "s2-begin" "s1-update_value_1_of_1_or_3_to_5" "s2-update_value_1_of_1_or_3_to_8" "s1-commit" "s2-commit"
+# multi-shard update affecting the different rows
+permutation "s1-begin" "s2-begin" "s2-update_value_1_of_1_or_3_to_8" "s1-update_value_1_of_2_or_4_to_5" "s2-commit" "s1-commit"
 
 # test with sequential connections, sequential tests should not block each other
 # if they are targeting different shards. If multiple connections updating the same
 # row, second one must wait for the first one.
 permutation "s1-begin" "s1-change_connection_mode_to_sequential" "s1-update_all_value_1" "s2-begin" "s2-change_connection_mode_to_sequential" "s2-update_all_value_1" "s1-commit" "s2-commit" "s2-select"
-permutation "s1-begin" "s1-change_connection_mode_to_sequential" "s1-update_value_1_of_1_or_3" "s2-begin" "s2-change_connection_mode_to_sequential" "s2-update_value_1_of_1_or_3" "s1-commit" "s2-commit" "s2-select"
-permutation "s1-begin" "s1-change_connection_mode_to_sequential" "s1-update_value_1_of_1_or_3" "s2-begin" "s2-change_connection_mode_to_sequential" "s2-update_value_1_of_4_or_6" "s1-commit" "s2-commit" "s2-select"
+permutation "s1-begin" "s1-change_connection_mode_to_sequential" "s1-update_value_1_of_1_or_3_to_5" "s2-begin" "s2-change_connection_mode_to_sequential" "s2-update_value_1_of_1_or_3_to_8" "s1-commit" "s2-commit" "s2-select"
+permutation "s1-begin" "s1-change_connection_mode_to_sequential" "s1-update_value_1_of_1_or_3_to_5" "s2-begin" "s2-change_connection_mode_to_sequential" "s2-update_value_1_of_4_or_6_to_4" "s1-commit" "s2-commit" "s2-select"
+# multi-shard update affecting the same rows
+permutation "s1-begin" "s2-begin" "s1-change_connection_mode_to_sequential" "s2-change_connection_mode_to_sequential" "s1-update_value_1_of_1_or_3_to_5" "s2-update_value_1_of_1_or_3_to_8" "s1-commit" "s2-commit"
+# multi-shard update affecting the different rows
+permutation "s1-begin" "s2-begin" "s1-change_connection_mode_to_sequential" "s2-change_connection_mode_to_sequential" "s2-update_value_1_of_1_or_3_to_8" "s1-update_value_1_of_2_or_4_to_5" "s1-commit" "s2-commit"
\ No newline at end of file

From 0722334e508d5a8147bc7036de20e2dda4940ffc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?mehmet=20furkan=20=C5=9Fahin?= <sahinffurkan@gmail.com>
Date: Tue, 14 Nov 2017 13:26:31 +0300
Subject: [PATCH 2/4] concurrent master_append_table test is added

---
 src/test/regress/isolation_schedule           |  1 +
 .../specs/isolation_master_append_table.spec  | 64 +++++++++++++++++++
 2 files changed, 65 insertions(+)
 create mode 100644 src/test/regress/specs/isolation_master_append_table.spec

diff --git a/src/test/regress/isolation_schedule b/src/test/regress/isolation_schedule
index b58d38cc8..e079f038c 100644
--- a/src/test/regress/isolation_schedule
+++ b/src/test/regress/isolation_schedule
@@ -38,3 +38,4 @@ test: isolation_delete_vs_all
 test: isolation_truncate_vs_all
 test: isolation_drop_vs_all
 test: isolation_ddl_vs_all
+test: isolation_master_append_table
\ No newline at end of file
diff --git a/src/test/regress/specs/isolation_master_append_table.spec b/src/test/regress/specs/isolation_master_append_table.spec
new file mode 100644
index 000000000..98b9220b9
--- /dev/null
+++ b/src/test/regress/specs/isolation_master_append_table.spec
@@ -0,0 +1,64 @@
+setup
+{	
+	ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 230000;
+
+  	CREATE TABLE table_to_append(id int);
+  	CREATE TABLE table_to_be_appended(id int);
+
+  	SELECT create_distributed_table('table_to_append', 'id', 'append');
+  	INSERT INTO table_to_be_appended VALUES(0),(1),(2),(3),(4),(5);
+
+  	COPY table_to_append FROM PROGRAM 'echo "1\n2\n3\n4\n5"';
+}
+
+teardown
+{
+	DROP TABLE table_to_append CASCADE;
+	DROP TABLE table_to_be_appended CASCADE;
+}
+
+session "s1"
+
+step "s1-begin"
+{
+    BEGIN;
+}
+
+step "s1-master_append_table_to_shard"
+{
+   	SELECT	
+   		master_append_table_to_shard(shardid, 'table_to_be_appended', 'localhost', 57636)
+	FROM
+		pg_dist_shard
+	WHERE
+		'table_to_append'::regclass::oid = logicalrelid;
+}
+
+step "s1-commit"
+{
+    COMMIT;
+}
+
+session "s2"
+
+step "s2-begin"
+{
+	BEGIN;
+}
+
+step "s2-master_append_table_to_shard"
+{
+
+   	SELECT	
+   		master_append_table_to_shard(shardid, 'table_to_be_appended', 'localhost', 57636)
+	FROM
+		pg_dist_shard
+	WHERE
+		'table_to_append'::regclass::oid = logicalrelid;
+}
+step "s2-commit"
+{
+	COMMIT;
+}
+
+permutation "s1-begin" "s2-begin" "s1-master_append_table_to_shard" "s2-master_append_table_to_shard" "s1-commit" "s2-commit"
\ No newline at end of file

From 636faadc4775b17ca7955f4447f136c3d248c6f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?mehmet=20furkan=20=C5=9Fahin?= <sahinffurkan@gmail.com>
Date: Tue, 14 Nov 2017 15:42:30 +0300
Subject: [PATCH 3/4] create_distributed_table vs create_distributed_table,
 master_append_table_to_shard vs master_apply_delete_command,
 master_apply_delete_command vs master_apply_delete_command are added

---
 .../isolation_master_append_table.out         | 150 ++++++++++++++++++
 src/test/regress/isolation_schedule           |   2 +-
 .../specs/isolation_master_append_table.spec  |  43 ++++-
 3 files changed, 189 insertions(+), 6 deletions(-)
 create mode 100644 src/test/regress/expected/isolation_master_append_table.out

diff --git a/src/test/regress/expected/isolation_master_append_table.out b/src/test/regress/expected/isolation_master_append_table.out
new file mode 100644
index 000000000..2e31b69d7
--- /dev/null
+++ b/src/test/regress/expected/isolation_master_append_table.out
@@ -0,0 +1,150 @@
+Parsed test spec with 2 sessions
+
+starting permutation: s1-begin s2-begin s1-master_append_table_to_shard s2-master_append_table_to_shard s1-commit s2-commit
+step s1-begin: 
+    BEGIN;
+
+step s2-begin: 
+	BEGIN;
+
+step s1-master_append_table_to_shard: 
+   	SELECT	
+   		master_append_table_to_shard(shardid, 'table_to_be_appended', 'localhost', 57636)
+	FROM
+		pg_dist_shard
+	WHERE
+		'table_to_append'::regclass::oid = logicalrelid;
+
+master_append_table_to_shard
+
+0.0266667      
+step s2-master_append_table_to_shard: 
+
+   	SELECT	
+   		master_append_table_to_shard(shardid, 'table_to_be_appended', 'localhost', 57636)
+	FROM
+		pg_dist_shard
+	WHERE
+		'table_to_append'::regclass::oid = logicalrelid;
+ <waiting ...>
+step s1-commit: 
+    COMMIT;
+
+step s2-master_append_table_to_shard: <... completed>
+master_append_table_to_shard
+
+0.0266667      
+step s2-commit: 
+	COMMIT;
+
+
+starting permutation: s1-begin s2-begin s1-create_distributed_table s2-create_distributed_table s1-commit s2-commit
+step s1-begin: 
+    BEGIN;
+
+step s2-begin: 
+	BEGIN;
+
+step s1-create_distributed_table: 
+	SELECT create_distributed_table('table_to_distribute', 'id');
+
+create_distributed_table
+
+               
+step s2-create_distributed_table: 
+	SELECT create_distributed_table('table_to_distribute', 'id');
+ <waiting ...>
+step s1-commit: 
+    COMMIT;
+
+step s2-create_distributed_table: <... completed>
+error in steps s1-commit s2-create_distributed_table: ERROR:  table "table_to_distribute" is already distributed
+step s2-commit: 
+	COMMIT;
+
+
+starting permutation: s1-begin s2-begin s1-master_append_table_to_shard s2-master_apply_delete_command s1-commit s2-commit
+step s1-begin: 
+    BEGIN;
+
+step s2-begin: 
+	BEGIN;
+
+step s1-master_append_table_to_shard: 
+   	SELECT	
+   		master_append_table_to_shard(shardid, 'table_to_be_appended', 'localhost', 57636)
+	FROM
+		pg_dist_shard
+	WHERE
+		'table_to_append'::regclass::oid = logicalrelid;
+
+master_append_table_to_shard
+
+0.0266667      
+step s2-master_apply_delete_command: 
+   	SELECT master_apply_delete_command($$DELETE FROM table_to_append WHERE id >= 0$$);
+
+ERROR:  canceling statement due to user request
+step s1-commit: 
+    COMMIT;
+
+step s2-commit: 
+	COMMIT;
+
+
+starting permutation: s1-begin s2-begin s1-master_apply_delete_command s2-master_append_table_to_shard s1-commit s2-commit
+step s1-begin: 
+    BEGIN;
+
+step s2-begin: 
+	BEGIN;
+
+step s1-master_apply_delete_command: 
+   	SELECT master_apply_delete_command($$DELETE FROM table_to_append WHERE id >= 0$$);
+
+master_apply_delete_command
+
+1              
+step s2-master_append_table_to_shard: 
+
+   	SELECT	
+   		master_append_table_to_shard(shardid, 'table_to_be_appended', 'localhost', 57636)
+	FROM
+		pg_dist_shard
+	WHERE
+		'table_to_append'::regclass::oid = logicalrelid;
+
+ERROR:  canceling statement due to user request
+step s1-commit: 
+    COMMIT;
+
+step s2-commit: 
+	COMMIT;
+
+
+starting permutation: s1-begin s2-begin s1-master_apply_delete_command s2-master_apply_delete_command s1-commit s2-commit
+step s1-begin: 
+    BEGIN;
+
+step s2-begin: 
+	BEGIN;
+
+step s1-master_apply_delete_command: 
+   	SELECT master_apply_delete_command($$DELETE FROM table_to_append WHERE id >= 0$$);
+
+master_apply_delete_command
+
+1              
+step s2-master_apply_delete_command: 
+   	SELECT master_apply_delete_command($$DELETE FROM table_to_append WHERE id >= 0$$);
+ <waiting ...>
+step s1-commit: 
+    COMMIT;
+
+step s2-master_apply_delete_command: <... completed>
+master_apply_delete_command
+
+0              
+step s2-commit: 
+	COMMIT;
+
diff --git a/src/test/regress/isolation_schedule b/src/test/regress/isolation_schedule
index e079f038c..595f9e1a8 100644
--- a/src/test/regress/isolation_schedule
+++ b/src/test/regress/isolation_schedule
@@ -23,6 +23,7 @@ test: isolation_distributed_deadlock_detection
 # writes, run this test serially.
 test: isolation_create_restore_point
 
+test: isolation_master_append_table
 test: isolation_multi_shard_modify_vs_all
 test: isolation_hash_copy_vs_all
 test: isolation_append_copy_vs_all
@@ -38,4 +39,3 @@ test: isolation_delete_vs_all
 test: isolation_truncate_vs_all
 test: isolation_drop_vs_all
 test: isolation_ddl_vs_all
-test: isolation_master_append_table
\ No newline at end of file
diff --git a/src/test/regress/specs/isolation_master_append_table.spec b/src/test/regress/specs/isolation_master_append_table.spec
index 98b9220b9..e85c2dbc1 100644
--- a/src/test/regress/specs/isolation_master_append_table.spec
+++ b/src/test/regress/specs/isolation_master_append_table.spec
@@ -1,20 +1,21 @@
 setup
 {	
-	ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 230000;
-
   	CREATE TABLE table_to_append(id int);
   	CREATE TABLE table_to_be_appended(id int);
+  	CREATE TABLE table_to_distribute(id int);
 
   	SELECT create_distributed_table('table_to_append', 'id', 'append');
-  	INSERT INTO table_to_be_appended VALUES(0),(1),(2),(3),(4),(5);
+  	INSERT INTO table_to_be_appended VALUES(1),(2),(3),(4),(5),(6);
+  	INSERT INTO table_to_distribute SELECT generate_series(1,100);
 
-  	COPY table_to_append FROM PROGRAM 'echo "1\n2\n3\n4\n5"';
+  	COPY table_to_append FROM PROGRAM 'echo "0\n7\n8\n9\n10"';
 }
 
 teardown
 {
 	DROP TABLE table_to_append CASCADE;
 	DROP TABLE table_to_be_appended CASCADE;
+	DROP TABLE table_to_distribute CASCADE;
 }
 
 session "s1"
@@ -24,6 +25,11 @@ step "s1-begin"
     BEGIN;
 }
 
+step "s1-create_distributed_table"
+{
+	SELECT create_distributed_table('table_to_distribute', 'id');
+}
+
 step "s1-master_append_table_to_shard"
 {
    	SELECT	
@@ -34,6 +40,11 @@ step "s1-master_append_table_to_shard"
 		'table_to_append'::regclass::oid = logicalrelid;
 }
 
+step "s1-master_apply_delete_command"
+{
+   	SELECT master_apply_delete_command($$DELETE FROM table_to_append WHERE id >= 0$$);
+}
+
 step "s1-commit"
 {
     COMMIT;
@@ -46,6 +57,11 @@ step "s2-begin"
 	BEGIN;
 }
 
+step "s2-create_distributed_table"
+{
+	SELECT create_distributed_table('table_to_distribute', 'id');
+}
+
 step "s2-master_append_table_to_shard"
 {
 
@@ -56,9 +72,26 @@ step "s2-master_append_table_to_shard"
 	WHERE
 		'table_to_append'::regclass::oid = logicalrelid;
 }
+
+step "s2-master_apply_delete_command"
+{
+   	SELECT master_apply_delete_command($$DELETE FROM table_to_append WHERE id >= 0$$);
+}
+
 step "s2-commit"
 {
 	COMMIT;
 }
 
-permutation "s1-begin" "s2-begin" "s1-master_append_table_to_shard" "s2-master_append_table_to_shard" "s1-commit" "s2-commit"
\ No newline at end of file
+# concurrent master_append_table_to_shard tests
+permutation "s1-begin" "s2-begin" "s1-master_append_table_to_shard" "s2-master_append_table_to_shard" "s1-commit" "s2-commit"
+
+#concurrent create_distributed_table
+permutation "s1-begin" "s2-begin" "s1-create_distributed_table" "s2-create_distributed_table" "s1-commit" "s2-commit"
+
+# concurrent master_append_table_to_shard vs master_apply_delete_command tests
+permutation "s1-begin" "s2-begin" "s1-master_append_table_to_shard" "s2-master_apply_delete_command" "s1-commit" "s2-commit"
+permutation "s1-begin" "s2-begin" "s1-master_apply_delete_command" "s2-master_append_table_to_shard" "s1-commit" "s2-commit"
+
+#concurrent master_apply_delete_command vs master_apply_delete_command
+permutation "s1-begin" "s2-begin" "s1-master_apply_delete_command" "s2-master_apply_delete_command" "s1-commit" "s2-commit"
\ No newline at end of file

From 8d55754b4dcf91109b8bf3d9be1c02a851a5cfb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?mehmet=20furkan=20=C5=9Fahin?= <sahinffurkan@gmail.com>
Date: Wed, 15 Nov 2017 15:16:27 +0300
Subject: [PATCH 4/4] the tests are separated and some more added

---
 .../isolation_create_distributed_table.out    | 102 ++++++++++++++++
 .../isolation_master_append_table.out         | 115 +-----------------
 .../isolation_master_apply_delete.out         | 109 +++++++++++++++++
 src/test/regress/isolation_schedule           |   2 +-
 .../isolation_create_distributed_table.spec   |  63 ++++++++++
 .../specs/isolation_master_append_table.spec  |  37 +-----
 .../specs/isolation_master_apply_delete.spec  |  63 ++++++++++
 7 files changed, 342 insertions(+), 149 deletions(-)
 create mode 100644 src/test/regress/expected/isolation_create_distributed_table.out
 create mode 100644 src/test/regress/expected/isolation_master_apply_delete.out
 create mode 100644 src/test/regress/specs/isolation_create_distributed_table.spec
 create mode 100644 src/test/regress/specs/isolation_master_apply_delete.spec

diff --git a/src/test/regress/expected/isolation_create_distributed_table.out b/src/test/regress/expected/isolation_create_distributed_table.out
new file mode 100644
index 000000000..bef65be94
--- /dev/null
+++ b/src/test/regress/expected/isolation_create_distributed_table.out
@@ -0,0 +1,102 @@
+Parsed test spec with 2 sessions
+
+starting permutation: s1-begin s2-begin s1-create_distributed_table s2-create_distributed_table s1-commit s2-commit
+step s1-begin: 
+    BEGIN;
+
+step s2-begin: 
+	BEGIN;
+
+step s1-create_distributed_table: 
+	SELECT create_distributed_table('table_to_distribute', 'id');
+
+create_distributed_table
+
+               
+step s2-create_distributed_table: 
+	SELECT create_distributed_table('table_to_distribute', 'id');
+ <waiting ...>
+step s1-commit: 
+    COMMIT;
+
+step s2-create_distributed_table: <... completed>
+error in steps s1-commit s2-create_distributed_table: ERROR:  table "table_to_distribute" is already distributed
+step s2-commit: 
+	COMMIT;
+
+
+starting permutation: s1-begin s2-begin s1-create_distributed_table s2-copy_to_local_table s1-commit s2-commit
+step s1-begin: 
+    BEGIN;
+
+step s2-begin: 
+	BEGIN;
+
+step s1-create_distributed_table: 
+	SELECT create_distributed_table('table_to_distribute', 'id');
+
+create_distributed_table
+
+               
+step s2-copy_to_local_table: 
+	COPY table_to_distribute FROM PROGRAM 'echo "0\n1\n2\n3\n4\n5\n6\n7\n8"';
+ <waiting ...>
+step s1-commit: 
+    COMMIT;
+
+step s2-copy_to_local_table: <... completed>
+step s2-commit: 
+	COMMIT;
+
+
+starting permutation: s1-begin s2-begin s2-copy_to_local_table s1-create_distributed_table s2-commit s1-commit
+step s1-begin: 
+    BEGIN;
+
+step s2-begin: 
+	BEGIN;
+
+step s2-copy_to_local_table: 
+	COPY table_to_distribute FROM PROGRAM 'echo "0\n1\n2\n3\n4\n5\n6\n7\n8"';
+
+step s1-create_distributed_table: 
+	SELECT create_distributed_table('table_to_distribute', 'id');
+ <waiting ...>
+step s2-commit: 
+	COMMIT;
+
+step s1-create_distributed_table: <... completed>
+create_distributed_table
+
+               
+step s1-commit: 
+    COMMIT;
+
+
+starting permutation: s1-copy_to_local_table s1-begin s2-begin s1-create_distributed_table s2-create_distributed_table s1-commit s2-commit
+step s1-copy_to_local_table: 
+	COPY table_to_distribute FROM PROGRAM 'echo "0\n1\n2\n3\n4\n5\n6\n7\n8"';
+
+step s1-begin: 
+    BEGIN;
+
+step s2-begin: 
+	BEGIN;
+
+step s1-create_distributed_table: 
+	SELECT create_distributed_table('table_to_distribute', 'id');
+
+create_distributed_table
+
+               
+step s2-create_distributed_table: 
+	SELECT create_distributed_table('table_to_distribute', 'id');
+ <waiting ...>
+step s1-commit: 
+    COMMIT;
+
+step s2-create_distributed_table: <... completed>
+error in steps s1-commit s2-create_distributed_table: ERROR:  table "table_to_distribute" is already distributed
+step s2-commit: 
+	COMMIT;
+
diff --git a/src/test/regress/expected/isolation_master_append_table.out b/src/test/regress/expected/isolation_master_append_table.out
index 2e31b69d7..40070bd32 100644
--- a/src/test/regress/expected/isolation_master_append_table.out
+++ b/src/test/regress/expected/isolation_master_append_table.out
@@ -17,7 +17,7 @@ step s1-master_append_table_to_shard:
 
 master_append_table_to_shard
 
-0.0266667      
+0.213333       
 step s2-master_append_table_to_shard: 
 
    	SELECT	
@@ -33,118 +33,7 @@ step s1-commit:
 step s2-master_append_table_to_shard: <... completed>
 master_append_table_to_shard
 
-0.0266667      
-step s2-commit: 
-	COMMIT;
-
-
-starting permutation: s1-begin s2-begin s1-create_distributed_table s2-create_distributed_table s1-commit s2-commit
-step s1-begin: 
-    BEGIN;
-
-step s2-begin: 
-	BEGIN;
-
-step s1-create_distributed_table: 
-	SELECT create_distributed_table('table_to_distribute', 'id');
-
-create_distributed_table
-
-               
-step s2-create_distributed_table: 
-	SELECT create_distributed_table('table_to_distribute', 'id');
- <waiting ...>
-step s1-commit: 
-    COMMIT;
-
-step s2-create_distributed_table: <... completed>
-error in steps s1-commit s2-create_distributed_table: ERROR:  table "table_to_distribute" is already distributed
-step s2-commit: 
-	COMMIT;
-
-
-starting permutation: s1-begin s2-begin s1-master_append_table_to_shard s2-master_apply_delete_command s1-commit s2-commit
-step s1-begin: 
-    BEGIN;
-
-step s2-begin: 
-	BEGIN;
-
-step s1-master_append_table_to_shard: 
-   	SELECT	
-   		master_append_table_to_shard(shardid, 'table_to_be_appended', 'localhost', 57636)
-	FROM
-		pg_dist_shard
-	WHERE
-		'table_to_append'::regclass::oid = logicalrelid;
-
-master_append_table_to_shard
-
-0.0266667      
-step s2-master_apply_delete_command: 
-   	SELECT master_apply_delete_command($$DELETE FROM table_to_append WHERE id >= 0$$);
-
-ERROR:  canceling statement due to user request
-step s1-commit: 
-    COMMIT;
-
-step s2-commit: 
-	COMMIT;
-
-
-starting permutation: s1-begin s2-begin s1-master_apply_delete_command s2-master_append_table_to_shard s1-commit s2-commit
-step s1-begin: 
-    BEGIN;
-
-step s2-begin: 
-	BEGIN;
-
-step s1-master_apply_delete_command: 
-   	SELECT master_apply_delete_command($$DELETE FROM table_to_append WHERE id >= 0$$);
-
-master_apply_delete_command
-
-1              
-step s2-master_append_table_to_shard: 
-
-   	SELECT	
-   		master_append_table_to_shard(shardid, 'table_to_be_appended', 'localhost', 57636)
-	FROM
-		pg_dist_shard
-	WHERE
-		'table_to_append'::regclass::oid = logicalrelid;
-
-ERROR:  canceling statement due to user request
-step s1-commit: 
-    COMMIT;
-
-step s2-commit: 
-	COMMIT;
-
-
-starting permutation: s1-begin s2-begin s1-master_apply_delete_command s2-master_apply_delete_command s1-commit s2-commit
-step s1-begin: 
-    BEGIN;
-
-step s2-begin: 
-	BEGIN;
-
-step s1-master_apply_delete_command: 
-   	SELECT master_apply_delete_command($$DELETE FROM table_to_append WHERE id >= 0$$);
-
-master_apply_delete_command
-
-1              
-step s2-master_apply_delete_command: 
-   	SELECT master_apply_delete_command($$DELETE FROM table_to_append WHERE id >= 0$$);
- <waiting ...>
-step s1-commit: 
-    COMMIT;
-
-step s2-master_apply_delete_command: <... completed>
-master_apply_delete_command
-
-0              
+0.32           
 step s2-commit: 
 	COMMIT;
 
diff --git a/src/test/regress/expected/isolation_master_apply_delete.out b/src/test/regress/expected/isolation_master_apply_delete.out
new file mode 100644
index 000000000..fe596eef3
--- /dev/null
+++ b/src/test/regress/expected/isolation_master_apply_delete.out
@@ -0,0 +1,109 @@
+Parsed test spec with 2 sessions
+
+starting permutation: s1-begin s2-begin s1-master_apply_delete_command_all_shard s2-master_apply_delete_command_all_shard s1-commit s2-commit
+step s1-begin: 
+    BEGIN;
+
+step s2-begin: 
+	BEGIN;
+
+step s1-master_apply_delete_command_all_shard: 
+   	SELECT master_apply_delete_command($$DELETE FROM table_to_delete_from WHERE id >= 0$$);
+
+master_apply_delete_command
+
+1              
+step s2-master_apply_delete_command_all_shard: 
+   	SELECT master_apply_delete_command($$DELETE FROM table_to_delete_from WHERE id >= 0$$);
+ <waiting ...>
+step s1-commit: 
+    COMMIT;
+
+step s2-master_apply_delete_command_all_shard: <... completed>
+master_apply_delete_command
+
+0              
+step s2-commit: 
+	COMMIT;
+
+
+starting permutation: s1-begin s2-begin s1-master_apply_delete_command_all_shard s2-master_apply_delete_command_row s1-commit s2-commit
+step s1-begin: 
+    BEGIN;
+
+step s2-begin: 
+	BEGIN;
+
+step s1-master_apply_delete_command_all_shard: 
+   	SELECT master_apply_delete_command($$DELETE FROM table_to_delete_from WHERE id >= 0$$);
+
+master_apply_delete_command
+
+1              
+step s2-master_apply_delete_command_row: 
+   	SELECT master_apply_delete_command($$DELETE FROM table_to_delete_from WHERE id >= 0 and id < 3$$);
+ <waiting ...>
+step s1-commit: 
+    COMMIT;
+
+step s2-master_apply_delete_command_row: <... completed>
+master_apply_delete_command
+
+0              
+step s2-commit: 
+	COMMIT;
+
+
+starting permutation: s1-begin s2-begin s1-master_apply_delete_command_row s2-master_apply_delete_command_all_shard s1-commit s2-commit
+step s1-begin: 
+    BEGIN;
+
+step s2-begin: 
+	BEGIN;
+
+step s1-master_apply_delete_command_row: 
+   	SELECT master_apply_delete_command($$DELETE FROM table_to_delete_from WHERE id >= 0 and id < 3$$);
+
+master_apply_delete_command
+
+0              
+step s2-master_apply_delete_command_all_shard: 
+   	SELECT master_apply_delete_command($$DELETE FROM table_to_delete_from WHERE id >= 0$$);
+ <waiting ...>
+step s1-commit: 
+    COMMIT;
+
+step s2-master_apply_delete_command_all_shard: <... completed>
+master_apply_delete_command
+
+1              
+step s2-commit: 
+	COMMIT;
+
+
+starting permutation: s1-begin s2-begin s1-master_apply_delete_command_row s2-master_apply_delete_command_row s1-commit s2-commit
+step s1-begin: 
+    BEGIN;
+
+step s2-begin: 
+	BEGIN;
+
+step s1-master_apply_delete_command_row: 
+   	SELECT master_apply_delete_command($$DELETE FROM table_to_delete_from WHERE id >= 0 and id < 3$$);
+
+master_apply_delete_command
+
+0              
+step s2-master_apply_delete_command_row: 
+   	SELECT master_apply_delete_command($$DELETE FROM table_to_delete_from WHERE id >= 0 and id < 3$$);
+ <waiting ...>
+step s1-commit: 
+    COMMIT;
+
+step s2-master_apply_delete_command_row: <... completed>
+master_apply_delete_command
+
+0              
+step s2-commit: 
+	COMMIT;
+
diff --git a/src/test/regress/isolation_schedule b/src/test/regress/isolation_schedule
index 595f9e1a8..220f24db6 100644
--- a/src/test/regress/isolation_schedule
+++ b/src/test/regress/isolation_schedule
@@ -23,7 +23,7 @@ test: isolation_distributed_deadlock_detection
 # writes, run this test serially.
 test: isolation_create_restore_point
 
-test: isolation_master_append_table
+test: isolation_create_distributed_table isolation_master_append_table isolation_master_apply_delete
 test: isolation_multi_shard_modify_vs_all
 test: isolation_hash_copy_vs_all
 test: isolation_append_copy_vs_all
diff --git a/src/test/regress/specs/isolation_create_distributed_table.spec b/src/test/regress/specs/isolation_create_distributed_table.spec
new file mode 100644
index 000000000..3b5fa3b6a
--- /dev/null
+++ b/src/test/regress/specs/isolation_create_distributed_table.spec
@@ -0,0 +1,63 @@
+setup
+{	
+  	CREATE TABLE table_to_distribute(id int);
+}
+
+teardown
+{
+	DROP TABLE table_to_distribute CASCADE;
+}
+
+session "s1"
+
+step "s1-begin"
+{
+    BEGIN;
+}
+
+step "s1-create_distributed_table"
+{
+	SELECT create_distributed_table('table_to_distribute', 'id');
+}
+
+step "s1-copy_to_local_table"
+{
+	COPY table_to_distribute FROM PROGRAM 'echo "0\n1\n2\n3\n4\n5\n6\n7\n8"';
+}
+
+step "s1-commit"
+{
+    COMMIT;
+}
+
+session "s2"
+
+step "s2-begin"
+{
+	BEGIN;
+}
+
+step "s2-create_distributed_table"
+{
+	SELECT create_distributed_table('table_to_distribute', 'id');
+}
+
+step "s2-copy_to_local_table"
+{
+	COPY table_to_distribute FROM PROGRAM 'echo "0\n1\n2\n3\n4\n5\n6\n7\n8"';
+}
+
+step "s2-commit"
+{
+	COMMIT;
+}
+
+#concurrent create_distributed_table on empty table
+permutation "s1-begin" "s2-begin" "s1-create_distributed_table" "s2-create_distributed_table" "s1-commit" "s2-commit"
+
+#concurrent create_distributed_table vs. copy to table
+permutation "s1-begin" "s2-begin" "s1-create_distributed_table" "s2-copy_to_local_table" "s1-commit" "s2-commit"
+permutation "s1-begin" "s2-begin" "s2-copy_to_local_table" "s1-create_distributed_table" "s2-commit" "s1-commit"
+
+#concurrent create_distributed_table on non-empty table
+permutation "s1-copy_to_local_table" "s1-begin" "s2-begin" "s1-create_distributed_table" "s2-create_distributed_table" "s1-commit" "s2-commit"
diff --git a/src/test/regress/specs/isolation_master_append_table.spec b/src/test/regress/specs/isolation_master_append_table.spec
index e85c2dbc1..fcbc8696a 100644
--- a/src/test/regress/specs/isolation_master_append_table.spec
+++ b/src/test/regress/specs/isolation_master_append_table.spec
@@ -2,20 +2,17 @@ setup
 {	
   	CREATE TABLE table_to_append(id int);
   	CREATE TABLE table_to_be_appended(id int);
-  	CREATE TABLE table_to_distribute(id int);
 
   	SELECT create_distributed_table('table_to_append', 'id', 'append');
-  	INSERT INTO table_to_be_appended VALUES(1),(2),(3),(4),(5),(6);
-  	INSERT INTO table_to_distribute SELECT generate_series(1,100);
+  	INSERT INTO table_to_be_appended SELECT generate_series(1,1000);
 
-  	COPY table_to_append FROM PROGRAM 'echo "0\n7\n8\n9\n10"';
+  	COPY table_to_append FROM PROGRAM 'echo "0\n7\n8\n9\n10000"';
 }
 
 teardown
 {
 	DROP TABLE table_to_append CASCADE;
 	DROP TABLE table_to_be_appended CASCADE;
-	DROP TABLE table_to_distribute CASCADE;
 }
 
 session "s1"
@@ -25,11 +22,6 @@ step "s1-begin"
     BEGIN;
 }
 
-step "s1-create_distributed_table"
-{
-	SELECT create_distributed_table('table_to_distribute', 'id');
-}
-
 step "s1-master_append_table_to_shard"
 {
    	SELECT	
@@ -40,11 +32,6 @@ step "s1-master_append_table_to_shard"
 		'table_to_append'::regclass::oid = logicalrelid;
 }
 
-step "s1-master_apply_delete_command"
-{
-   	SELECT master_apply_delete_command($$DELETE FROM table_to_append WHERE id >= 0$$);
-}
-
 step "s1-commit"
 {
     COMMIT;
@@ -57,11 +44,6 @@ step "s2-begin"
 	BEGIN;
 }
 
-step "s2-create_distributed_table"
-{
-	SELECT create_distributed_table('table_to_distribute', 'id');
-}
-
 step "s2-master_append_table_to_shard"
 {
 
@@ -73,11 +55,6 @@ step "s2-master_append_table_to_shard"
 		'table_to_append'::regclass::oid = logicalrelid;
 }
 
-step "s2-master_apply_delete_command"
-{
-   	SELECT master_apply_delete_command($$DELETE FROM table_to_append WHERE id >= 0$$);
-}
-
 step "s2-commit"
 {
 	COMMIT;
@@ -85,13 +62,3 @@ step "s2-commit"
 
 # concurrent master_append_table_to_shard tests
 permutation "s1-begin" "s2-begin" "s1-master_append_table_to_shard" "s2-master_append_table_to_shard" "s1-commit" "s2-commit"
-
-#concurrent create_distributed_table
-permutation "s1-begin" "s2-begin" "s1-create_distributed_table" "s2-create_distributed_table" "s1-commit" "s2-commit"
-
-# concurrent master_append_table_to_shard vs master_apply_delete_command tests
-permutation "s1-begin" "s2-begin" "s1-master_append_table_to_shard" "s2-master_apply_delete_command" "s1-commit" "s2-commit"
-permutation "s1-begin" "s2-begin" "s1-master_apply_delete_command" "s2-master_append_table_to_shard" "s1-commit" "s2-commit"
-
-#concurrent master_apply_delete_command vs master_apply_delete_command
-permutation "s1-begin" "s2-begin" "s1-master_apply_delete_command" "s2-master_apply_delete_command" "s1-commit" "s2-commit"
\ No newline at end of file
diff --git a/src/test/regress/specs/isolation_master_apply_delete.spec b/src/test/regress/specs/isolation_master_apply_delete.spec
new file mode 100644
index 000000000..632e2874f
--- /dev/null
+++ b/src/test/regress/specs/isolation_master_apply_delete.spec
@@ -0,0 +1,63 @@
+setup
+{	
+  	CREATE TABLE table_to_delete_from(id int);
+
+  	SELECT create_distributed_table('table_to_delete_from', 'id', 'append');
+
+  	COPY table_to_delete_from FROM PROGRAM 'echo "0\n1\n2\n3\n4\n5\n6\n7\n8\n9\n10"';
+}
+
+teardown
+{
+	DROP TABLE table_to_delete_from CASCADE;
+}
+
+session "s1"
+
+step "s1-begin"
+{
+    BEGIN;
+}
+
+step "s1-master_apply_delete_command_all_shard"
+{
+   	SELECT master_apply_delete_command($$DELETE FROM table_to_delete_from WHERE id >= 0$$);
+}
+
+step "s1-master_apply_delete_command_row"
+{
+   	SELECT master_apply_delete_command($$DELETE FROM table_to_delete_from WHERE id >= 0 and id < 3$$);
+}
+
+step "s1-commit"
+{
+    COMMIT;
+}
+
+session "s2"
+
+step "s2-begin"
+{
+	BEGIN;
+}
+
+step "s2-master_apply_delete_command_all_shard"
+{
+   	SELECT master_apply_delete_command($$DELETE FROM table_to_delete_from WHERE id >= 0$$);
+}
+
+step "s2-master_apply_delete_command_row"
+{
+   	SELECT master_apply_delete_command($$DELETE FROM table_to_delete_from WHERE id >= 0 and id < 3$$);
+}
+
+step "s2-commit"
+{
+	COMMIT;
+}
+
+#concurrent master_apply_delete_command vs master_apply_delete_command
+permutation "s1-begin" "s2-begin" "s1-master_apply_delete_command_all_shard" "s2-master_apply_delete_command_all_shard" "s1-commit" "s2-commit"
+permutation "s1-begin" "s2-begin" "s1-master_apply_delete_command_all_shard" "s2-master_apply_delete_command_row" "s1-commit" "s2-commit"
+permutation "s1-begin" "s2-begin" "s1-master_apply_delete_command_row" "s2-master_apply_delete_command_all_shard" "s1-commit" "s2-commit"
+permutation "s1-begin" "s2-begin" "s1-master_apply_delete_command_row" "s2-master_apply_delete_command_row" "s1-commit" "s2-commit"