From 0ba2f9e4e47a49982a79f35a7f766dc8d0edc985 Mon Sep 17 00:00:00 2001 From: Onder Kalaci Date: Fri, 11 Aug 2017 13:28:47 +0300 Subject: [PATCH] Add regression tests for distributed deadlock detection --- ...olation_distributed_deadlock_detection.out | 876 ++++++++++++++++++ src/test/regress/isolation_schedule | 1 + src/test/regress/pg_regress_multi.pl | 11 + ...lation_distributed_deadlock_detection.spec | 408 ++++++++ 4 files changed, 1296 insertions(+) create mode 100644 src/test/regress/expected/isolation_distributed_deadlock_detection.out create mode 100644 src/test/regress/specs/isolation_distributed_deadlock_detection.spec diff --git a/src/test/regress/expected/isolation_distributed_deadlock_detection.out b/src/test/regress/expected/isolation_distributed_deadlock_detection.out new file mode 100644 index 000000000..0d21c17e6 --- /dev/null +++ b/src/test/regress/expected/isolation_distributed_deadlock_detection.out @@ -0,0 +1,876 @@ +Parsed test spec with 7 sessions + +starting permutation: s1-begin s2-begin s1-update-1 s2-update-2 s2-update-1 deadlock-checker-call s1-update-2 deadlock-checker-call s1-finish s2-finish +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-update-1: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 1; + +step s2-update-2: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 2; + +step s2-update-1: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 1; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +f +step s1-update-2: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 2; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +t +step s2-update-1: <... completed> +step s1-update-2: <... completed> +error in steps deadlock-checker-call s2-update-1 s1-update-2: ERROR: distributed deadlock detected and to resolve the deadlock this transaction is cancelled +step s1-finish: + COMMIT; + +step s2-finish: + COMMIT; + + +starting permutation: s1-begin s2-begin s1-update-1-rep-2 s2-update-2-rep-2 s2-update-1-rep-2 deadlock-checker-call s1-update-2-rep-2 deadlock-checker-call s1-finish s2-finish +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-update-1-rep-2: + UPDATE deadlock_detection_test_rep_2 SET some_val = 1 WHERE user_id = 1; + +step s2-update-2-rep-2: + UPDATE deadlock_detection_test_rep_2 SET some_val = 1 WHERE user_id = 2; + +step s2-update-1-rep-2: + UPDATE deadlock_detection_test_rep_2 SET some_val = 1 WHERE user_id = 1; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +f +step s1-update-2-rep-2: + UPDATE deadlock_detection_test_rep_2 SET some_val = 1 WHERE user_id = 2; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +t +step s2-update-1-rep-2: <... completed> +step s1-update-2-rep-2: <... completed> +error in steps deadlock-checker-call s2-update-1-rep-2 s1-update-2-rep-2: ERROR: distributed deadlock detected and to resolve the deadlock this transaction is cancelled +step s1-finish: + COMMIT; + +step s2-finish: + COMMIT; + + +starting permutation: s1-begin s2-begin s1-set-2pc s2-set-2pc s1-update-1 s2-update-2 s2-update-1 deadlock-checker-call s1-update-2 deadlock-checker-call s1-finish s2-finish +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-set-2pc: + set citus.multi_shard_commit_protocol TO '2pc'; + +step s2-set-2pc: + set citus.multi_shard_commit_protocol TO '2pc'; + +step s1-update-1: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 1; + +step s2-update-2: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 2; + +step s2-update-1: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 1; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +f +step s1-update-2: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 2; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +t +step s2-update-1: <... completed> +step s1-update-2: <... completed> +error in steps deadlock-checker-call s2-update-1 s1-update-2: ERROR: distributed deadlock detected and to resolve the deadlock this transaction is cancelled +step s1-finish: + COMMIT; + +step s2-finish: + COMMIT; + + +starting permutation: s1-begin s2-begin s1-update-1 s2-update-2 s1-update-2 deadlock-checker-call s2-upsert-select-all deadlock-checker-call s1-finish s2-finish +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-update-1: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 1; + +step s2-update-2: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 2; + +step s1-update-2: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 2; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +f +step s2-upsert-select-all: + INSERT INTO deadlock_detection_test SELECT * FROM deadlock_detection_test ON CONFLICT(user_id) DO UPDATE SET some_val = deadlock_detection_test.some_val + 5 RETURNING *; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +t +step s1-update-2: <... completed> +step s2-upsert-select-all: <... completed> +error in steps deadlock-checker-call s1-update-2 s2-upsert-select-all: ERROR: distributed deadlock detected and to resolve the deadlock this transaction is cancelled +step s1-finish: + COMMIT; + +step s2-finish: + COMMIT; + + +starting permutation: s1-begin s2-begin s1-update-1 s2-update-2 s1-update-2 deadlock-checker-call s2-ddl deadlock-checker-call s1-finish s2-finish +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-update-1: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 1; + +step s2-update-2: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 2; + +step s1-update-2: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 2; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +f +step s2-ddl: + ALTER TABLE deadlock_detection_test ADD COLUMN test_col INT; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +t +step s1-update-2: <... completed> +step s2-ddl: <... completed> +error in steps deadlock-checker-call s1-update-2 s2-ddl: ERROR: distributed deadlock detected and to resolve the deadlock this transaction is cancelled +step s1-finish: + COMMIT; + +step s2-finish: + COMMIT; + + +starting permutation: s1-begin s2-begin s1-insert-dist-10 s2-insert-local-10 s2-insert-dist-10 s1-insert-local-10 deadlock-checker-call s1-finish s2-finish +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-insert-dist-10: + INSERT INTO deadlock_detection_test VALUES (10, 10); + +step s2-insert-local-10: + INSERT INTO local_deadlock_table VALUES (10, 10); + +step s2-insert-dist-10: + INSERT INTO deadlock_detection_test VALUES (10, 10); + +step s1-insert-local-10: + INSERT INTO local_deadlock_table VALUES (10, 10); + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +t +step s2-insert-dist-10: <... completed> +step s1-insert-local-10: <... completed> +error in steps deadlock-checker-call s2-insert-dist-10 s1-insert-local-10: ERROR: distributed deadlock detected and to resolve the deadlock this transaction is cancelled +step s1-finish: + COMMIT; + +step s2-finish: + COMMIT; + + +starting permutation: s1-begin s2-begin s2-insert-ref-10 s1-insert-ref-11 s2-insert-ref-11 s1-insert-ref-10 deadlock-checker-call s1-finish s2-finish +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s2-insert-ref-10: + INSERT INTO deadlock_detection_reference VALUES (10, 10); + +step s1-insert-ref-11: + INSERT INTO deadlock_detection_reference VALUES (11, 11); + +step s2-insert-ref-11: + INSERT INTO deadlock_detection_reference VALUES (11, 11); + +step s1-insert-ref-10: + INSERT INTO deadlock_detection_reference VALUES (10, 10); + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +t +step s2-insert-ref-11: <... completed> +step s1-insert-ref-10: <... completed> +error in steps deadlock-checker-call s2-insert-ref-11 s1-insert-ref-10: ERROR: distributed deadlock detected and to resolve the deadlock this transaction is cancelled +step s1-finish: + COMMIT; + +step s2-finish: + COMMIT; + + +starting permutation: s1-begin s2-begin s2-insert-ref-10 s1-update-1 deadlock-checker-call s2-update-1 s1-insert-ref-10 deadlock-checker-call s1-finish s2-finish +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s2-insert-ref-10: + INSERT INTO deadlock_detection_reference VALUES (10, 10); + +step s1-update-1: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 1; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +f +step s2-update-1: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 1; + +step s1-insert-ref-10: + INSERT INTO deadlock_detection_reference VALUES (10, 10); + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +t +step s2-update-1: <... completed> +step s1-insert-ref-10: <... completed> +error in steps deadlock-checker-call s2-update-1 s1-insert-ref-10: ERROR: distributed deadlock detected and to resolve the deadlock this transaction is cancelled +step s1-finish: + COMMIT; + +step s2-finish: + COMMIT; + + +starting permutation: s1-begin s2-begin s3-begin s1-update-1 s2-update-2 s3-update-3 deadlock-checker-call s1-update-2 s2-update-3 s3-update-1 deadlock-checker-call s3-finish s2-finish s1-finish +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s3-begin: + BEGIN; + +step s1-update-1: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 1; + +step s2-update-2: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 2; + +step s3-update-3: + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 3; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +f +step s1-update-2: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 2; + +step s2-update-3: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 3; + +step s3-update-1: + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 1; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +t +step s2-update-3: <... completed> +step s3-update-1: <... completed> +error in steps deadlock-checker-call s2-update-3 s3-update-1: ERROR: distributed deadlock detected and to resolve the deadlock this transaction is cancelled +step s3-finish: + COMMIT; + +step s2-finish: + COMMIT; + +step s1-update-2: <... completed> +step s1-finish: + COMMIT; + + +starting permutation: s1-begin s2-begin s3-begin s2-update-1 s1-update-1 s2-update-2 s3-update-3 s3-update-2 deadlock-checker-call s2-update-3 deadlock-checker-call s3-finish s2-finish s1-finish +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s3-begin: + BEGIN; + +step s2-update-1: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 1; + +step s1-update-1: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 1; + +step s2-update-2: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 2; + +step s3-update-3: + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 3; + +step s3-update-2: + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 2; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +f +step s2-update-3: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 3; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +t +step s3-update-2: <... completed> +step s2-update-3: <... completed> +error in steps deadlock-checker-call s3-update-2 s2-update-3: ERROR: distributed deadlock detected and to resolve the deadlock this transaction is cancelled +step s3-finish: + COMMIT; + +step s2-finish: + COMMIT; + +step s1-update-1: <... completed> +step s1-finish: + COMMIT; + + +starting permutation: s1-begin s2-begin s3-begin s4-begin s1-update-1 s2-update-2 s3-update-3 s3-update-2 deadlock-checker-call s4-update-4 s2-update-3 deadlock-checker-call s3-finish s2-finish s1-finish s4-finish +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s3-begin: + BEGIN; + +step s4-begin: + BEGIN; + +step s1-update-1: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 1; + +step s2-update-2: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 2; + +step s3-update-3: + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 3; + +step s3-update-2: + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 2; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +f +step s4-update-4: + UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 4; + +step s2-update-3: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 3; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +t +step s3-update-2: <... completed> +step s2-update-3: <... completed> +error in steps deadlock-checker-call s3-update-2 s2-update-3: ERROR: distributed deadlock detected and to resolve the deadlock this transaction is cancelled +step s3-finish: + COMMIT; + +step s2-finish: + COMMIT; + +step s1-finish: + COMMIT; + +step s4-finish: + COMMIT; + + +starting permutation: s1-begin s2-begin s3-begin s4-begin s4-update-1 s1-update-1 deadlock-checker-call s2-update-2 s3-update-3 s2-update-3 s3-update-2 deadlock-checker-call s3-finish s2-finish s4-finish s1-finish +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s3-begin: + BEGIN; + +step s4-begin: + BEGIN; + +step s4-update-1: + UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 1; + +step s1-update-1: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 1; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +f +step s2-update-2: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 2; + +step s3-update-3: + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 3; + +step s2-update-3: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 3; + +step s3-update-2: + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 2; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +t +step s2-update-3: <... completed> +step s3-update-2: <... completed> +error in steps deadlock-checker-call s2-update-3 s3-update-2: ERROR: distributed deadlock detected and to resolve the deadlock this transaction is cancelled +step s3-finish: + COMMIT; + +step s2-finish: + COMMIT; + +step s4-finish: + COMMIT; + +step s1-update-1: <... completed> +step s1-finish: + COMMIT; + + +starting permutation: s1-begin s2-begin s3-begin s4-begin s1-update-1 s4-update-4 s2-update-2 s3-update-3 s3-update-2 s4-update-1 s1-update-4 deadlock-checker-call s1-finish s4-finish s2-update-3 deadlock-checker-call s2-finish s3-finish +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s3-begin: + BEGIN; + +step s4-begin: + BEGIN; + +step s1-update-1: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 1; + +step s4-update-4: + UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 4; + +step s2-update-2: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 2; + +step s3-update-3: + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 3; + +step s3-update-2: + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 2; + +step s4-update-1: + UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 1; + +step s1-update-4: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 4; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +t +step s4-update-1: <... completed> +step s1-update-4: <... completed> +error in steps deadlock-checker-call s4-update-1 s1-update-4: ERROR: distributed deadlock detected and to resolve the deadlock this transaction is cancelled +step s1-finish: + COMMIT; + +step s4-finish: + COMMIT; + +step s2-update-3: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 3; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +t +step s3-update-2: <... completed> +step s2-update-3: <... completed> +error in steps deadlock-checker-call s3-update-2 s2-update-3: ERROR: distributed deadlock detected and to resolve the deadlock this transaction is cancelled +step s2-finish: + COMMIT; + +step s3-finish: + COMMIT; + + +starting permutation: s1-begin s2-begin s3-begin s4-begin s5-begin s6-begin s1-update-1 s5-update-5 s3-update-2 s2-update-3 s4-update-4 s3-update-4 deadlock-checker-call s6-update-6 s4-update-6 s1-update-5 s5-update-1 deadlock-checker-call s1-finish s5-finish s6-finish s4-finish s3-finish s2-finish +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s3-begin: + BEGIN; + +step s4-begin: + BEGIN; + +step s5-begin: + BEGIN; + +step s6-begin: + BEGIN; + +step s1-update-1: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 1; + +step s5-update-5: + UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 5; + +step s3-update-2: + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 2; + +step s2-update-3: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 3; + +step s4-update-4: + UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 4; + +step s3-update-4: + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 4; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +f +step s6-update-6: + UPDATE deadlock_detection_test SET some_val = 6 WHERE user_id = 6; + +step s4-update-6: + UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 6; + +step s1-update-5: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 5; + +step s5-update-1: + UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 1; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +t +step s1-update-5: <... completed> +step s5-update-1: <... completed> +error in steps deadlock-checker-call s1-update-5 s5-update-1: ERROR: distributed deadlock detected and to resolve the deadlock this transaction is cancelled +step s1-finish: + COMMIT; + +step s5-finish: + COMMIT; + +step s6-finish: + COMMIT; + +step s4-update-6: <... completed> +step s4-finish: + COMMIT; + +step s3-update-4: <... completed> +step s3-finish: + COMMIT; + +step s2-finish: + COMMIT; + + +starting permutation: s1-begin s2-begin s3-begin s4-begin s5-begin s6-begin s6-update-6 s5-update-5 s5-update-6 s4-update-4 s1-update-4 s4-update-5 deadlock-checker-call s2-update-3 s3-update-2 s2-update-2 s3-update-3 deadlock-checker-call s6-finish s5-finish s4-finish s1-finish s3-finish s2-finish +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s3-begin: + BEGIN; + +step s4-begin: + BEGIN; + +step s5-begin: + BEGIN; + +step s6-begin: + BEGIN; + +step s6-update-6: + UPDATE deadlock_detection_test SET some_val = 6 WHERE user_id = 6; + +step s5-update-5: + UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 5; + +step s5-update-6: + UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 6; + +step s4-update-4: + UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 4; + +step s1-update-4: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 4; + +step s4-update-5: + UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 5; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +f +step s2-update-3: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 3; + +step s3-update-2: + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 2; + +step s2-update-2: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 2; + +step s3-update-3: + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 3; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +t +step s2-update-2: <... completed> +step s3-update-3: <... completed> +error in steps deadlock-checker-call s2-update-2 s3-update-3: ERROR: distributed deadlock detected and to resolve the deadlock this transaction is cancelled +step s6-finish: + COMMIT; + +step s5-update-6: <... completed> +step s5-finish: + COMMIT; + +step s4-update-5: <... completed> +step s4-finish: + COMMIT; + +step s1-update-4: <... completed> +step s1-finish: + COMMIT; + +step s3-finish: + COMMIT; + +step s2-finish: + COMMIT; + + +starting permutation: s1-begin s2-begin s3-begin s4-begin s5-begin s6-begin s5-update-5 s3-update-2 s2-update-2 s4-update-4 s3-update-4 s4-update-5 s1-update-4 deadlock-checker-call s6-update-6 s5-update-6 s6-update-5 deadlock-checker-call s5-finish s6-finish s4-finish s3-finish s1-finish s2-finish +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s3-begin: + BEGIN; + +step s4-begin: + BEGIN; + +step s5-begin: + BEGIN; + +step s6-begin: + BEGIN; + +step s5-update-5: + UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 5; + +step s3-update-2: + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 2; + +step s2-update-2: + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 2; + +step s4-update-4: + UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 4; + +step s3-update-4: + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 4; + +step s4-update-5: + UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 5; + +step s1-update-4: + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 4; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +f +step s6-update-6: + UPDATE deadlock_detection_test SET some_val = 6 WHERE user_id = 6; + +step s5-update-6: + UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 6; + +step s6-update-5: + UPDATE deadlock_detection_test SET some_val = 6 WHERE user_id = 5; + +step deadlock-checker-call: + SELECT check_distributed_deadlocks(); + +check_distributed_deadlocks + +t +step s5-update-6: <... completed> +step s6-update-5: <... completed> +error in steps deadlock-checker-call s5-update-6 s6-update-5: ERROR: distributed deadlock detected and to resolve the deadlock this transaction is cancelled +step s5-finish: + COMMIT; + +step s4-update-5: <... completed> +step s6-finish: + COMMIT; + +step s4-finish: + COMMIT; + +step s3-update-4: <... completed> +step s3-finish: + COMMIT; + +step s2-update-2: <... completed> +step s1-update-4: <... completed> +step s1-finish: + COMMIT; + +step s2-finish: + COMMIT; + diff --git a/src/test/regress/isolation_schedule b/src/test/regress/isolation_schedule index 5e31d931c..edc6c7440 100644 --- a/src/test/regress/isolation_schedule +++ b/src/test/regress/isolation_schedule @@ -16,6 +16,7 @@ test: isolation_distributed_transaction_id isolation_progress_monitoring test: isolation_dump_local_wait_edges isolation_dump_global_wait_edges test: isolation_replace_wait_function +test: isolation_distributed_deadlock_detection # creating a restore point briefly blocks all # writes, run this test serially. diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index 1a503427d..9080eea3d 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -253,6 +253,17 @@ if ($followercluster) push(@pgOptions, '-c', "wal_level=replica"); } +# disable automatic distributed deadlock detection during the isolation testing +# to make sure that we always get consistent test outputs. If we don't manually +# (i.e., calling a UDF) detect the deadlocks, some sessions that do not participate +# in the deadlock may interleave with the deadlock detection, which results in non- +# consistent test outputs. +if($isolationtester) +{ + push(@pgOptions, '-c', "citus.log_distributed_deadlock_detection=on"); + push(@pgOptions, '-c', "citus.distributed_deadlock_detection_factor=-1"); +} + # Add externally added options last, so they overwrite the default ones above for my $option (@userPgOptions) { diff --git a/src/test/regress/specs/isolation_distributed_deadlock_detection.spec b/src/test/regress/specs/isolation_distributed_deadlock_detection.spec new file mode 100644 index 000000000..9068133af --- /dev/null +++ b/src/test/regress/specs/isolation_distributed_deadlock_detection.spec @@ -0,0 +1,408 @@ +setup +{ + SELECT citus.replace_isolation_tester_func(); + SELECT citus.refresh_isolation_tester_prepared_statement(); + + CREATE TABLE deadlock_detection_reference (user_id int UNIQUE, some_val int); + SELECT create_reference_table('deadlock_detection_reference'); + + CREATE TABLE deadlock_detection_test (user_id int UNIQUE, some_val int); + INSERT INTO deadlock_detection_test SELECT i, i FROM generate_series(1,7) i; + SELECT create_distributed_table('deadlock_detection_test', 'user_id'); + + CREATE TABLE local_deadlock_table (user_id int UNIQUE, some_val int); + + CREATE TABLE deadlock_detection_test_rep_2 (user_id int UNIQUE, some_val int); + SET citus.shard_replication_factor = 2; + SELECT create_distributed_table('deadlock_detection_test_rep_2', 'user_id'); + + INSERT INTO deadlock_detection_test_rep_2 VALUES (1,1); + INSERT INTO deadlock_detection_test_rep_2 VALUES (2,2); +} + +teardown +{ + DROP TABLE deadlock_detection_test; + DROP TABLE local_deadlock_table; + DROP TABLE deadlock_detection_test_rep_2; + DROP TABLE deadlock_detection_reference; + SELECT citus.restore_isolation_tester_func(); + SET citus.shard_replication_factor = 1; +} + +session "s1" + +step "s1-begin" +{ + BEGIN; +} + +step "s1-update-1" +{ + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 1; +} + +step "s1-update-2" +{ + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 2; +} + +step "s1-update-3" +{ + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 3; +} + +step "s1-update-4" +{ + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 4; +} + +step "s1-update-5" +{ + UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 5; +} + +step "s1-insert-dist-10" +{ + INSERT INTO deadlock_detection_test VALUES (10, 10); +} + +step "s1-insert-local-10" +{ + INSERT INTO local_deadlock_table VALUES (10, 10); +} + +step "s1-set-2pc" +{ + set citus.multi_shard_commit_protocol TO '2pc'; +} + +step "s1-update-1-rep-2" +{ + UPDATE deadlock_detection_test_rep_2 SET some_val = 1 WHERE user_id = 1; +} + +step "s1-update-2-rep-2" +{ + UPDATE deadlock_detection_test_rep_2 SET some_val = 1 WHERE user_id = 2; +} + +step "s1-insert-ref-10" +{ + INSERT INTO deadlock_detection_reference VALUES (10, 10); +} + +step "s1-insert-ref-11" +{ + INSERT INTO deadlock_detection_reference VALUES (11, 11); +} + +step "s1-finish" +{ + COMMIT; +} + +session "s2" + +step "s2-begin" +{ + BEGIN; +} + +step "s2-update-1" +{ + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 1; +} + +step "s2-update-2" +{ + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 2; +} + +step "s2-update-3" +{ + UPDATE deadlock_detection_test SET some_val = 2 WHERE user_id = 3; +} + +step "s2-upsert-select-all" +{ + INSERT INTO deadlock_detection_test SELECT * FROM deadlock_detection_test ON CONFLICT(user_id) DO UPDATE SET some_val = deadlock_detection_test.some_val + 5 RETURNING *; +} + +step "s2-ddl" +{ + ALTER TABLE deadlock_detection_test ADD COLUMN test_col INT; +} + +step "s2-insert-dist-10" +{ + INSERT INTO deadlock_detection_test VALUES (10, 10); +} + +step "s2-insert-local-10" +{ + INSERT INTO local_deadlock_table VALUES (10, 10); +} + +step "s2-set-2pc" +{ + set citus.multi_shard_commit_protocol TO '2pc'; +} + +step "s2-update-1-rep-2" +{ + UPDATE deadlock_detection_test_rep_2 SET some_val = 1 WHERE user_id = 1; +} + +step "s2-update-2-rep-2" +{ + UPDATE deadlock_detection_test_rep_2 SET some_val = 1 WHERE user_id = 2; +} + +step "s2-insert-ref-10" +{ + INSERT INTO deadlock_detection_reference VALUES (10, 10); +} + +step "s2-insert-ref-11" +{ + INSERT INTO deadlock_detection_reference VALUES (11, 11); +} + + +step "s2-finish" +{ + COMMIT; +} + +session "s3" + +step "s3-begin" +{ + BEGIN; +} + +step "s3-update-1" +{ + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 1; +} + +step "s3-update-2" +{ + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 2; +} + +step "s3-update-3" +{ + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 3; +} + +step "s3-update-4" +{ + UPDATE deadlock_detection_test SET some_val = 3 WHERE user_id = 4; +} + +step "s3-finish" +{ + COMMIT; +} + +session "s4" + +step "s4-begin" +{ + BEGIN; +} + +step "s4-update-1" +{ + UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 1; +} + +step "s4-update-2" +{ + UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 2; +} + +step "s4-update-3" +{ + UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 3; +} + +step "s4-update-4" +{ + UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 4; +} + +step "s4-update-5" +{ + UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 5; +} + +step "s4-update-6" +{ + UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 6; +} + +step "s4-update-7" +{ + UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 7; +} + +step "s4-finish" +{ + COMMIT; +} + +session "s5" + +step "s5-begin" +{ + BEGIN; +} + +step "s5-update-1" +{ + UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 1; +} + +step "s5-update-2" +{ + UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 2; +} + +step "s5-update-3" +{ + UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 3; +} + +step "s5-update-4" +{ + UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 4; +} + +step "s5-update-5" +{ + UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 5; +} + +step "s5-update-6" +{ + UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 6; +} + +step "s5-update-7" +{ + UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 7; +} + +step "s5-finish" +{ + COMMIT; +} + +session "s6" + +step "s6-begin" +{ + BEGIN; +} + +step "s6-update-1" +{ + UPDATE deadlock_detection_test SET some_val = 6 WHERE user_id = 1; +} + +step "s6-update-2" +{ + UPDATE deadlock_detection_test SET some_val = 6 WHERE user_id = 2; +} + +step "s6-update-3" +{ + UPDATE deadlock_detection_test SET some_val = 6 WHERE user_id = 3; +} + +step "s6-update-4" +{ + UPDATE deadlock_detection_test SET some_val = 6 WHERE user_id = 4; +} + +step "s6-update-5" +{ + UPDATE deadlock_detection_test SET some_val = 6 WHERE user_id = 5; +} + +step "s6-update-6" +{ + UPDATE deadlock_detection_test SET some_val = 6 WHERE user_id = 6; +} + +step "s6-update-7" +{ + UPDATE deadlock_detection_test SET some_val = 6 WHERE user_id = 7; +} + +step "s6-finish" +{ + COMMIT; +} + +# we disable the deamon during the regression tests in order to get consistent results +# thus we manually issue the deadlock detection +session "deadlock-checker" + +# we issue the checker not only when there are deadlocks to ensure that we never cancel +# backend inappropriately +step "deadlock-checker-call" +{ + SELECT check_distributed_deadlocks(); +} + +# simplest case, loop with two nodes +permutation "s1-begin" "s2-begin" "s1-update-1" "s2-update-2" "s2-update-1" "deadlock-checker-call" "s1-update-2" "deadlock-checker-call" "s1-finish" "s2-finish" + +# simplest case with replication factor 2 +permutation "s1-begin" "s2-begin" "s1-update-1-rep-2" "s2-update-2-rep-2" "s2-update-1-rep-2" "deadlock-checker-call" "s1-update-2-rep-2" "deadlock-checker-call" "s1-finish" "s2-finish" + +# simplest case with 2pc enabled +permutation "s1-begin" "s2-begin" "s1-set-2pc" "s2-set-2pc" "s1-update-1" "s2-update-2" "s2-update-1" "deadlock-checker-call" "s1-update-2" "deadlock-checker-call" "s1-finish" "s2-finish" + +# simplest case with multi-shard query is cancelled +permutation "s1-begin" "s2-begin" "s1-update-1" "s2-update-2" "s1-update-2" "deadlock-checker-call" "s2-upsert-select-all" "deadlock-checker-call" "s1-finish" "s2-finish" + +# simplest case with DDL is cancelled +permutation "s1-begin" "s2-begin" "s1-update-1" "s2-update-2" "s1-update-2" "deadlock-checker-call" "s2-ddl" "deadlock-checker-call" "s1-finish" "s2-finish" + +# daedlock with local table +permutation "s1-begin" "s2-begin" "s1-insert-dist-10" "s2-insert-local-10" "s2-insert-dist-10" "s1-insert-local-10" "deadlock-checker-call" "s1-finish" "s2-finish" + +# daedlock with reference tables only +permutation "s1-begin" "s2-begin" "s2-insert-ref-10" "s1-insert-ref-11" "s2-insert-ref-11" "s1-insert-ref-10" "deadlock-checker-call" "s1-finish" "s2-finish" + +# deadlock with referecen + distributed tables +permutation "s1-begin" "s2-begin" "s2-insert-ref-10" "s1-update-1" "deadlock-checker-call" "s2-update-1" "s1-insert-ref-10" "deadlock-checker-call" "s1-finish" "s2-finish" + +# slightly more complex case, loop with three nodes +permutation "s1-begin" "s2-begin" "s3-begin" "s1-update-1" "s2-update-2" "s3-update-3" "deadlock-checker-call" "s1-update-2" "s2-update-3" "s3-update-1" "deadlock-checker-call" "s3-finish" "s2-finish" "s1-finish" + +# similar to the above (i.e., 3 nodes), but the cycle starts from the second node +permutation "s1-begin" "s2-begin" "s3-begin" "s2-update-1" "s1-update-1" "s2-update-2" "s3-update-3" "s3-update-2" "deadlock-checker-call" "s2-update-3" "deadlock-checker-call" "s3-finish" "s2-finish" "s1-finish" + +# not connected graph +permutation "s1-begin" "s2-begin" "s3-begin" "s4-begin" "s1-update-1" "s2-update-2" "s3-update-3" "s3-update-2" "deadlock-checker-call" "s4-update-4" "s2-update-3" "deadlock-checker-call" "s3-finish" "s2-finish" "s1-finish" "s4-finish" + +# still a not connected graph, but each smaller graph contains dependencies, one of which is a distributed deadlock +permutation "s1-begin" "s2-begin" "s3-begin" "s4-begin" "s4-update-1" "s1-update-1" "deadlock-checker-call" "s2-update-2" "s3-update-3" "s2-update-3" "s3-update-2" "deadlock-checker-call" "s3-finish" "s2-finish" "s4-finish" "s1-finish" + +# multiple deadlocks on a not connected graph +permutation "s1-begin" "s2-begin" "s3-begin" "s4-begin" "s1-update-1" "s4-update-4" "s2-update-2" "s3-update-3" "s3-update-2" "s4-update-1" "s1-update-4" "deadlock-checker-call" "s1-finish" "s4-finish" "s2-update-3" "deadlock-checker-call" "s2-finish" "s3-finish" + +# a larger graph where the first node is in the distributed deadlock +permutation "s1-begin" "s2-begin" "s3-begin" "s4-begin" "s5-begin" "s6-begin" "s1-update-1" "s5-update-5" "s3-update-2" "s2-update-3" "s4-update-4" "s3-update-4" "deadlock-checker-call" "s6-update-6" "s4-update-6" "s1-update-5" "s5-update-1" "deadlock-checker-call" "s1-finish" "s5-finish" "s6-finish" "s4-finish" "s3-finish" "s2-finish" + +# a larger graph where the deadlock starts from a middle node +permutation "s1-begin" "s2-begin" "s3-begin" "s4-begin" "s5-begin" "s6-begin" "s6-update-6" "s5-update-5" "s5-update-6" "s4-update-4" "s1-update-4" "s4-update-5" "deadlock-checker-call" "s2-update-3" "s3-update-2" "s2-update-2" "s3-update-3" "deadlock-checker-call" "s6-finish" "s5-finish" "s4-finish" "s1-finish" "s3-finish" "s2-finish" + +# a larger graph where the deadlock starts from the last node +permutation "s1-begin" "s2-begin" "s3-begin" "s4-begin" "s5-begin" "s6-begin" "s5-update-5" "s3-update-2" "s2-update-2" "s4-update-4" "s3-update-4" "s4-update-5" "s1-update-4" "deadlock-checker-call" "s6-update-6" "s5-update-6" "s6-update-5" "deadlock-checker-call" "s5-finish" "s6-finish" "s4-finish" "s3-finish" "s1-finish" "s2-finish"