mirror of https://github.com/citusdata/citus.git
Add more tests for concurrent operations
Hanefi Onaldi
parent
2f7cf843c9
commit
acecdf8827
|
|
@ -0,0 +1,335 @@
|
|||
Parsed test spec with 4 sessions
|
||||
|
||||
starting permutation: w1-start-session-level-connection w1-begin-on-worker w2-start-session-level-connection w2-begin-on-worker w1-create-named-index w2-create-named-index w1-commit-worker w2-commit-worker w1-stop-connection w2-stop-connection coord-print-index-count
|
||||
step w1-start-session-level-connection:
|
||||
SELECT start_session_level_connection_to_node('localhost', 57637);
|
||||
|
||||
start_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w1-begin-on-worker:
|
||||
SELECT run_commands_on_session_level_connection_to_node('BEGIN');
|
||||
|
||||
run_commands_on_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w2-start-session-level-connection:
|
||||
SELECT start_session_level_connection_to_node('localhost', 57638);
|
||||
|
||||
start_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w2-begin-on-worker:
|
||||
SELECT run_commands_on_session_level_connection_to_node('BEGIN');
|
||||
|
||||
run_commands_on_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w1-create-named-index:
|
||||
SELECT run_commands_on_session_level_connection_to_node('CREATE INDEX dist_table_index ON dist_table(id)');
|
||||
|
||||
run_commands_on_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w2-create-named-index:
|
||||
SELECT run_commands_on_session_level_connection_to_node('CREATE INDEX dist_table_index_2 ON dist_table(id)');
|
||||
|
||||
run_commands_on_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w1-commit-worker:
|
||||
SELECT run_commands_on_session_level_connection_to_node('COMMIT');
|
||||
|
||||
run_commands_on_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w2-commit-worker:
|
||||
SELECT run_commands_on_session_level_connection_to_node('COMMIT');
|
||||
|
||||
run_commands_on_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w1-stop-connection:
|
||||
SELECT stop_session_level_connection_to_node();
|
||||
|
||||
stop_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w2-stop-connection:
|
||||
SELECT stop_session_level_connection_to_node();
|
||||
|
||||
stop_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step coord-print-index-count:
|
||||
SELECT
|
||||
result
|
||||
FROM
|
||||
run_command_on_placements('dist_table', 'select count(*) from pg_indexes WHERE tablename = ''%s''')
|
||||
ORDER BY
|
||||
nodeport;
|
||||
|
||||
result
|
||||
---------------------------------------------------------------------
|
||||
2
|
||||
2
|
||||
2
|
||||
2
|
||||
(4 rows)
|
||||
|
||||
|
||||
starting permutation: w1-start-session-level-connection w1-begin-on-worker w2-start-session-level-connection w2-begin-on-worker w1-create-unnamed-index w2-create-unnamed-index w1-commit-worker w1-stop-connection w2-stop-connection coord-print-index-count
|
||||
step w1-start-session-level-connection:
|
||||
SELECT start_session_level_connection_to_node('localhost', 57637);
|
||||
|
||||
start_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w1-begin-on-worker:
|
||||
SELECT run_commands_on_session_level_connection_to_node('BEGIN');
|
||||
|
||||
run_commands_on_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w2-start-session-level-connection:
|
||||
SELECT start_session_level_connection_to_node('localhost', 57638);
|
||||
|
||||
start_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w2-begin-on-worker:
|
||||
SELECT run_commands_on_session_level_connection_to_node('BEGIN');
|
||||
|
||||
run_commands_on_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w1-create-unnamed-index:
|
||||
SELECT run_commands_on_session_level_connection_to_node('CREATE INDEX ON dist_table(id,data)');
|
||||
|
||||
run_commands_on_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w2-create-unnamed-index:
|
||||
SELECT run_commands_on_session_level_connection_to_node('CREATE INDEX ON dist_table(id,data)');
|
||||
<waiting ...>
|
||||
step w1-commit-worker:
|
||||
SELECT run_commands_on_session_level_connection_to_node('COMMIT');
|
||||
|
||||
run_commands_on_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w2-create-unnamed-index: <... completed>
|
||||
ERROR: duplicate key value violates unique constraint "pg_class_relname_nsp_index"
|
||||
step w1-stop-connection:
|
||||
SELECT stop_session_level_connection_to_node();
|
||||
|
||||
stop_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w2-stop-connection:
|
||||
SELECT stop_session_level_connection_to_node();
|
||||
|
||||
stop_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step coord-print-index-count:
|
||||
SELECT
|
||||
result
|
||||
FROM
|
||||
run_command_on_placements('dist_table', 'select count(*) from pg_indexes WHERE tablename = ''%s''')
|
||||
ORDER BY
|
||||
nodeport;
|
||||
|
||||
result
|
||||
---------------------------------------------------------------------
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
(4 rows)
|
||||
|
||||
|
||||
starting permutation: w1-start-session-level-connection w2-start-session-level-connection w1-create-index-concurrently w2-create-index-concurrently w1-empty w1-stop-connection w2-stop-connection coord-print-index-count
|
||||
step w1-start-session-level-connection:
|
||||
SELECT start_session_level_connection_to_node('localhost', 57637);
|
||||
|
||||
start_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w2-start-session-level-connection:
|
||||
SELECT start_session_level_connection_to_node('localhost', 57638);
|
||||
|
||||
start_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w1-create-index-concurrently:
|
||||
SELECT run_commands_on_session_level_connection_to_node('CREATE INDEX CONCURRENTLY dist_table_index ON dist_table(id)');
|
||||
<waiting ...>
|
||||
step w2-create-index-concurrently:
|
||||
SELECT run_commands_on_session_level_connection_to_node('CREATE INDEX CONCURRENTLY dist_table_index_2 ON dist_table(id)');
|
||||
<waiting ...>
|
||||
step w1-create-index-concurrently: <... completed>
|
||||
run_commands_on_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w1-empty:
|
||||
step w2-create-index-concurrently: <... completed>
|
||||
run_commands_on_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w1-stop-connection:
|
||||
SELECT stop_session_level_connection_to_node();
|
||||
|
||||
stop_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w2-stop-connection:
|
||||
SELECT stop_session_level_connection_to_node();
|
||||
|
||||
stop_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step coord-print-index-count:
|
||||
SELECT
|
||||
result
|
||||
FROM
|
||||
run_command_on_placements('dist_table', 'select count(*) from pg_indexes WHERE tablename = ''%s''')
|
||||
ORDER BY
|
||||
nodeport;
|
||||
|
||||
result
|
||||
---------------------------------------------------------------------
|
||||
2
|
||||
2
|
||||
2
|
||||
2
|
||||
(4 rows)
|
||||
|
||||
|
||||
starting permutation: w1-start-session-level-connection w1-create-named-index w1-begin-on-worker w1-delete coord-begin coord-short-statement-timeout coord-take-lock w1-reindex deadlock-checker-call coord-rollback w1-commit-worker w1-stop-connection
|
||||
step w1-start-session-level-connection:
|
||||
SELECT start_session_level_connection_to_node('localhost', 57637);
|
||||
|
||||
start_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w1-create-named-index:
|
||||
SELECT run_commands_on_session_level_connection_to_node('CREATE INDEX dist_table_index ON dist_table(id)');
|
||||
|
||||
run_commands_on_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w1-begin-on-worker:
|
||||
SELECT run_commands_on_session_level_connection_to_node('BEGIN');
|
||||
|
||||
run_commands_on_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w1-delete:
|
||||
SELECT run_commands_on_session_level_connection_to_node('DELETE FROM dist_table');
|
||||
|
||||
run_commands_on_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step coord-begin:
|
||||
BEGIN;
|
||||
|
||||
step coord-short-statement-timeout:
|
||||
SET statement_timeout = 100;
|
||||
|
||||
step coord-take-lock:
|
||||
LOCK TABLE dist_table IN ACCESS EXCLUSIVE MODE;
|
||||
<waiting ...>
|
||||
step w1-reindex:
|
||||
SELECT run_commands_on_session_level_connection_to_node('REINDEX INDEX dist_table_index');
|
||||
|
||||
run_commands_on_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step deadlock-checker-call:
|
||||
SELECT check_distributed_deadlocks();
|
||||
|
||||
check_distributed_deadlocks
|
||||
---------------------------------------------------------------------
|
||||
f
|
||||
(1 row)
|
||||
|
||||
step coord-take-lock: <... completed>
|
||||
ERROR: canceling statement due to statement timeout
|
||||
step coord-rollback:
|
||||
ROLLBACK;
|
||||
|
||||
step w1-commit-worker:
|
||||
SELECT run_commands_on_session_level_connection_to_node('COMMIT');
|
||||
|
||||
run_commands_on_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
step w1-stop-connection:
|
||||
SELECT stop_session_level_connection_to_node();
|
||||
|
||||
stop_session_level_connection_to_node
|
||||
---------------------------------------------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
|
|
@ -102,6 +102,7 @@ test: isolation_insert_select_vs_all_on_mx
|
|||
test: isolation_ref_select_for_update_vs_all_on_mx
|
||||
test: isolation_ref_update_delete_upsert_vs_all_on_mx
|
||||
test: isolation_dis2ref_foreign_keys_on_mx
|
||||
test: isolation_index_vs_all_on_mx
|
||||
test: isolation_metadata_sync_deadlock
|
||||
test: isolation_replicated_dist_on_mx
|
||||
test: isolation_replicate_reference_tables_to_coordinator
|
||||
|
|
|
|||
|
|
@ -0,0 +1,176 @@
|
|||
#include "isolation_mx_common.include.spec"
|
||||
|
||||
setup
|
||||
{
|
||||
CREATE TABLE dist_table(id int, data int);
|
||||
SELECT create_distributed_table('dist_table', 'id');
|
||||
COPY dist_table FROM PROGRAM 'echo 1, 10 && echo 2, 20 && echo 3, 30 && echo 4, 40 && echo 5, 50' WITH CSV;
|
||||
}
|
||||
|
||||
teardown
|
||||
{
|
||||
DROP TABLE IF EXISTS dist_table CASCADE;
|
||||
}
|
||||
|
||||
session "w1"
|
||||
|
||||
step "w1-start-session-level-connection"
|
||||
{
|
||||
SELECT start_session_level_connection_to_node('localhost', 57637);
|
||||
}
|
||||
|
||||
step "w1-begin-on-worker"
|
||||
{
|
||||
SELECT run_commands_on_session_level_connection_to_node('BEGIN');
|
||||
}
|
||||
|
||||
step "w1-create-named-index"
|
||||
{
|
||||
SELECT run_commands_on_session_level_connection_to_node('CREATE INDEX dist_table_index ON dist_table(id)');
|
||||
}
|
||||
|
||||
step "w1-create-unnamed-index"
|
||||
{
|
||||
SELECT run_commands_on_session_level_connection_to_node('CREATE INDEX ON dist_table(id,data)');
|
||||
}
|
||||
|
||||
step "w1-create-index-concurrently"
|
||||
{
|
||||
SELECT run_commands_on_session_level_connection_to_node('CREATE INDEX CONCURRENTLY dist_table_index ON dist_table(id)');
|
||||
}
|
||||
|
||||
// an empty step to have consistent output for CONCURRENTLY
|
||||
step "w1-empty" {}
|
||||
|
||||
step "w1-reindex"
|
||||
{
|
||||
SELECT run_commands_on_session_level_connection_to_node('REINDEX INDEX dist_table_index');
|
||||
}
|
||||
|
||||
step "w1-delete"
|
||||
{
|
||||
SELECT run_commands_on_session_level_connection_to_node('DELETE FROM dist_table');
|
||||
}
|
||||
|
||||
step "w1-commit-worker"
|
||||
{
|
||||
SELECT run_commands_on_session_level_connection_to_node('COMMIT');
|
||||
}
|
||||
|
||||
step "w1-stop-connection"
|
||||
{
|
||||
SELECT stop_session_level_connection_to_node();
|
||||
}
|
||||
|
||||
|
||||
session "w2"
|
||||
|
||||
step "w2-start-session-level-connection"
|
||||
{
|
||||
SELECT start_session_level_connection_to_node('localhost', 57638);
|
||||
}
|
||||
|
||||
step "w2-begin-on-worker"
|
||||
{
|
||||
SELECT run_commands_on_session_level_connection_to_node('BEGIN');
|
||||
}
|
||||
|
||||
step "w2-create-named-index"
|
||||
{
|
||||
SELECT run_commands_on_session_level_connection_to_node('CREATE INDEX dist_table_index_2 ON dist_table(id)');
|
||||
}
|
||||
|
||||
step "w2-create-unnamed-index"
|
||||
{
|
||||
SELECT run_commands_on_session_level_connection_to_node('CREATE INDEX ON dist_table(id,data)');
|
||||
}
|
||||
|
||||
step "w2-create-index-concurrently"
|
||||
{
|
||||
SELECT run_commands_on_session_level_connection_to_node('CREATE INDEX CONCURRENTLY dist_table_index_2 ON dist_table(id)');
|
||||
}
|
||||
|
||||
step "w2-commit-worker"
|
||||
{
|
||||
SELECT run_commands_on_session_level_connection_to_node('COMMIT');
|
||||
}
|
||||
|
||||
step "w2-stop-connection"
|
||||
{
|
||||
SELECT stop_session_level_connection_to_node();
|
||||
}
|
||||
|
||||
session "coord"
|
||||
|
||||
step "coord-begin"
|
||||
{
|
||||
BEGIN;
|
||||
}
|
||||
|
||||
step "coord-rollback"
|
||||
{
|
||||
ROLLBACK;
|
||||
}
|
||||
|
||||
step "coord-print-index-count"
|
||||
{
|
||||
SELECT
|
||||
result
|
||||
FROM
|
||||
run_command_on_placements('dist_table', 'select count(*) from pg_indexes WHERE tablename = ''%s''')
|
||||
ORDER BY
|
||||
nodeport;
|
||||
}
|
||||
|
||||
step "coord-take-lock"
|
||||
{
|
||||
LOCK TABLE dist_table IN ACCESS EXCLUSIVE MODE;
|
||||
}
|
||||
|
||||
step "coord-short-statement-timeout"
|
||||
{
|
||||
SET statement_timeout = 100;
|
||||
}
|
||||
|
||||
session "deadlock-checker"
|
||||
|
||||
step "deadlock-checker-call"
|
||||
{
|
||||
SELECT check_distributed_deadlocks();
|
||||
}
|
||||
|
||||
permutation "w1-start-session-level-connection" "w1-begin-on-worker" // open transaction on worker 1
|
||||
"w2-start-session-level-connection" "w2-begin-on-worker" // open transaction on worker 2
|
||||
"w1-create-named-index" "w2-create-named-index" // create indexes with unique names on workers
|
||||
"w1-commit-worker" "w2-commit-worker" // commit transactions on workers
|
||||
"w1-stop-connection" "w2-stop-connection" // close connections to workers
|
||||
"coord-print-index-count" // show indexes on coordinator
|
||||
|
||||
permutation "w1-start-session-level-connection" "w1-begin-on-worker" // open transaction on worker 1
|
||||
"w2-start-session-level-connection" "w2-begin-on-worker" // open transaction on worker 2
|
||||
"w1-create-unnamed-index" "w2-create-unnamed-index" // create unnamed indexes on workers
|
||||
"w1-commit-worker" // commit transactions on worker 1 and see error on worker 2 due to name clash
|
||||
"w1-stop-connection" "w2-stop-connection" // close connections to workers
|
||||
"coord-print-index-count" // show indexes on coordinator
|
||||
|
||||
permutation "w1-start-session-level-connection" // start session on worker 1
|
||||
"w2-start-session-level-connection" // start session on worker 2
|
||||
"w1-create-index-concurrently"(*) // create both indexes with concurrently option
|
||||
"w2-create-index-concurrently"(*)
|
||||
"w1-empty" // empty steps to have consistent output for CONCURRENTLY
|
||||
"w1-stop-connection" "w2-stop-connection" // close connections to workers
|
||||
"coord-print-index-count" // show indexes on coordinator
|
||||
|
||||
// the following permutation is expected to fail with a distributed deadlock. However, we do not detect the deadlock, and get blocked until statement_timeout.
|
||||
permutation "w1-start-session-level-connection" // start session on worker 1 only
|
||||
"w1-create-named-index" // create index on worker 1
|
||||
"w1-begin-on-worker" // open transaction block on worker 1
|
||||
"w1-delete" // delete from table on worker 1
|
||||
"coord-begin" // open transaction on coordinator to test distributed deadlock
|
||||
"coord-short-statement-timeout" // set statement timeout on coordinator to early abort deadlock check
|
||||
"coord-take-lock" // take ACCESS EXCLUSIVE lock on table on coordinator, get blocked on worker 1
|
||||
"w1-reindex" // reindex on worker that will acquire ACCESS EXCLUSIVE lock on table, create distributed deadlock
|
||||
"deadlock-checker-call" // check that distributed deadlock is detected properly
|
||||
"coord-rollback" // rollback transaction on coordinator to unblock
|
||||
"w1-commit-worker" // commit transaction on worker 1
|
||||
"w1-stop-connection" // close connection to worker 1
|
||||
Loading…
Reference in New Issue