From c3e1d49e345da9678d145bedea5faf0bdcdefd27 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Wed, 26 Oct 2016 06:51:16 -0700 Subject: [PATCH 1/4] Don't try to shutdown servers that have not been started in regression tests. This avoids spurious output from failing shutdowns and uninitialized variable warnings if pg_regress_multi.pl fails before starting servers. --- src/test/regress/pg_regress_multi.pl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index 170f32225..a38e810b6 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -47,6 +47,8 @@ my %fdwServers = (); my %functions = (); my %operators = (); +my $serversAreShutdown = "TRUE"; + GetOptions( 'bindir=s' => \$bindir, 'libdir=s' => \$libdir, @@ -170,7 +172,6 @@ for my $port (@workerPorts) } # Routine to shutdown servers at failure/exit -my $serversAreShutdown = "FALSE"; sub ShutdownServers() { if ($serversAreShutdown eq "FALSE") @@ -204,6 +205,9 @@ END } } +# Signal that servers should be shutdown +$serversAreShutdown = "FALSE"; + # Start servers system("$bindir/pg_ctl", ('start', '-w', From ce73ffdf2e2febba54d4e0bad9b5c1e35a318452 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Wed, 26 Oct 2016 05:07:08 -0700 Subject: [PATCH 2/4] Identify build and source directory of postgres we're compiling against. That's useful when trying to rely on files only present in source and/or build directories, not in the normal installation. E.g. the isolationtester binary, or the valgrind suppression files. --- Makefile.global.in | 3 +++ configure | 14 ++++++++++++++ configure.in | 10 ++++++++++ 3 files changed, 27 insertions(+) diff --git a/Makefile.global.in b/Makefile.global.in index 03c21416f..7cdbcd5d9 100644 --- a/Makefile.global.in +++ b/Makefile.global.in @@ -11,6 +11,9 @@ citus_abs_srcdir:=@abs_top_srcdir@/${citus_subdir} citus_abs_top_srcdir:=@abs_top_srcdir@ +postgres_abs_srcdir:=@POSTGRES_SRCDIR@ +postgres_abs_builddir:=@POSTGRES_BUILDDIR@ + PG_CONFIG:=@PG_CONFIG@ PGXS:=$(shell $(PG_CONFIG) --pgxs) diff --git a/configure b/configure index e122b8702..3325bb70e 100755 --- a/configure +++ b/configure @@ -586,6 +586,8 @@ PACKAGE_URL='' ac_subst_vars='LTLIBOBJS LIBOBJS +POSTGRES_BUILDDIR +POSTGRES_SRCDIR CITUS_CFLAGS OBJEXT EXEEXT @@ -2540,6 +2542,14 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu CFLAGS="$SAVE_CFLAGS" + +# Locate source and build directory of the postgres we're building +# against. Can't rely on either still being present, but e.g. optional +# test infrastructure can rely on it. +POSTGRES_SRCDIR=$(grep ^abs_top_srcdir $(dirname $($PG_CONFIG --pgxs))/../Makefile.global|cut -d ' ' -f3-) +POSTGRES_BUILDDIR=$(grep ^abs_top_builddir $(dirname $($PG_CONFIG --pgxs))/../Makefile.global|cut -d ' ' -f3-) + + # check for a number of CFLAGS that make development easier # CITUSAC_PROG_CC_CFLAGS_OPT @@ -2938,6 +2948,10 @@ fi CITUS_CFLAGS="$CITUS_CFLAGS" +POSTGRES_SRCDIR="$POSTGRES_SRCDIR" + +POSTGRES_BUILDDIR="$POSTGRES_BUILDDIR" + ac_config_files="$ac_config_files Makefile.global" diff --git a/configure.in b/configure.in index e0903260d..faf6721d1 100644 --- a/configure.in +++ b/configure.in @@ -63,6 +63,14 @@ SAVE_CFLAGS="$CFLAGS" AC_PROG_CC([$($PG_CONFIG --cc)]) CFLAGS="$SAVE_CFLAGS" + +# Locate source and build directory of the postgres we're building +# against. Can't rely on either still being present, but e.g. optional +# test infrastructure can rely on it. +POSTGRES_SRCDIR=$(grep ^abs_top_srcdir $(dirname $($PG_CONFIG --pgxs))/../Makefile.global|cut -d ' ' -f3-) +POSTGRES_BUILDDIR=$(grep ^abs_top_builddir $(dirname $($PG_CONFIG --pgxs))/../Makefile.global|cut -d ' ' -f3-) + + # check for a number of CFLAGS that make development easier # CITUSAC_PROG_CC_CFLAGS_OPT @@ -102,6 +110,8 @@ CITUSAC_PROG_CC_CFLAGS_OPT([-Wmissing-declarations]) CITUSAC_PROG_CC_CFLAGS_OPT([-Wmissing-prototypes]) AC_SUBST(CITUS_CFLAGS, "$CITUS_CFLAGS") +AC_SUBST(POSTGRES_SRCDIR, "$POSTGRES_SRCDIR") +AC_SUBST(POSTGRES_BUILDDIR, "$POSTGRES_BUILDDIR") AC_CONFIG_FILES([Makefile.global]) AC_CONFIG_HEADERS([src/include/citus_config.h]) From 121b868da546d37faf01f4c859f2b56c946fc0a6 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Wed, 26 Oct 2016 05:09:12 -0700 Subject: [PATCH 3/4] Add very basic isolationtester infrastructure including a trivial test. --- src/test/regress/Makefile | 6 ++- .../expected/isolation_cluster_management.out | 13 ++++++ .../expected/isolation_concurrent_dml.out | 30 ++++++++++++++ src/test/regress/isolation_schedule | 2 + src/test/regress/pg_regress_multi.pl | 41 ++++++++++++++++++- .../specs/isolation_cluster_management.spec | 8 ++++ .../specs/isolation_concurrent_dml.spec | 40 ++++++++++++++++++ 7 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 src/test/regress/expected/isolation_cluster_management.out create mode 100644 src/test/regress/expected/isolation_concurrent_dml.out create mode 100644 src/test/regress/isolation_schedule create mode 100644 src/test/regress/specs/isolation_cluster_management.spec create mode 100644 src/test/regress/specs/isolation_concurrent_dml.spec diff --git a/src/test/regress/Makefile b/src/test/regress/Makefile index 4d72f20ac..7ad661a36 100644 --- a/src/test/regress/Makefile +++ b/src/test/regress/Makefile @@ -14,7 +14,7 @@ endif ## Citus regression support ## MULTI_INSTALLDIR=$(CURDIR)/tmp_check/install -pg_regress_multi_check = $(PERL) $(citus_abs_srcdir)/pg_regress_multi.pl --pgxsdir="$(pgxsdir)" --bindir="$(bindir)" --libdir="$(libdir)" --majorversion="$(MAJORVERSION)" +pg_regress_multi_check = $(PERL) $(citus_abs_srcdir)/pg_regress_multi.pl --pgxsdir="$(pgxsdir)" --bindir="$(bindir)" --libdir="$(libdir)" --majorversion="$(MAJORVERSION)" --postgres-builddir="$(postgres_abs_builddir)" --postgres-srcdir="$(postgres_abs_srcdir)" MULTI_REGRESS_OPTS = --inputdir=$(citus_abs_srcdir) $(pg_regress_locale_flags) # XXX: Can't actually do useful testruns against install - $libdir @@ -49,6 +49,10 @@ check-multi: all tempinstall-main $(pg_regress_multi_check) --load-extension=citus \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule $(EXTRA_TESTS) +check-isolation: all tempinstall-main + $(pg_regress_multi_check) --load-extension=citus --isolationtester \ + -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/isolation_schedule $(EXTRA_TESTS) + check-multi-task-tracker-extra: all tempinstall-main $(pg_regress_multi_check) --load-extension=citus \ --server-option=citus.task_executor_type=task-tracker \ diff --git a/src/test/regress/expected/isolation_cluster_management.out b/src/test/regress/expected/isolation_cluster_management.out new file mode 100644 index 000000000..8ee09c894 --- /dev/null +++ b/src/test/regress/expected/isolation_cluster_management.out @@ -0,0 +1,13 @@ +Parsed test spec with 1 sessions + +starting permutation: s1a +step s1a: + SELECT master_add_node('localhost', 57637); + SELECT master_add_node('localhost', 57638); + +master_add_node + +(1,1,localhost,57637,default,f) +master_add_node + +(2,2,localhost,57638,default,f) diff --git a/src/test/regress/expected/isolation_concurrent_dml.out b/src/test/regress/expected/isolation_concurrent_dml.out new file mode 100644 index 000000000..a5e6a1737 --- /dev/null +++ b/src/test/regress/expected/isolation_concurrent_dml.out @@ -0,0 +1,30 @@ +Parsed test spec with 2 sessions + +starting permutation: s1-begin s1-insert s2-update s1-commit +master_create_worker_shards + + +step s1-begin: + BEGIN; + +step s1-insert: + INSERT INTO test_table VALUES(1); + +step s2-update: + UPDATE test_table SET data = 'blarg' WHERE test_id = 1; + +step s1-commit: + COMMIT; + +step s2-update: <... completed> + +starting permutation: s1-insert s2-update +master_create_worker_shards + + +step s1-insert: + INSERT INTO test_table VALUES(1); + +step s2-update: + UPDATE test_table SET data = 'blarg' WHERE test_id = 1; + diff --git a/src/test/regress/isolation_schedule b/src/test/regress/isolation_schedule new file mode 100644 index 000000000..d702c065f --- /dev/null +++ b/src/test/regress/isolation_schedule @@ -0,0 +1,2 @@ +test: isolation_cluster_management +test: isolation_concurrent_dml diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index a38e810b6..99407ac51 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -26,8 +26,11 @@ sub Usage() print " pg_regress_multi [MULTI OPTIONS] -- [PG REGRESS OPTS]\n"; print "\n"; print "Multi Options:\n"; + print " --isolationtester Run isolationtester tests instead of plain tests\n"; print " --bindir Path to postgres binary directory\n"; print " --libdir Path to postgres library directory\n"; + print " --postgres-builddir Path to postgres build directory\n"; + print " --postgres-srcdir Path to postgres build directory\n"; print " --pgxsdir Path to the PGXS directory\n"; print " --load-extension Extensions to install in all nodes\n"; print " --server-option Config option to pass to the server\n"; @@ -35,9 +38,12 @@ sub Usage() } # Option parsing +my $isolationtester = 0; my $bindir = ""; my $libdir = undef; my $pgxsdir = ""; +my $postgresBuilddir = ""; +my $postgresSrcdir = ""; my $majorversion = ""; my @extensions = (); my @userPgOptions = (); @@ -50,9 +56,12 @@ my %operators = (); my $serversAreShutdown = "TRUE"; GetOptions( + 'isolationtester' => \$isolationtester, 'bindir=s' => \$bindir, 'libdir=s' => \$libdir, 'pgxsdir=s' => \$pgxsdir, + 'postgres-builddir=s' => \$postgresBuilddir, + 'postgres-srcdir=s' => \$postgresSrcdir, 'majorversion=s' => \$majorversion, 'load-extension=s' => \@extensions, 'server-option=s' => \@userPgOptions, @@ -73,6 +82,25 @@ if (defined $libdir) $ENV{PATH} = "$libdir:".($ENV{PATH} || ''); } +my $plainRegress = "$pgxsdir/src/test/regress/pg_regress"; +my $isolationRegress = "${postgresBuilddir}/src/test/isolation/pg_isolation_regress"; +if ($isolationtester && ! -f "$isolationRegress") +{ + die <<"MESSAGE"; + +isolationtester not found at $isolationRegress. + +isolationtester tests can only be run when source (detected as ${postgresSrcdir}) +and build (detected as ${postgresBuilddir}) directory corresponding to $bindir +are present. + +Additionally isolationtester in src/test/isolation needs to be built, +which it is not by default if tests have not been run. If the build +directory is present locally +"make -C ${postgresBuilddir} all" should do the trick. +MESSAGE +} + # Set some default configuration options my $masterPort = 57636; my $workerCount = 2; @@ -313,8 +341,17 @@ push(@arguments, @ARGV); my $startTime = time(); # Finally run the tests -system("$pgxsdir/src/test/regress/pg_regress", @arguments) == 0 - or die "Could not run regression tests"; +if (!$isolationtester) +{ + system("$plainRegress", @arguments) == 0 + or die "Could not run regression tests"; +} +else +{ + push(@arguments, "--dbname=regression"); + system("$isolationRegress", @arguments) == 0 + or die "Could not run isolation tests"; +} my $endTime = time(); diff --git a/src/test/regress/specs/isolation_cluster_management.spec b/src/test/regress/specs/isolation_cluster_management.spec new file mode 100644 index 000000000..19c9d8001 --- /dev/null +++ b/src/test/regress/specs/isolation_cluster_management.spec @@ -0,0 +1,8 @@ +session "s1" +step "s1a" +{ + SELECT master_add_node('localhost', 57637); + SELECT master_add_node('localhost', 57638); +} + +permutation "s1a" diff --git a/src/test/regress/specs/isolation_concurrent_dml.spec b/src/test/regress/specs/isolation_concurrent_dml.spec new file mode 100644 index 000000000..3288d7521 --- /dev/null +++ b/src/test/regress/specs/isolation_concurrent_dml.spec @@ -0,0 +1,40 @@ +setup +{ + CREATE TABLE test_table (test_id integer NOT NULL, data text); + SELECT master_create_distributed_table('test_table', 'test_id', 'hash'); + SELECT master_create_worker_shards('test_table', 4, 2); +} + +teardown +{ + DROP TABLE IF EXISTS test_table CASCADE; +} + +session "s1" + +step "s1-begin" +{ + BEGIN; +} + +step "s1-insert" +{ + INSERT INTO test_table VALUES(1); +} + +step "s1-commit" +{ + COMMIT; +} + +session "s2" + +step "s2-update" +{ + UPDATE test_table SET data = 'blarg' WHERE test_id = 1; +} + +# verify that an in-progress insert blocks concurrent updates +permutation "s1-begin" "s1-insert" "s2-update" "s1-commit" +# but an insert without xact will not block +permutation "s1-insert" "s2-update" From dfe7b357c5fb2f2d61a0faeccd1b182627162ef6 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Wed, 26 Oct 2016 06:41:02 -0700 Subject: [PATCH 4/4] Simple isolationtester dml vs. repair tests. --- .../expected/isolation_dml_vs_repair.out | 200 ++++++++++++++++++ src/test/regress/isolation_schedule | 1 + .../specs/isolation_dml_vs_repair.spec | 110 ++++++++++ 3 files changed, 311 insertions(+) create mode 100644 src/test/regress/expected/isolation_dml_vs_repair.out create mode 100644 src/test/regress/specs/isolation_dml_vs_repair.spec diff --git a/src/test/regress/expected/isolation_dml_vs_repair.out b/src/test/regress/expected/isolation_dml_vs_repair.out new file mode 100644 index 000000000..0730dd370 --- /dev/null +++ b/src/test/regress/expected/isolation_dml_vs_repair.out @@ -0,0 +1,200 @@ +Parsed test spec with 2 sessions + +starting permutation: s2-invalidate-57637 s1-begin s1-insertone s2-repair s1-commit +master_create_worker_shards + + +step s2-invalidate-57637: + UPDATE pg_dist_shard_placement SET shardstate = '3' WHERE shardid = 102008 AND nodeport = 57637; + +step s1-begin: + BEGIN; + +step s1-insertone: + INSERT INTO test_table VALUES(1, 1); + +step s2-repair: + SELECT master_copy_shard_placement(102008, 'localhost', 57638, 'localhost', 57637); + +step s1-commit: + COMMIT; + +step s2-repair: <... completed> +master_copy_shard_placement + + + +starting permutation: s1-insertone s2-invalidate-57637 s1-begin s1-insertall s2-repair s1-commit +master_create_worker_shards + + +step s1-insertone: + INSERT INTO test_table VALUES(1, 1); + +step s2-invalidate-57637: + UPDATE pg_dist_shard_placement SET shardstate = '3' WHERE shardid = 102008 AND nodeport = 57637; + +step s1-begin: + BEGIN; + +step s1-insertall: + INSERT INTO test_table SELECT test_id, data+1 FROM test_table; + +step s2-repair: + SELECT master_copy_shard_placement(102008, 'localhost', 57638, 'localhost', 57637); + +step s1-commit: + COMMIT; + +step s2-repair: <... completed> +master_copy_shard_placement + + + +starting permutation: s2-invalidate-57637 s2-begin s2-repair s1-insertone s2-commit s2-invalidate-57638 s1-display s2-invalidate-57637 s2-revalidate-57638 s1-display +master_create_worker_shards + + +step s2-invalidate-57637: + UPDATE pg_dist_shard_placement SET shardstate = '3' WHERE shardid = 102008 AND nodeport = 57637; + +step s2-begin: + BEGIN; + +step s2-repair: + SELECT master_copy_shard_placement(102008, 'localhost', 57638, 'localhost', 57637); + +master_copy_shard_placement + + +step s1-insertone: + INSERT INTO test_table VALUES(1, 1); + +step s2-commit: + COMMIT; + +step s1-insertone: <... completed> +step s2-invalidate-57638: + UPDATE pg_dist_shard_placement SET shardstate = '3' WHERE shardid = 102008 AND nodeport = 57638; + +step s1-display: + SELECT * FROM test_table WHERE test_id = 1; + +test_id data + +1 1 +step s2-invalidate-57637: + UPDATE pg_dist_shard_placement SET shardstate = '3' WHERE shardid = 102008 AND nodeport = 57637; + +step s2-revalidate-57638: + UPDATE pg_dist_shard_placement SET shardstate = '1' WHERE shardid = 102008 AND nodeport = 57638; + +step s1-display: + SELECT * FROM test_table WHERE test_id = 1; + +test_id data + +1 1 + +starting permutation: s2-invalidate-57637 s1-prepared-insertone s2-begin s2-repair s1-prepared-insertone s2-commit s2-invalidate-57638 s1-display s2-invalidate-57637 s2-revalidate-57638 s1-display +master_create_worker_shards + + +step s2-invalidate-57637: + UPDATE pg_dist_shard_placement SET shardstate = '3' WHERE shardid = 102008 AND nodeport = 57637; + +step s1-prepared-insertone: + EXECUTE insertone; + +step s2-begin: + BEGIN; + +step s2-repair: + SELECT master_copy_shard_placement(102008, 'localhost', 57638, 'localhost', 57637); + +master_copy_shard_placement + + +step s1-prepared-insertone: + EXECUTE insertone; + +step s2-commit: + COMMIT; + +step s1-prepared-insertone: <... completed> +error in steps s2-commit s1-prepared-insertone: ERROR: prepared modifications cannot be executed on a shard while it is being copied +step s2-invalidate-57638: + UPDATE pg_dist_shard_placement SET shardstate = '3' WHERE shardid = 102008 AND nodeport = 57638; + +step s1-display: + SELECT * FROM test_table WHERE test_id = 1; + +test_id data + +1 1 +step s2-invalidate-57637: + UPDATE pg_dist_shard_placement SET shardstate = '3' WHERE shardid = 102008 AND nodeport = 57637; + +step s2-revalidate-57638: + UPDATE pg_dist_shard_placement SET shardstate = '1' WHERE shardid = 102008 AND nodeport = 57638; + +step s1-display: + SELECT * FROM test_table WHERE test_id = 1; + +test_id data + +1 1 + +starting permutation: s2-invalidate-57637 s1-insertone s1-prepared-insertall s2-begin s2-repair s1-prepared-insertall s2-commit s2-invalidate-57638 s1-display s2-invalidate-57637 s2-revalidate-57638 s1-display +master_create_worker_shards + + +step s2-invalidate-57637: + UPDATE pg_dist_shard_placement SET shardstate = '3' WHERE shardid = 102008 AND nodeport = 57637; + +step s1-insertone: + INSERT INTO test_table VALUES(1, 1); + +step s1-prepared-insertall: + EXECUTE insertall; + +step s2-begin: + BEGIN; + +step s2-repair: + SELECT master_copy_shard_placement(102008, 'localhost', 57638, 'localhost', 57637); + +master_copy_shard_placement + + +step s1-prepared-insertall: + EXECUTE insertall; + +step s2-commit: + COMMIT; + +step s1-prepared-insertall: <... completed> +error in steps s2-commit s1-prepared-insertall: ERROR: prepared modifications cannot be executed on a shard while it is being copied +step s2-invalidate-57638: + UPDATE pg_dist_shard_placement SET shardstate = '3' WHERE shardid = 102008 AND nodeport = 57638; + +step s1-display: + SELECT * FROM test_table WHERE test_id = 1; + +test_id data + +1 1 +1 2 +step s2-invalidate-57637: + UPDATE pg_dist_shard_placement SET shardstate = '3' WHERE shardid = 102008 AND nodeport = 57637; + +step s2-revalidate-57638: + UPDATE pg_dist_shard_placement SET shardstate = '1' WHERE shardid = 102008 AND nodeport = 57638; + +step s1-display: + SELECT * FROM test_table WHERE test_id = 1; + +test_id data + +1 1 +1 2 diff --git a/src/test/regress/isolation_schedule b/src/test/regress/isolation_schedule index d702c065f..93a2f5ab8 100644 --- a/src/test/regress/isolation_schedule +++ b/src/test/regress/isolation_schedule @@ -1,2 +1,3 @@ test: isolation_cluster_management test: isolation_concurrent_dml +test: isolation_dml_vs_repair diff --git a/src/test/regress/specs/isolation_dml_vs_repair.spec b/src/test/regress/specs/isolation_dml_vs_repair.spec new file mode 100644 index 000000000..e4e9219ee --- /dev/null +++ b/src/test/regress/specs/isolation_dml_vs_repair.spec @@ -0,0 +1,110 @@ +setup +{ + ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 102008; + CREATE TABLE test_table (test_id integer NOT NULL, data int); + SELECT master_create_distributed_table('test_table', 'test_id', 'hash'); + SELECT master_create_worker_shards('test_table', 1, 2); +} + +teardown +{ + DROP TABLE IF EXISTS test_table CASCADE; +} + +session "s1" + +setup +{ + DEALLOCATE all; + TRUNCATE test_table; + PREPARE insertone AS INSERT INTO test_table VALUES(1, 1); + PREPARE insertall AS INSERT INTO test_table SELECT test_id, data+1 FROM test_table; +} + +step "s1-begin" +{ + BEGIN; +} + +step "s1-insertone" +{ + INSERT INTO test_table VALUES(1, 1); +} + +step "s1-prepared-insertone" +{ + EXECUTE insertone; +} + +step "s1-insertall" +{ + INSERT INTO test_table SELECT test_id, data+1 FROM test_table; +} + +step "s1-prepared-insertall" +{ + EXECUTE insertall; +} + +step "s1-display" +{ + SELECT * FROM test_table WHERE test_id = 1; +} + +step "s1-commit" +{ + COMMIT; +} + +session "s2" + + +step "s2-begin" +{ + BEGIN; +} + +step "s2-invalidate-57637" +{ + UPDATE pg_dist_shard_placement SET shardstate = '3' WHERE shardid = 102008 AND nodeport = 57637; +} + +step "s2-revalidate-57637" +{ + UPDATE pg_dist_shard_placement SET shardstate = '1' WHERE shardid = 102008 AND nodeport = 57637; +} + +step "s2-invalidate-57638" +{ + UPDATE pg_dist_shard_placement SET shardstate = '3' WHERE shardid = 102008 AND nodeport = 57638; +} + +step "s2-revalidate-57638" +{ + UPDATE pg_dist_shard_placement SET shardstate = '1' WHERE shardid = 102008 AND nodeport = 57638; +} + +step "s2-repair" +{ + SELECT master_copy_shard_placement(102008, 'localhost', 57638, 'localhost', 57637); +} + +step "s2-commit" +{ + COMMIT; +} + +# verify that repair is blocked by ongoing modifying simple transaction +permutation "s2-invalidate-57637" "s1-begin" "s1-insertone" "s2-repair" "s1-commit" + +# verify that repair is blocked by ongoing modifying insert...select transaction +permutation "s1-insertone" "s2-invalidate-57637" "s1-begin" "s1-insertall" "s2-repair" "s1-commit" + +# verify that modifications wait for shard repair +permutation "s2-invalidate-57637" "s2-begin" "s2-repair" "s1-insertone" "s2-commit" "s2-invalidate-57638" "s1-display" "s2-invalidate-57637" "s2-revalidate-57638" "s1-display" + +# verify that prepared plain modifications wait for shard repair (and then fail to avoid race) +permutation "s2-invalidate-57637" "s1-prepared-insertone" "s2-begin" "s2-repair" "s1-prepared-insertone" "s2-commit" "s2-invalidate-57638" "s1-display" "s2-invalidate-57637" "s2-revalidate-57638" "s1-display" + +# verify that prepared INSERT ... SELECT waits for shard repair (and then fail to avoid race) +permutation "s2-invalidate-57637" "s1-insertone" "s1-prepared-insertall" "s2-begin" "s2-repair" "s1-prepared-insertall" "s2-commit" "s2-invalidate-57638" "s1-display" "s2-invalidate-57637" "s2-revalidate-57638" "s1-display"