external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make pg_regress_multi.pl platform independent 

- don't hardcode path names
- replace system calls for rm/mkdir/rm -rf with perl equivalents
- force utf-8 encoding
- the Windows shell uses different quoting and escape rules
pull/2056/head^2
Brian Cloutier 2017-11-09 11:09:55 -08:00 committed by Brian Cloutier
parent adb4669d34
commit 2140b5d82d
1 changed files with 142 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

215
src/test/regress/pg_regress_multi.pl
View File

@ -16,7 +16,9 @@ use warnings;
use Fcntl; use Fcntl;
use Getopt::Long; use Getopt::Long;
use File::Spec::Functions;
use File::Path qw(make_path remove_tree);
use Config;
sub Usage() sub Usage()
{ {
@ -67,6 +69,12 @@ my $pgCtlTimeout = undef;
my $connectionTimeout = 5000; my $connectionTimeout = 5000;
my $serversAreShutdown = "TRUE"; my $serversAreShutdown = "TRUE";
my $usingWindows = 0;
if ($Config{osname} eq "MSWin32")
{
$usingWindows = 1;
};
GetOptions( GetOptions(
'isolationtester' => \$isolationtester, 'isolationtester' => \$isolationtester,
@ -117,8 +125,20 @@ if (defined $bindir)
# a bit more context to make it easier to locate failed test sections. # a bit more context to make it easier to locate failed test sections.
$ENV{PG_REGRESS_DIFF_OPTS} = '-dU10'; $ENV{PG_REGRESS_DIFF_OPTS} = '-dU10';
my $plainRegress = "$pgxsdir/src/test/regress/pg_regress"; my $plainRegress = "";
my $isolationRegress = "${postgresBuilddir}/src/test/isolation/pg_isolation_regress"; my $isolationRegress = "";
if ($usingWindows)
{
$plainRegress = "$bindir\\pg_regress.exe";
$isolationRegress = "$bindir\\pg_isolation_regress.exe";
}
else
{
$plainRegress = "$pgxsdir/src/test/regress/pg_regress";
$isolationRegress = "${postgresBuilddir}/src/test/isolation/pg_isolation_regress";
}
if ($isolationtester && ! -f "$isolationRegress") if ($isolationtester && ! -f "$isolationRegress")
{ {
die <<"MESSAGE"; die <<"MESSAGE";
@ -136,7 +156,7 @@ directory is present locally
MESSAGE MESSAGE
} }
my $vanillaRegress = "${postgresBuilddir}/src/test/regress/pg_regress"; my $vanillaRegress = catfile("${postgresBuilddir}", "src", "test", "regress", "pg_regress");
if ($vanillatest && ! -f "$vanillaRegress") if ($vanillatest && ! -f "$vanillaRegress")
{ {
die <<"MESSAGE"; die <<"MESSAGE";
@ -164,18 +184,18 @@ if (defined $pgCtlTimeout)
# only reliable way to do this. # only reliable way to do this.
sub replace_postgres sub replace_postgres
{ {
if (-e "$bindir/postgres.orig") if (-e catfile("$bindir", "postgres.orig"))
{ {
print "wrapper exists\n"; print "wrapper exists\n";
} }
else else
{ {
print "moving $bindir/postgres to $bindir/postgres.orig\n"; print "moving $bindir/postgres to $bindir/postgres.orig\n";
rename "$bindir/postgres", "$bindir/postgres.orig" rename catfile("$bindir", "postgres"), catfile("$bindir", "postgres.orig")
or die "Could not move postgres out of the way"; or die "Could not move postgres out of the way";
} }
sysopen my $fh, "$bindir/postgres", O_CREAT|O_TRUNC|O_RDWR, 0700 sysopen my $fh, catfile("$bindir", "postgres"), O_CREAT|O_TRUNC|O_RDWR, 0700
or die "Could not create postgres wrapper at $bindir/postgres"; or die "Could not create postgres wrapper at $bindir/postgres";
print $fh <<"END"; print $fh <<"END";
#!/bin/bash #!/bin/bash
@ -195,11 +215,11 @@ END
# revert changes replace_postgres() performed # revert changes replace_postgres() performed
sub revert_replace_postgres sub revert_replace_postgres
{ {
if (-e "$bindir/postgres.orig") if (-e catfile("$bindir", "postgres.orig"))
{ {
print "wrapper exists, removing\n"; print "wrapper exists, removing\n";
print "moving $bindir/postgres.orig to $bindir/postgres\n"; print "moving $bindir/postgres.orig to $bindir/postgres\n";
rename "$bindir/postgres.orig", "$bindir/postgres" rename catfile("$bindir", "postgres.orig"), catfile("$bindir", "postgres")
or die "Could not move postgres back"; or die "Could not move postgres back";
} }
} }
@ -230,7 +250,7 @@ my $user = "postgres";
my @pgOptions = (); my @pgOptions = ();
# Postgres options set for the tests # Postgres options set for the tests
push(@pgOptions, '-c', "listen_addresses='${host}'"); push(@pgOptions, '-c', "listen_addresses=${host}");
# not required, and we don't necessarily have access to the default directory # not required, and we don't necessarily have access to the default directory
push(@pgOptions, '-c', "unix_socket_directories="); push(@pgOptions, '-c', "unix_socket_directories=");
push(@pgOptions, '-c', "fsync=off"); push(@pgOptions, '-c', "fsync=off");
@ -295,27 +315,53 @@ for my $option (@userPgOptions)
%fdwServers = ('fake_fdw_server', 'fake_fdw'); %fdwServers = ('fake_fdw_server', 'fake_fdw');
# Cleanup leftovers and prepare directories for the run # Cleanup leftovers and prepare directories for the run
system("rm", ('-rf', 'tmp_check/tmp-bin')) == 0 or die "Could not remove tmp-bin directory"; if (-e catfile('tmp_check', 'tmp-bin'))
system("rm", ('-rf', 'tmp_check/master')) == 0 or die "Could not remove master directory";
for my $port (@workerPorts)
{ {
system("rm", ('-rf', "tmp_check/worker.$port")) == 0 or die "Could not remove worker directory"; remove_tree(catfile('tmp_check', 'tmp-bin')) or die "Could not remove tmp-bin directory";
}
if (-e catfile('tmp_check', 'master'))
{
remove_tree(catfile('tmp_check', 'master')) or die "Could not remove master directory";
}
for my $port (@workerPorts)
{
if (-e catfile("tmp_check", "worker.$port"))
{
remove_tree(catfile("tmp_check", "worker.$port")) or die "Could not remove worker directory";
}
}
if (-e catfile("tmp_check", "master-follower"))
{
remove_tree(catfile("tmp_check", "master-follower")) or die "Could not remove master directory";
} }
system("rm", ('-rf', 'tmp_check/master-follower')) == 0 or die "Could not remove master directory";
for my $port (@followerWorkerPorts) for my $port (@followerWorkerPorts)
{ {
system("rm", ('-rf', "tmp_check/follower.$port")) == 0 or die "Could not remove worker directory"; if (-e catfile("tmp_check", "follower.$port"))
{
remove_tree(catfile("tmp_check", "follower.$port")) or die "Could not remove worker directory";
}
} }
# Prepare directory in which 'psql' has some helpful variables for locating the workers # Prepare directory in which 'psql' has some helpful variables for locating the workers
system("mkdir", ('-p', "tmp_check/tmp-bin")) == 0 make_path(catfile("tmp_check", "tmp-bin")) or die "Could not create tmp_bin directory $!\n";
or die "Could not create tmp-bin directory";
sysopen my $fh, "tmp_check/tmp-bin/psql", O_CREAT|O_TRUNC|O_RDWR, 0700 my $psql_name = "psql";
if ($usingWindows)
{
$psql_name = "psql.cmd";
}
sysopen my $fh, catfile("tmp_check", "tmp-bin", $psql_name), O_CREAT|O_TRUNC|O_RDWR, 0700
or die "Could not create psql wrapper"; or die "Could not create psql wrapper";
print $fh "#!/bin/bash\n"; if ($usingWindows)
print $fh "exec psql "; {
print $fh "\@echo off\n";
}
print $fh catfile($bindir, "psql")." ";
print $fh "--variable=master_port=$masterPort "; print $fh "--variable=master_port=$masterPort ";
print $fh "--variable=follower_master_port=$followerCoordPort "; print $fh "--variable=follower_master_port=$followerCoordPort ";
print $fh "--variable=default_user=$user "; print $fh "--variable=default_user=$user ";
@ -330,71 +376,91 @@ for my $workeroff (0 .. $#followerWorkerPorts)
my $port = $followerWorkerPorts[$workeroff]; my $port = $followerWorkerPorts[$workeroff];
print $fh "--variable=follower_worker_".($workeroff+1)."_port=$port "; print $fh "--variable=follower_worker_".($workeroff+1)."_port=$port ";
} }
print $fh "\"\$@\"\n"; # pass on the commandline arguments
if ($usingWindows)
{
print $fh "%*\n"; # pass on the commandline arguments
}
else
{
print $fh "\"\$@\"\n"; # pass on the commandline arguments
}
close $fh; close $fh;
system("mkdir", ('-p', 'tmp_check/master/log')) == 0 or die "Could not create master directory"; make_path(catfile('tmp_check', 'master', 'log')) or die 'Could not create master directory';
for my $port (@workerPorts) for my $port (@workerPorts)
{ {
system("mkdir", ('-p', "tmp_check/worker.$port/log")) == 0 make_path(catfile("tmp_check", "worker.$port", "log"))
or die "Could not create worker directory"; or die "Could not create worker directory";
} }
if ($followercluster) if ($followercluster)
{ {
system("mkdir", ('-p', 'tmp_check/master-follower/log')) == 0 or die "Could not create follower directory"; make_path(catfile('tmp_check', 'master-follower', 'log')) or die "Could not create follower directory";
for my $port (@followerWorkerPorts) for my $port (@followerWorkerPorts)
{ {
system("mkdir", ('-p', "tmp_check/follower.$port/log")) == 0 make_path(catfile("tmp_check", "follower.$port", "log")) == 0
or die "Could not create worker directory"; or die "Could not create worker directory";
} }
} }
# Create new data directories, copy workers for speed # Create new data directories, copy workers for speed
system("$bindir/initdb", ("--nosync", "-U", $user, "tmp_check/master/data")) == 0 system(catfile("$bindir", "initdb"), ("--nosync", "-U", $user, "--encoding", "UTF8", catfile("tmp_check", "master", "data"))) == 0
or die "Could not create master data directory"; or die "Could not create master data directory";
if ($followercluster) if ($followercluster)
{ {
# This is only necessary on PG 9.6 but it doesn't hurt PG 10 # This is only necessary on PG 9.6 but it doesn't hurt PG 10
open(my $fd, ">>", "tmp_check/master/data/pg_hba.conf") open(my $fd, ">>", catfile("tmp_check", "master", "data", "pg_hba.conf"))
or die "could not open pg_hba.conf"; or die "could not open pg_hba.conf";
print $fd "\nhost replication postgres 127.0.0.1/32 trust"; print $fd "\nhost replication postgres 127.0.0.1/32 trust";
close $fd; close $fd;
} }
for my $port (@workerPorts) if ($usingWindows)
{ {
system("cp -a tmp_check/master/data tmp_check/worker.$port/data") == 0 for my $port (@workerPorts)
{
system(catfile("$bindir", "initdb"), ("--nosync", "-U", $user, "--encoding", "UTF8", catfile("tmp_check", "worker.$port", "data"))) == 0
or die "Could not create worker data directory"; or die "Could not create worker data directory";
}
} }
else
{
for my $port (@workerPorts)
{
system("cp", ("-a", catfile("tmp_check", "master", "data"), catfile("tmp_check", "worker.$port", "data"))) == 0
or die "Could not create worker data directory";
}
}
# Routine to shutdown servers at failure/exit # Routine to shutdown servers at failure/exit
sub ShutdownServers() sub ShutdownServers()
{ {
if ($serversAreShutdown eq "FALSE") if ($serversAreShutdown eq "FALSE")
{ {
system("$bindir/pg_ctl", system(catfile("$bindir", "pg_ctl"),
('stop', '-w', '-D', 'tmp_check/master/data')) == 0 ('stop', '-w', '-D', catfile('tmp_check', 'master', 'data'))) == 0
or warn "Could not shutdown worker server"; or warn "Could not shutdown worker server";
for my $port (@workerPorts) for my $port (@workerPorts)
{ {
system("$bindir/pg_ctl", system(catfile("$bindir", "pg_ctl"),
('stop', '-w', '-D', "tmp_check/worker.$port/data")) == 0 ('stop', '-w', '-D', catfile("tmp_check", "worker.$port", "data"))) == 0
or warn "Could not shutdown worker server"; or warn "Could not shutdown worker server";
} }
if ($followercluster) if ($followercluster)
{ {
system("$bindir/pg_ctl", system(catfile("$bindir", "pg_ctl"),
('stop', '-w', '-D', 'tmp_check/master-follower/data')) == 0 ('stop', '-w', '-D', catfile('tmp_check', 'master-follower', 'data'))) == 0
or warn "Could not shutdown worker server"; or warn "Could not shutdown worker server";
for my $port (@followerWorkerPorts) for my $port (@followerWorkerPorts)
{ {
system("$bindir/pg_ctl", system(catfile("$bindir", "pg_ctl"),
('stop', '-w', '-D', "tmp_check/follower.$port/data")) == 0 ('stop', '-w', '-D', catfile("tmp_check", "follower.$port", "data"))) == 0
or warn "Could not shutdown worker server"; or warn "Could not shutdown worker server";
} }
} }
@ -433,24 +499,24 @@ if ($valgrind)
$serversAreShutdown = "FALSE"; $serversAreShutdown = "FALSE";
# Start servers # Start servers
if(system("$bindir/pg_ctl", if(system(catfile("$bindir", "pg_ctl"),
('start', '-w', ('start', '-w',
'-o', join(" ", @pgOptions)." -c port=$masterPort", '-o', join(" ", @pgOptions)." -c port=$masterPort",
'-D', 'tmp_check/master/data', '-l', 'tmp_check/master/log/postmaster.log')) != 0) '-D', catfile('tmp_check', 'master', 'data'), '-l', catfile('tmp_check', 'master', 'log', 'postmaster.log'))) != 0)
{ {
system("tail", ("-n20", "tmp_check/master/log/postmaster.log")); system("tail", ("-n20", catfile("tmp_check", "master", "log", "postmaster.log")));
die "Could not start master server"; die "Could not start master server";
} }
for my $port (@workerPorts) for my $port (@workerPorts)
{ {
if(system("$bindir/pg_ctl", if(system(catfile("$bindir", "pg_ctl"),
('start', '-w', ('start', '-w',
'-o', join(" ", @pgOptions)." -c port=$port", '-o', join(" ", @pgOptions)." -c port=$port",
'-D', "tmp_check/worker.$port/data", '-D', catfile("tmp_check", "worker.$port", "data"),
'-l', "tmp_check/worker.$port/log/postmaster.log")) != 0) '-l', catfile("tmp_check", "worker.$port", "log", "postmaster.log"))) != 0)
{ {
system("tail", ("-n20", "tmp_check/worker.$port/log/postmaster.log")); system("tail", ("-n20", catfile("tmp_check", "worker.$port", "log", "postmaster.log")));
die "Could not start worker server"; die "Could not start worker server";
} }
} }
@ -460,8 +526,8 @@ if ($followercluster)
{ {
# This test would run faster on PG10 if we could pass --no-sync here but that flag # This test would run faster on PG10 if we could pass --no-sync here but that flag
# isn't supported on PG 9.6. In a year when we drop support for PG9.6 add that flag! # isn't supported on PG 9.6. In a year when we drop support for PG9.6 add that flag!
system("$bindir/pg_basebackup", system(catfile("$bindir", "pg_basebackup"),
("-D", "tmp_check/master-follower/data", "--host=$host", "--port=$masterPort", ("-D", catfile("tmp_check", "master-follower", "data"), "--host=$host", "--port=$masterPort",
"--username=$user", "-R", "-X", "stream")) == 0 "--username=$user", "-R", "-X", "stream")) == 0
or die 'could not take basebackup'; or die 'could not take basebackup';
@ -469,30 +535,30 @@ if ($followercluster)
{ {
my $workerPort = $workerPorts[$offset]; my $workerPort = $workerPorts[$offset];
my $followerPort = $followerWorkerPorts[$offset]; my $followerPort = $followerWorkerPorts[$offset];
system("$bindir/pg_basebackup", system(catfile("$bindir", "pg_basebackup"),
("-D", "tmp_check/follower.$followerPort/data", "--host=$host", "--port=$workerPort", ("-D", catfile("tmp_check", "follower.$followerPort", "data"), "--host=$host", "--port=$workerPort",
"--username=$user", "-R", "-X", "stream")) == 0 "--username=$user", "-R", "-X", "stream")) == 0
or die "Could not take basebackup"; or die "Could not take basebackup";
} }
if(system("$bindir/pg_ctl", if(system(catfile("$bindir", "pg_ctl"),
('start', '-w', ('start', '-w',
'-o', join(" ", @pgOptions)." -c port=$followerCoordPort", '-o', join(" ", @pgOptions)." -c port=$followerCoordPort",
'-D', 'tmp_check/master-follower/data', '-l', 'tmp_check/master-follower/log/postmaster.log')) != 0) '-D', catfile('tmp_check', 'master-follower', 'data'), '-l', catfile('tmp_check', 'master-follower', 'log', 'postmaster.log'))) != 0)
{ {
system("tail", ("-n20", "tmp_check/master-follower/log/postmaster.log")); system("tail", ("-n20", catfile("tmp_check", "master-follower", "log", "postmaster.log")));
die "Could not start master follower server"; die "Could not start master follower server";
} }
for my $port (@followerWorkerPorts) for my $port (@followerWorkerPorts)
{ {
if(system("$bindir/pg_ctl", if(system(catfile("$bindir", "pg_ctl"),
('start', '-w', ('start', '-w',
'-o', join(" ", @pgOptions)." -c port=$port", '-o', join(" ", @pgOptions)." -c port=$port",
'-D', "tmp_check/follower.$port/data", '-D', catfile("tmp_check", "follower.$port", "data"),
'-l', "tmp_check/follower.$port/log/postmaster.log")) != 0) '-l', catfile("tmp_check", "follower.$port", "log", "postmaster.log"))) != 0)
{ {
system("tail", ("-n20", "tmp_check/follower.$port/log/postmaster.log")); system("tail", ("-n20", catfile("tmp_check", "follower.$port", "log", "postmaster.log")));
die "Could not start follower server"; die "Could not start follower server";
} }
} }
@ -504,55 +570,55 @@ if ($followercluster)
### ###
for my $port (@workerPorts) for my $port (@workerPorts)
{ {
system("psql", '-X', system(catfile($bindir, "psql"),
('-h', $host, '-p', $port, '-U', $user, "postgres", ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "postgres",
'-c', "CREATE DATABASE regression;")) == 0 '-c', "CREATE DATABASE regression;")) == 0
or die "Could not create regression database on worker"; or die "Could not create regression database on worker";
for my $extension (@extensions) for my $extension (@extensions)
{ {
system("psql", '-X', system(catfile($bindir, "psql"),
('-h', $host, '-p', $port, '-U', $user, "regression", ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression",
'-c', "CREATE EXTENSION IF NOT EXISTS \"$extension\";")) == 0 '-c', "CREATE EXTENSION IF NOT EXISTS $extension;")) == 0
or die "Could not create extension on worker"; or die "Could not create extension on worker";
} }
foreach my $dataType (keys %dataTypes) foreach my $dataType (keys %dataTypes)
{ {
system("psql", '-X', system(catfile($bindir, "psql"),
('-h', $host, '-p', $port, '-U', $user, "regression", ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression",
'-c', "CREATE TYPE $dataType AS $dataTypes{$dataType};")) == 0 '-c', "CREATE TYPE $dataType AS $dataTypes{$dataType};")) == 0
or die "Could not create TYPE $dataType on worker"; or die "Could not create TYPE $dataType on worker";
} }
foreach my $function (keys %functions) foreach my $function (keys %functions)
{ {
system("psql", '-X', system(catfile($bindir, "psql"),
('-h', $host, '-p', $port, '-U', $user, "regression", ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression",
'-c', "CREATE FUNCTION $function RETURNS $functions{$function};")) == 0 '-c', "CREATE FUNCTION $function RETURNS $functions{$function};")) == 0
or die "Could not create FUNCTION $function on worker"; or die "Could not create FUNCTION $function on worker";
} }
foreach my $operator (keys %operators) foreach my $operator (keys %operators)
{ {
system("psql", '-X', system(catfile($bindir, "psql"),
('-h', $host, '-p', $port, '-U', $user, "regression", ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression",
'-c', "CREATE OPERATOR $operator $operators{$operator};")) == 0 '-c', "CREATE OPERATOR $operator $operators{$operator};")) == 0
or die "Could not create OPERATOR $operator on worker"; or die "Could not create OPERATOR $operator on worker";
} }
foreach my $fdw (keys %fdws) foreach my $fdw (keys %fdws)
{ {
system("psql", '-X', system(catfile($bindir, "psql"),
('-h', $host, '-p', $port, '-U', $user, "regression", ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression",
'-c', "CREATE FOREIGN DATA WRAPPER $fdw HANDLER $fdws{$fdw};")) == 0 '-c', "CREATE FOREIGN DATA WRAPPER $fdw HANDLER $fdws{$fdw};")) == 0
or die "Could not create foreign data wrapper $fdw on worker"; or die "Could not create foreign data wrapper $fdw on worker";
} }
foreach my $fdwServer (keys %fdwServers) foreach my $fdwServer (keys %fdwServers)
{ {
system("psql", '-X', system(catfile($bindir, "psql"),
('-h', $host, '-p', $port, '-U', $user, "regression", ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression",
'-c', "CREATE SERVER $fdwServer FOREIGN DATA WRAPPER $fdwServers{$fdwServer};")) == 0 '-c', "CREATE SERVER $fdwServer FOREIGN DATA WRAPPER $fdwServers{$fdwServer};")) == 0
or die "Could not create server $fdwServer on worker"; or die "Could not create server $fdwServer on worker";
} }
@ -563,7 +629,8 @@ my @arguments = (
"--host", $host, "--host", $host,
'--port', $masterPort, '--port', $masterPort,
'--user', $user, '--user', $user,
'--bindir', "tmp_check/tmp-bin" # '--bindir', 'C:\Users\Administrator\Downloads\pg-64\bin',
'--bindir', catfile("tmp_check", "tmp-bin")
); );
# Add load extension parameters to the argument list # Add load extension parameters to the argument list
@ -584,7 +651,7 @@ if ($vanillatest)
$ENV{PGPORT} = $masterPort; $ENV{PGPORT} = $masterPort;
$ENV{PGUSER} = $user; $ENV{PGUSER} = $user;
system("make -C $postgresBuilddir/src/test/regress installcheck-parallel") == 0 system("make", ("-C", catfile("$postgresBuilddir", "src", "test", "regress"), "installcheck-parallel")) == 0
or die "Could not run vanilla tests"; or die "Could not run vanilla tests";
} }
elsif ($isolationtester) elsif ($isolationtester)