From 2140b5d82d678a3ba687391ba97625faf21cb673 Mon Sep 17 00:00:00 2001 From: Brian Cloutier Date: Thu, 9 Nov 2017 11:09:55 -0800 Subject: [PATCH] Make pg_regress_multi.pl platform independent - don't hardcode path names - replace system calls for rm/mkdir/rm -rf with perl equivalents - force utf-8 encoding - the Windows shell uses different quoting and escape rules --- src/test/regress/pg_regress_multi.pl | 217 ++++++++++++++++++--------- 1 file changed, 142 insertions(+), 75 deletions(-) diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index cc0731827..4b57e6d26 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -16,7 +16,9 @@ use warnings; use Fcntl; use Getopt::Long; - +use File::Spec::Functions; +use File::Path qw(make_path remove_tree); +use Config; sub Usage() { @@ -67,6 +69,12 @@ my $pgCtlTimeout = undef; my $connectionTimeout = 5000; my $serversAreShutdown = "TRUE"; +my $usingWindows = 0; + +if ($Config{osname} eq "MSWin32") +{ + $usingWindows = 1; +}; GetOptions( 'isolationtester' => \$isolationtester, @@ -117,8 +125,20 @@ if (defined $bindir) # a bit more context to make it easier to locate failed test sections. $ENV{PG_REGRESS_DIFF_OPTS} = '-dU10'; -my $plainRegress = "$pgxsdir/src/test/regress/pg_regress"; -my $isolationRegress = "${postgresBuilddir}/src/test/isolation/pg_isolation_regress"; +my $plainRegress = ""; +my $isolationRegress = ""; + +if ($usingWindows) +{ + $plainRegress = "$bindir\\pg_regress.exe"; + $isolationRegress = "$bindir\\pg_isolation_regress.exe"; +} +else +{ + $plainRegress = "$pgxsdir/src/test/regress/pg_regress"; + $isolationRegress = "${postgresBuilddir}/src/test/isolation/pg_isolation_regress"; +} + if ($isolationtester && ! -f "$isolationRegress") { die <<"MESSAGE"; @@ -136,7 +156,7 @@ directory is present locally MESSAGE } -my $vanillaRegress = "${postgresBuilddir}/src/test/regress/pg_regress"; +my $vanillaRegress = catfile("${postgresBuilddir}", "src", "test", "regress", "pg_regress"); if ($vanillatest && ! -f "$vanillaRegress") { die <<"MESSAGE"; @@ -164,18 +184,18 @@ if (defined $pgCtlTimeout) # only reliable way to do this. sub replace_postgres { - if (-e "$bindir/postgres.orig") + if (-e catfile("$bindir", "postgres.orig")) { print "wrapper exists\n"; } else { print "moving $bindir/postgres to $bindir/postgres.orig\n"; - rename "$bindir/postgres", "$bindir/postgres.orig" + rename catfile("$bindir", "postgres"), catfile("$bindir", "postgres.orig") or die "Could not move postgres out of the way"; } - sysopen my $fh, "$bindir/postgres", O_CREAT|O_TRUNC|O_RDWR, 0700 + sysopen my $fh, catfile("$bindir", "postgres"), O_CREAT|O_TRUNC|O_RDWR, 0700 or die "Could not create postgres wrapper at $bindir/postgres"; print $fh <<"END"; #!/bin/bash @@ -195,11 +215,11 @@ END # revert changes replace_postgres() performed sub revert_replace_postgres { - if (-e "$bindir/postgres.orig") + if (-e catfile("$bindir", "postgres.orig")) { print "wrapper exists, removing\n"; print "moving $bindir/postgres.orig to $bindir/postgres\n"; - rename "$bindir/postgres.orig", "$bindir/postgres" + rename catfile("$bindir", "postgres.orig"), catfile("$bindir", "postgres") or die "Could not move postgres back"; } } @@ -230,7 +250,7 @@ my $user = "postgres"; my @pgOptions = (); # Postgres options set for the tests -push(@pgOptions, '-c', "listen_addresses='${host}'"); +push(@pgOptions, '-c', "listen_addresses=${host}"); # not required, and we don't necessarily have access to the default directory push(@pgOptions, '-c', "unix_socket_directories="); push(@pgOptions, '-c', "fsync=off"); @@ -295,27 +315,53 @@ for my $option (@userPgOptions) %fdwServers = ('fake_fdw_server', 'fake_fdw'); # Cleanup leftovers and prepare directories for the run -system("rm", ('-rf', 'tmp_check/tmp-bin')) == 0 or die "Could not remove tmp-bin directory"; - -system("rm", ('-rf', 'tmp_check/master')) == 0 or die "Could not remove master directory"; -for my $port (@workerPorts) +if (-e catfile('tmp_check', 'tmp-bin')) { - system("rm", ('-rf', "tmp_check/worker.$port")) == 0 or die "Could not remove worker directory"; + remove_tree(catfile('tmp_check', 'tmp-bin')) or die "Could not remove tmp-bin directory"; +} + +if (-e catfile('tmp_check', 'master')) +{ + remove_tree(catfile('tmp_check', 'master')) or die "Could not remove master directory"; +} + +for my $port (@workerPorts) +{ + if (-e catfile("tmp_check", "worker.$port")) + { + remove_tree(catfile("tmp_check", "worker.$port")) or die "Could not remove worker directory"; + } +} + +if (-e catfile("tmp_check", "master-follower")) +{ + remove_tree(catfile("tmp_check", "master-follower")) or die "Could not remove master directory"; } -system("rm", ('-rf', 'tmp_check/master-follower')) == 0 or die "Could not remove master directory"; for my $port (@followerWorkerPorts) { - system("rm", ('-rf', "tmp_check/follower.$port")) == 0 or die "Could not remove worker directory"; + if (-e catfile("tmp_check", "follower.$port")) + { + remove_tree(catfile("tmp_check", "follower.$port")) or die "Could not remove worker directory"; + } } # Prepare directory in which 'psql' has some helpful variables for locating the workers -system("mkdir", ('-p', "tmp_check/tmp-bin")) == 0 - or die "Could not create tmp-bin directory"; -sysopen my $fh, "tmp_check/tmp-bin/psql", O_CREAT|O_TRUNC|O_RDWR, 0700 +make_path(catfile("tmp_check", "tmp-bin")) or die "Could not create tmp_bin directory $!\n"; + +my $psql_name = "psql"; +if ($usingWindows) +{ + $psql_name = "psql.cmd"; +} + +sysopen my $fh, catfile("tmp_check", "tmp-bin", $psql_name), O_CREAT|O_TRUNC|O_RDWR, 0700 or die "Could not create psql wrapper"; -print $fh "#!/bin/bash\n"; -print $fh "exec psql "; +if ($usingWindows) +{ + print $fh "\@echo off\n"; +} +print $fh catfile($bindir, "psql")." "; print $fh "--variable=master_port=$masterPort "; print $fh "--variable=follower_master_port=$followerCoordPort "; print $fh "--variable=default_user=$user "; @@ -330,71 +376,91 @@ for my $workeroff (0 .. $#followerWorkerPorts) my $port = $followerWorkerPorts[$workeroff]; print $fh "--variable=follower_worker_".($workeroff+1)."_port=$port "; } -print $fh "\"\$@\"\n"; # pass on the commandline arguments + +if ($usingWindows) +{ + print $fh "%*\n"; # pass on the commandline arguments +} +else +{ + print $fh "\"\$@\"\n"; # pass on the commandline arguments +} close $fh; -system("mkdir", ('-p', 'tmp_check/master/log')) == 0 or die "Could not create master directory"; +make_path(catfile('tmp_check', 'master', 'log')) or die 'Could not create master directory'; for my $port (@workerPorts) { - system("mkdir", ('-p', "tmp_check/worker.$port/log")) == 0 + make_path(catfile("tmp_check", "worker.$port", "log")) or die "Could not create worker directory"; } if ($followercluster) { - system("mkdir", ('-p', 'tmp_check/master-follower/log')) == 0 or die "Could not create follower directory"; + make_path(catfile('tmp_check', 'master-follower', 'log')) or die "Could not create follower directory"; for my $port (@followerWorkerPorts) { - system("mkdir", ('-p', "tmp_check/follower.$port/log")) == 0 + make_path(catfile("tmp_check", "follower.$port", "log")) == 0 or die "Could not create worker directory"; } } # Create new data directories, copy workers for speed -system("$bindir/initdb", ("--nosync", "-U", $user, "tmp_check/master/data")) == 0 +system(catfile("$bindir", "initdb"), ("--nosync", "-U", $user, "--encoding", "UTF8", catfile("tmp_check", "master", "data"))) == 0 or die "Could not create master data directory"; if ($followercluster) { # This is only necessary on PG 9.6 but it doesn't hurt PG 10 - open(my $fd, ">>", "tmp_check/master/data/pg_hba.conf") + open(my $fd, ">>", catfile("tmp_check", "master", "data", "pg_hba.conf")) or die "could not open pg_hba.conf"; print $fd "\nhost replication postgres 127.0.0.1/32 trust"; close $fd; } -for my $port (@workerPorts) +if ($usingWindows) { - system("cp -a tmp_check/master/data tmp_check/worker.$port/data") == 0 - or die "Could not create worker data directory"; + for my $port (@workerPorts) + { + system(catfile("$bindir", "initdb"), ("--nosync", "-U", $user, "--encoding", "UTF8", catfile("tmp_check", "worker.$port", "data"))) == 0 + or die "Could not create worker data directory"; + } } +else +{ + for my $port (@workerPorts) + { + system("cp", ("-a", catfile("tmp_check", "master", "data"), catfile("tmp_check", "worker.$port", "data"))) == 0 + or die "Could not create worker data directory"; + } +} + # Routine to shutdown servers at failure/exit sub ShutdownServers() { if ($serversAreShutdown eq "FALSE") { - system("$bindir/pg_ctl", - ('stop', '-w', '-D', 'tmp_check/master/data')) == 0 + system(catfile("$bindir", "pg_ctl"), + ('stop', '-w', '-D', catfile('tmp_check', 'master', 'data'))) == 0 or warn "Could not shutdown worker server"; for my $port (@workerPorts) { - system("$bindir/pg_ctl", - ('stop', '-w', '-D', "tmp_check/worker.$port/data")) == 0 + system(catfile("$bindir", "pg_ctl"), + ('stop', '-w', '-D', catfile("tmp_check", "worker.$port", "data"))) == 0 or warn "Could not shutdown worker server"; } if ($followercluster) { - system("$bindir/pg_ctl", - ('stop', '-w', '-D', 'tmp_check/master-follower/data')) == 0 + system(catfile("$bindir", "pg_ctl"), + ('stop', '-w', '-D', catfile('tmp_check', 'master-follower', 'data'))) == 0 or warn "Could not shutdown worker server"; for my $port (@followerWorkerPorts) { - system("$bindir/pg_ctl", - ('stop', '-w', '-D', "tmp_check/follower.$port/data")) == 0 + system(catfile("$bindir", "pg_ctl"), + ('stop', '-w', '-D', catfile("tmp_check", "follower.$port", "data"))) == 0 or warn "Could not shutdown worker server"; } } @@ -433,24 +499,24 @@ if ($valgrind) $serversAreShutdown = "FALSE"; # Start servers -if(system("$bindir/pg_ctl", +if(system(catfile("$bindir", "pg_ctl"), ('start', '-w', '-o', join(" ", @pgOptions)." -c port=$masterPort", - '-D', 'tmp_check/master/data', '-l', 'tmp_check/master/log/postmaster.log')) != 0) + '-D', catfile('tmp_check', 'master', 'data'), '-l', catfile('tmp_check', 'master', 'log', 'postmaster.log'))) != 0) { - system("tail", ("-n20", "tmp_check/master/log/postmaster.log")); + system("tail", ("-n20", catfile("tmp_check", "master", "log", "postmaster.log"))); die "Could not start master server"; } for my $port (@workerPorts) { - if(system("$bindir/pg_ctl", + if(system(catfile("$bindir", "pg_ctl"), ('start', '-w', '-o', join(" ", @pgOptions)." -c port=$port", - '-D', "tmp_check/worker.$port/data", - '-l', "tmp_check/worker.$port/log/postmaster.log")) != 0) + '-D', catfile("tmp_check", "worker.$port", "data"), + '-l', catfile("tmp_check", "worker.$port", "log", "postmaster.log"))) != 0) { - system("tail", ("-n20", "tmp_check/worker.$port/log/postmaster.log")); + system("tail", ("-n20", catfile("tmp_check", "worker.$port", "log", "postmaster.log"))); die "Could not start worker server"; } } @@ -460,8 +526,8 @@ if ($followercluster) { # This test would run faster on PG10 if we could pass --no-sync here but that flag # isn't supported on PG 9.6. In a year when we drop support for PG9.6 add that flag! - system("$bindir/pg_basebackup", - ("-D", "tmp_check/master-follower/data", "--host=$host", "--port=$masterPort", + system(catfile("$bindir", "pg_basebackup"), + ("-D", catfile("tmp_check", "master-follower", "data"), "--host=$host", "--port=$masterPort", "--username=$user", "-R", "-X", "stream")) == 0 or die 'could not take basebackup'; @@ -469,30 +535,30 @@ if ($followercluster) { my $workerPort = $workerPorts[$offset]; my $followerPort = $followerWorkerPorts[$offset]; - system("$bindir/pg_basebackup", - ("-D", "tmp_check/follower.$followerPort/data", "--host=$host", "--port=$workerPort", + system(catfile("$bindir", "pg_basebackup"), + ("-D", catfile("tmp_check", "follower.$followerPort", "data"), "--host=$host", "--port=$workerPort", "--username=$user", "-R", "-X", "stream")) == 0 or die "Could not take basebackup"; } - if(system("$bindir/pg_ctl", + if(system(catfile("$bindir", "pg_ctl"), ('start', '-w', '-o', join(" ", @pgOptions)." -c port=$followerCoordPort", - '-D', 'tmp_check/master-follower/data', '-l', 'tmp_check/master-follower/log/postmaster.log')) != 0) + '-D', catfile('tmp_check', 'master-follower', 'data'), '-l', catfile('tmp_check', 'master-follower', 'log', 'postmaster.log'))) != 0) { - system("tail", ("-n20", "tmp_check/master-follower/log/postmaster.log")); + system("tail", ("-n20", catfile("tmp_check", "master-follower", "log", "postmaster.log"))); die "Could not start master follower server"; } for my $port (@followerWorkerPorts) { - if(system("$bindir/pg_ctl", + if(system(catfile("$bindir", "pg_ctl"), ('start', '-w', '-o', join(" ", @pgOptions)." -c port=$port", - '-D', "tmp_check/follower.$port/data", - '-l', "tmp_check/follower.$port/log/postmaster.log")) != 0) + '-D', catfile("tmp_check", "follower.$port", "data"), + '-l', catfile("tmp_check", "follower.$port", "log", "postmaster.log"))) != 0) { - system("tail", ("-n20", "tmp_check/follower.$port/log/postmaster.log")); + system("tail", ("-n20", catfile("tmp_check", "follower.$port", "log", "postmaster.log"))); die "Could not start follower server"; } } @@ -504,55 +570,55 @@ if ($followercluster) ### for my $port (@workerPorts) { - system("psql", '-X', - ('-h', $host, '-p', $port, '-U', $user, "postgres", + system(catfile($bindir, "psql"), + ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "postgres", '-c', "CREATE DATABASE regression;")) == 0 or die "Could not create regression database on worker"; for my $extension (@extensions) { - system("psql", '-X', - ('-h', $host, '-p', $port, '-U', $user, "regression", - '-c', "CREATE EXTENSION IF NOT EXISTS \"$extension\";")) == 0 + system(catfile($bindir, "psql"), + ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression", + '-c', "CREATE EXTENSION IF NOT EXISTS $extension;")) == 0 or die "Could not create extension on worker"; } foreach my $dataType (keys %dataTypes) { - system("psql", '-X', - ('-h', $host, '-p', $port, '-U', $user, "regression", + system(catfile($bindir, "psql"), + ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression", '-c', "CREATE TYPE $dataType AS $dataTypes{$dataType};")) == 0 or die "Could not create TYPE $dataType on worker"; } foreach my $function (keys %functions) { - system("psql", '-X', - ('-h', $host, '-p', $port, '-U', $user, "regression", + system(catfile($bindir, "psql"), + ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression", '-c', "CREATE FUNCTION $function RETURNS $functions{$function};")) == 0 or die "Could not create FUNCTION $function on worker"; } foreach my $operator (keys %operators) { - system("psql", '-X', - ('-h', $host, '-p', $port, '-U', $user, "regression", + system(catfile($bindir, "psql"), + ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression", '-c', "CREATE OPERATOR $operator $operators{$operator};")) == 0 or die "Could not create OPERATOR $operator on worker"; } foreach my $fdw (keys %fdws) { - system("psql", '-X', - ('-h', $host, '-p', $port, '-U', $user, "regression", + system(catfile($bindir, "psql"), + ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression", '-c', "CREATE FOREIGN DATA WRAPPER $fdw HANDLER $fdws{$fdw};")) == 0 or die "Could not create foreign data wrapper $fdw on worker"; } foreach my $fdwServer (keys %fdwServers) { - system("psql", '-X', - ('-h', $host, '-p', $port, '-U', $user, "regression", + system(catfile($bindir, "psql"), + ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression", '-c', "CREATE SERVER $fdwServer FOREIGN DATA WRAPPER $fdwServers{$fdwServer};")) == 0 or die "Could not create server $fdwServer on worker"; } @@ -563,7 +629,8 @@ my @arguments = ( "--host", $host, '--port', $masterPort, '--user', $user, - '--bindir', "tmp_check/tmp-bin" +# '--bindir', 'C:\Users\Administrator\Downloads\pg-64\bin', + '--bindir', catfile("tmp_check", "tmp-bin") ); # Add load extension parameters to the argument list @@ -584,7 +651,7 @@ if ($vanillatest) $ENV{PGPORT} = $masterPort; $ENV{PGUSER} = $user; - system("make -C $postgresBuilddir/src/test/regress installcheck-parallel") == 0 + system("make", ("-C", catfile("$postgresBuilddir", "src", "test", "regress"), "installcheck-parallel")) == 0 or die "Could not run vanilla tests"; } elsif ($isolationtester)