Add support for proper valgrind tests

This change allows valgrind tests (`make check-multi-vg`) to be
run seamlessly without test output errors and timeout problems.

src/test/regress/Makefile

@@ -51,6 +51,7 @@ check-multi: all tempinstall-main
 
 check-multi-vg: all tempinstall-main
 	$(pg_regress_multi_check) --load-extension=citus --valgrind \
+	--pg_ctl-timeout=360 --connection-timeout=500000 --valgrind-path=valgrind \
 	-- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule $(EXTRA_TESTS)
 
 check-isolation: all tempinstall-main
@@ -70,7 +71,6 @@ check-multi-task-tracker-extra: all tempinstall-main
 	--server-option=citus.large_table_shard_count=1 \
 	-- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_task_tracker_extra_schedule $(EXTRA_TESTS)
 
-
 check-multi-binary: all tempinstall-main
 	$(pg_regress_multi_check) --load-extension=citus \
 	--server-option=citus.binary_worker_copy_format=on \

src/test/regress/input/multi_load_data.source

@@ -23,3 +23,5 @@ ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 290000;
 \copy lineitem_hash_part FROM '@abs_srcdir@/data/lineitem.2.data' with delimiter '|'
 \copy orders_hash_part FROM '@abs_srcdir@/data/orders.1.data' with delimiter '|'
 \copy orders_hash_part FROM '@abs_srcdir@/data/orders.2.data' with delimiter '|'
+
+VACUUM ANALYZE;

src/test/regress/input/multi_load_more_data.source

@@ -13,3 +13,5 @@ ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 280000;
 \copy customer FROM '@abs_srcdir@/data/customer.2.data' with delimiter '|'
 \copy customer FROM '@abs_srcdir@/data/customer.3.data' with delimiter '|'
 \copy part FROM '@abs_srcdir@/data/part.more.data' with delimiter '|'
+
+VACUUM ANALYZE;

src/test/regress/output/multi_load_data.source

@@ -19,3 +19,6 @@ ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 290000;
 \copy lineitem_hash_part FROM '@abs_srcdir@/data/lineitem.2.data' with delimiter '|'
 \copy orders_hash_part FROM '@abs_srcdir@/data/orders.1.data' with delimiter '|'
 \copy orders_hash_part FROM '@abs_srcdir@/data/orders.2.data' with delimiter '|'
+VACUUM ANALYZE;
+WARNING:  not propagating VACUUM command to worker nodes
+HINT:  Provide a specific table in order to VACUUM distributed tables.

src/test/regress/output/multi_load_more_data.source

@@ -8,3 +8,6 @@ ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 280000;
 \copy customer FROM '@abs_srcdir@/data/customer.2.data' with delimiter '|'
 \copy customer FROM '@abs_srcdir@/data/customer.3.data' with delimiter '|'
 \copy part FROM '@abs_srcdir@/data/part.more.data' with delimiter '|'
+VACUUM ANALYZE;
+WARNING:  not propagating VACUUM command to worker nodes
+HINT:  Provide a specific table in order to VACUUM distributed tables.

src/test/regress/pg_regress_multi.pl

@@ -36,6 +36,9 @@ sub Usage()
     print "  --load-extension    	Extensions to install in all nodes\n";
     print "  --server-option     	Config option to pass to the server\n";
     print "  --valgrind          	Run server via valgrind\n";
+    print "  --valgrind-path     	Path to the valgrind executable\n";
+    print "  --pg_ctl-timeout    	Timeout for pg_ctl\n";
+    print "  --connection-timeout	Timeout for connecting to worker nodes\n";
     exit 1;
 }
 
@@ -56,6 +59,9 @@ my %fdwServers = ();
 my %functions = ();
 my %operators = ();
 my $valgrind = 0;
+my $valgrind_path = "valgrind";
+my $pg_ctl_timeout = undef;
+my $connection_timeout = 5000;
 
 my $serversAreShutdown = "TRUE";
 
@@ -71,6 +77,9 @@ GetOptions(
     'load-extension=s' => \@extensions,
     'server-option=s' => \@userPgOptions,
     'valgrind' => \$valgrind,
+    'valgrind-path=s' => \$valgrind_path,
+    'pg_ctl-timeout=s' => \$pg_ctl_timeout,
+    'connection-timeout=s' => \$connection_timeout,
     'help' => sub { Usage() });
 
 # Update environment to include [DY]LD_LIBRARY_PATH/LIBDIR/etc -
@@ -136,9 +145,9 @@ MESSAGE
 }
 
 # valgrind starts slow, need to increase timeout
-if ($valgrind)
+if (defined $pg_ctl_timeout)
 {
-    $ENV{PGCTLTIMEOUT} = '360';
+    $ENV{PGCTLTIMEOUT} = "$pg_ctl_timeout";
 }
 
 # We don't want valgrind to run pg_ctl itself, as that'd trigger a lot
@@ -163,7 +172,7 @@ sub replace_postgres
 	or die "Could not create postgres wrapper at $bindir/postgres";
     print $fh <<"END";
 #!/bin/bash
-exec valgrind \\
+exec $valgrind_path \\
     --quiet \\
     --suppressions=${postgresSrcdir}/src/tools/valgrind.supp \\
     --trace-children=yes --track-origins=yes --read-var-info=yes \\
@@ -220,6 +229,7 @@ push(@pgOptions, '-c', "citus.expire_cached_shards=on");
 push(@pgOptions, '-c', "citus.task_tracker_delay=10ms");
 push(@pgOptions, '-c', "citus.remote_task_check_interval=1ms");
 push(@pgOptions, '-c', "citus.shard_replication_factor=2");
+push(@pgOptions, '-c', "citus.node_connection_timeout=${connection_timeout}");
 
 # Add externally added options last, so they overwrite the default ones above
 for my $option (@userPgOptions)