Test two previous commits which did not include tests

https://github.com/citusdata/citus/pull/3400
https://github.com/citusdata/citus/pull/3382

src/backend/distributed/test/system_utils.c

@@ -0,0 +1,138 @@
+/*-------------------------------------------------------------------------
+ *
+ * system_utils.c
+ *	  Methods for managing databases during regression tests.
+ *
+ * Copyright (c) Citus Data, Inc.
+ *
+ * $Id$
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+
+#include "libpq-fe.h"
+#include "pgstat.h"
+#include "port.h"
+#include "storage/fd.h"
+
+#include "distributed/master_metadata_utility.h"
+#include "distributed/version_compat.h"
+
+PG_FUNCTION_INFO_V1(citus_stop_test_worker);
+PG_FUNCTION_INFO_V1(citus_start_test_worker);
+
+
+/*
+ * Invokes pg_ctl stop for the worker using the given port.
+ */
+Datum
+citus_stop_test_worker(PG_FUNCTION_ARGS)
+{
+	EnsureSuperUser();
+
+	int port = PG_GETARG_INT32(0);
+
+	StringInfoData commandString;
+	initStringInfo(&commandString);
+	appendStringInfo(&commandString, "pg_ctl stop -D ../../worker.%d/data", port);
+
+	PG_RETURN_INT32(system(commandString.data));
+}
+
+
+/*
+ * Starts worker using the given port.
+ */
+Datum
+citus_start_test_worker(PG_FUNCTION_ARGS)
+{
+	EnsureSuperUser();
+
+	int port = PG_GETARG_INT32(0);
+	const int fileFlags = O_RDONLY;
+	const int fileMode = S_IRUSR;
+	StringInfoData optionsPath;
+	initStringInfo(&optionsPath);
+	appendStringInfo(&optionsPath, "../../worker.%d/data/postmaster.opts", port);
+	File file =
+		PathNameOpenFilePerm(optionsPath.data, fileFlags, fileMode);
+	FileCompat fileCompat = FileCompatFromFileStart(file);
+
+	StringInfoData optionsString;
+	initStringInfo(&optionsString);
+
+	while (true)
+	{
+		char chunk[1025];
+		int amount = FileReadCompat(&fileCompat, chunk, 1024, PG_WAIT_IO);
+		if (amount <= 0)
+		{
+			break;
+		}
+
+		chunk[amount] = 0;
+		appendStringInfoString(&optionsString, chunk);
+	}
+
+	FileClose(file);
+
+	char *tmpCheckPtr = strstr(optionsString.data, " \"tmp_check/");
+	if (tmpCheckPtr == NULL)
+	{
+		/* If we've already been through this logic previously, the fix will've persisted. */
+		char *tmpCheckAlreadyCorrectPtr = strstr(optionsString.data,
+												 " \"../../../tmp_check");
+		if (tmpCheckAlreadyCorrectPtr == NULL)
+		{
+			elog(ERROR, "Could not find directory in command string: %s",
+				 optionsString.data);
+		}
+	}
+	else
+	{
+		/* replace " with NUL */
+		tmpCheckPtr[1] = 0;
+	}
+
+	StringInfoData commandString;
+	initStringInfo(&commandString);
+
+	appendStringInfoString(&commandString, optionsString.data);
+	if (tmpCheckPtr != NULL)
+	{
+		appendStringInfoString(&commandString, "\"../../../");
+		appendStringInfoString(&commandString, tmpCheckPtr + 2);
+	}
+	Assert(commandString.data[commandString.len - 1] == '\n');
+	commandString.data[commandString.len - 1] = '&';
+
+	int result = system(commandString.data);
+	if (result != 0)
+	{
+		elog(ERROR, "Failed to start worker, exit code: %d command: %s", result,
+			 commandString.data);
+	}
+
+	StringInfoData connectionString;
+	initStringInfo(&connectionString);
+	appendStringInfo(&connectionString, "host=localhost port=%d dbname=regression", port);
+
+	/* poll for at least 10 seconds */
+	int i = 10000;
+	while (PQping(connectionString.data) != PQPING_OK)
+	{
+		pg_usleep(100);
+		if (!i--)
+		{
+			elog(ERROR, "could not connect");
+		}
+	}
+
+	PG_RETURN_VOID();
+}

src/test/regress/expected/multi_multiuser.out

@@ -651,6 +651,14 @@ ERROR:  permission denied for function worker_cleanup_job_schema_cache
 SET ROLE no_access;
 SELECT worker_cleanup_job_schema_cache();
 ERROR:  permission denied for function worker_cleanup_job_schema_cache
+-- test other functions only superuser can call
+SET ROLE full_access;
+SELECT citus_stop_test_worker(:worker_1_port);
+ERROR:  operation is not allowed
+HINT:  Run the command with a superuser.
+SELECT citus_start_test_worker(:worker_1_port);
+ERROR:  operation is not allowed
+HINT:  Run the command with a superuser.
 RESET ROLE;
 -- to test access to files created during repartition we will create some on worker 1
 \c - - - :worker_1_port

src/test/regress/expected/multi_subtransactions.out

@@ -403,29 +403,26 @@ SELECT * FROM researchers WHERE lab_id=10;
  32 |     10 | Raymond Smullyan
 (2 rows)
 
--- Verify that we don't have a memory leak in subtransactions
+-- Test https://github.com/citusdata/citus/issues/2179
--- See https://github.com/citusdata/citus/pull/4000
+CREATE TABLE ref(a int unique, b int);
-CREATE FUNCTION text2number(v_value text) RETURNS numeric
+SELECT create_reference_table('ref');
-    LANGUAGE plpgsql VOLATILE
+ create_reference_table
-    AS $$
-BEGIN
- RETURN v_value::numeric;
-exception
-    when others then
-        return null;
-END;
-$$;
--- if we leak at least an integer in each subxact, then size of TopTransactionSize
--- will be way beyond the 50k limit. If issue #3999 happens, then this will also take
--- a long time, since for each row we will create a memory context that is not destroyed
--- until the end of command.
-SELECT max(text2number('1234')), max(public.top_transaction_context_size()) > 50000 AS leaked
-FROM generate_series(1, 20000);
- max  | leaked
 ---------------------------------------------------------------------
- 1234 | f
+
 (1 row)
 
--- Clean-up
+BEGIN;
-SET client_min_messages TO ERROR;
+SAVEPOINT start;
-DROP SCHEMA multi_subtransactions CASCADE;
+INSERT INTO ref VALUES (1001,2);
+SELECT public.citus_stop_test_worker(:worker_2_port);
+ citus_stop_test_worker
+---------------------------------------------------------------------
+                      0
+(1 row)
+
+SELECT * FROM ref;
+WARNING:  connection to the remote node localhost:xxxxx failed
+WARNING:  connection to the remote node localhost:xxxxx failed
+FATAL:  terminating connection due to administrator command
+SSL connection has been closed unexpectedly
+connection to server was lost

src/test/regress/expected/multi_test_helpers_superuser.out

@@ -53,3 +53,7 @@ CREATE OR REPLACE FUNCTION pg_catalog.partition_task_list_results(resultIdPrefix
 CREATE OR REPLACE FUNCTION top_transaction_context_size() RETURNS BIGINT
 LANGUAGE C STRICT VOLATILE
 AS 'citus', $$top_transaction_context_size$$;
+CREATE FUNCTION citus_stop_test_worker(int)
+    RETURNS int AS 'citus' LANGUAGE C;
+CREATE FUNCTION citus_start_test_worker(int)
+    RETURNS void AS 'citus' LANGUAGE C;

src/test/regress/sql/multi_multiuser.sql

@@ -387,6 +387,12 @@ SET ROLE read_access;
 SELECT worker_cleanup_job_schema_cache();
 SET ROLE no_access;
 SELECT worker_cleanup_job_schema_cache();
+
+-- test other functions only superuser can call
+SET ROLE full_access;
+SELECT citus_stop_test_worker(:worker_1_port);
+SELECT citus_start_test_worker(:worker_1_port);
+
 RESET ROLE;
 
 -- to test access to files created during repartition we will create some on worker 1

src/test/regress/sql/multi_subtransactions.sql

@@ -308,9 +308,39 @@ COMMIT;
 
 SELECT * FROM researchers WHERE lab_id=10;
 
+-- Test https://github.com/citusdata/citus/issues/2179
+CREATE TABLE ref(a int unique, b int);
+SELECT create_reference_table('ref');
+BEGIN;
+SAVEPOINT start;
+INSERT INTO ref VALUES (1001,2);
+SELECT public.citus_stop_test_worker(:worker_2_port);
+SELECT * FROM ref;
+ROLLBACK TO SAVEPOINT start;
+SELECT * FROM ref;
+END;
+SELECT public.citus_start_test_worker(:worker_2_port);
+SELECT shardid FROM pg_dist_shard
+JOIN pg_dist_placement USING (shardid)
+WHERE shardstate = 3 AND logicalrelid = 'ref'::regclass;
+SELECT * FROM ref ORDER BY 1;
+
+-- Test https://github.com/citusdata/citus/issues/3360
+BEGIN;
+SELECT * FROM artists JOIN ref ON id = a;
+SELECT public.citus_stop_test_worker(:worker_2_port);
+INSERT INTO ref VALUES (1002,2);
+END;
+SELECT public.citus_start_test_worker(:worker_2_port);
+SELECT * FROM ref ORDER BY 1;
+
+-- Clean-up
+DROP TABLE artists;
+DROP TABLE researchers;
+DROP TABLE ref;
+
 -- Verify that we don't have a memory leak in subtransactions
 -- See https://github.com/citusdata/citus/pull/4000
-
 CREATE FUNCTION text2number(v_value text) RETURNS numeric
     LANGUAGE plpgsql VOLATILE
     AS $$

src/test/regress/sql/multi_test_helpers_superuser.sql

@@ -54,3 +54,9 @@ CREATE OR REPLACE FUNCTION pg_catalog.partition_task_list_results(resultIdPrefix
 CREATE OR REPLACE FUNCTION top_transaction_context_size() RETURNS BIGINT
 LANGUAGE C STRICT VOLATILE
 AS 'citus', $$top_transaction_context_size$$;
+
+CREATE FUNCTION citus_stop_test_worker(int)
+    RETURNS int AS 'citus' LANGUAGE C;
+CREATE FUNCTION citus_start_test_worker(int)
+    RETURNS void AS 'citus' LANGUAGE C;
+