Rethrow original concurrent index creation failure message (#4469)

* Rethrow original concurrent index creation failure message

* Alter test outputs for concurrent index creation

* Detect duplicate table failure in concurrent index creation

* Add test for conc. index creation w/out duplicates

src/backend/distributed/commands/utility_hook.c

@@ -705,6 +705,7 @@ ExecuteDistributedDDLJob(DDLJob *ddlJob)
 		Assert(SavedMultiShardCommitProtocol == COMMIT_PROTOCOL_BARE);
 		SavedMultiShardCommitProtocol = MultiShardCommitProtocol;
 		MultiShardCommitProtocol = COMMIT_PROTOCOL_BARE;
+		MemoryContext savedContext = CurrentMemoryContext;
 
 		PG_TRY();
 		{
@@ -731,12 +732,33 @@ ExecuteDistributedDDLJob(DDLJob *ddlJob)
 		}
 		PG_CATCH();
 		{
-			ereport(ERROR,
+			/* CopyErrorData() requires (CurrentMemoryContext != ErrorContext) */
-					(errmsg("CONCURRENTLY-enabled index command failed"),
+			MemoryContextSwitchTo(savedContext);
-					 errdetail("CONCURRENTLY-enabled index commands can fail partially, "
+			ErrorData *edata = CopyErrorData();
-							   "leaving behind an INVALID index."),
+
-					 errhint("Use DROP INDEX CONCURRENTLY IF EXISTS to remove the "
+			/*
-							 "invalid index, then retry the original command.")));
+			 * In concurrent index creation, if a worker index with the same name already
+			 * exists, prompt to DROP the current index and retry the original command
+			 */
+			if (edata->sqlerrcode == ERRCODE_DUPLICATE_TABLE)
+			{
+				ereport(ERROR,
+						(errmsg("CONCURRENTLY-enabled index command failed"),
+						 errdetail(
+							 "CONCURRENTLY-enabled index commands can fail partially, "
+							 "leaving behind an INVALID index."),
+						 errhint("Use DROP INDEX CONCURRENTLY IF EXISTS to remove the "
+								 "invalid index, then retry the original command.")));
+			}
+			else
+			{
+				ereport(WARNING,
+						(errmsg(
+							 "CONCURRENTLY-enabled index commands can fail partially, "
+							 "leaving behind an INVALID index.\n Use DROP INDEX "
+							 "CONCURRENTLY IF EXISTS to remove the invalid index.")));
+				PG_RE_THROW();
+			}
 		}
 		PG_END_TRY();
 	}

src/test/regress/expected/failure_create_index_concurrently.out

@@ -26,9 +26,11 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE").kill()');
 (1 row)
 
 CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1);
-ERROR:  CONCURRENTLY-enabled index command failed
+WARNING:  CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index.
-DETAIL:  CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index.
+Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index.
-HINT:  Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index, then retry the original command.
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
+	   This probably means the server terminated abnormally
+	   before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy
 ---------------------------------------------------------------------
@@ -59,9 +61,11 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE").kill()');
 (1 row)
 
 CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1);
-ERROR:  CONCURRENTLY-enabled index command failed
+WARNING:  CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index.
-DETAIL:  CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index.
+Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index.
-HINT:  Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index, then retry the original command.
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
+	   This probably means the server terminated abnormally
+	   before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy
 ---------------------------------------------------------------------
@@ -86,9 +90,9 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE").cancel(' || pg_backend_pid(
 (1 row)
 
 CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1);
-ERROR:  CONCURRENTLY-enabled index command failed
+WARNING:  CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index.
-DETAIL:  CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index.
+Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index.
-HINT:  Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index, then retry the original command.
+ERROR:  canceling statement due to user request
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy
 ---------------------------------------------------------------------
@@ -111,9 +115,9 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE").cancel(' || pg_backend_pid(
 (1 row)
 
 CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1);
-ERROR:  CONCURRENTLY-enabled index command failed
+WARNING:  CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index.
-DETAIL:  CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index.
+Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index.
-HINT:  Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index, then retry the original command.
+ERROR:  canceling statement due to user request
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy
 ---------------------------------------------------------------------
@@ -137,9 +141,11 @@ SELECT citus.mitmproxy('conn.onQuery(query="DROP INDEX CONCURRENTLY").kill()');
 (1 row)
 
 DROP INDEX CONCURRENTLY IF EXISTS idx_index_test;
-ERROR:  CONCURRENTLY-enabled index command failed
+WARNING:  CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index.
-DETAIL:  CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index.
+Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index.
-HINT:  Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index, then retry the original command.
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
+	   This probably means the server terminated abnormally
+	   before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy
 ---------------------------------------------------------------------
@@ -154,9 +160,31 @@ WHERE nodeport = :worker_2_proxy_port;
  localhost |     9060 | t       | 4
 (1 row)
 
+-- test unique concurrent index creation failure when there are duplicates
+CREATE TABLE index_test_2 (a int, b int);
+SELECT create_distributed_table('index_test_2', 'a');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+INSERT INTO index_test_2 VALUES (1, 1), (1, 2);
+CREATE UNIQUE INDEX CONCURRENTLY index_test_2_a_idx ON index_test_2(a);
+WARNING:  CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index.
+Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index.
+ERROR:  could not create unique index "index_test_2_a_idx_1880019"
+DETAIL:  Key (a)=(1) is duplicated.
+CONTEXT:  while executing command on localhost:xxxxx
+DROP INDEX CONCURRENTLY IF EXISTS index_test_2_a_idx;
+-- verify that index creation doesn't fail when duplicates are removed
+DELETE FROM index_test_2 WHERE a = 1 AND b = 2;
+CREATE UNIQUE INDEX CONCURRENTLY index_test_2_a_idx ON index_test_2(a);
+DROP INDEX CONCURRENTLY IF EXISTS index_test_2_a_idx;
 RESET SEARCH_PATH;
 DROP SCHEMA index_schema CASCADE;
-NOTICE:  drop cascades to table index_schema.index_test
+NOTICE:  drop cascades to 2 other objects
+DETAIL:  drop cascades to table index_schema.index_test
+drop cascades to table index_schema.index_test_2
 -- verify index is not at worker 2 upon cleanup
 SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$)
 WHERE nodeport = :worker_2_proxy_port;

src/test/regress/expected/multi_index_statements.out

@@ -497,7 +497,7 @@ DROP INDEX f1;
 DROP INDEX ix_test_index_creation2;
 DROP INDEX ix_test_index_creation1_ix_test_index_creation1_ix_test_index_creation1_ix_test_index_creation1_ix_test_index_creation1;
 DROP INDEX CONCURRENTLY ith_b_idx;
-ERROR:  CONCURRENTLY-enabled index command failed
+ERROR:  index "ith_b_idx_102089" does not exist
 -- the failure results in an INVALID index
 SELECT indisvalid AS "Index Valid?" FROM pg_index WHERE indexrelid='ith_b_idx'::regclass;
  Index Valid?

src/test/regress/sql/failure_create_index_concurrently.sql

@@ -81,6 +81,18 @@ SELECT citus.mitmproxy('conn.allow()');
 SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$)
 WHERE nodeport = :worker_2_proxy_port;
 
+-- test unique concurrent index creation failure when there are duplicates
+CREATE TABLE index_test_2 (a int, b int);
+SELECT create_distributed_table('index_test_2', 'a');
+INSERT INTO index_test_2 VALUES (1, 1), (1, 2);
+CREATE UNIQUE INDEX CONCURRENTLY index_test_2_a_idx ON index_test_2(a);
+DROP INDEX CONCURRENTLY IF EXISTS index_test_2_a_idx;
+
+-- verify that index creation doesn't fail when duplicates are removed
+DELETE FROM index_test_2 WHERE a = 1 AND b = 2;
+CREATE UNIQUE INDEX CONCURRENTLY index_test_2_a_idx ON index_test_2(a);
+DROP INDEX CONCURRENTLY IF EXISTS index_test_2_a_idx;
+
 RESET SEARCH_PATH;
 DROP SCHEMA index_schema CASCADE;