From 0c5d0cfee97282d93330244c50f923f28095b39c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Hanefi=20=C3=96nald=C4=B1?= <Hanefi.Onaldi@Microsoft.com>
Date: Fri, 17 Apr 2020 13:20:30 +0300
Subject: [PATCH] Notice message to help truncate local data after distribution

---
 .../commands/create_distributed_table.c       | 12 +++++
 src/test/regress/expected/ch_bench_having.out |  3 ++
 .../regress/expected/ch_bench_having_mx.out   |  3 ++
 .../expected/coordinator_shouldhaveshards.out |  6 +++
 .../expected/cte_nested_modification.out      |  6 +++
 .../regress/expected/cte_prepared_modify.out  |  6 +++
 .../expected/distributed_collations.out       |  3 ++
 .../expected/expression_reference_join.out    |  6 +++
 .../failure_connection_establishment.out      |  3 ++
 ...ure_create_distributed_table_non_empty.out |  6 +++
 .../failure_create_reference_table.out        | 12 +++++
 .../failure_multi_shard_update_delete.out     |  6 +++
 .../foreign_key_restriction_enforcement.out   |  6 +++
 .../foreign_key_to_reference_table.out        |  6 +++
 .../regress/expected/local_shard_copy.out     |  3 ++
 .../local_shard_utility_command_execution.out |  2 +
 .../regress/expected/materialized_view.out    |  6 +++
 .../multi_alter_table_add_constraints.out     | 11 +++--
 .../regress/expected/multi_create_table.out   | 12 +++++
 .../expected/multi_create_table_superuser.out | 45 +++++++++++++++++++
 src/test/regress/expected/multi_multiuser.out |  9 ++++
 .../expected/multi_mx_partitioning.out        |  9 ++++
 .../regress/expected/multi_partitioning.out   | 23 +++++++++-
 .../expected/multi_real_time_transaction.out  |  6 +++
 .../expected/multi_reference_table.out        |  3 ++
 .../expected/multi_shard_update_delete.out    |  9 ++++
 src/test/regress/expected/multi_truncate.out  | 18 ++++++++
 src/test/regress/expected/pg12.out            | 15 +++++++
 src/test/regress/expected/pg_dump.out         |  6 +++
 .../expected/relation_access_tracking.out     |  3 ++
 .../expected/replicated_partitioned_table.out |  9 ++++
 .../expected/sequential_modifications.out     |  3 ++
 .../expected/subquery_partitioning.out        |  6 +++
 .../regress/expected/upgrade_basic_after.out  |  3 ++
 .../regress/expected/with_partitioning.out    |  6 +++
 .../regress/output/hyperscale_tutorial.source | 15 +++++++
 src/test/regress/output/multi_copy.source     |  3 ++
 37 files changed, 304 insertions(+), 5 deletions(-)

diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c
index eb23c5c76..4584afda8 100644
--- a/src/backend/distributed/commands/create_distributed_table.c
+++ b/src/backend/distributed/commands/create_distributed_table.c
@@ -1340,6 +1340,18 @@ DoCopyFromLocalTableIntoShards(Relation distributedRelation,
 		ereport(DEBUG1, (errmsg("Copied " UINT64_FORMAT " rows", rowsCopied)));
 	}
 
+	if (rowsCopied > 0)
+	{
+		char *qualifiedRelationName =
+			generate_qualified_relation_name(RelationGetRelid(distributedRelation));
+		ereport(NOTICE, (errmsg("copying the data has completed"),
+						 errdetail("The local data in the table is longer visible, "
+								   "but is still on disk."),
+						 errhint("To remove the local data, run: SELECT "
+								 "truncate_local_data_after_distributing_table($$%s$$)",
+								 qualifiedRelationName)));
+	}
+
 	MemoryContextSwitchTo(oldContext);
 
 	/* finish reading from the local table */
diff --git a/src/test/regress/expected/ch_bench_having.out b/src/test/regress/expected/ch_bench_having.out
index 51c6e85f5..eb641c309 100644
--- a/src/test/regress/expected/ch_bench_having.out
+++ b/src/test/regress/expected/ch_bench_having.out
@@ -315,6 +315,9 @@ insert into stock VALUES
 (32, 1, 1, 1, 1, 1, '', '','','','','','','','','','');
 SELECT create_distributed_table('stock','s_w_id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$ch_bench_having.stock$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/ch_bench_having_mx.out b/src/test/regress/expected/ch_bench_having_mx.out
index 372155303..0dbc14e40 100644
--- a/src/test/regress/expected/ch_bench_having_mx.out
+++ b/src/test/regress/expected/ch_bench_having_mx.out
@@ -323,6 +323,9 @@ insert into stock VALUES
 (32, 1, 1, 1, 1, 1, '', '','','','','','','','','','');
 SELECT create_distributed_table('stock','s_w_id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$ch_bench_having.stock$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/coordinator_shouldhaveshards.out b/src/test/regress/expected/coordinator_shouldhaveshards.out
index ac0813ef0..fe88d9372 100644
--- a/src/test/regress/expected/coordinator_shouldhaveshards.out
+++ b/src/test/regress/expected/coordinator_shouldhaveshards.out
@@ -145,6 +145,9 @@ NOTICE:  executing the command locally: SELECT worker_apply_shard_ddl_command (1
 NOTICE:  executing the copy locally for shard xxxxx
 NOTICE:  Copying data from local table...
 NOTICE:  executing the copy locally for shard xxxxx
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$coordinator_shouldhaveshards.dist_table$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -178,6 +181,9 @@ NOTICE:  executing the command locally: SELECT worker_apply_shard_ddl_command (1
 NOTICE:  executing the copy locally for shard xxxxx
 NOTICE:  Copying data from local table...
 NOTICE:  executing the copy locally for shard xxxxx
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$coordinator_shouldhaveshards.dist_table$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/cte_nested_modification.out b/src/test/regress/expected/cte_nested_modification.out
index 49f478f01..1f31f7bf9 100644
--- a/src/test/regress/expected/cte_nested_modification.out
+++ b/src/test/regress/expected/cte_nested_modification.out
@@ -4,6 +4,9 @@ CREATE TABLE tt1(id int, value_1 int);
 INSERT INTO tt1 VALUES(1,2),(2,3),(3,4);
 SELECT create_distributed_table('tt1','id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$cte_nested_modifications.tt1$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -13,6 +16,9 @@ CREATE TABLE tt2(id int, value_1 int);
 INSERT INTO tt2 VALUES(3,3),(4,4),(5,5);
 SELECT create_distributed_table('tt2','id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$cte_nested_modifications.tt2$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/cte_prepared_modify.out b/src/test/regress/expected/cte_prepared_modify.out
index 9460ab275..6da772f4f 100644
--- a/src/test/regress/expected/cte_prepared_modify.out
+++ b/src/test/regress/expected/cte_prepared_modify.out
@@ -4,6 +4,9 @@ CREATE TABLE tt1(id int, value_1 int);
 INSERT INTO tt1 VALUES(1,2),(2,3),(3,4);
 SELECT create_distributed_table('tt1','id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$cte_prepared_modify.tt1$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -13,6 +16,9 @@ CREATE TABLE tt2(id int, value_1 int);
 INSERT INTO tt2 VALUES(3,3),(4,4),(5,5);
 SELECT create_distributed_table('tt2','id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$cte_prepared_modify.tt2$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/distributed_collations.out b/src/test/regress/expected/distributed_collations.out
index 521147e15..b22b1b193 100644
--- a/src/test/regress/expected/distributed_collations.out
+++ b/src/test/regress/expected/distributed_collations.out
@@ -35,6 +35,9 @@ CREATE TABLE test_propagate(id int, t1 text COLLATE german_phonebook,
 INSERT INTO test_propagate VALUES (1, 'aesop', U&'\00E4sop'), (2, U&'Vo\1E9Er', 'Vossr');
 SELECT create_distributed_table('test_propagate', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$collation_tests.test_propagate$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/expression_reference_join.out b/src/test/regress/expected/expression_reference_join.out
index 8076d9200..e0b8a09a8 100644
--- a/src/test/regress/expected/expression_reference_join.out
+++ b/src/test/regress/expected/expression_reference_join.out
@@ -13,6 +13,9 @@ INSERT INTO test VALUES
     (2,2);
 SELECT create_reference_table('ref');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$expression_reference_join.ref$$)
  create_reference_table
 ---------------------------------------------------------------------
 
@@ -20,6 +23,9 @@ NOTICE:  Copying data from local table...
 
 SELECT create_distributed_table('test', 'x');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$expression_reference_join.test$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/failure_connection_establishment.out b/src/test/regress/expected/failure_connection_establishment.out
index 778fcad8e..30beab4ef 100644
--- a/src/test/regress/expected/failure_connection_establishment.out
+++ b/src/test/regress/expected/failure_connection_establishment.out
@@ -61,6 +61,9 @@ INSERT INTO r1 (id, name) VALUES
 (3,'baz');
 SELECT create_reference_table('r1');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$fail_connect.r1$$)
  create_reference_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/failure_create_distributed_table_non_empty.out b/src/test/regress/expected/failure_create_distributed_table_non_empty.out
index 9402dde3e..2fea6e43a 100644
--- a/src/test/regress/expected/failure_create_distributed_table_non_empty.out
+++ b/src/test/regress/expected/failure_create_distributed_table_non_empty.out
@@ -227,6 +227,9 @@ SELECT citus.mitmproxy('conn.onCommandComplete(command="COPY").kill()');
 
 SELECT create_distributed_table('test_table', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$create_distributed_table_non_empty_failure.test_table$$)
 ERROR:  failed to COPY to shard xxxxx on localhost:xxxxx
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
  count
@@ -259,6 +262,9 @@ SELECT citus.mitmproxy('conn.onCommandComplete(command="COPY").cancel(' ||  pg_b
 
 SELECT create_distributed_table('test_table', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$create_distributed_table_non_empty_failure.test_table$$)
 ERROR:  canceling statement due to user request
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
  count
diff --git a/src/test/regress/expected/failure_create_reference_table.out b/src/test/regress/expected/failure_create_reference_table.out
index a9d15187c..13b383b8b 100644
--- a/src/test/regress/expected/failure_create_reference_table.out
+++ b/src/test/regress/expected/failure_create_reference_table.out
@@ -110,6 +110,9 @@ SELECT citus.mitmproxy('conn.onCommandComplete(command="COPY 3").kill()');
 
 SELECT create_reference_table('ref_table');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$failure_reference_table.ref_table$$)
 ERROR:  failed to COPY to shard xxxxx on localhost:xxxxx
 SELECT count(*) FROM pg_dist_shard_placement;
  count
@@ -126,6 +129,9 @@ SELECT citus.mitmproxy('conn.onCommandComplete(command="COPY 3").cancel(' || pg_
 
 SELECT create_reference_table('ref_table');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$failure_reference_table.ref_table$$)
 ERROR:  canceling statement due to user request
 SELECT count(*) FROM pg_dist_shard_placement;
  count
@@ -215,6 +221,9 @@ SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
 BEGIN;
 SELECT create_reference_table('ref_table');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$failure_reference_table.ref_table$$)
  create_reference_table
 ---------------------------------------------------------------------
 
@@ -239,6 +248,9 @@ SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").cancel(' || pg_backend_p
 BEGIN;
 SELECT create_reference_table('ref_table');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$failure_reference_table.ref_table$$)
  create_reference_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/failure_multi_shard_update_delete.out b/src/test/regress/expected/failure_multi_shard_update_delete.out
index ffc08aef7..68860e5b0 100644
--- a/src/test/regress/expected/failure_multi_shard_update_delete.out
+++ b/src/test/regress/expected/failure_multi_shard_update_delete.out
@@ -436,6 +436,9 @@ SELECT citus.mitmproxy('conn.allow()');
 CREATE TABLE t3 AS SELECT * FROM t2;
 SELECT create_distributed_table('t3', 'a');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$multi_shard.t3$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -534,6 +537,9 @@ DROP TABLE t3;
 CREATE TABLE t3 AS SELECT * FROM t2;
 SELECT create_distributed_table('t3', 'a');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$multi_shard.t3$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/foreign_key_restriction_enforcement.out b/src/test/regress/expected/foreign_key_restriction_enforcement.out
index 261e00096..41a6aa0cf 100644
--- a/src/test/regress/expected/foreign_key_restriction_enforcement.out
+++ b/src/test/regress/expected/foreign_key_restriction_enforcement.out
@@ -1272,6 +1272,9 @@ BEGIN;
 	INSERT INTO test_table_2 SELECT i, i FROM generate_series(0,100) i;
 	SELECT create_reference_table('test_table_1');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$test_fkey_to_ref_in_tx.test_table_1$$)
  create_reference_table
 ---------------------------------------------------------------------
 
@@ -1296,6 +1299,9 @@ BEGIN;
 	INSERT INTO test_table_2 SELECT i, i FROM generate_series(0,100) i;
 	SELECT create_reference_table('test_table_1');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$test_fkey_to_ref_in_tx.test_table_1$$)
  create_reference_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/foreign_key_to_reference_table.out b/src/test/regress/expected/foreign_key_to_reference_table.out
index 37547b3f5..d8be4cc53 100644
--- a/src/test/regress/expected/foreign_key_to_reference_table.out
+++ b/src/test/regress/expected/foreign_key_to_reference_table.out
@@ -746,6 +746,9 @@ INSERT INTO referenced_table VALUES (1,1), (2,2), (3,3);
 INSERT INTO referencing_table VALUES (1,1), (2,2), (3,3);
 SELECT create_reference_table('referenced_table');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$fkey_reference_table.referenced_table$$)
  create_reference_table
 ---------------------------------------------------------------------
 
@@ -1305,6 +1308,9 @@ BEGIN;
   INSERT INTO test_table_2 SELECT i, i FROM generate_series(0,100) i;
   SELECT create_reference_table('test_table_1');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$fkey_reference_table.test_table_1$$)
  create_reference_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/local_shard_copy.out b/src/test/regress/expected/local_shard_copy.out
index a0cc25c86..06963f237 100644
--- a/src/test/regress/expected/local_shard_copy.out
+++ b/src/test/regress/expected/local_shard_copy.out
@@ -502,6 +502,9 @@ SELECT create_reference_table('ref_table');
 NOTICE:  executing the command locally: SELECT worker_apply_shard_ddl_command (1330000, 'local_shard_copy', 'CREATE TABLE local_shard_copy.ref_table (a integer)');SELECT worker_apply_shard_ddl_command (1330000, 'local_shard_copy', 'ALTER TABLE local_shard_copy.ref_table OWNER TO postgres')
 NOTICE:  executing the copy locally for shard xxxxx
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$local_shard_copy.ref_table$$)
  create_reference_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/local_shard_utility_command_execution.out b/src/test/regress/expected/local_shard_utility_command_execution.out
index b49bee224..76f0c291d 100644
--- a/src/test/regress/expected/local_shard_utility_command_execution.out
+++ b/src/test/regress/expected/local_shard_utility_command_execution.out
@@ -698,7 +698,9 @@ INSERT INTO partitioning_test VALUES (6, '2012-07-07');
 INSERT INTO partitioning_test VALUES (5, '2013-06-06');
 SELECT create_distributed_table('partitioning_test', 'id', colocate_with:='dist_table');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/materialized_view.out b/src/test/regress/expected/materialized_view.out
index b8cb80529..f2d17804f 100644
--- a/src/test/regress/expected/materialized_view.out
+++ b/src/test/regress/expected/materialized_view.out
@@ -138,6 +138,9 @@ FROM lineitem_local_to_hash_part, orders_local_to_hash_part, (SELECT SUM(l_exten
 WHERE lineitem_local_to_hash_part.l_orderkey=orders_local_to_hash_part.o_orderkey;
 SELECT create_distributed_table('lineitem_local_to_hash_part', 'l_orderkey');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$materialized_view.lineitem_local_to_hash_part$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -145,6 +148,9 @@ NOTICE:  Copying data from local table...
 
 SELECT create_distributed_table('orders_local_to_hash_part', 'o_orderkey');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$materialized_view.orders_local_to_hash_part$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/multi_alter_table_add_constraints.out b/src/test/regress/expected/multi_alter_table_add_constraints.out
index ea3321867..7949a39e0 100644
--- a/src/test/regress/expected/multi_alter_table_add_constraints.out
+++ b/src/test/regress/expected/multi_alter_table_add_constraints.out
@@ -89,7 +89,7 @@ ALTER TABLE products_append ADD CONSTRAINT p_key PRIMARY KEY(product_no);
 WARNING:  table "products_append" has a UNIQUE or EXCLUDE constraint
 DETAIL:  UNIQUE constraints, EXCLUDE constraints, and PRIMARY KEYs on append-partitioned tables cannot be enforced.
 HINT:  Consider using hash partitioning.
---- Error out since first and third rows have the same product_no		
+--- Error out since first and third rows have the same product_no
 \COPY products_append FROM STDIN DELIMITER AS ',';
 ERROR:  duplicate key value violates unique constraint "p_key_1450033"
 DETAIL:  Key (product_no)=(1) already exists.
@@ -176,8 +176,8 @@ ALTER TABLE unique_test_table_append ADD CONSTRAINT unn_id UNIQUE(id);
 WARNING:  table "unique_test_table_append" has a UNIQUE or EXCLUDE constraint
 DETAIL:  UNIQUE constraints, EXCLUDE constraints, and PRIMARY KEYs on append-partitioned tables cannot be enforced.
 HINT:  Consider using hash partitioning.
--- Error out. Table can not have two rows with the same id.		
-\COPY unique_test_table_append FROM STDIN DELIMITER AS ',';		
+-- Error out. Table can not have two rows with the same id.
+\COPY unique_test_table_append FROM STDIN DELIMITER AS ',';
 ERROR:  duplicate key value violates unique constraint "unn_id_1450067"
 DETAIL:  Key (id)=(X) already exists.
 DROP TABLE unique_test_table_append;
@@ -336,7 +336,7 @@ ALTER TABLE products_append ADD CONSTRAINT exc_pno_name EXCLUDE USING btree (pro
 WARNING:  table "products_append" has a UNIQUE or EXCLUDE constraint
 DETAIL:  UNIQUE constraints, EXCLUDE constraints, and PRIMARY KEYs on append-partitioned tables cannot be enforced.
 HINT:  Consider using hash partitioning.
--- Error out since first and third can not pass the exclusion check.		
+-- Error out since first and third can not pass the exclusion check.
 \COPY products_append FROM STDIN DELIMITER AS ',';
 ERROR:  conflicting key value violates exclusion constraint "exc_pno_name_1450135"
 DETAIL:  Key (product_no, name)=(1,  Product_1) conflicts with existing key (product_no, name)=(1,  Product_1).
@@ -560,6 +560,9 @@ INSERT INTO sc3.alter_add_prim_key(x) SELECT generate_series(1,100);
 SET search_path TO 'sc3';
 SELECT create_distributed_table('alter_add_prim_key', 'x');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$sc3.alter_add_prim_key$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/multi_create_table.out b/src/test/regress/expected/multi_create_table.out
index a6038d1d7..4bd2a2f16 100644
--- a/src/test/regress/expected/multi_create_table.out
+++ b/src/test/regress/expected/multi_create_table.out
@@ -201,6 +201,9 @@ HINT:  Empty your table before distributing it.
 -- create_distributed_table creates shards and copies data into the distributed table
 SELECT create_distributed_table('data_load_test', 'col1');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.data_load_test$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -260,6 +263,9 @@ CREATE TABLE data_load_test (col1 int, col2 text, col3 serial);
 INSERT INTO data_load_test VALUES (132, 'hello');
 SELECT create_distributed_table('data_load_test', 'col1');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.data_load_test$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -281,6 +287,9 @@ CREATE TABLE data_load_test1 (col1 int, col2 text, col3 serial);
 INSERT INTO data_load_test1 VALUES (132, 'hello');
 SELECT create_distributed_table('data_load_test1', 'col1');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.data_load_test1$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -290,6 +299,9 @@ CREATE TABLE data_load_test2 (col1 int, col2 text, col3 serial);
 INSERT INTO data_load_test2 VALUES (132, 'world');
 SELECT create_distributed_table('data_load_test2', 'col1');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.data_load_test2$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/multi_create_table_superuser.out b/src/test/regress/expected/multi_create_table_superuser.out
index dd6d3225c..d2f978150 100644
--- a/src/test/regress/expected/multi_create_table_superuser.out
+++ b/src/test/regress/expected/multi_create_table_superuser.out
@@ -237,6 +237,9 @@ CREATE TABLE data_load_test (col1 int, col2 text, col3 serial);
 INSERT INTO data_load_test VALUES (132, 'hello');
 SELECT create_distributed_table('data_load_test', 'col1');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.data_load_test$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -251,6 +254,9 @@ CREATE TABLE data_load_test (col1 int, col2 text, col3 serial);
 INSERT INTO data_load_test VALUES (132, 'hello');
 SELECT create_distributed_table('data_load_test', 'col1');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.data_load_test$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -264,6 +270,9 @@ CREATE TABLE data_load_test (col1 int, col2 text, col3 serial);
 INSERT INTO data_load_test VALUES (132, 'hello');
 SELECT create_distributed_table('data_load_test', 'col1');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.data_load_test$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -279,6 +288,9 @@ INSERT INTO data_load_test VALUES (243, 'world', 'hello');
 ALTER TABLE data_load_test DROP COLUMN col1;
 SELECT create_distributed_table('data_load_test', 'col3');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.data_load_test$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -371,6 +383,9 @@ INSERT INTO rollback_table VALUES(2, 'Name_2');
 INSERT INTO rollback_table VALUES(3, 'Name_3');
 SELECT create_distributed_table('rollback_table','id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.rollback_table$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -498,6 +513,9 @@ CREATE TABLE tt1(id int);
 INSERT INTO tt1 VALUES(1);
 SELECT create_distributed_table('tt1','id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.tt1$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -554,6 +572,9 @@ CREATE TABLE stage_table (LIKE sample_table);
 \COPY stage_table FROM stdin; -- Note that this operation is a local copy
 SELECT create_distributed_table('stage_table', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.stage_table$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -625,6 +646,9 @@ CREATE TABLE sc.ref(a int);
 insert into sc.ref SELECT s FROM generate_series(0, 100) s;
 SELECT create_reference_table('sc.ref');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$sc.ref$$)
  create_reference_table
 ---------------------------------------------------------------------
 
@@ -634,6 +658,9 @@ CREATE TABLE sc.hash(a int);
 insert into sc.hash SELECT s FROM generate_series(0, 100) s;
 SELECT create_distributed_table('sc.hash', 'a');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$sc.hash$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -647,6 +674,9 @@ CREATE TABLE sc2.hash(a int);
 insert into sc2.hash SELECT s FROM generate_series(0, 100) s;
 SELECT create_distributed_table('sc2.hash', 'a');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$sc2.hash$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -656,6 +686,9 @@ CREATE TABLE sc2.ref(a int);
 insert into sc2.ref SELECT s FROM generate_series(0, 100) s;
 SELECT create_reference_table('sc2.ref');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$sc2.ref$$)
  create_reference_table
 ---------------------------------------------------------------------
 
@@ -697,6 +730,9 @@ SET search_path = 'sc4';
 ALTER TABLE alter_replica_table REPLICA IDENTITY USING INDEX alter_replica_table_pkey;
 SELECT create_distributed_table('alter_replica_table', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$sc4.alter_replica_table$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -722,6 +758,9 @@ INSERT INTO sc5.alter_replica_table(id) SELECT generate_series(1,100);
 ALTER TABLE sc5.alter_replica_table REPLICA IDENTITY FULL;
 SELECT create_distributed_table('sc5.alter_replica_table', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$sc5.alter_replica_table$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -747,6 +786,9 @@ CREATE UNIQUE INDEX unique_idx ON sc6.alter_replica_table(id);
 ALTER TABLE sc6.alter_replica_table REPLICA IDENTITY USING INDEX unique_idx;
 SELECT create_distributed_table('sc6.alter_replica_table', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$sc6.alter_replica_table$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -771,6 +813,9 @@ CREATE UNIQUE INDEX unique_idx ON alter_replica_table(id);
 ALTER TABLE alter_replica_table REPLICA IDENTITY USING INDEX unique_idx;
 SELECT create_distributed_table('alter_replica_table', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.alter_replica_table$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/multi_multiuser.out b/src/test/regress/expected/multi_multiuser.out
index d1cee6f75..fe624e513 100644
--- a/src/test/regress/expected/multi_multiuser.out
+++ b/src/test/regress/expected/multi_multiuser.out
@@ -378,6 +378,9 @@ INSERT INTO my_table_with_data VALUES (1,2);
 RESET ROLE;
 SELECT create_distributed_table('my_table_with_data', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.my_table_with_data$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -405,6 +408,9 @@ RESET ROLE;
 SET ROLE read_access;
 SELECT create_distributed_table('my_role_table_with_data', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.my_role_table_with_data$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -561,6 +567,9 @@ RESET ROLE;
 -- now we distribute the table as super user
 SELECT create_distributed_table('full_access_user_schema.t1', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$full_access_user_schema.t1$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/multi_mx_partitioning.out b/src/test/regress/expected/multi_mx_partitioning.out
index b985bc146..54381d50b 100644
--- a/src/test/regress/expected/multi_mx_partitioning.out
+++ b/src/test/regress/expected/multi_mx_partitioning.out
@@ -26,7 +26,13 @@ INSERT INTO partitioning_test_2010 VALUES (4, '2010-03-03');
 -- distribute partitioned table
 SELECT create_distributed_table('partitioning_test', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.partitioning_test_2009$$)
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.partitioning_test_2010$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -134,6 +140,9 @@ INSERT INTO partitioning_test_2012 VALUES (5, '2012-06-06');
 INSERT INTO partitioning_test_2012 VALUES (6, '2012-07-07');
 ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_2012 FOR VALUES FROM ('2012-01-01') TO ('2013-01-01');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.partitioning_test_2012$$)
 -- see from MX node, attached partition is distributed as well
 \c - - - :worker_1_port
 SELECT
diff --git a/src/test/regress/expected/multi_partitioning.out b/src/test/regress/expected/multi_partitioning.out
index ff130bc36..db192e41e 100644
--- a/src/test/regress/expected/multi_partitioning.out
+++ b/src/test/regress/expected/multi_partitioning.out
@@ -28,7 +28,13 @@ INSERT INTO partitioning_hash_test VALUES (4, 4);
 -- distribute partitioned table
 SELECT create_distributed_table('partitioning_test', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.partitioning_test_2009$$)
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.partitioning_test_2010$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -36,7 +42,13 @@ NOTICE:  Copying data from local table...
 
 SELECT create_distributed_table('partitioning_hash_test', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.partitioning_hash_test_0$$)
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.partitioning_hash_test_1$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -157,6 +169,9 @@ INSERT INTO partitioning_test_2012 VALUES (5, '2012-06-06');
 INSERT INTO partitioning_test_2012 VALUES (6, '2012-07-07');
 ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_2012 FOR VALUES FROM ('2012-01-01') TO ('2013-01-01');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.partitioning_test_2012$$)
 -- attached partition is distributed as well
 SELECT
 	logicalrelid
@@ -199,6 +214,9 @@ CREATE TABLE partitioning_hash_test_2 (id int, subid int);
 INSERT INTO partitioning_hash_test_2 VALUES (8, 5);
 ALTER TABLE partitioning_hash_test ATTACH PARTITION partitioning_hash_test_2 FOR VALUES WITH (MODULUS 3, REMAINDER 2);
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.partitioning_hash_test_2$$)
 INSERT INTO partitioning_hash_test VALUES (9, 12);
 -- see the data is loaded to shards
 SELECT * FROM partitioning_test ORDER BY 1;
@@ -412,7 +430,7 @@ SELECT * FROM partitioning_test WHERE id = 9 OR id = 10 ORDER BY 1;
 -- create default partition
 CREATE TABLE partitioning_test_default PARTITION OF partitioning_test DEFAULT;
 \d+ partitioning_test
-                             Table "public.partitioning_test"
+                       Table "public.partitioning_test"
  Column |  Type   | Collation | Nullable | Default | Storage | Stats target | Description
 ---------------------------------------------------------------------
  id     | integer |           |          |         | plain   |              |
@@ -692,6 +710,9 @@ CREATE TABLE partitioning_test_reference(id int PRIMARY KEY, subid int);
 INSERT INTO partitioning_test_reference SELECT a, a FROM generate_series(1, 50) a;
 SELECT create_reference_table('partitioning_test_reference');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.partitioning_test_reference$$)
  create_reference_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/multi_real_time_transaction.out b/src/test/regress/expected/multi_real_time_transaction.out
index 663d8c8d2..ae836d170 100644
--- a/src/test/regress/expected/multi_real_time_transaction.out
+++ b/src/test/regress/expected/multi_real_time_transaction.out
@@ -207,7 +207,13 @@ INSERT INTO partitioning_test VALUES (1, '2009-06-06');
 INSERT INTO partitioning_test VALUES (2, '2010-07-07');
 SELECT create_distributed_table('partitioning_test', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$multi_real_time_transaction.partitioning_test_2009$$)
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$multi_real_time_transaction.partitioning_test_2010$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/multi_reference_table.out b/src/test/regress/expected/multi_reference_table.out
index 0ac22313a..68c30c422 100644
--- a/src/test/regress/expected/multi_reference_table.out
+++ b/src/test/regress/expected/multi_reference_table.out
@@ -5,6 +5,9 @@ INSERT INTO reference_table_test VALUES (1, 1.0, '1', '2016-12-01');
 -- create the reference table
 SELECT create_reference_table('reference_table_test');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.reference_table_test$$)
  create_reference_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/multi_shard_update_delete.out b/src/test/regress/expected/multi_shard_update_delete.out
index f00fe3570..1119cde9b 100644
--- a/src/test/regress/expected/multi_shard_update_delete.out
+++ b/src/test/regress/expected/multi_shard_update_delete.out
@@ -227,6 +227,9 @@ CREATE TABLE tt1_1120 partition of tt1 for VALUES FROM (11) to (20);
 INSERT INTO tt1 VALUES (1,11), (3,15), (5,17), (6,19), (8,17), (2,12);
 SELECT create_distributed_table('tt1','id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.tt1_1120$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -341,6 +344,9 @@ INSERT INTO test_table_1 VALUES(2, '2015-02-01 08:31:16', 7);
 INSERT INTO test_table_1 VALUES(3, '2111-01-12 08:35:19', 9);
 SELECT create_distributed_table('test_table_1', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.test_table_1$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -793,6 +799,9 @@ INSERT INTO test_table_2 VALUES(2, random());
 INSERT INTO test_table_2 VALUES(3, random());
 SELECT create_distributed_table('test_table_2', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.test_table_2$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/multi_truncate.out b/src/test/regress/expected/multi_truncate.out
index 0b03a2d8b..98519b2d3 100644
--- a/src/test/regress/expected/multi_truncate.out
+++ b/src/test/regress/expected/multi_truncate.out
@@ -307,6 +307,9 @@ CREATE TABLE test_local_truncate (x int, y int);
 INSERT INTO test_local_truncate VALUES (1,2);
 SELECT create_distributed_table('test_local_truncate', 'x', colocate_with => 'none');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$multi_truncate.test_local_truncate$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -343,6 +346,9 @@ CREATE TABLE test_local_truncate (x int, y int);
 INSERT INTO test_local_truncate VALUES (1,2);
 SELECT create_distributed_table('test_local_truncate', 'x', colocate_with => 'none');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$multi_truncate.test_local_truncate$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -388,6 +394,9 @@ DETAIL:  This UDF only truncates local records of distributed tables.
 -- We can not truncate local tables until all the local foreign keys are removed.
 SELECT create_distributed_table('referenced_table', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$multi_truncate.referenced_table$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -400,6 +409,9 @@ DETAIL:  Table "referencing_table" references "referenced_table"
 -- Test foreign keys between distributed tables
 SELECT create_distributed_table('referencing_table', 'ref_id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$multi_truncate.referencing_table$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -472,6 +484,9 @@ CREATE TABLE ref(id int UNIQUE, data int);
 INSERT INTO ref SELECT x,x FROM generate_series(1,10000) x;
 SELECT create_reference_table('ref');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$multi_truncate.ref$$)
  create_reference_table
 ---------------------------------------------------------------------
 
@@ -509,6 +524,9 @@ CREATE TABLE t1(a int, b int);
 INSERT INTO t1 VALUES(1,1);
 SELECT create_distributed_table('t1', 'a');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$multi_truncate.t1$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/pg12.out b/src/test/regress/expected/pg12.out
index f353ac2a9..f7d51e098 100644
--- a/src/test/regress/expected/pg12.out
+++ b/src/test/regress/expected/pg12.out
@@ -36,6 +36,9 @@ insert into gen1 (id, val1) values (1,4),(3,6),(5,2),(7,2);
 insert into gen2 (id, val1) values (1,4),(3,6),(5,2),(7,2);
 select create_distributed_table('gen1', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$test_pg12.gen1$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -267,6 +270,9 @@ ERROR:  insert or update on table "collection_users" violates foreign key constr
 DETAIL:  Key (key, collection_id)=(1, 1000) is not present in table "collections_list".
 SELECT create_distributed_table('collections_list', 'key');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$test_pg12.collections_list_0$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -274,6 +280,9 @@ NOTICE:  Copying data from local table...
 
 SELECT create_distributed_table('collection_users', 'key');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$test_pg12.collection_users$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -292,6 +301,9 @@ CREATE TABLE test (x int, y int);
 INSERT INTO test (x,y) SELECT i,i*3 from generate_series(1, 100) i;
 SELECT create_distributed_table('test', 'x');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$test_pg12.test$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -379,6 +391,9 @@ select create_distributed_table('col_test', 'val');
 ERROR:  Hash distributed partition columns may not use a non deterministic collation
 select create_distributed_table('col_test', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$test_pg12.col_test$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/pg_dump.out b/src/test/regress/expected/pg_dump.out
index 157c95f35..ab5cf5b33 100644
--- a/src/test/regress/expected/pg_dump.out
+++ b/src/test/regress/expected/pg_dump.out
@@ -58,6 +58,9 @@ drop cascades to table "weird.table"
 -- redistribute the schema
 SELECT create_distributed_table('data', 'key');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$dumper.data$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -65,6 +68,9 @@ NOTICE:  Copying data from local table...
 
 SELECT create_distributed_table('"weird.table"', 'key,');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$dumper."weird.table"$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/relation_access_tracking.out b/src/test/regress/expected/relation_access_tracking.out
index 241716558..f6664f408 100644
--- a/src/test/regress/expected/relation_access_tracking.out
+++ b/src/test/regress/expected/relation_access_tracking.out
@@ -971,6 +971,9 @@ INSERT INTO table_3 SELECT i, i FROM generate_series(0,100) i;
 BEGIN;
 	SELECT create_distributed_table('table_3', 'key');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$access_tracking.table_3$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/replicated_partitioned_table.out b/src/test/regress/expected/replicated_partitioned_table.out
index 19e0b740e..e57fe41c3 100644
--- a/src/test/regress/expected/replicated_partitioned_table.out
+++ b/src/test/regress/expected/replicated_partitioned_table.out
@@ -27,7 +27,13 @@ INSERT INTO collections (key, ts, collection_id, value) VALUES (4, '2009-01-01',
 -- already existing partitioninong hierarcy
 SELECT create_distributed_table('collections', 'key');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$partitioned_table_replicated.collections_1$$)
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$partitioned_table_replicated.collections_2$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
@@ -41,6 +47,9 @@ CREATE TABLE collections_4 AS SELECT * FROM collections LIMIT 0;
 INSERT INTO collections_4 SELECT i, '2009-01-01', 4, i FROM generate_series (0, 10) i;
 ALTER TABLE collections ATTACH PARTITION collections_4 FOR VALUES IN ( 4 );
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$partitioned_table_replicated.collections_4$$)
 -- finally attach a distributed table to a distributed table
 CREATE TABLE collections_5 AS SELECT * FROM collections LIMIT 0;
 SELECT create_distributed_table('collections_5', 'key');
diff --git a/src/test/regress/expected/sequential_modifications.out b/src/test/regress/expected/sequential_modifications.out
index a2a4d8b43..4c58f8e3b 100644
--- a/src/test/regress/expected/sequential_modifications.out
+++ b/src/test/regress/expected/sequential_modifications.out
@@ -293,6 +293,9 @@ CREATE TABLE test_seq_truncate (a int);
 INSERT INTO test_seq_truncate SELECT i FROM generate_series(0, 100) i;
 SELECT create_distributed_table('test_seq_truncate', 'a');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$test_seq_ddl.test_seq_truncate$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/subquery_partitioning.out b/src/test/regress/expected/subquery_partitioning.out
index 5e4c09cc4..116368009 100644
--- a/src/test/regress/expected/subquery_partitioning.out
+++ b/src/test/regress/expected/subquery_partitioning.out
@@ -18,7 +18,13 @@ INSERT INTO partitioning_test_2010 VALUES (4, 4, '2010-03-03');
 SET citus.shard_replication_factor TO 1;
 SELECT create_distributed_table('partitioning_test', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$subquery_and_partitioning.partitioning_test_2010$$)
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$subquery_and_partitioning.partitioning_test_2017$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/upgrade_basic_after.out b/src/test/regress/expected/upgrade_basic_after.out
index 88e60c649..cf3eeb880 100644
--- a/src/test/regress/expected/upgrade_basic_after.out
+++ b/src/test/regress/expected/upgrade_basic_after.out
@@ -207,6 +207,9 @@ SELECT * FROM t2 ORDER BY a;
 
 SELECT create_distributed_table('t2', 'a');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$upgrade_basic.t2$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/expected/with_partitioning.out b/src/test/regress/expected/with_partitioning.out
index 0f7d40e4e..77d71294a 100644
--- a/src/test/regress/expected/with_partitioning.out
+++ b/src/test/regress/expected/with_partitioning.out
@@ -15,7 +15,13 @@ INSERT INTO partitioning_test_2010 VALUES (4, '2010-03-03');
 -- distribute partitioned table
 SELECT create_distributed_table('with_partitioning.partitioning_test', 'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$with_partitioning.partitioning_test_2010$$)
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$with_partitioning.partitioning_test_2017$$)
  create_distributed_table
 ---------------------------------------------------------------------
 
diff --git a/src/test/regress/output/hyperscale_tutorial.source b/src/test/regress/output/hyperscale_tutorial.source
index eed99efee..fc2cce1a2 100644
--- a/src/test/regress/output/hyperscale_tutorial.source
+++ b/src/test/regress/output/hyperscale_tutorial.source
@@ -205,6 +205,9 @@ CREATE TABLE impressions (
 \copy impressions from '@abs_srcdir@/data/impressions.csv' with csv
 SELECT create_distributed_table('companies',   'id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.companies$$)
  create_distributed_table 
 --------------------------
  
@@ -212,6 +215,9 @@ NOTICE:  Copying data from local table...
 
 SELECT create_distributed_table('campaigns',   'company_id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.campaigns$$)
  create_distributed_table 
 --------------------------
  
@@ -219,6 +225,9 @@ NOTICE:  Copying data from local table...
 
 SELECT create_distributed_table('ads',         'company_id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.ads$$)
  create_distributed_table 
 --------------------------
  
@@ -226,6 +235,9 @@ NOTICE:  Copying data from local table...
 
 SELECT create_distributed_table('clicks',      'company_id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.clicks$$)
  create_distributed_table 
 --------------------------
  
@@ -233,6 +245,9 @@ NOTICE:  Copying data from local table...
 
 SELECT create_distributed_table('impressions', 'company_id');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.impressions$$)
  create_distributed_table 
 --------------------------
  
diff --git a/src/test/regress/output/multi_copy.source b/src/test/regress/output/multi_copy.source
index fce99bc54..99c5a3d9f 100644
--- a/src/test/regress/output/multi_copy.source
+++ b/src/test/regress/output/multi_copy.source
@@ -988,6 +988,9 @@ CREATE UNLOGGED TABLE trigger_flush AS
 SELECT 1 AS a, s AS b, s AS c, s AS d, s AS e, s AS f, s AS g, s AS h FROM generate_series(1,150000) s;
 SELECT create_distributed_table('trigger_flush','a');
 NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.trigger_flush$$)
  create_distributed_table 
 --------------------------