From b2e01d0745c65a6cbd5506f8b493cb74d2d08a0a Mon Sep 17 00:00:00 2001
From: Onder Kalaci <onder@citusdata.com>
Date: Mon, 5 Aug 2019 18:12:59 +0200
Subject: [PATCH 1/3] Refactor switching to sequential mode

We don't need to wait until the execution. As soon as we realize
that we need sequential execution, we should do it.
---
 src/backend/distributed/commands/table.c                  | 6 +++++-
 src/backend/distributed/commands/utility_hook.c           | 8 +++-----
 src/backend/distributed/commands/vacuum.c                 | 2 +-
 src/backend/distributed/executor/adaptive_executor.c      | 6 ++----
 src/backend/distributed/master/master_truncate.c          | 3 +--
 src/include/distributed/commands/utility_hook.h           | 1 -
 src/include/distributed/multi_executor.h                  | 3 +--
 .../expected/foreign_key_restriction_enforcement.out      | 2 --
 .../expected/foreign_key_restriction_enforcement_0.out    | 2 --
 9 files changed, 13 insertions(+), 20 deletions(-)

diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c
index 73f10cea7..5e4ad1fbc 100644
--- a/src/backend/distributed/commands/table.c
+++ b/src/backend/distributed/commands/table.c
@@ -418,11 +418,15 @@ PlanAlterTableStmt(AlterTableStmt *alterTableStatement, const char *alterTableCo
 															   command);
 	}
 
+	if (executeSequentially)
+	{
+		SetLocalMultiShardModifyModeToSequential();
+	}
+
 	ddlJob = palloc0(sizeof(DDLJob));
 	ddlJob->targetRelationId = leftRelationId;
 	ddlJob->concurrentIndexCmd = false;
 	ddlJob->commandString = alterTableCommand;
-	ddlJob->executeSequentially = executeSequentially;
 
 	if (rightRelationId)
 	{
diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c
index 5806dc647..54f6de839 100644
--- a/src/backend/distributed/commands/utility_hook.c
+++ b/src/backend/distributed/commands/utility_hook.c
@@ -553,7 +553,7 @@ static void
 ExecuteDistributedDDLJob(DDLJob *ddlJob)
 {
 	bool shouldSyncMetadata = ShouldSyncTableMetadata(ddlJob->targetRelationId);
-	int targetPoolSize = ddlJob->executeSequentially ? 1 : MaxAdaptiveExecutorPoolSize;
+	int targetPoolSize = MaxAdaptiveExecutorPoolSize;
 
 	EnsureCoordinator();
 	EnsurePartitionTableNotReplicated(ddlJob->targetRelationId);
@@ -579,8 +579,7 @@ ExecuteDistributedDDLJob(DDLJob *ddlJob)
 		}
 
 		/* use adaptive executor when enabled */
-		ExecuteUtilityTaskListWithoutResults(ddlJob->taskList, targetPoolSize,
-											 ddlJob->executeSequentially);
+		ExecuteUtilityTaskListWithoutResults(ddlJob->taskList, targetPoolSize);
 	}
 	else
 	{
@@ -592,8 +591,7 @@ ExecuteDistributedDDLJob(DDLJob *ddlJob)
 		PG_TRY();
 		{
 			/* use adaptive executor when enabled */
-			ExecuteUtilityTaskListWithoutResults(ddlJob->taskList, targetPoolSize,
-												 ddlJob->executeSequentially);
+			ExecuteUtilityTaskListWithoutResults(ddlJob->taskList, targetPoolSize);
 
 			if (shouldSyncMetadata)
 			{
diff --git a/src/backend/distributed/commands/vacuum.c b/src/backend/distributed/commands/vacuum.c
index 8f69f769e..ee9926cbf 100644
--- a/src/backend/distributed/commands/vacuum.c
+++ b/src/backend/distributed/commands/vacuum.c
@@ -94,7 +94,7 @@ ProcessVacuumStmt(VacuumStmt *vacuumStmt, const char *vacuumCommand)
 			taskList = VacuumTaskList(relationId, vacuumStmt->options, vacuumColumnList);
 
 			/* use adaptive executor when enabled */
-			ExecuteUtilityTaskListWithoutResults(taskList, targetPoolSize, false);
+			ExecuteUtilityTaskListWithoutResults(taskList, targetPoolSize);
 			executedVacuumCount++;
 		}
 		relationIndex++;
diff --git a/src/backend/distributed/executor/adaptive_executor.c b/src/backend/distributed/executor/adaptive_executor.c
index 64310c836..ca42fe0f5 100644
--- a/src/backend/distributed/executor/adaptive_executor.c
+++ b/src/backend/distributed/executor/adaptive_executor.c
@@ -657,8 +657,7 @@ AdaptiveExecutor(CustomScanState *node)
  * through router executor.
  */
 void
-ExecuteUtilityTaskListWithoutResults(List *taskList, int targetPoolSize,
-									 bool forceSequentialExecution)
+ExecuteUtilityTaskListWithoutResults(List *taskList, int targetPoolSize)
 {
 	if (TaskExecutorType == MULTI_EXECUTOR_ADAPTIVE)
 	{
@@ -666,8 +665,7 @@ ExecuteUtilityTaskListWithoutResults(List *taskList, int targetPoolSize,
 	}
 	else
 	{
-		if (MultiShardConnectionType == SEQUENTIAL_CONNECTION ||
-			forceSequentialExecution)
+		if (MultiShardConnectionType == SEQUENTIAL_CONNECTION)
 		{
 			ExecuteModifyTasksSequentiallyWithoutResults(taskList, ROW_MODIFY_NONE);
 		}
diff --git a/src/backend/distributed/master/master_truncate.c b/src/backend/distributed/master/master_truncate.c
index dff775e0a..ed91ad747 100644
--- a/src/backend/distributed/master/master_truncate.c
+++ b/src/backend/distributed/master/master_truncate.c
@@ -77,8 +77,7 @@ citus_truncate_trigger(PG_FUNCTION_ARGS)
 	{
 		List *taskList = TruncateTaskList(relationId);
 
-		ExecuteUtilityTaskListWithoutResults(taskList, MaxAdaptiveExecutorPoolSize,
-											 false);
+		ExecuteUtilityTaskListWithoutResults(taskList, MaxAdaptiveExecutorPoolSize);
 	}
 
 	PG_RETURN_DATUM(PointerGetDatum(NULL));
diff --git a/src/include/distributed/commands/utility_hook.h b/src/include/distributed/commands/utility_hook.h
index 64792f3a1..113e82eb0 100644
--- a/src/include/distributed/commands/utility_hook.h
+++ b/src/include/distributed/commands/utility_hook.h
@@ -37,7 +37,6 @@ typedef struct DDLJob
 {
 	Oid targetRelationId;      /* oid of the target distributed relation */
 	bool concurrentIndexCmd;   /* related to a CONCURRENTLY index command? */
-	bool executeSequentially;
 	const char *commandString; /* initial (coordinator) DDL command string */
 	List *taskList;            /* worker DDL tasks to execute */
 } DDLJob;
diff --git a/src/include/distributed/multi_executor.h b/src/include/distributed/multi_executor.h
index ed8989bd5..a1d54e160 100644
--- a/src/include/distributed/multi_executor.h
+++ b/src/include/distributed/multi_executor.h
@@ -42,8 +42,7 @@ extern uint64 ExecuteTaskListExtended(RowModifyLevel modLevel, List *taskList,
 									  TupleDesc tupleDescriptor,
 									  Tuplestorestate *tupleStore,
 									  bool hasReturning, int targetPoolSize);
-extern void ExecuteUtilityTaskListWithoutResults(List *taskList, int targetPoolSize,
-												 bool forceSequentialExecution);
+extern void ExecuteUtilityTaskListWithoutResults(List *taskList, int targetPoolSize);
 extern uint64 ExecuteTaskList(RowModifyLevel modLevel, List *taskList, int
 							  targetPoolSize);
 extern TupleTableSlot * CitusExecScan(CustomScanState *node);
diff --git a/src/test/regress/expected/foreign_key_restriction_enforcement.out b/src/test/regress/expected/foreign_key_restriction_enforcement.out
index b7df1f54f..27f9f5bd3 100644
--- a/src/test/regress/expected/foreign_key_restriction_enforcement.out
+++ b/src/test/regress/expected/foreign_key_restriction_enforcement.out
@@ -1064,8 +1064,6 @@ ADD CONSTRAINT
 	fkey_delete FOREIGN KEY(value_1) 
 REFERENCES 
 	reference_table(id) ON DELETE CASCADE;
-DEBUG:  switching to sequential query execution mode
-DETAIL:  Reference relation "reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode
 INSERT INTO reference_table SELECT i FROM generate_series(0, 10) i;
 DEBUG:  distributed INSERT ... SELECT can only select from distributed tables
 DEBUG:  Collecting INSERT ... SELECT results on coordinator
diff --git a/src/test/regress/expected/foreign_key_restriction_enforcement_0.out b/src/test/regress/expected/foreign_key_restriction_enforcement_0.out
index 695961aae..e96db795e 100644
--- a/src/test/regress/expected/foreign_key_restriction_enforcement_0.out
+++ b/src/test/regress/expected/foreign_key_restriction_enforcement_0.out
@@ -1064,8 +1064,6 @@ ADD CONSTRAINT
 	fkey_delete FOREIGN KEY(value_1) 
 REFERENCES 
 	reference_table(id) ON DELETE CASCADE;
-DEBUG:  switching to sequential query execution mode
-DETAIL:  Reference relation "reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode
 INSERT INTO reference_table SELECT i FROM generate_series(0, 10) i;
 DEBUG:  distributed INSERT ... SELECT can only select from distributed tables
 DEBUG:  Collecting INSERT ... SELECT results on coordinator

From 35ee896f3d4e675a0956dcdf7364bd3ed967c3b0 Mon Sep 17 00:00:00 2001
From: Onder Kalaci <onder@citusdata.com>
Date: Tue, 6 Aug 2019 12:23:03 +0200
Subject: [PATCH 2/3] Get rid of an unnecessary parameter

targetPoolSize parameter for ExecuteUtilityTaskListWithoutResults
becomes obsolete, just remove it.
---
 src/backend/distributed/commands/utility_hook.c      | 5 ++---
 src/backend/distributed/commands/vacuum.c            | 3 +--
 src/backend/distributed/executor/adaptive_executor.c | 4 ++--
 src/backend/distributed/master/master_truncate.c     | 2 +-
 src/include/distributed/multi_executor.h             | 2 +-
 5 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c
index 54f6de839..7d684aad1 100644
--- a/src/backend/distributed/commands/utility_hook.c
+++ b/src/backend/distributed/commands/utility_hook.c
@@ -553,7 +553,6 @@ static void
 ExecuteDistributedDDLJob(DDLJob *ddlJob)
 {
 	bool shouldSyncMetadata = ShouldSyncTableMetadata(ddlJob->targetRelationId);
-	int targetPoolSize = MaxAdaptiveExecutorPoolSize;
 
 	EnsureCoordinator();
 	EnsurePartitionTableNotReplicated(ddlJob->targetRelationId);
@@ -579,7 +578,7 @@ ExecuteDistributedDDLJob(DDLJob *ddlJob)
 		}
 
 		/* use adaptive executor when enabled */
-		ExecuteUtilityTaskListWithoutResults(ddlJob->taskList, targetPoolSize);
+		ExecuteUtilityTaskListWithoutResults(ddlJob->taskList);
 	}
 	else
 	{
@@ -591,7 +590,7 @@ ExecuteDistributedDDLJob(DDLJob *ddlJob)
 		PG_TRY();
 		{
 			/* use adaptive executor when enabled */
-			ExecuteUtilityTaskListWithoutResults(ddlJob->taskList, targetPoolSize);
+			ExecuteUtilityTaskListWithoutResults(ddlJob->taskList);
 
 			if (shouldSyncMetadata)
 			{
diff --git a/src/backend/distributed/commands/vacuum.c b/src/backend/distributed/commands/vacuum.c
index ee9926cbf..3eb20f58a 100644
--- a/src/backend/distributed/commands/vacuum.c
+++ b/src/backend/distributed/commands/vacuum.c
@@ -74,7 +74,6 @@ ProcessVacuumStmt(VacuumStmt *vacuumStmt, const char *vacuumCommand)
 		{
 			List *vacuumColumnList = NIL;
 			List *taskList = NIL;
-			int targetPoolSize = MaxAdaptiveExecutorPoolSize;
 
 			/*
 			 * VACUUM commands cannot run inside a transaction block, so we use
@@ -94,7 +93,7 @@ ProcessVacuumStmt(VacuumStmt *vacuumStmt, const char *vacuumCommand)
 			taskList = VacuumTaskList(relationId, vacuumStmt->options, vacuumColumnList);
 
 			/* use adaptive executor when enabled */
-			ExecuteUtilityTaskListWithoutResults(taskList, targetPoolSize);
+			ExecuteUtilityTaskListWithoutResults(taskList);
 			executedVacuumCount++;
 		}
 		relationIndex++;
diff --git a/src/backend/distributed/executor/adaptive_executor.c b/src/backend/distributed/executor/adaptive_executor.c
index ca42fe0f5..80d574341 100644
--- a/src/backend/distributed/executor/adaptive_executor.c
+++ b/src/backend/distributed/executor/adaptive_executor.c
@@ -657,11 +657,11 @@ AdaptiveExecutor(CustomScanState *node)
  * through router executor.
  */
 void
-ExecuteUtilityTaskListWithoutResults(List *taskList, int targetPoolSize)
+ExecuteUtilityTaskListWithoutResults(List *taskList)
 {
 	if (TaskExecutorType == MULTI_EXECUTOR_ADAPTIVE)
 	{
-		ExecuteTaskList(ROW_MODIFY_NONE, taskList, targetPoolSize);
+		ExecuteTaskList(ROW_MODIFY_NONE, taskList, MaxAdaptiveExecutorPoolSize);
 	}
 	else
 	{
diff --git a/src/backend/distributed/master/master_truncate.c b/src/backend/distributed/master/master_truncate.c
index ed91ad747..c6ffcf2fb 100644
--- a/src/backend/distributed/master/master_truncate.c
+++ b/src/backend/distributed/master/master_truncate.c
@@ -77,7 +77,7 @@ citus_truncate_trigger(PG_FUNCTION_ARGS)
 	{
 		List *taskList = TruncateTaskList(relationId);
 
-		ExecuteUtilityTaskListWithoutResults(taskList, MaxAdaptiveExecutorPoolSize);
+		ExecuteUtilityTaskListWithoutResults(taskList);
 	}
 
 	PG_RETURN_DATUM(PointerGetDatum(NULL));
diff --git a/src/include/distributed/multi_executor.h b/src/include/distributed/multi_executor.h
index a1d54e160..e530701ac 100644
--- a/src/include/distributed/multi_executor.h
+++ b/src/include/distributed/multi_executor.h
@@ -42,7 +42,7 @@ extern uint64 ExecuteTaskListExtended(RowModifyLevel modLevel, List *taskList,
 									  TupleDesc tupleDescriptor,
 									  Tuplestorestate *tupleStore,
 									  bool hasReturning, int targetPoolSize);
-extern void ExecuteUtilityTaskListWithoutResults(List *taskList, int targetPoolSize);
+extern void ExecuteUtilityTaskListWithoutResults(List *taskList);
 extern uint64 ExecuteTaskList(RowModifyLevel modLevel, List *taskList, int
 							  targetPoolSize);
 extern TupleTableSlot * CitusExecScan(CustomScanState *node);

From 060ac1147611c1dcccd99391b85f2f4f2a62422b Mon Sep 17 00:00:00 2001
From: Onder Kalaci <onder@citusdata.com>
Date: Mon, 5 Aug 2019 10:08:44 +0200
Subject: [PATCH 3/3] Do not record relation accessess unnecessarily

Before this commit, we've recorded the relation accesses in 3 different
places
    - FindPlacementListConnection         -- applies all executor in tx block
    - StartPlacementExecutionOnSession()  -- adaptive executor only
    - StartPlacementListConnection()      -- router/real-time only

This is different than Citus 8.2, and could lead to query execution times
increase considerably on multi-shard commands in transaction block
that are on partitioned tables.

Benchmarks:

```
1+8 c5.4xlarge cluster

Empty distributed partitioned table with 365 partitions: https://gist.github.com/onderkalaci/1edace4ed6bd6f061c8a15594865bb51#file-partitions_365-sql

./pgbench -f /tmp/multi_shard.sql -c10 -j10 -P 1 -T 120 postgres://citus:w3r6KLJpv3mxe9E-NIUeJw@c.fy5fkjcv45vcepaogqcaskmmkee.db.citusdata.com:5432/citus?sslmode=require

cat  /tmp/multi_shard.sql
BEGIN;
	DELETE FROM collections_list;
	DELETE FROM collections_list;
	DELETE FROM collections_list;
COMMIT;
cat  /tmp/single_shard.sql
BEGIN;
	DELETE FROM collections_list WHERE key = :aid;
	DELETE FROM collections_list WHERE key = :aid;
	DELETE FROM collections_list WHERE key = :aid;
COMMIT;

cat  /tmp/mix.sql
BEGIN;
	DELETE FROM collections_list WHERE key = :aid;
	DELETE FROM collections_list WHERE key = :aid;
	DELETE FROM collections_list WHERE key = :aid;

	DELETE FROM collections_list;
	DELETE FROM collections_list;
	DELETE FROM collections_list;
COMMIT;
```

The table shows `latency average` of pgbench runs explained above, so we have a pretty solid improvement even over 8.2.2.

| Test  | Citus 8.2.2  |  Citus 8.3.1   | Citus 8.3.2 (this branch)  | Citus 8.3.1 (FKEYs disabled via GUC)  |
| ------------- | ------------- | ------------- |------------- | ------------- |
|multi_shard |  2370.083 ms  |3605.040 ms |1324.094 ms |1247.255 ms  |
| single_shard  | 85.338 ms  |120.934 ms  |73.216 ms  | 78.765 ms |
| mix  | 2434.459 ms | 3727.080 ms  |1306.456 ms  | 1280.326 ms |
---
 src/backend/distributed/commands/multi_copy.c |   2 +
 .../connection/placement_connection.c         |  10 +-
 .../transaction/relation_access_tracking.c    |  39 +-
 .../distributed/relation_access_tracking.h    |  11 +-
 .../expected/relation_access_tracking.out     | 517 +++++++++---------
 .../regress/sql/relation_access_tracking.sql  |  29 +-
 6 files changed, 329 insertions(+), 279 deletions(-)

diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c
index 9bf20556e..5b5464680 100644
--- a/src/backend/distributed/commands/multi_copy.c
+++ b/src/backend/distributed/commands/multi_copy.c
@@ -2346,6 +2346,8 @@ CitusCopyDestReceiverStartup(DestReceiver *dest, int operation,
 
 	copyDest->shardStateHash = CreateShardStateHash(TopTransactionContext);
 	copyDest->connectionStateHash = CreateConnectionStateHash(TopTransactionContext);
+
+	RecordRelationAccessIfReferenceTable(tableId, PLACEMENT_ACCESS_DML);
 }
 
 
diff --git a/src/backend/distributed/connection/placement_connection.c b/src/backend/distributed/connection/placement_connection.c
index a9292036d..8e946403b 100644
--- a/src/backend/distributed/connection/placement_connection.c
+++ b/src/backend/distributed/connection/placement_connection.c
@@ -366,6 +366,8 @@ AssignPlacementListToConnection(List *placementAccessList, MultiConnection *conn
 		ConnectionPlacementHashEntry *placementEntry = NULL;
 		ConnectionReference *placementConnection = NULL;
 
+		Oid relationId = InvalidOid;
+
 		if (placement->shardId == INVALID_SHARD_ID)
 		{
 			/*
@@ -452,8 +454,9 @@ AssignPlacementListToConnection(List *placementAccessList, MultiConnection *conn
 			placementConnection->hadDML = true;
 		}
 
-		/* record the relation access mapping */
-		AssociatePlacementAccessWithRelation(placement, accessType);
+		/* record the relation access */
+		relationId = RelationIdForShard(placement->shardId);
+		RecordRelationAccessIfReferenceTable(relationId, accessType);
 	}
 }
 
@@ -700,9 +703,6 @@ FindPlacementListConnection(int flags, List *placementAccessList, const char *us
 			Assert(!placementConnection->hadDML);
 			Assert(accessType != PLACEMENT_ACCESS_DDL);
 		}
-
-		/* record the relation access mapping */
-		AssociatePlacementAccessWithRelation(placement, accessType);
 	}
 
 	return chosenConnection;
diff --git a/src/backend/distributed/transaction/relation_access_tracking.c b/src/backend/distributed/transaction/relation_access_tracking.c
index f790a4355..88fb500ce 100644
--- a/src/backend/distributed/transaction/relation_access_tracking.c
+++ b/src/backend/distributed/transaction/relation_access_tracking.c
@@ -82,7 +82,7 @@ static HTAB *RelationAccessHash;
 
 
 /* functions related to access recording */
-static void RecordRelationAccess(Oid relationId, ShardPlacementAccessType accessType);
+static void RecordRelationAccessBase(Oid relationId, ShardPlacementAccessType accessType);
 static void RecordPlacementAccessToCache(Oid relationId,
 										 ShardPlacementAccessType accessType);
 static void RecordRelationParallelSelectAccessForTask(Task *task);
@@ -148,25 +148,32 @@ AllocateRelationAccessHash(void)
 
 
 /*
- * AssociatePlacementAccessWithRelation associates the placement access to the
- * distributed relation that the placement belongs to.
+ * RecordRelationAccessIfReferenceTable marks the relation accessed if it is a
+ * reference relation.
+ *
+ * The function is a wrapper around RecordRelationAccessBase().
  */
 void
-AssociatePlacementAccessWithRelation(ShardPlacement *placement,
-									 ShardPlacementAccessType accessType)
+RecordRelationAccessIfReferenceTable(Oid relationId, ShardPlacementAccessType accessType)
 {
-	Oid relationId = InvalidOid;
-	uint64 shardId = INVALID_SHARD_ID;
-
 	if (!ShouldRecordRelationAccess())
 	{
 		return;
 	}
 
-	shardId = placement->shardId;
-	relationId = RelationIdForShard(shardId);
+	/*
+	 * We keep track of relation accesses for the purposes of foreign keys to
+	 * reference tables. So, other distributed tables are not relevant for now.
+	 * Additionally, partitioned tables with lots of partitions might require
+	 * recursively calling RecordRelationAccessBase(), so becareful about
+	 * removing this check.
+	 */
+	if (PartitionMethod(relationId) != DISTRIBUTE_BY_NONE)
+	{
+		return;
+	}
 
-	RecordRelationAccess(relationId, accessType);
+	RecordRelationAccessBase(relationId, accessType);
 }
 
 
@@ -205,15 +212,15 @@ PlacementAccessTypeToText(ShardPlacementAccessType accessType)
 
 
 /*
- * RecordRelationAccess associates the access to the distributed relation. The
+ * RecordRelationAccessBase associates the access to the distributed relation. The
  * function takes partitioned relations into account as well.
  *
  * We implemented this function to prevent accessing placement metadata during
  * recursive calls of the function itself (e.g., avoid
- * AssociatePlacementAccessWithRelation()).
+ * RecordRelationAccessBase()).
  */
 static void
-RecordRelationAccess(Oid relationId, ShardPlacementAccessType accessType)
+RecordRelationAccessBase(Oid relationId, ShardPlacementAccessType accessType)
 {
 	/* make sure that this is not a conflicting access */
 	CheckConflictingRelationAccesses(relationId, accessType);
@@ -244,7 +251,7 @@ RecordRelationAccess(Oid relationId, ShardPlacementAccessType accessType)
 			}
 
 			/* recursively call the function to cover multi-level partitioned tables */
-			RecordRelationAccess(partitionOid, accessType);
+			RecordRelationAccessBase(partitionOid, accessType);
 		}
 	}
 	else if (PartitionTableNoLock(relationId))
@@ -680,7 +687,7 @@ GetRelationAccessMode(Oid relationId, ShardPlacementAccessType accessType)
 	}
 	else
 	{
-		return RELATION_SEQUENTIAL_ACCESSED;
+		return RELATION_REFERENCE_ACCESSED;
 	}
 }
 
diff --git a/src/include/distributed/relation_access_tracking.h b/src/include/distributed/relation_access_tracking.h
index 2327c27f5..49427d15c 100644
--- a/src/include/distributed/relation_access_tracking.h
+++ b/src/include/distributed/relation_access_tracking.h
@@ -23,13 +23,20 @@ struct ShardPlacement;
 typedef enum RelationAccessMode
 {
 	RELATION_NOT_ACCESSED,
-	RELATION_SEQUENTIAL_ACCESSED,
+
+	/* only valid for reference tables */
+	RELATION_REFERENCE_ACCESSED,
+
+	/*
+	 * Only valid for distributed tables and set
+	 * if table is accessed in parallel mode
+	 */
 	RELATION_PARALLEL_ACCESSED
 } RelationAccessMode;
 
 extern void AllocateRelationAccessHash(void);
 extern void ResetRelationAccessHash(void);
-extern void AssociatePlacementAccessWithRelation(ShardPlacement *placement,
+extern void RecordRelationAccessIfReferenceTable(Oid relationId,
 												 ShardPlacementAccessType accessType);
 extern void RecordParallelRelationAccessForTaskList(List *taskList);
 extern void RecordParallelSelectAccess(Oid relationId);
diff --git a/src/test/regress/expected/relation_access_tracking.out b/src/test/regress/expected/relation_access_tracking.out
index 76c94b192..269bac822 100644
--- a/src/test/regress/expected/relation_access_tracking.out
+++ b/src/test/regress/expected/relation_access_tracking.out
@@ -22,14 +22,30 @@ CREATE OR REPLACE FUNCTION relation_ddl_access_mode(relationId Oid)
     RETURNS int
     LANGUAGE C STABLE STRICT
     AS 'citus', $$relation_ddl_access_mode$$;
-CREATE OR REPLACE FUNCTION relation_access_mode_to_text(relationShardAccess int)
+CREATE OR REPLACE FUNCTION distributed_relation(relation_name text)
+RETURNS bool AS
+$$
+DECLARE
+	part_method char;
+BEGIN
+		 select partmethod INTO part_method from pg_dist_partition  WHERE logicalrelid = relation_name::regclass;
+         IF part_method = 'h' THEN
+                RETURN true;
+         ELSE
+                RETURN false;
+         END IF;
+END;
+$$ LANGUAGE 'plpgsql' IMMUTABLE;
+CREATE OR REPLACE FUNCTION relation_access_mode_to_text(relation_name text, relationShardAccess int)
 RETURNS text AS
 $$
 BEGIN
-	 IF relationShardAccess = 0 THEN
+	 IF relationShardAccess = 0 and distributed_relation(relation_name) THEN
+	 	RETURN 'not_parallel_accessed';
+	 ELSIF relationShardAccess = 0 and NOT distributed_relation(relation_name) THEN
 	 	RETURN 'not_accessed';
 	 ELSIF relationShardAccess = 1 THEN
-	 	RETURN 'sequential_access';
+	 	RETURN 'reference_table_access';
 	 ELSE 
 		RETURN 'parallel_access';
 	 END IF;
@@ -37,9 +53,9 @@ END;
 $$ LANGUAGE 'plpgsql' IMMUTABLE;
 CREATE VIEW relation_acesses AS 
 	SELECT table_name, 
-			relation_access_mode_to_text(relation_select_access_mode(table_name::regclass)) as select_access,
-			relation_access_mode_to_text(relation_dml_access_mode(table_name::regclass)) as dml_access,
-			relation_access_mode_to_text(relation_ddl_access_mode(table_name::regclass)) as ddl_access
+			relation_access_mode_to_text(table_name, relation_select_access_mode(table_name::regclass)) as select_access,
+			relation_access_mode_to_text(table_name, relation_dml_access_mode(table_name::regclass)) as dml_access,
+			relation_access_mode_to_text(table_name, relation_ddl_access_mode(table_name::regclass)) as ddl_access
 	FROM 
 		((SELECT 'table_' || i as table_name FROM generate_series(1, 7) i) UNION (SELECT 'partitioning_test') UNION (SELECT 'partitioning_test_2009') UNION (SELECT 'partitioning_test_2010')) tables;
 SET citus.shard_replication_factor TO 1;
@@ -101,9 +117,9 @@ BEGIN;
 (1 row)
 
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_7')  ORDER BY 1;
- table_name | select_access |  dml_access  |   ddl_access    
-------------+---------------+--------------+-----------------
- table_7    | not_accessed  | not_accessed | parallel_access
+ table_name |     select_access     |      dml_access       |   ddl_access    
+------------+-----------------------+-----------------------+-----------------
+ table_7    | not_parallel_accessed | not_parallel_accessed | parallel_access
 (1 row)
 
 COMMIT;
@@ -115,18 +131,18 @@ SELECT count(*) FROM table_1;
 (1 row)
 
 SELECT * FROM relation_acesses WHERE table_name = 'table_1';
- table_name | select_access |  dml_access  |  ddl_access  
-------------+---------------+--------------+--------------
- table_1    | not_accessed  | not_accessed | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 -- a very simple test that first checks sequential 
 -- and parallel SELECTs,DMLs, and DDLs
 BEGIN;
 	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
- table_name | select_access |  dml_access  |  ddl_access  
-------------+---------------+--------------+--------------
- table_1    | not_accessed  | not_accessed | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 	SELECT count(*) FROM table_1 WHERE key = 1;
@@ -136,9 +152,9 @@ BEGIN;
 (1 row)
 
 	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
- table_name |   select_access   |  dml_access  |  ddl_access  
-------------+-------------------+--------------+--------------
- table_1    | sequential_access | not_accessed | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 	SELECT count(*) FROM table_1 WHERE key = 1 OR key = 2;
@@ -148,31 +164,31 @@ BEGIN;
 (1 row)
 
 	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
- table_name |  select_access  |  dml_access  |  ddl_access  
-------------+-----------------+--------------+--------------
- table_1    | parallel_access | not_accessed | not_accessed
+ table_name |  select_access  |      dml_access       |      ddl_access       
+------------+-----------------+-----------------------+-----------------------
+ table_1    | parallel_access | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 	INSERT INTO table_1 VALUES (1,1);
 	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
- table_name |  select_access  |    dml_access     |  ddl_access  
-------------+-----------------+-------------------+--------------
- table_1    | parallel_access | sequential_access | not_accessed
+ table_name |  select_access  |      dml_access       |      ddl_access       
+------------+-----------------+-----------------------+-----------------------
+ table_1    | parallel_access | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 	INSERT INTO table_1 VALUES (1,1), (2,2);
 	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
- table_name |  select_access  |    dml_access     |  ddl_access  
-------------+-----------------+-------------------+--------------
- table_1    | parallel_access | sequential_access | not_accessed
+ table_name |  select_access  |      dml_access       |      ddl_access       
+------------+-----------------+-----------------------+-----------------------
+ table_1    | parallel_access | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 	ALTER TABLE table_1 ADD COLUMN test_col INT;
 	-- now see that the other tables are not accessed at all
 	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
- table_name |  select_access  |    dml_access     |   ddl_access    
-------------+-----------------+-------------------+-----------------
- table_1    | parallel_access | sequential_access | parallel_access
+ table_name |  select_access  |      dml_access       |   ddl_access    
+------------+-----------------+-----------------------+-----------------
+ table_1    | parallel_access | not_parallel_accessed | parallel_access
 (1 row)
 
 ROLLBACK;
@@ -186,9 +202,9 @@ BEGIN;
 (1 row)
 
 	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
- table_name |   select_access   |  dml_access  |  ddl_access  
-------------+-------------------+--------------+--------------
- table_1    | sequential_access | not_accessed | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 	SELECT count(*) FROM table_1 WHERE key = 2;
@@ -198,23 +214,23 @@ BEGIN;
 (1 row)
 
 	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
- table_name |   select_access   |  dml_access  |  ddl_access  
-------------+-------------------+--------------+--------------
- table_1    | sequential_access | not_accessed | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 	INSERT INTO table_1 VALUES (1,1);
 	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
- table_name |   select_access   |    dml_access     |  ddl_access  
-------------+-------------------+-------------------+--------------
- table_1    | sequential_access | sequential_access | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 	INSERT INTO table_1 VALUES (2,2);
 	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
- table_name |   select_access   |    dml_access     |  ddl_access  
-------------+-------------------+-------------------+--------------
- table_1    | sequential_access | sequential_access | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 ROLLBACK;
@@ -222,9 +238,9 @@ ROLLBACK;
 BEGIN;
 	ALTER TABLE table_1 ADD CONSTRAINT table_1_u UNIQUE (key);
 	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
- table_name | select_access |  dml_access  |   ddl_access    
-------------+---------------+--------------+-----------------
- table_1    | not_accessed  | not_accessed | parallel_access
+ table_name |     select_access     |      dml_access       |   ddl_access    
+------------+-----------------------+-----------------------+-----------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | parallel_access
 (1 row)
 
 ROLLBACK;
@@ -244,15 +260,15 @@ BEGIN;
 (1 row)
 
 		SELECT * FROM relation_acesses WHERE table_name LIKE 'table_%' ORDER BY 1;
- table_name |   select_access   |  dml_access  |  ddl_access  
-------------+-------------------+--------------+--------------
- table_1    | sequential_access | not_accessed | not_accessed
- table_2    | sequential_access | not_accessed | not_accessed
- table_3    | sequential_access | not_accessed | not_accessed
- table_4    | sequential_access | not_accessed | not_accessed
- table_5    | sequential_access | not_accessed | not_accessed
- table_6    | not_accessed      | not_accessed | not_accessed
- table_7    | not_accessed      | not_accessed | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ table_2    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ table_3    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ table_4    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ table_5    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ table_6    | not_accessed          | not_accessed          | not_accessed
+ table_7    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (7 rows)
 
 ROLLBACK;
@@ -270,10 +286,10 @@ BEGIN;
 (1 row)
 
 		SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
- table_name |  select_access  |  dml_access  |  ddl_access  
-------------+-----------------+--------------+--------------
- table_1    | parallel_access | not_accessed | not_accessed
- table_2    | parallel_access | not_accessed | not_accessed
+ table_name |  select_access  |      dml_access       |      ddl_access       
+------------+-----------------+-----------------------+-----------------------
+ table_1    | parallel_access | not_parallel_accessed | not_parallel_accessed
+ table_2    | parallel_access | not_parallel_accessed | not_parallel_accessed
 (2 rows)
 
 ROLLBACK;
@@ -293,10 +309,10 @@ BEGIN;
 (1 row)
 
 		SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
- table_name |   select_access   |  dml_access  |  ddl_access  
-------------+-------------------+--------------+--------------
- table_1    | sequential_access | not_accessed | not_accessed
- table_2    | sequential_access | not_accessed | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ table_2    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (2 rows)
 
 ROLLBACK;
@@ -320,15 +336,15 @@ BEGIN;
 (1 row)
 
 	SELECT * FROM relation_acesses WHERE table_name LIKE 'table_%' ORDER BY 1;
- table_name |  select_access  |  dml_access  |  ddl_access  
-------------+-----------------+--------------+--------------
- table_1    | parallel_access | not_accessed | not_accessed
- table_2    | parallel_access | not_accessed | not_accessed
- table_3    | parallel_access | not_accessed | not_accessed
- table_4    | parallel_access | not_accessed | not_accessed
- table_5    | parallel_access | not_accessed | not_accessed
- table_6    | not_accessed    | not_accessed | not_accessed
- table_7    | not_accessed    | not_accessed | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | parallel_access       | not_parallel_accessed | not_parallel_accessed
+ table_2    | parallel_access       | not_parallel_accessed | not_parallel_accessed
+ table_3    | parallel_access       | not_parallel_accessed | not_parallel_accessed
+ table_4    | parallel_access       | not_parallel_accessed | not_parallel_accessed
+ table_5    | parallel_access       | not_parallel_accessed | not_parallel_accessed
+ table_6    | not_accessed          | not_accessed          | not_accessed
+ table_7    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (7 rows)
 
 ROLLBACK;
@@ -338,18 +354,18 @@ ROLLBACK;
 BEGIN;
 	UPDATE table_1 SET value = 15;
 	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
- table_name |  select_access  |   dml_access    |  ddl_access  
-------------+-----------------+-----------------+--------------
- table_1    | parallel_access | parallel_access | not_accessed
+ table_name |  select_access  |   dml_access    |      ddl_access       
+------------+-----------------+-----------------+-----------------------
+ table_1    | parallel_access | parallel_access | not_parallel_accessed
 (1 row)
 
 	SET LOCAL citus.multi_shard_modify_mode = 'sequential';
 	UPDATE table_2 SET value = 15;
 	SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
- table_name |   select_access   |    dml_access     |  ddl_access  
-------------+-------------------+-------------------+--------------
- table_1    | parallel_access   | parallel_access   | not_accessed
- table_2    | sequential_access | sequential_access | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | parallel_access       | parallel_access       | not_parallel_accessed
+ table_2    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (2 rows)
 
 ROLLBACK;
@@ -359,11 +375,11 @@ BEGIN;
 		table_1 SET value = 15 
 	WHERE key IN (SELECT key FROM table_2 JOIN table_3 USING (key) WHERE table_2.value = 15);
 	SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2', 'table_3')  ORDER BY 1;
- table_name |  select_access  |   dml_access    |  ddl_access  
-------------+-----------------+-----------------+--------------
- table_1    | parallel_access | parallel_access | not_accessed
- table_2    | parallel_access | not_accessed    | not_accessed
- table_3    | parallel_access | not_accessed    | not_accessed
+ table_name |  select_access  |      dml_access       |      ddl_access       
+------------+-----------------+-----------------------+-----------------------
+ table_1    | parallel_access | parallel_access       | not_parallel_accessed
+ table_2    | parallel_access | not_parallel_accessed | not_parallel_accessed
+ table_3    | parallel_access | not_parallel_accessed | not_parallel_accessed
 (3 rows)
 
 ROLLBACK;
@@ -371,10 +387,10 @@ ROLLBACK;
 BEGIN;
 	INSERT INTO table_2 SELECT * FROM table_1;
 	SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
- table_name |  select_access  |   dml_access    |  ddl_access  
-------------+-----------------+-----------------+--------------
- table_1    | parallel_access | not_accessed    | not_accessed
- table_2    | not_accessed    | parallel_access | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | parallel_access       | not_parallel_accessed | not_parallel_accessed
+ table_2    | not_parallel_accessed | parallel_access       | not_parallel_accessed
 (2 rows)
 
 ROLLBACK;
@@ -383,10 +399,10 @@ BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode = 'sequential';
 	INSERT INTO table_2 SELECT * FROM table_1;
 	SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
- table_name |   select_access   |    dml_access     |  ddl_access  
-------------+-------------------+-------------------+--------------
- table_1    | sequential_access | not_accessed      | not_accessed
- table_2    | not_accessed      | sequential_access | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ table_2    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (2 rows)
 
 ROLLBACK;
@@ -394,10 +410,10 @@ ROLLBACK;
 BEGIN;
 	INSERT INTO table_2 SELECT * FROM table_1 OFFSET 0;
 	SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
- table_name |  select_access  |   dml_access    |  ddl_access  
-------------+-----------------+-----------------+--------------
- table_1    | parallel_access | not_accessed    | not_accessed
- table_2    | not_accessed    | parallel_access | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | parallel_access       | not_parallel_accessed | not_parallel_accessed
+ table_2    | not_parallel_accessed | parallel_access       | not_parallel_accessed
 (2 rows)
 
 ROLLBACK;
@@ -422,10 +438,10 @@ BEGIN;
 (1 row)
 
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
- table_name |  select_access  |  dml_access  |  ddl_access  
-------------+-----------------+--------------+--------------
- table_1    | parallel_access | not_accessed | not_accessed
- table_2    | parallel_access | not_accessed | not_accessed
+ table_name |  select_access  |      dml_access       |      ddl_access       
+------------+-----------------+-----------------------+-----------------------
+ table_1    | parallel_access | not_parallel_accessed | not_parallel_accessed
+ table_2    | parallel_access | not_parallel_accessed | not_parallel_accessed
 (2 rows)
 
 ROLLBACK;
@@ -445,11 +461,11 @@ BEGIN;
 			OFFSET 0
 	) as foo;
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2', 'table_3')  ORDER BY 1;
- table_name |  select_access  |   dml_access    |  ddl_access  
-------------+-----------------+-----------------+--------------
- table_1    | parallel_access | not_accessed    | not_accessed
- table_2    | parallel_access | not_accessed    | not_accessed
- table_3    | not_accessed    | parallel_access | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | parallel_access       | not_parallel_accessed | not_parallel_accessed
+ table_2    | parallel_access       | not_parallel_accessed | not_parallel_accessed
+ table_3    | not_parallel_accessed | parallel_access       | not_parallel_accessed
 (3 rows)
 
 ROLLBACK;
@@ -471,11 +487,11 @@ BEGIN;
 			OFFSET 0
 	) as foo;
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2', 'table_3')  ORDER BY 1;
- table_name |   select_access   |    dml_access     |  ddl_access  
-------------+-------------------+-------------------+--------------
- table_1    | sequential_access | not_accessed      | not_accessed
- table_2    | sequential_access | not_accessed      | not_accessed
- table_3    | not_accessed      | sequential_access | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ table_2    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ table_3    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (3 rows)
 
 ROLLBACK;
@@ -497,12 +513,12 @@ BEGIN;
 		) as foo
 	) AND value IN (SELECT key FROM table_4);
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2', 'table_3', 'table_4')  ORDER BY 1;
- table_name |  select_access  |   dml_access    |  ddl_access  
-------------+-----------------+-----------------+--------------
- table_1    | parallel_access | not_accessed    | not_accessed
- table_2    | parallel_access | not_accessed    | not_accessed
- table_3    | parallel_access | parallel_access | not_accessed
- table_4    | parallel_access | not_accessed    | not_accessed
+ table_name |  select_access  |      dml_access       |      ddl_access       
+------------+-----------------+-----------------------+-----------------------
+ table_1    | parallel_access | not_parallel_accessed | not_parallel_accessed
+ table_2    | parallel_access | not_parallel_accessed | not_parallel_accessed
+ table_3    | parallel_access | parallel_access       | not_parallel_accessed
+ table_4    | parallel_access | not_parallel_accessed | not_parallel_accessed
 (4 rows)
 
 ROLLBACK;
@@ -513,9 +529,9 @@ BEGIN;
 2	2
 3	3
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
- table_name |  select_access  |  dml_access  |  ddl_access  
-------------+-----------------+--------------+--------------
- table_1    | parallel_access | not_accessed | not_accessed
+ table_name |  select_access  |      dml_access       |      ddl_access       
+------------+-----------------+-----------------------+-----------------------
+ table_1    | parallel_access | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 ROLLBACK;
@@ -523,9 +539,9 @@ ROLLBACK;
 BEGIN;
 	COPY table_1 FROM STDIN WITH CSV;
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
- table_name | select_access |   dml_access    |  ddl_access  
-------------+---------------+-----------------+--------------
- table_1    | not_accessed  | parallel_access | not_accessed
+ table_name |     select_access     |   dml_access    |      ddl_access       
+------------+-----------------------+-----------------+-----------------------
+ table_1    | not_parallel_accessed | parallel_access | not_parallel_accessed
 (1 row)
 
 ROLLBACK;
@@ -533,9 +549,9 @@ ROLLBACK;
 BEGIN;
 	COPY table_1 FROM STDIN WITH CSV;
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
- table_name | select_access |    dml_access     |  ddl_access  
-------------+---------------+-------------------+--------------
- table_1    | not_accessed  | sequential_access | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 ROLLBACK;
@@ -548,23 +564,23 @@ BEGIN;
 (1 row)
 
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_6');
- table_name |   select_access   |  dml_access  |  ddl_access  
-------------+-------------------+--------------+--------------
- table_6    | sequential_access | not_accessed | not_accessed
+ table_name |     select_access      |  dml_access  |  ddl_access  
+------------+------------------------+--------------+--------------
+ table_6    | reference_table_access | not_accessed | not_accessed
 (1 row)
 
 	UPDATE table_6 SET value = 15;
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_6');
- table_name |   select_access   |    dml_access     |  ddl_access  
-------------+-------------------+-------------------+--------------
- table_6    | sequential_access | sequential_access | not_accessed
+ table_name |     select_access      |       dml_access       |  ddl_access  
+------------+------------------------+------------------------+--------------
+ table_6    | reference_table_access | reference_table_access | not_accessed
 (1 row)
 
 	ALTER TABLE table_6 ADD COLUMN x INT;	
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_6');
- table_name |   select_access   |    dml_access     |    ddl_access     
-------------+-------------------+-------------------+-------------------
- table_6    | sequential_access | sequential_access | sequential_access
+ table_name |     select_access      |       dml_access       |       ddl_access       
+------------+------------------------+------------------------+------------------------
+ table_6    | reference_table_access | reference_table_access | reference_table_access
 (1 row)
 
 ROLLBACK;
@@ -577,10 +593,10 @@ BEGIN;
 (1 row)
 
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_6', 'table_1') ORDER BY 1,2;
- table_name |  select_access  |  dml_access  |  ddl_access  
-------------+-----------------+--------------+--------------
- table_1    | parallel_access | not_accessed | not_accessed
- table_6    | parallel_access | not_accessed | not_accessed
+ table_name |  select_access  |      dml_access       |      ddl_access       
+------------+-----------------+-----------------------+-----------------------
+ table_1    | parallel_access | not_parallel_accessed | not_parallel_accessed
+ table_6    | parallel_access | not_accessed          | not_accessed
 (2 rows)
 
 ROLLBACK;
@@ -588,9 +604,9 @@ ROLLBACK;
 BEGIN;
 	TRUNCATE table_1;
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
- table_name | select_access |  dml_access  |   ddl_access    
-------------+---------------+--------------+-----------------
- table_1    | not_accessed  | not_accessed | parallel_access
+ table_name |     select_access     |      dml_access       |   ddl_access    
+------------+-----------------------+-----------------------+-----------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | parallel_access
 (1 row)
 
 ROLLBACK;
@@ -599,9 +615,9 @@ BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode = 'sequential';
 	TRUNCATE table_1;
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
- table_name | select_access |  dml_access  |    ddl_access     
-------------+---------------+--------------+-------------------
- table_1    | not_accessed  | not_accessed | sequential_access
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 ROLLBACK;
@@ -609,9 +625,9 @@ ROLLBACK;
 BEGIN;
 	TRUNCATE table_6;
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_6')  ORDER BY 1;
- table_name | select_access |  dml_access  |    ddl_access     
-------------+---------------+--------------+-------------------
- table_6    | not_accessed  | not_accessed | sequential_access
+ table_name | select_access |  dml_access  |       ddl_access       
+------------+---------------+--------------+------------------------
+ table_6    | not_accessed  | not_accessed | reference_table_access
 (1 row)
 
 ROLLBACK;
@@ -620,10 +636,10 @@ ALTER TABLE table_1 ADD CONSTRAINT table_1_u UNIQUE (key);
 BEGIN;
 	ALTER TABLE table_2 ADD CONSTRAINT table_2_u FOREIGN KEY (key) REFERENCES table_1(key);
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
- table_name | select_access |  dml_access  |   ddl_access    
-------------+---------------+--------------+-----------------
- table_1    | not_accessed  | not_accessed | parallel_access
- table_2    | not_accessed  | not_accessed | parallel_access
+ table_name |     select_access     |      dml_access       |   ddl_access    
+------------+-----------------------+-----------------------+-----------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | parallel_access
+ table_2    | not_parallel_accessed | not_parallel_accessed | parallel_access
 (2 rows)
 
 ROLLBACK;
@@ -633,10 +649,10 @@ BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode = 'sequential';
 	ALTER TABLE table_2 ADD CONSTRAINT table_2_u FOREIGN KEY (key) REFERENCES table_1(key);
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
- table_name | select_access |  dml_access  |    ddl_access     
-------------+---------------+--------------+-------------------
- table_1    | not_accessed  | not_accessed | sequential_access
- table_2    | not_accessed  | not_accessed | sequential_access
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ table_2    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (2 rows)
 
 ROLLBACK;
@@ -651,10 +667,10 @@ SELECT create_distributed_table('partitioning_test', 'id');
 BEGIN;
 	CREATE TABLE partitioning_test_2009 PARTITION OF partitioning_test FOR VALUES FROM ('2009-01-01') TO ('2010-01-01');
 	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009')  ORDER BY 1;
-       table_name       | select_access |  dml_access  |   ddl_access    
-------------------------+---------------+--------------+-----------------
- partitioning_test      | not_accessed  | not_accessed | parallel_access
- partitioning_test_2009 | not_accessed  | not_accessed | parallel_access
+       table_name       |     select_access     |      dml_access       |   ddl_access    
+------------------------+-----------------------+-----------------------+-----------------
+ partitioning_test      | not_parallel_accessed | not_parallel_accessed | parallel_access
+ partitioning_test_2009 | not_parallel_accessed | not_parallel_accessed | parallel_access
 (2 rows)
 
 ROLLBACK;
@@ -663,10 +679,10 @@ CREATE TABLE partitioning_test_2009 AS SELECT * FROM partitioning_test;
 BEGIN;
 	ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_2009 FOR VALUES FROM ('2009-01-01') TO ('2010-01-01');
 	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009')  ORDER BY 1;
-       table_name       | select_access |  dml_access  |   ddl_access    
-------------------------+---------------+--------------+-----------------
- partitioning_test      | not_accessed  | not_accessed | parallel_access
- partitioning_test_2009 | not_accessed  | not_accessed | parallel_access
+       table_name       |     select_access     |      dml_access       |   ddl_access    
+------------------------+-----------------------+-----------------------+-----------------
+ partitioning_test      | not_parallel_accessed | not_parallel_accessed | parallel_access
+ partitioning_test_2009 | not_parallel_accessed | not_parallel_accessed | parallel_access
 (2 rows)
 
 COMMIT;
@@ -681,10 +697,10 @@ SELECT create_distributed_table('partitioning_test_2010', 'id');
 BEGIN;
 	ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_2010 FOR VALUES FROM ('2010-01-01') TO ('2011-01-01');
 	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2010')  ORDER BY 1;
-       table_name       | select_access |  dml_access  |   ddl_access    
-------------------------+---------------+--------------+-----------------
- partitioning_test      | not_accessed  | not_accessed | parallel_access
- partitioning_test_2010 | not_accessed  | not_accessed | parallel_access
+       table_name       |     select_access     |      dml_access       |   ddl_access    
+------------------------+-----------------------+-----------------------+-----------------
+ partitioning_test      | not_parallel_accessed | not_parallel_accessed | parallel_access
+ partitioning_test_2010 | not_parallel_accessed | not_parallel_accessed | parallel_access
 (2 rows)
 
 COMMIT;
@@ -697,11 +713,11 @@ BEGIN;
 (1 row)
 
 	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
-       table_name       |  select_access  |  dml_access  |  ddl_access  
-------------------------+-----------------+--------------+--------------
- partitioning_test      | parallel_access | not_accessed | not_accessed
- partitioning_test_2009 | parallel_access | not_accessed | not_accessed
- partitioning_test_2010 | parallel_access | not_accessed | not_accessed
+       table_name       |  select_access  |      dml_access       |      ddl_access       
+------------------------+-----------------+-----------------------+-----------------------
+ partitioning_test      | parallel_access | not_parallel_accessed | not_parallel_accessed
+ partitioning_test_2009 | parallel_access | not_parallel_accessed | not_parallel_accessed
+ partitioning_test_2010 | parallel_access | not_parallel_accessed | not_parallel_accessed
 (3 rows)
 
 COMMIT;
@@ -715,11 +731,11 @@ BEGIN;
 (1 row)
 
 	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
-       table_name       |   select_access   |  dml_access  |  ddl_access  
-------------------------+-------------------+--------------+--------------
- partitioning_test      | sequential_access | not_accessed | not_accessed
- partitioning_test_2009 | sequential_access | not_accessed | not_accessed
- partitioning_test_2010 | sequential_access | not_accessed | not_accessed
+       table_name       |     select_access     |      dml_access       |      ddl_access       
+------------------------+-----------------------+-----------------------+-----------------------
+ partitioning_test      | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ partitioning_test_2009 | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ partitioning_test_2010 | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (3 rows)
 
 COMMIT;
@@ -727,11 +743,11 @@ COMMIT;
 BEGIN;
 	UPDATE partitioning_test SET time = now();
 	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
-       table_name       |  select_access  |   dml_access    |  ddl_access  
-------------------------+-----------------+-----------------+--------------
- partitioning_test      | parallel_access | parallel_access | not_accessed
- partitioning_test_2009 | parallel_access | parallel_access | not_accessed
- partitioning_test_2010 | parallel_access | parallel_access | not_accessed
+       table_name       |  select_access  |   dml_access    |      ddl_access       
+------------------------+-----------------+-----------------+-----------------------
+ partitioning_test      | parallel_access | parallel_access | not_parallel_accessed
+ partitioning_test_2009 | parallel_access | parallel_access | not_parallel_accessed
+ partitioning_test_2010 | parallel_access | parallel_access | not_parallel_accessed
 (3 rows)
 
 COMMIT;
@@ -740,11 +756,11 @@ BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
 	UPDATE partitioning_test SET time = now();
 	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
-       table_name       |   select_access   |    dml_access     |  ddl_access  
-------------------------+-------------------+-------------------+--------------
- partitioning_test      | sequential_access | sequential_access | not_accessed
- partitioning_test_2009 | sequential_access | sequential_access | not_accessed
- partitioning_test_2010 | sequential_access | sequential_access | not_accessed
+       table_name       |     select_access     |      dml_access       |      ddl_access       
+------------------------+-----------------------+-----------------------+-----------------------
+ partitioning_test      | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ partitioning_test_2009 | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ partitioning_test_2010 | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (3 rows)
 
 COMMIT;
@@ -752,11 +768,11 @@ COMMIT;
 BEGIN;
 	ALTER TABLE partitioning_test ADD COLUMN X INT;
 	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
-       table_name       | select_access |  dml_access  |   ddl_access    
-------------------------+---------------+--------------+-----------------
- partitioning_test      | not_accessed  | not_accessed | parallel_access
- partitioning_test_2009 | not_accessed  | not_accessed | parallel_access
- partitioning_test_2010 | not_accessed  | not_accessed | parallel_access
+       table_name       |     select_access     |      dml_access       |   ddl_access    
+------------------------+-----------------------+-----------------------+-----------------
+ partitioning_test      | not_parallel_accessed | not_parallel_accessed | parallel_access
+ partitioning_test_2009 | not_parallel_accessed | not_parallel_accessed | parallel_access
+ partitioning_test_2010 | not_parallel_accessed | not_parallel_accessed | parallel_access
 (3 rows)
 
 ROLLBACK;
@@ -765,11 +781,11 @@ BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
 	ALTER TABLE partitioning_test ADD COLUMN X INT;
 	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
-       table_name       | select_access |  dml_access  |    ddl_access     
-------------------------+---------------+--------------+-------------------
- partitioning_test      | not_accessed  | not_accessed | sequential_access
- partitioning_test_2009 | not_accessed  | not_accessed | sequential_access
- partitioning_test_2010 | not_accessed  | not_accessed | sequential_access
+       table_name       |     select_access     |      dml_access       |      ddl_access       
+------------------------+-----------------------+-----------------------+-----------------------
+ partitioning_test      | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ partitioning_test_2009 | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ partitioning_test_2010 | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (3 rows)
 
 ROLLBACK;
@@ -782,11 +798,11 @@ BEGIN;
 (1 row)
 
 	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
-       table_name       |  select_access  |  dml_access  |  ddl_access  
-------------------------+-----------------+--------------+--------------
- partitioning_test      | parallel_access | not_accessed | not_accessed
- partitioning_test_2009 | parallel_access | not_accessed | not_accessed
- partitioning_test_2010 | not_accessed    | not_accessed | not_accessed
+       table_name       |     select_access     |      dml_access       |      ddl_access       
+------------------------+-----------------------+-----------------------+-----------------------
+ partitioning_test      | parallel_access       | not_parallel_accessed | not_parallel_accessed
+ partitioning_test_2009 | parallel_access       | not_parallel_accessed | not_parallel_accessed
+ partitioning_test_2010 | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (3 rows)
 
 COMMIT;
@@ -800,11 +816,11 @@ BEGIN;
 (1 row)
 
 	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
-       table_name       |   select_access   |  dml_access  |  ddl_access  
-------------------------+-------------------+--------------+--------------
- partitioning_test      | sequential_access | not_accessed | not_accessed
- partitioning_test_2009 | sequential_access | not_accessed | not_accessed
- partitioning_test_2010 | not_accessed      | not_accessed | not_accessed
+       table_name       |     select_access     |      dml_access       |      ddl_access       
+------------------------+-----------------------+-----------------------+-----------------------
+ partitioning_test      | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ partitioning_test_2009 | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ partitioning_test_2010 | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (3 rows)
 
 COMMIT;
@@ -812,11 +828,11 @@ COMMIT;
 BEGIN;
 	UPDATE partitioning_test_2009 SET time = now();
 	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
-       table_name       |  select_access  |   dml_access    |  ddl_access  
-------------------------+-----------------+-----------------+--------------
- partitioning_test      | parallel_access | parallel_access | not_accessed
- partitioning_test_2009 | parallel_access | parallel_access | not_accessed
- partitioning_test_2010 | not_accessed    | not_accessed    | not_accessed
+       table_name       |     select_access     |      dml_access       |      ddl_access       
+------------------------+-----------------------+-----------------------+-----------------------
+ partitioning_test      | parallel_access       | parallel_access       | not_parallel_accessed
+ partitioning_test_2009 | parallel_access       | parallel_access       | not_parallel_accessed
+ partitioning_test_2010 | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (3 rows)
 
 COMMIT;
@@ -825,11 +841,11 @@ BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
 	UPDATE partitioning_test_2009 SET time = now();
 	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
-       table_name       |   select_access   |    dml_access     |  ddl_access  
-------------------------+-------------------+-------------------+--------------
- partitioning_test      | sequential_access | sequential_access | not_accessed
- partitioning_test_2009 | sequential_access | sequential_access | not_accessed
- partitioning_test_2010 | not_accessed      | not_accessed      | not_accessed
+       table_name       |     select_access     |      dml_access       |      ddl_access       
+------------------------+-----------------------+-----------------------+-----------------------
+ partitioning_test      | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ partitioning_test_2009 | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ partitioning_test_2010 | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (3 rows)
 
 COMMIT;
@@ -837,11 +853,11 @@ COMMIT;
 BEGIN;
 	CREATE INDEX i1000000 ON partitioning_test_2009 (id);
 	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
-       table_name       | select_access |  dml_access  |   ddl_access    
-------------------------+---------------+--------------+-----------------
- partitioning_test      | not_accessed  | not_accessed | parallel_access
- partitioning_test_2009 | not_accessed  | not_accessed | parallel_access
- partitioning_test_2010 | not_accessed  | not_accessed | not_accessed
+       table_name       |     select_access     |      dml_access       |      ddl_access       
+------------------------+-----------------------+-----------------------+-----------------------
+ partitioning_test      | not_parallel_accessed | not_parallel_accessed | parallel_access
+ partitioning_test_2009 | not_parallel_accessed | not_parallel_accessed | parallel_access
+ partitioning_test_2010 | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (3 rows)
 
 ROLLBACK;
@@ -850,11 +866,11 @@ BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
 	CREATE INDEX i1000000 ON partitioning_test_2009 (id);
 	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
-       table_name       | select_access |  dml_access  |    ddl_access     
-------------------------+---------------+--------------+-------------------
- partitioning_test      | not_accessed  | not_accessed | sequential_access
- partitioning_test_2009 | not_accessed  | not_accessed | sequential_access
- partitioning_test_2010 | not_accessed  | not_accessed | not_accessed
+       table_name       |     select_access     |      dml_access       |      ddl_access       
+------------------------+-----------------------+-----------------------+-----------------------
+ partitioning_test      | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ partitioning_test_2009 | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+ partitioning_test_2010 | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (3 rows)
 
 ROLLBACK;
@@ -864,10 +880,10 @@ BEGIN;
 	TRUNCATE table_1 CASCADE;
 NOTICE:  truncate cascades to table "table_2"
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
- table_name | select_access |  dml_access  |   ddl_access    
-------------+---------------+--------------+-----------------
- table_1    | not_accessed  | not_accessed | parallel_access
- table_2    | not_accessed  | not_accessed | parallel_access
+ table_name |     select_access     |      dml_access       |   ddl_access    
+------------+-----------------------+-----------------------+-----------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | parallel_access
+ table_2    | not_parallel_accessed | not_parallel_accessed | parallel_access
 (2 rows)
 
 ROLLBACK;
@@ -882,9 +898,9 @@ BEGIN;
 (1 row)
 
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
- table_name |  select_access  |  dml_access  |  ddl_access  
-------------+-----------------+--------------+--------------
- table_1    | parallel_access | not_accessed | not_accessed
+ table_name |  select_access  |      dml_access       |      ddl_access       
+------------+-----------------+-----------------------+-----------------------
+ table_1    | parallel_access | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 COMMIT;
@@ -899,9 +915,9 @@ BEGIN;
 (1 row)
 
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
- table_name |   select_access   |  dml_access  |  ddl_access  
-------------+-------------------+--------------+--------------
- table_1    | sequential_access | not_accessed | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 COMMIT;
@@ -918,9 +934,9 @@ BEGIN;
 (3 rows)
 
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
- table_name | select_access |    dml_access     |  ddl_access  
-------------+---------------+-------------------+--------------
- table_1    | not_accessed  | sequential_access | not_accessed
+ table_name |     select_access     |      dml_access       |      ddl_access       
+------------+-----------------------+-----------------------+-----------------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 ROLLBACK;
@@ -935,9 +951,9 @@ BEGIN;
 (1 row)
 
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
- table_name |  select_access  |   dml_access    |  ddl_access  
-------------+-----------------+-----------------+--------------
- table_1    | parallel_access | parallel_access | not_accessed
+ table_name |  select_access  |   dml_access    |      ddl_access       
+------------+-----------------+-----------------+-----------------------
+ table_1    | parallel_access | parallel_access | not_parallel_accessed
 (1 row)
 
 ROLLBACK;
@@ -952,9 +968,9 @@ BEGIN;
 (1 row)
 
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
- table_name |  select_access  |   dml_access    |  ddl_access  
-------------+-----------------+-----------------+--------------
- table_1    | parallel_access | parallel_access | not_accessed
+ table_name |  select_access  |   dml_access    |      ddl_access       
+------------+-----------------+-----------------+-----------------------
+ table_1    | parallel_access | parallel_access | not_parallel_accessed
 (1 row)
 
 ROLLBACK;
@@ -972,19 +988,20 @@ NOTICE:  Copying data from local table...
 (1 row)
 
 	SELECT * FROM relation_acesses  WHERE table_name IN ('table_3')  ORDER BY 1;
- table_name | select_access |   dml_access    |   ddl_access    
-------------+---------------+-----------------+-----------------
- table_3    | not_accessed  | parallel_access | parallel_access
+ table_name |     select_access     |   dml_access    |   ddl_access    
+------------+-----------------------+-----------------+-----------------
+ table_3    | not_parallel_accessed | parallel_access | parallel_access
 (1 row)
 
 COMMIT;
 SET search_path TO 'public';
 DROP SCHEMA access_tracking CASCADE;
-NOTICE:  drop cascades to 13 other objects
+NOTICE:  drop cascades to 14 other objects
 DETAIL:  drop cascades to function access_tracking.relation_select_access_mode(oid)
 drop cascades to function access_tracking.relation_dml_access_mode(oid)
 drop cascades to function access_tracking.relation_ddl_access_mode(oid)
-drop cascades to function access_tracking.relation_access_mode_to_text(integer)
+drop cascades to function access_tracking.distributed_relation(text)
+drop cascades to function access_tracking.relation_access_mode_to_text(text,integer)
 drop cascades to view access_tracking.relation_acesses
 drop cascades to table access_tracking.table_1
 drop cascades to table access_tracking.table_2
diff --git a/src/test/regress/sql/relation_access_tracking.sql b/src/test/regress/sql/relation_access_tracking.sql
index 5196cf523..13cb5dc92 100644
--- a/src/test/regress/sql/relation_access_tracking.sql
+++ b/src/test/regress/sql/relation_access_tracking.sql
@@ -23,15 +23,32 @@ CREATE OR REPLACE FUNCTION relation_ddl_access_mode(relationId Oid)
     LANGUAGE C STABLE STRICT
     AS 'citus', $$relation_ddl_access_mode$$;
 
+CREATE OR REPLACE FUNCTION distributed_relation(relation_name text)
+RETURNS bool AS
+$$
+DECLARE
+	part_method char;
+BEGIN
+		 select partmethod INTO part_method from pg_dist_partition  WHERE logicalrelid = relation_name::regclass;
+         IF part_method = 'h' THEN
+                RETURN true;
+         ELSE
+                RETURN false;
+         END IF;
+END;
+$$ LANGUAGE 'plpgsql' IMMUTABLE;
 
-CREATE OR REPLACE FUNCTION relation_access_mode_to_text(relationShardAccess int)
+
+CREATE OR REPLACE FUNCTION relation_access_mode_to_text(relation_name text, relationShardAccess int)
 RETURNS text AS
 $$
 BEGIN
-	 IF relationShardAccess = 0 THEN
+	 IF relationShardAccess = 0 and distributed_relation(relation_name) THEN
+	 	RETURN 'not_parallel_accessed';
+	 ELSIF relationShardAccess = 0 and NOT distributed_relation(relation_name) THEN
 	 	RETURN 'not_accessed';
 	 ELSIF relationShardAccess = 1 THEN
-	 	RETURN 'sequential_access';
+	 	RETURN 'reference_table_access';
 	 ELSE 
 		RETURN 'parallel_access';
 	 END IF;
@@ -42,9 +59,9 @@ $$ LANGUAGE 'plpgsql' IMMUTABLE;
 
 CREATE VIEW relation_acesses AS 
 	SELECT table_name, 
-			relation_access_mode_to_text(relation_select_access_mode(table_name::regclass)) as select_access,
-			relation_access_mode_to_text(relation_dml_access_mode(table_name::regclass)) as dml_access,
-			relation_access_mode_to_text(relation_ddl_access_mode(table_name::regclass)) as ddl_access
+			relation_access_mode_to_text(table_name, relation_select_access_mode(table_name::regclass)) as select_access,
+			relation_access_mode_to_text(table_name, relation_dml_access_mode(table_name::regclass)) as dml_access,
+			relation_access_mode_to_text(table_name, relation_ddl_access_mode(table_name::regclass)) as ddl_access
 	FROM 
 		((SELECT 'table_' || i as table_name FROM generate_series(1, 7) i) UNION (SELECT 'partitioning_test') UNION (SELECT 'partitioning_test_2009') UNION (SELECT 'partitioning_test_2010')) tables;