Merge pull request #3606 from citusdata/improve_error_messages

Improve connection error message from the worker nodes
pull/3782/head
Önder Kalacı 2020-04-20 13:47:43 +02:00 committed by GitHub
commit 793c65b539
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 425 additions and 522 deletions

View File

@ -277,10 +277,23 @@ ReportConnectionError(MultiConnection *connection, int elevel)
messageDetail = pchomp(PQerrorMessage(pgConn));
}
ereport(elevel, (errcode(ERRCODE_CONNECTION_FAILURE),
errmsg("connection error: %s:%d", nodeName, nodePort),
messageDetail != NULL ?
errdetail("%s", ApplyLogRedaction(messageDetail)) : 0));
if (messageDetail)
{
/*
* We don't use ApplyLogRedaction(messageDetail) as we expect any error
* detail that requires log reduction should have done it locally.
*/
ereport(elevel, (errcode(ERRCODE_CONNECTION_FAILURE),
errmsg("connection to the remote node %s:%d failed with the "
"following error: %s", nodeName, nodePort,
messageDetail)));
}
else
{
ereport(elevel, (errcode(ERRCODE_CONNECTION_FAILURE),
errmsg("connection to the remote node %s:%d failed",
nodeName, nodePort)));
}
}

View File

@ -168,8 +168,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").kill()');
(1 row)
SELECT count(1) FROM copy_test;
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
count

View File

@ -219,8 +219,7 @@ SELECT citus.mitmproxy('conn.killall()');
(1 row)
COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV;
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: COPY copy_test, line 1: "0, 0"

View File

@ -227,7 +227,7 @@ RESET client_min_messages;
-- verify get_global_active_transactions works when a timeout happens on a connection
SELECT get_global_active_transactions();
WARNING: could not establish connection after 400 ms
WARNING: connection error: localhost:xxxxx
WARNING: connection to the remote node localhost:xxxxx failed
get_global_active_transactions
---------------------------------------------------------------------
(0 rows)

View File

@ -36,8 +36,7 @@ SELECT citus.mitmproxy('conn.kill()');
(1 row)
\COPY test_table FROM stdin delimiter ',';
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: COPY test_table, line 1: "1,2"
@ -276,23 +275,19 @@ SELECT citus.mitmproxy('conn.kill()');
(1 row)
\COPY test_table_2 FROM stdin delimiter ',';
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: COPY test_table_2, line 1: "1,2"
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: COPY test_table_2, line 2: "3,4"
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: COPY test_table_2, line 3: "6,7"
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: COPY test_table_2, line 5: "9,10"

View File

@ -26,8 +26,7 @@ SELECT citus.mitmproxy('conn.kill()');
(1 row)
SELECT create_distributed_table('test_table', 'id');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
@ -125,8 +124,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
(1 row)
SELECT create_distributed_table('test_table', 'id');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
@ -190,8 +188,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
(1 row)
SELECT create_distributed_table('test_table', 'id');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
@ -473,8 +470,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
(1 row)
SELECT create_distributed_table('test_table', 'id', colocate_with => 'colocated_table');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
@ -561,8 +557,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT worker_apply_shard_ddl_comma
(1 row)
SELECT create_distributed_table('test_table', 'id', colocate_with => 'colocated_table');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
@ -599,8 +594,7 @@ SELECT citus.mitmproxy('conn.kill()');
(1 row)
SELECT create_distributed_table('test_table', 'id');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
@ -667,8 +661,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
(1 row)
SELECT create_distributed_table('test_table', 'id');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
@ -732,8 +725,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
(1 row)
SELECT create_distributed_table('test_table', 'id');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
@ -946,8 +938,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
(1 row)
SELECT create_distributed_table('test_table', 'id', colocate_with => 'colocated_table');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;

View File

@ -25,8 +25,7 @@ SELECT citus.mitmproxy('conn.onQuery().kill()');
(1 row)
SELECT create_reference_table('ref_table');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT count(*) FROM pg_dist_shard_placement;
@ -43,8 +42,7 @@ SELECT citus.mitmproxy('conn.onCommandComplete(command="BEGIN").kill()');
(1 row)
SELECT create_reference_table('ref_table');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT count(*) FROM pg_dist_shard_placement;
@ -76,8 +74,7 @@ SELECT citus.mitmproxy('conn.onCommandComplete(command="SELECT 1").kill()');
(1 row)
SELECT create_reference_table('ref_table');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT count(*) FROM pg_dist_shard_placement;

View File

@ -20,8 +20,7 @@ SELECT citus.mitmproxy('conn.kill()');
(1 row)
SELECT create_distributed_table('test_table','id');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
@ -89,8 +88,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
(1 row)
SELECT create_distributed_table('test_table','id');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
@ -120,8 +118,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_apply_shard_ddl_comman
(1 row)
SELECT create_distributed_table('test_table','id');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
@ -154,8 +151,7 @@ BEGIN;
(1 row)
SELECT create_distributed_table('test_table', 'id');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
COMMIT;
@ -225,8 +221,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
(1 row)
SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
@ -347,8 +342,7 @@ SELECT citus.mitmproxy('conn.kill()');
BEGIN;
SELECT create_distributed_table('test_table','id');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
ROLLBACK;
@ -384,8 +378,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
BEGIN;
SELECT create_distributed_table('test_table','id');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
ROLLBACK;
@ -463,8 +456,7 @@ SELECT citus.mitmproxy('conn.kill()');
BEGIN;
SELECT create_distributed_table('test_table','id');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
ROLLBACK;
@ -496,8 +488,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
BEGIN;
SELECT create_distributed_table('test_table','id');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
ROLLBACK;
@ -533,8 +524,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
BEGIN;
SELECT create_distributed_table('test_table','id');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
ROLLBACK;
@ -615,8 +605,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
(1 row)
SELECT master_create_worker_shards('test_table_2', 4, 2);
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT count(*) FROM pg_dist_shard;

View File

@ -88,8 +88,7 @@ FROM
ORDER BY 1 DESC LIMIT 5
) as foo
WHERE foo.user_id = cte.user_id;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- kill at the third copy (pull)
@ -122,8 +121,7 @@ FROM
ORDER BY 1 DESC LIMIT 5
) as foo
WHERE foo.user_id = cte.user_id;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- cancel at the first copy (push)
@ -262,8 +260,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^DELETE FROM").kill()');
WITH cte_delete as (DELETE FROM users_table WHERE user_name in ('A', 'D') RETURNING *)
INSERT INTO users_table SELECT * FROM cte_delete;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- verify contents are the same
@ -378,8 +375,7 @@ BEGIN;
SET LOCAL citus.multi_shard_modify_mode = 'sequential';
WITH cte_delete as (DELETE FROM users_table WHERE user_name in ('A', 'D') RETURNING *)
INSERT INTO users_table SELECT * FROM cte_delete;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
END;

View File

@ -37,8 +37,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()');
(1 row)
ALTER TABLE test_table ADD COLUMN new_column INT;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@ -71,8 +70,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
(1 row)
ALTER TABLE test_table ADD COLUMN new_column INT;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@ -104,8 +102,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil
(1 row)
ALTER TABLE test_table ADD COLUMN new_column INT;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- show that we've never commited the changes
@ -349,8 +346,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()');
(1 row)
ALTER TABLE test_table DROP COLUMN new_column;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@ -383,8 +379,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
(1 row)
ALTER TABLE test_table DROP COLUMN new_column;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@ -416,8 +411,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil
(1 row)
ALTER TABLE test_table DROP COLUMN new_column;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@ -720,8 +714,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()');
(1 row)
ALTER TABLE test_table ADD COLUMN new_column INT;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@ -754,8 +747,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
(1 row)
ALTER TABLE test_table ADD COLUMN new_column INT;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@ -787,8 +779,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil
(1 row)
ALTER TABLE test_table ADD COLUMN new_column INT;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@ -1048,8 +1039,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
(1 row)
ALTER TABLE test_table ADD COLUMN new_column INT;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@ -1081,8 +1071,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil
(1 row)
ALTER TABLE test_table ADD COLUMN new_column INT;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- kill as soon as the coordinator after it sends worker_apply_shard_ddl_command 2nd time
@ -1093,8 +1082,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").aft
(1 row)
ALTER TABLE test_table ADD COLUMN new_column INT;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- cancel as soon as the coordinator after it sends worker_apply_shard_ddl_command 2nd time

View File

@ -85,8 +85,7 @@ CREATE TABLE distributed_result_info AS
SELECT resultId, nodeport, rowcount, targetShardId, targetShardIndex
FROM partition_task_list_results('test', $$ SELECT * FROM source_table $$, 'target_table')
NATURAL JOIN pg_dist_node;
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT * FROM distributed_result_info ORDER BY resultId;

View File

@ -44,8 +44,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT INTO insert_select_pushdown"
(1 row)
INSERT INTO events_summary SELECT user_id, event_id, count(*) FROM events_table GROUP BY 1,2;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
--verify nothing is modified
@ -98,8 +97,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT INTO insert_select_pushdown"
(1 row)
INSERT INTO events_table SELECT * FROM events_table;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
--verify nothing is modified

View File

@ -54,8 +54,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_partition_query_result").kill
(1 row)
INSERT INTO target_table SELECT * FROM source_table;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT * FROM target_table ORDER BY a;
@ -70,12 +69,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_partition_query_result").kill
(1 row)
INSERT INTO target_table SELECT * FROM replicated_source_table;
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT * FROM target_table ORDER BY a;
@ -106,8 +103,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="fetch_intermediate_results").kill()'
(1 row)
INSERT INTO target_table SELECT * FROM source_table;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT * FROM target_table ORDER BY a;
@ -122,8 +118,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="fetch_intermediate_results").kill()'
(1 row)
INSERT INTO target_table SELECT * FROM replicated_source_table;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT * FROM target_table ORDER BY a;
@ -143,8 +138,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()')
(1 row)
INSERT INTO target_table SELECT * FROM source_table;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT * FROM target_table ORDER BY a;
@ -159,8 +153,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()')
(1 row)
INSERT INTO target_table SELECT * FROM replicated_source_table;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT * FROM target_table ORDER BY a;
@ -179,8 +172,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()')
(1 row)
INSERT INTO replicated_target_table SELECT * FROM source_table;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT * FROM replicated_target_table;

View File

@ -33,8 +33,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^DELETE").kill()');
BEGIN;
DELETE FROM dml_test WHERE id = 1;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
DELETE FROM dml_test WHERE id = 2;
@ -96,8 +95,7 @@ BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
@ -154,8 +152,7 @@ DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
UPDATE dml_test SET name = 'gamma' WHERE id = 3;

View File

@ -43,8 +43,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
(1 row)
INSERT INTO distributed_table VALUES (1,1), (1,2), (1,3);
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- this test is broken, see https://github.com/citusdata/citus/issues/2460
@ -58,8 +57,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
(1 row)
INSERT INTO distributed_table VALUES (1,7), (5,8);
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- this test is broken, see https://github.com/citusdata/citus/issues/2460
@ -73,8 +71,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
(1 row)
INSERT INTO distributed_table VALUES (1,11), (6,12);
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')');
@ -93,8 +90,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").after(1).kill()');
(1 row)
INSERT INTO distributed_table VALUES (1,15), (6,16);
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").after(1).cancel(' || :pid || ')');
@ -113,8 +109,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
(1 row)
INSERT INTO distributed_table VALUES (2,19),(1,20);
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')');

View File

@ -64,8 +64,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
-- issue a multi shard delete
DELETE FROM t2 WHERE b = 2;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- verify nothing is deleted
@ -83,8 +82,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM multi_shard.t2_201005").
(1 row)
DELETE FROM t2 WHERE b = 2;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- verify nothing is deleted
@ -146,8 +144,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
-- issue a multi shard update
UPDATE t2 SET c = 4 WHERE b = 2;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- verify nothing is updated
@ -165,8 +162,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t2_201005").kill(
(1 row)
UPDATE t2 SET c = 4 WHERE b = 2;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- verify nothing is updated
@ -222,8 +218,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
-- issue a multi shard delete
DELETE FROM t2 WHERE b = 2;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- verify nothing is deleted
@ -241,8 +236,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM multi_shard.t2_201005").
(1 row)
DELETE FROM t2 WHERE b = 2;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- verify nothing is deleted
@ -304,8 +298,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
-- issue a multi shard update
UPDATE t2 SET c = 4 WHERE b = 2;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- verify nothing is updated
@ -323,8 +316,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t2_201005").kill(
(1 row)
UPDATE t2 SET c = 4 WHERE b = 2;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- verify nothing is updated
@ -397,8 +389,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
(1 row)
DELETE FROM r1 WHERE a = 2;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- verify nothing is deleted
@ -415,8 +406,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
(1 row)
DELETE FROM t2 WHERE b = 2;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- verify nothing is deleted
@ -500,8 +490,7 @@ UPDATE t3 SET c = q.c FROM (
SELECT b, max(c) as c FROM t2 GROUP BY b) q
WHERE t3.b = q.b
RETURNING *;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
--- verify nothing is updated
@ -559,8 +548,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t3_201013").kill(
(1 row)
UPDATE t3 SET b = 2 WHERE b = 1;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- verify nothing is updated
@ -594,8 +582,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO
-- following will fail
UPDATE t3 SET b = 2 WHERE b = 1;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
END;
@ -613,8 +600,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO
(1 row)
UPDATE t3 SET b = 1 WHERE b = 2 RETURNING *;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- verify nothing is updated
@ -633,8 +619,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO
(1 row)
UPDATE t3 SET b = 2 WHERE b = 1;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- verify nothing is updated
@ -668,8 +653,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO
-- following will fail
UPDATE t3 SET b = 2 WHERE b = 1;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
END;

View File

@ -33,8 +33,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
(1 row)
INSERT INTO ref_table VALUES (5, 6);
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT COUNT(*) FROM ref_table WHERE key=5;
@ -51,8 +50,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
(1 row)
UPDATE ref_table SET key=7 RETURNING value;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT COUNT(*) FROM ref_table WHERE key=7;
@ -71,8 +69,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
BEGIN;
DELETE FROM ref_table WHERE key=5;
UPDATE ref_table SET key=value;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
COMMIT;

View File

@ -29,8 +29,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
(1 row)
INSERT INTO partitioned_table VALUES (0, 0);
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- use both placements

View File

@ -40,8 +40,7 @@ INSERT INTO artists VALUES (5, 'Asher Lev');
SAVEPOINT s1;
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection error: localhost:xxxxx
DETAIL: connection not open
WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection not open
@ -74,8 +73,7 @@ RELEASE SAVEPOINT s1;
WARNING: AbortSubTransaction while in COMMIT state
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection error: localhost:xxxxx
DETAIL: connection not open
WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection not open
@ -132,8 +130,7 @@ RELEASE SAVEPOINT s2;
WARNING: AbortSubTransaction while in COMMIT state
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection error: localhost:xxxxx
DETAIL: connection not open
WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection not open
@ -213,7 +210,7 @@ ROLLBACK TO SAVEPOINT s1;
WARNING: connection not open
WARNING: connection not open
WARNING: connection not open
WARNING: connection error: localhost:xxxxx
WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open
WARNING: connection not open
WARNING: connection not open
COMMIT;
@ -248,7 +245,7 @@ BEGIN;
INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
SAVEPOINT s1;
WARNING: connection not open
WARNING: connection error: localhost:xxxxx
WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open
WARNING: connection not open
WARNING: connection not open
ERROR: connection not open
@ -321,7 +318,7 @@ ROLLBACK TO s1;
RELEASE SAVEPOINT s1;
WARNING: AbortSubTransaction while in COMMIT state
WARNING: connection not open
WARNING: connection error: localhost:xxxxx
WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open
WARNING: connection not open
WARNING: connection not open
WARNING: savepoint "savepoint_3" does not exist
@ -370,10 +367,10 @@ WARNING: connection not open
WARNING: connection not open
SELECT * FROM ref;
WARNING: connection not open
WARNING: connection error: localhost:xxxxx
WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open
WARNING: connection not open
WARNING: connection not open
ERROR: connection error: localhost:xxxxx
ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open
END;
-- clean up
SELECT citus.mitmproxy('conn.allow()');

View File

@ -27,8 +27,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
(1 row)
INSERT INTO mod_test VALUES (2, 6);
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT COUNT(*) FROM mod_test WHERE key=2;
@ -63,8 +62,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
(1 row)
UPDATE mod_test SET value='ok' WHERE key=2 RETURNING key;
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
key
@ -102,8 +100,7 @@ INSERT INTO mod_test VALUES (2, 6);
INSERT INTO mod_test VALUES (2, 7);
DELETE FROM mod_test WHERE key=2 AND value = '7';
UPDATE mod_test SET value='ok' WHERE key=2;
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
COMMIT;

View File

@ -28,8 +28,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()');
(1 row)
SELECT * FROM select_test WHERE key = 3;
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
key | value
@ -38,8 +37,7 @@ DETAIL: server closed the connection unexpectedly
(1 row)
SELECT * FROM select_test WHERE key = 3;
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
key | value
@ -57,8 +55,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()');
BEGIN;
INSERT INTO select_test VALUES (3, 'more data');
SELECT * FROM select_test WHERE key = 3;
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
key | value
@ -69,8 +66,7 @@ DETAIL: server closed the connection unexpectedly
INSERT INTO select_test VALUES (3, 'even more data');
SELECT * FROM select_test WHERE key = 3;
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
key | value
@ -160,8 +156,7 @@ SELECT * FROM select_test WHERE key = 3;
INSERT INTO select_test VALUES (3, 'even more data');
SELECT * FROM select_test WHERE key = 3;
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
key | value
@ -215,8 +210,7 @@ SELECT * FROM select_test WHERE key = 1;
(1 row)
SELECT * FROM select_test WHERE key = 1;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- now the same test with query cancellation

View File

@ -44,8 +44,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()');
(1 row)
TRUNCATE test_table;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
@ -102,8 +101,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
(1 row)
TRUNCATE test_table;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
@ -159,8 +157,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="TRUNCATE TABLE truncate_failure.test
(1 row)
TRUNCATE test_table;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
@ -433,8 +430,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^TRUNCATE TABLE").after(2).kill()');
(1 row)
TRUNCATE reference_table CASCADE;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
@ -577,8 +573,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()');
(1 row)
TRUNCATE test_table;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
@ -635,8 +630,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
(1 row)
TRUNCATE test_table;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
@ -692,8 +686,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^TRUNCATE TABLE truncate_failure.tes
(1 row)
TRUNCATE test_table;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
@ -956,8 +949,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()');
(1 row)
TRUNCATE test_table;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
@ -1014,8 +1006,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
(1 row)
TRUNCATE test_table;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
@ -1071,8 +1062,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="TRUNCATE TABLE truncate_failure.test
(1 row)
TRUNCATE test_table;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');

View File

@ -31,8 +31,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM").kill()');
(1 row)
VACUUM vacuum_test;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()');
@ -42,8 +41,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()');
(1 row)
ANALYZE vacuum_test;
WARNING: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
@ -111,8 +109,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").kill()');
(1 row)
VACUUM vacuum_test, other_vacuum_test;
ERROR: connection error: localhost:xxxxx
DETAIL: server closed the connection unexpectedly
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").cancel(' || pg_backend_pid() || ')');

View File

@ -1178,15 +1178,15 @@ NOTICE: not propagating ALTER ROLE ... RENAME TO commands to worker nodes
\c - test_user - :master_port
-- should fail since the worker doesn't have test_user anymore
INSERT INTO reference_failure_test VALUES (1, '1');
ERROR: connection error: localhost:xxxxx
ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist
-- the same as the above, but wrapped within a transaction
BEGIN;
INSERT INTO reference_failure_test VALUES (1, '1');
ERROR: connection error: localhost:xxxxx
ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist
COMMIT;
BEGIN;
COPY reference_failure_test FROM STDIN WITH (FORMAT 'csv');
ERROR: connection error: localhost:xxxxx
ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist
COMMIT;
-- show that no data go through the table and shard states are good
SET client_min_messages to 'ERROR';
@ -1211,8 +1211,8 @@ ORDER BY s.logicalrelid, sp.shardstate;
BEGIN;
COPY numbers_hash_failure_test FROM STDIN WITH (FORMAT 'csv');
WARNING: connection error: localhost:xxxxx
WARNING: connection error: localhost:xxxxx
WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist
WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist
-- some placements are invalid before abort
SELECT shardid, shardstate, nodename, nodeport
FROM pg_dist_shard_placement JOIN pg_dist_shard USING (shardid)
@ -1233,8 +1233,8 @@ ORDER BY shardid, nodeport;
ABORT;
-- verify nothing is inserted
SELECT count(*) FROM numbers_hash_failure_test;
WARNING: connection error: localhost:xxxxx
WARNING: connection error: localhost:xxxxx
WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist
WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist
count
---------------------------------------------------------------------
0
@ -1259,8 +1259,8 @@ ORDER BY shardid, nodeport;
BEGIN;
COPY numbers_hash_failure_test FROM STDIN WITH (FORMAT 'csv');
WARNING: connection error: localhost:xxxxx
WARNING: connection error: localhost:xxxxx
WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist
WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist
-- check shard states before commit
SELECT shardid, shardstate, nodename, nodeport
FROM pg_dist_shard_placement JOIN pg_dist_shard USING (shardid)
@ -1298,8 +1298,8 @@ ORDER BY shardid, nodeport;
-- verify data is inserted
SELECT count(*) FROM numbers_hash_failure_test;
WARNING: connection error: localhost:xxxxx
WARNING: connection error: localhost:xxxxx
WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist
WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist
count
---------------------------------------------------------------------
2
@ -1312,7 +1312,7 @@ NOTICE: not propagating ALTER ROLE ... RENAME TO commands to worker nodes
\c - test_user - :master_port
-- fails on all shard placements
INSERT INTO numbers_hash_failure_test VALUES (2,2);
ERROR: connection error: localhost:xxxxx
ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist
-- connect back to the master with the proper user to continue the tests
\c - :default_user - :master_port
SET citus.next_shard_id TO 1200020;

View File

@ -9,6 +9,7 @@ SET citus.replication_model TO streaming;
SET citus.shard_count TO 8;
SET citus.shard_replication_factor TO 1;
SET citus.replicate_reference_tables_on_activate TO off;
\set VERBOSITY terse
-- Simulates a readonly node by setting default_transaction_read_only.
CREATE FUNCTION mark_node_readonly(hostname TEXT, port INTEGER, isreadonly BOOLEAN)
RETURNS TEXT
@ -319,7 +320,6 @@ SET citus.log_remote_commands = ON;
-- trigger caching for prepared statements
EXECUTE foo;
NOTICE: issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
count
---------------------------------------------------------------------
0
@ -327,7 +327,6 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
EXECUTE foo;
NOTICE: issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
count
---------------------------------------------------------------------
0
@ -335,7 +334,6 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
EXECUTE foo;
NOTICE: issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
count
---------------------------------------------------------------------
0
@ -343,7 +341,6 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
EXECUTE foo;
NOTICE: issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
count
---------------------------------------------------------------------
0
@ -351,7 +348,6 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
EXECUTE foo;
NOTICE: issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
count
---------------------------------------------------------------------
0
@ -359,7 +355,6 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
EXECUTE foo;
NOTICE: issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
count
---------------------------------------------------------------------
0
@ -367,7 +362,6 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
EXECUTE foo;
NOTICE: issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
count
---------------------------------------------------------------------
0
@ -381,7 +375,6 @@ SELECT master_update_node(:nodeid_1, '127.0.0.1', :worker_1_port);
SELECT wait_until_metadata_sync(30000);
NOTICE: issuing LISTEN metadata_sync
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
wait_until_metadata_sync
---------------------------------------------------------------------
@ -390,7 +383,6 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
-- make sure the nodename changed.
EXECUTE foo;
NOTICE: issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
DETAIL: on server postgres@127.0.0.1:57637 connectionId: xxxxxxx
count
---------------------------------------------------------------------
0
@ -500,8 +492,6 @@ UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid IN (:nodeid_1, :nodei
-- should error out
SELECT 1 FROM master_disable_node('localhost', 1);
ERROR: Disabling localhost:xxxxx failed
DETAIL: connection error: localhost:xxxxx
HINT: If you are using MX, try stop_metadata_sync_to_node(hostname, port) for nodes that are down before disabling them.
-- try again after stopping metadata sync
SELECT stop_metadata_sync_to_node('localhost', 1);
stop_metadata_sync_to_node
@ -566,8 +556,6 @@ UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid IN (:nodeid_1, :nodei
-- should error out
SELECT 1 FROM master_disable_node('localhost', :worker_2_port);
ERROR: Disabling localhost:xxxxx failed
DETAIL: connection error: localhost:xxxxx
HINT: If you are using MX, try stop_metadata_sync_to_node(hostname, port) for nodes that are down before disabling them.
-- try again after stopping metadata sync
SELECT stop_metadata_sync_to_node('localhost', 1);
stop_metadata_sync_to_node

View File

@ -1088,10 +1088,24 @@ END;
-- test adding an invalid node while we have reference tables to replicate
-- set client message level to ERROR and verbosity to terse to supporess
-- OS-dependent host name resolution warnings
SET client_min_messages to ERROR;
\set VERBOSITY terse
SELECT master_add_node('invalid-node-name', 9999);
ERROR: connection error: invalid-node-name:9999
SET client_min_messages to ERROR;
DO $$
DECLARE
errors_received INTEGER;
BEGIN
errors_received := 0;
BEGIN
SELECT master_add_node('invalid-node-name', 9999);
EXCEPTION WHEN OTHERS THEN
IF SQLERRM LIKE 'connection to the remote node%%' THEN
errors_received := errors_received + 1;
END IF;
END;
RAISE '(%/1) failed to add node', errors_received;
END;
$$;
ERROR: (1/1) failed to add node
-- drop unnecassary tables
DROP TABLE initially_not_replicated_reference_table;
-- reload pg_dist_shard_placement table

View File

@ -2442,8 +2442,7 @@ GRANT INSERT ON ALL TABLES IN SCHEMA public TO router_user;
-- we will fail to connect to worker 2, since the user does not exist
BEGIN;
INSERT INTO failure_test VALUES (1, 1);
WARNING: connection error: localhost:xxxxx
DETAIL: FATAL: role "router_user" does not exist
WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "router_user" does not exist
SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement
WHERE shardid IN (
SELECT shardid FROM pg_dist_shard
@ -2460,8 +2459,7 @@ SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement
ROLLBACK;
INSERT INTO failure_test VALUES (2, 1);
WARNING: connection error: localhost:xxxxx
DETAIL: FATAL: role "router_user" does not exist
WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "router_user" does not exist
SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement
WHERE shardid IN (
SELECT shardid FROM pg_dist_shard

View File

@ -46,8 +46,7 @@ show citus.node_conninfo;
-- Should give a connection error because of bad sslmode
select count(*) from test where a = 0;
ERROR: connection error: localhost:xxxxx
DETAIL: invalid sslmode value: "doesnotexist"
ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-- Reset it again
ALTER SYSTEM RESET citus.node_conninfo;
select pg_reload_conf();
@ -114,8 +113,7 @@ select count(*) from test where a = 0;
COMMIT;
-- Should fail now with connection error, when transaction is finished
select count(*) from test where a = 0;
ERROR: connection error: localhost:xxxxx
DETAIL: invalid sslmode value: "doesnotexist"
ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-- Reset it again
ALTER SYSTEM RESET citus.node_conninfo;
select pg_reload_conf();
@ -177,8 +175,7 @@ select count(*) from test where a = 0;
COMMIT;
-- Should fail now, when transaction is finished
select count(*) from test where a = 0;
ERROR: connection error: localhost:xxxxx
DETAIL: invalid sslmode value: "doesnotexist"
ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-- Reset it again
ALTER SYSTEM RESET citus.node_conninfo;
select pg_reload_conf();
@ -231,13 +228,11 @@ show citus.node_conninfo;
-- Should fail since a different shard is accessed and thus a new connection
-- will to be created.
select count(*) from test where a = 0;
ERROR: connection error: localhost:xxxxx
DETAIL: invalid sslmode value: "doesnotexist"
ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
COMMIT;
-- Should still fail now, when transaction is finished
select count(*) from test where a = 0;
ERROR: connection error: localhost:xxxxx
DETAIL: invalid sslmode value: "doesnotexist"
ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-- Reset it again
ALTER SYSTEM RESET citus.node_conninfo;
select pg_reload_conf();
@ -298,8 +293,7 @@ select count(*) from test;
COMMIT;
-- Should fail now, when transaction is finished
select count(*) from test;
ERROR: connection error: localhost:xxxxx
DETAIL: invalid sslmode value: "doesnotexist"
ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-- Reset it again
ALTER SYSTEM RESET citus.node_conninfo;
select pg_reload_conf();
@ -355,8 +349,7 @@ ERROR: division by zero
ROLLBACK;
-- Should fail now, when transaction is finished
select count(*) from test;
ERROR: connection error: localhost:xxxxx
DETAIL: invalid sslmode value: "doesnotexist"
ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-- Reset it again
ALTER SYSTEM RESET citus.node_conninfo;
select pg_reload_conf();

File diff suppressed because it is too large Load Diff

View File

@ -13,6 +13,8 @@ SET citus.shard_count TO 8;
SET citus.shard_replication_factor TO 1;
SET citus.replicate_reference_tables_on_activate TO off;
\set VERBOSITY terse
-- Simulates a readonly node by setting default_transaction_read_only.
CREATE FUNCTION mark_node_readonly(hostname TEXT, port INTEGER, isreadonly BOOLEAN)
RETURNS TEXT

View File

@ -669,10 +669,23 @@ END;
-- test adding an invalid node while we have reference tables to replicate
-- set client message level to ERROR and verbosity to terse to supporess
-- OS-dependent host name resolution warnings
SET client_min_messages to ERROR;
\set VERBOSITY terse
SELECT master_add_node('invalid-node-name', 9999);
SET client_min_messages to ERROR;
DO $$
DECLARE
errors_received INTEGER;
BEGIN
errors_received := 0;
BEGIN
SELECT master_add_node('invalid-node-name', 9999);
EXCEPTION WHEN OTHERS THEN
IF SQLERRM LIKE 'connection to the remote node%%' THEN
errors_received := errors_received + 1;
END IF;
END;
RAISE '(%/1) failed to add node', errors_received;
END;
$$;
-- drop unnecassary tables
DROP TABLE initially_not_replicated_reference_table;