external
/
postgres-decoderbufs
mirror of https://github.com/debezium/postgres-decoderbufs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Don't use the WARNING log level for situations that can occur perfectly normally, 

as WARNING messages are transmitted all the way to the client (i.e. are costly).
In my case I have lots of tables with types that aren't yet understood by
decoderbufs.  These tables aren't being monitored, but because logical decoding
works on the level of the entire database, not just the tables monitored, rows
from them are still reaching decoderbufs, generating vast numbers of warnings
  "Encountered unknown typid..."
Likewise, I have many unmonitored materialized views.  Every row deleted when
doing a concurrent refresh generates the warning
  "...no PK is present..."
These warnings cause two types of trouble when they hit the debezium postgres
connector: a memory leak and increased CPU usage.  The memory leak is because
the JDBC layer carefully preserves these warnings in a list, and the list is
never being cleared (this should be fixed in the connector).  As for the CPU
usage, I don't fully understand why the processing time increases so much.
It's normal that it should double, as there are twice as many packets to
process (one warning for every row), but it seems to take way more than twice
as long.
pull/7/head
Duncan Sands 2017-06-02 21:56:12 +02:00
parent 8a6a8305c3
commit e248eda6a1
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/decoderbufs.c
View File

@ -529,7 +529,7 @@ static void set_datum_value(Decoderbufs__DatumMessage *datum_msg, Oid typid,
datum_msg->datum_case = DECODERBUFS__DATUM_MESSAGE__DATUM_DATUM_POINT; datum_msg->datum_case = DECODERBUFS__DATUM_MESSAGE__DATUM_DATUM_POINT;
} else { } else {
int len; int len;
elog(WARNING, "Encountered unknown typid: %d, using bytes", typid); elog(DEBUG1, "Encountered unknown typid: %d, using bytes", typid);
output = OidOutputFunctionCall(typoutput, datum); output = OidOutputFunctionCall(typoutput, datum);
len = strlen(output); len = strlen(output);
size = sizeof(char) * len; size = sizeof(char) * len;
@ -601,7 +601,7 @@ static void tuple_to_tuple_msg(Decoderbufs__DatumMessage **tmsg,
if (!isnull) { if (!isnull) {
if (typisvarlena && VARATT_IS_EXTERNAL_ONDISK(origval)) { if (typisvarlena && VARATT_IS_EXTERNAL_ONDISK(origval)) {
// TODO: Is there a way we can handle this? // TODO: Is there a way we can handle this?
elog(WARNING, "Not handling external on disk varlena at the moment."); elog(DEBUG1, "Not handling external on disk varlena at the moment.");
} else if (!typisvarlena) { } else if (!typisvarlena) {
set_datum_value(&datum_msg, attr->atttypid, typoutput, origval); set_datum_value(&datum_msg, attr->atttypid, typoutput, origval);
} else { } else {
@ -713,7 +713,7 @@ static void pg_decode_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn,
tuple_to_tuple_msg(rmsg.old_tuple, relation, tuple_to_tuple_msg(rmsg.old_tuple, relation,
&change->data.tp.oldtuple->tuple, tupdesc); &change->data.tp.oldtuple->tuple, tupdesc);
} else { } else {
elog(WARNING, "no information to decode from DELETE because either no PK is present or REPLICA IDENTITY NOTHING or invalid "); elog(DEBUG1, "no information to decode from DELETE because either no PK is present or REPLICA IDENTITY NOTHING or invalid ");
} }
break; break;
default: default: