mirror of https://github.com/citusdata/citus.git
Makefile changes to build CDC in builddir for pgoutput and wal2json. (#6827)
DESCRIPTION: Makefile changes to build different versions of CDC decoder for different base decoders like pgoutput and wal2json with the same name and copy it to $packagelib/cdc_decoders dir. This helps the user to use logical replication slots normally with pgoutput without being aware of CDC decoder. 1) Changed src/backend/distributed/cdc/Makefile to setup a build directory for CDC in build-cdc-$(DECODER) dir and copy the source files (.c.h and Makefile.decoder) to the build dir and build it for each base decoder. 2) copy the pgoutput.so and wal2json.so into the above build dir and install them in PG packagelibdir/citus_decoders directory. 3)Added a testcase 016_cdc_wal2json.pl for testing the wal2json decoder using pg_recv_logical_changes function.pull/6841/head
rajeshkt78
GitHub
parent
84f2d8685a
commit
29c8d9633a
|
|
@ -6,7 +6,7 @@ orbs:
|
|||
parameters:
|
||||
image_suffix:
|
||||
type: string
|
||||
default: '-v087ecd7'
|
||||
default: '-v3417e8d'
|
||||
pg13_version:
|
||||
type: string
|
||||
default: '13.10'
|
||||
|
|
@ -490,10 +490,6 @@ jobs:
|
|||
pg_major: << parameters.pg_major >>
|
||||
- configure
|
||||
- enable_core
|
||||
- run:
|
||||
name: 'Install DBI.pm'
|
||||
command: |
|
||||
apt-get update && apt-get install libdbi-perl && apt-get install libdbd-pg-perl
|
||||
- run:
|
||||
name: 'Run Test'
|
||||
command: |
|
||||
|
|
|
|||
|
|
@ -37,8 +37,7 @@ OBJS += \
|
|||
all: cdc
|
||||
|
||||
cdc:
|
||||
echo "running cdc make"
|
||||
$(MAKE) DECODER=pgoutput -C cdc all
|
||||
$(MAKE) -C cdc all
|
||||
|
||||
NO_PGXS = 1
|
||||
|
||||
|
|
@ -85,12 +84,13 @@ ifneq (,$(SQL_Po_files))
|
|||
include $(SQL_Po_files)
|
||||
endif
|
||||
|
||||
.PHONY: clean-full install install-downgrades install-all
|
||||
|
||||
.PHONY: clean-full install install-downgrades install-all install-cdc clean-cdc
|
||||
|
||||
clean: clean-cdc
|
||||
|
||||
clean-cdc:
|
||||
$(MAKE) DECODER=pgoutput -C cdc clean
|
||||
$(MAKE) -C cdc clean
|
||||
|
||||
cleanup-before-install:
|
||||
rm -f $(DESTDIR)$(datadir)/$(datamoduledir)/citus.control
|
||||
|
|
@ -99,7 +99,7 @@ cleanup-before-install:
|
|||
install: cleanup-before-install install-cdc
|
||||
|
||||
install-cdc:
|
||||
$(MAKE) DECODER=pgoutput -C cdc install
|
||||
$(MAKE) -C cdc install
|
||||
|
||||
# install and install-downgrades should be run sequentially
|
||||
install-all: install
|
||||
|
|
|
|||
|
|
@ -1,26 +1,45 @@
|
|||
ifndef DECODER
|
||||
DECODER = pgoutput
|
||||
endif
|
||||
|
||||
MODULE_big = citus_$(DECODER)
|
||||
citus_subdir = src/backend/distributed/cdc
|
||||
citus_top_builddir = ../../../..
|
||||
citus_decoders_dir = $(DESTDIR)$(pkglibdir)/citus_decoders
|
||||
|
||||
OBJS += cdc_decoder.o cdc_decoder_utils.o
|
||||
|
||||
include $(citus_top_builddir)/Makefile.global
|
||||
|
||||
override CFLAGS += -DDECODER=\"$(DECODER)\" -I$(citus_abs_top_srcdir)/include
|
||||
override CPPFLAGS += -DDECODER=\"$(DECODER)\" -I$(citus_abs_top_srcdir)/include
|
||||
citus_subdir = src/backend/distributed/cdc
|
||||
SRC_DIR = $(citus_abs_top_srcdir)/$(citus_subdir)
|
||||
|
||||
install: install-cdc
|
||||
#List of supported based decoders. Add new decoders here.
|
||||
cdc_base_decoders :=pgoutput wal2json
|
||||
|
||||
clean: clean-cdc
|
||||
all: build-cdc-decoders
|
||||
|
||||
install-cdc:
|
||||
mkdir -p '$(citus_decoders_dir)'
|
||||
$(INSTALL_SHLIB) citus_$(DECODER).so '$(citus_decoders_dir)/$(DECODER).so'
|
||||
copy-decoder-files-to-build-dir:
|
||||
$(eval DECODER_BUILD_DIR=build-cdc-$(DECODER))
|
||||
mkdir -p $(DECODER_BUILD_DIR)
|
||||
@for file in $(SRC_DIR)/*.c $(SRC_DIR)/*.h; do \
|
||||
if [ -f $$file ]; then \
|
||||
if [ -f $(DECODER_BUILD_DIR)/$$(basename $$file) ]; then \
|
||||
if ! diff -q $$file $(DECODER_BUILD_DIR)/$$(basename $$file); then \
|
||||
cp $$file $(DECODER_BUILD_DIR)/$$(basename $$file); \
|
||||
fi \
|
||||
else \
|
||||
cp $$file $(DECODER_BUILD_DIR)/$$(basename $$file); \
|
||||
fi \
|
||||
fi \
|
||||
done
|
||||
cp $(SRC_DIR)/Makefile.decoder $(DECODER_BUILD_DIR)/Makefile
|
||||
|
||||
build-cdc-decoders:
|
||||
$(foreach base_decoder,$(cdc_base_decoders),$(MAKE) DECODER=$(base_decoder) build-cdc-decoder;)
|
||||
|
||||
install-cdc-decoders:
|
||||
$(foreach base_decoder,$(cdc_base_decoders),$(MAKE) DECODER=$(base_decoder) -C build-cdc-$(base_decoder) install;)
|
||||
|
||||
clean-cdc-decoders:
|
||||
$(foreach base_decoder,$(cdc_base_decoders),rm -rf build-cdc-$(base_decoder);)
|
||||
|
||||
|
||||
build-cdc-decoder:
|
||||
$(MAKE) DECODER=$(DECODER) copy-decoder-files-to-build-dir
|
||||
$(MAKE) DECODER=$(DECODER) -C build-cdc-$(DECODER)
|
||||
|
||||
install: install-cdc-decoders
|
||||
|
||||
clean: clean-cdc-decoders
|
||||
|
||||
clean-cdc:
|
||||
rm -f '$(DESTDIR)$(datadir)/$(datamoduledir)/citus_decoders/$(DECODER).so'
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
MODULE_big = citus_$(DECODER)
|
||||
|
||||
citus_decoders_dir = $(DESTDIR)$(pkglibdir)/citus_decoders
|
||||
|
||||
citus_top_builddir = ../../../../..
|
||||
citus_subdir = src/backend/distributed/cdc/cdc_$(DECODER)
|
||||
|
||||
OBJS += cdc_decoder.o cdc_decoder_utils.o
|
||||
|
||||
include $(citus_top_builddir)/Makefile.global
|
||||
|
||||
override CFLAGS += -DDECODER=\"$(DECODER)\" -I$(citus_abs_top_srcdir)/include
|
||||
override CPPFLAGS += -DDECODER=\"$(DECODER)\" -I$(citus_abs_top_srcdir)/include
|
||||
|
||||
install: install-cdc
|
||||
|
||||
clean: clean-cdc
|
||||
|
||||
install-cdc:
|
||||
mkdir -p '$(citus_decoders_dir)'
|
||||
$(INSTALL_SHLIB) citus_$(DECODER).so '$(citus_decoders_dir)/$(DECODER).so'
|
||||
|
||||
clean-cdc:
|
||||
rm -f '$(DESTDIR)$(datadir)/$(datamoduledir)/citus_decoders/$(DECODER).so'
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
# Schema change CDC test for Citus
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use Test::More;
|
||||
|
||||
use lib './t';
|
||||
use cdctestlib;
|
||||
|
||||
use threads;
|
||||
|
||||
# Initialize co-ordinator node
|
||||
my $select_stmt = qq(SELECT * FROM data_100008 ORDER BY id;);
|
||||
my $result = 0;
|
||||
|
||||
### Create the citus cluster with coordinator and two worker nodes
|
||||
our ($node_coordinator, @workers) = create_citus_cluster(1,"localhost",57636);
|
||||
|
||||
print("coordinator port: " . $node_coordinator->port() . "\n");
|
||||
print("worker0 port:" . $workers[0]->port() . "\n");
|
||||
|
||||
my $initial_schema = "
|
||||
CREATE TABLE data_100008(
|
||||
id integer,
|
||||
data text,
|
||||
PRIMARY KEY (data));";
|
||||
|
||||
$node_coordinator->safe_psql('postgres',$initial_schema);
|
||||
$node_coordinator->safe_psql('postgres','ALTER TABLE data_100008 REPLICA IDENTITY FULL;');
|
||||
$node_coordinator->safe_psql('postgres',"SELECT pg_catalog.pg_create_logical_replication_slot('cdc_replication_slot','wal2json');");
|
||||
|
||||
#insert data into the data_100008 table in the coordinator node before distributing the table.
|
||||
$node_coordinator->safe_psql('postgres',"
|
||||
INSERT INTO data_100008
|
||||
SELECT i, 'my test data ' || i
|
||||
FROM generate_series(-1,1)i;");
|
||||
|
||||
my $output = $node_coordinator->safe_psql('postgres',"SELECT * FROM pg_logical_slot_get_changes('cdc_replication_slot', NULL, NULL);");
|
||||
print($output);
|
||||
|
||||
my $change_string_expected = '[0,"my test data 0"]';
|
||||
if ($output =~ /$change_string_expected/) {
|
||||
$result = 1;
|
||||
} else {
|
||||
$result = 0;
|
||||
}
|
||||
|
||||
is($result, 1, 'CDC create_distributed_table - wal2json test');
|
||||
$node_coordinator->safe_psql('postgres',"SELECT pg_drop_replication_slot('cdc_replication_slot');");
|
||||
|
||||
done_testing();
|
||||
Loading…
Reference in New Issue