diff --git a/.circleci/config.yml b/.circleci/config.yml index d5eadd94f..225195d27 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ orbs: parameters: image_suffix: type: string - default: '-v087ecd7' + default: '-v3417e8d' pg13_version: type: string default: '13.10' @@ -490,10 +490,6 @@ jobs: pg_major: << parameters.pg_major >> - configure - enable_core - - run: - name: 'Install DBI.pm' - command: | - apt-get update && apt-get install libdbi-perl && apt-get install libdbd-pg-perl - run: name: 'Run Test' command: | diff --git a/src/backend/distributed/Makefile b/src/backend/distributed/Makefile index 1cefb5769..8f28b04b0 100644 --- a/src/backend/distributed/Makefile +++ b/src/backend/distributed/Makefile @@ -37,8 +37,7 @@ OBJS += \ all: cdc cdc: - echo "running cdc make" - $(MAKE) DECODER=pgoutput -C cdc all + $(MAKE) -C cdc all NO_PGXS = 1 @@ -85,12 +84,13 @@ ifneq (,$(SQL_Po_files)) include $(SQL_Po_files) endif -.PHONY: clean-full install install-downgrades install-all + +.PHONY: clean-full install install-downgrades install-all install-cdc clean-cdc clean: clean-cdc clean-cdc: - $(MAKE) DECODER=pgoutput -C cdc clean + $(MAKE) -C cdc clean cleanup-before-install: rm -f $(DESTDIR)$(datadir)/$(datamoduledir)/citus.control @@ -99,7 +99,7 @@ cleanup-before-install: install: cleanup-before-install install-cdc install-cdc: - $(MAKE) DECODER=pgoutput -C cdc install + $(MAKE) -C cdc install # install and install-downgrades should be run sequentially install-all: install diff --git a/src/backend/distributed/cdc/Makefile b/src/backend/distributed/cdc/Makefile index 76aa28726..8a7449476 100644 --- a/src/backend/distributed/cdc/Makefile +++ b/src/backend/distributed/cdc/Makefile @@ -1,26 +1,45 @@ -ifndef DECODER - DECODER = pgoutput -endif - -MODULE_big = citus_$(DECODER) -citus_subdir = src/backend/distributed/cdc citus_top_builddir = ../../../.. -citus_decoders_dir = $(DESTDIR)$(pkglibdir)/citus_decoders - -OBJS += cdc_decoder.o cdc_decoder_utils.o - include $(citus_top_builddir)/Makefile.global -override CFLAGS += -DDECODER=\"$(DECODER)\" -I$(citus_abs_top_srcdir)/include -override CPPFLAGS += -DDECODER=\"$(DECODER)\" -I$(citus_abs_top_srcdir)/include +citus_subdir = src/backend/distributed/cdc +SRC_DIR = $(citus_abs_top_srcdir)/$(citus_subdir) -install: install-cdc +#List of supported based decoders. Add new decoders here. +cdc_base_decoders :=pgoutput wal2json -clean: clean-cdc +all: build-cdc-decoders -install-cdc: - mkdir -p '$(citus_decoders_dir)' - $(INSTALL_SHLIB) citus_$(DECODER).so '$(citus_decoders_dir)/$(DECODER).so' +copy-decoder-files-to-build-dir: + $(eval DECODER_BUILD_DIR=build-cdc-$(DECODER)) + mkdir -p $(DECODER_BUILD_DIR) + @for file in $(SRC_DIR)/*.c $(SRC_DIR)/*.h; do \ + if [ -f $$file ]; then \ + if [ -f $(DECODER_BUILD_DIR)/$$(basename $$file) ]; then \ + if ! diff -q $$file $(DECODER_BUILD_DIR)/$$(basename $$file); then \ + cp $$file $(DECODER_BUILD_DIR)/$$(basename $$file); \ + fi \ + else \ + cp $$file $(DECODER_BUILD_DIR)/$$(basename $$file); \ + fi \ + fi \ + done + cp $(SRC_DIR)/Makefile.decoder $(DECODER_BUILD_DIR)/Makefile + +build-cdc-decoders: + $(foreach base_decoder,$(cdc_base_decoders),$(MAKE) DECODER=$(base_decoder) build-cdc-decoder;) + +install-cdc-decoders: + $(foreach base_decoder,$(cdc_base_decoders),$(MAKE) DECODER=$(base_decoder) -C build-cdc-$(base_decoder) install;) + +clean-cdc-decoders: + $(foreach base_decoder,$(cdc_base_decoders),rm -rf build-cdc-$(base_decoder);) + + +build-cdc-decoder: + $(MAKE) DECODER=$(DECODER) copy-decoder-files-to-build-dir + $(MAKE) DECODER=$(DECODER) -C build-cdc-$(DECODER) + +install: install-cdc-decoders + +clean: clean-cdc-decoders -clean-cdc: - rm -f '$(DESTDIR)$(datadir)/$(datamoduledir)/citus_decoders/$(DECODER).so' diff --git a/src/backend/distributed/cdc/Makefile.decoder b/src/backend/distributed/cdc/Makefile.decoder new file mode 100644 index 000000000..963f8bf15 --- /dev/null +++ b/src/backend/distributed/cdc/Makefile.decoder @@ -0,0 +1,24 @@ +MODULE_big = citus_$(DECODER) + +citus_decoders_dir = $(DESTDIR)$(pkglibdir)/citus_decoders + +citus_top_builddir = ../../../../.. +citus_subdir = src/backend/distributed/cdc/cdc_$(DECODER) + +OBJS += cdc_decoder.o cdc_decoder_utils.o + +include $(citus_top_builddir)/Makefile.global + +override CFLAGS += -DDECODER=\"$(DECODER)\" -I$(citus_abs_top_srcdir)/include +override CPPFLAGS += -DDECODER=\"$(DECODER)\" -I$(citus_abs_top_srcdir)/include + +install: install-cdc + +clean: clean-cdc + +install-cdc: + mkdir -p '$(citus_decoders_dir)' + $(INSTALL_SHLIB) citus_$(DECODER).so '$(citus_decoders_dir)/$(DECODER).so' + +clean-cdc: + rm -f '$(DESTDIR)$(datadir)/$(datamoduledir)/citus_decoders/$(DECODER).so' diff --git a/src/test/cdc/t/016_cdc_wal2json.pl b/src/test/cdc/t/016_cdc_wal2json.pl new file mode 100644 index 000000000..ab384df64 --- /dev/null +++ b/src/test/cdc/t/016_cdc_wal2json.pl @@ -0,0 +1,51 @@ +# Schema change CDC test for Citus +use strict; +use warnings; + +use Test::More; + +use lib './t'; +use cdctestlib; + +use threads; + +# Initialize co-ordinator node +my $select_stmt = qq(SELECT * FROM data_100008 ORDER BY id;); +my $result = 0; + +### Create the citus cluster with coordinator and two worker nodes +our ($node_coordinator, @workers) = create_citus_cluster(1,"localhost",57636); + +print("coordinator port: " . $node_coordinator->port() . "\n"); +print("worker0 port:" . $workers[0]->port() . "\n"); + +my $initial_schema = " + CREATE TABLE data_100008( + id integer, + data text, + PRIMARY KEY (data));"; + +$node_coordinator->safe_psql('postgres',$initial_schema); +$node_coordinator->safe_psql('postgres','ALTER TABLE data_100008 REPLICA IDENTITY FULL;'); +$node_coordinator->safe_psql('postgres',"SELECT pg_catalog.pg_create_logical_replication_slot('cdc_replication_slot','wal2json');"); + +#insert data into the data_100008 table in the coordinator node before distributing the table. +$node_coordinator->safe_psql('postgres'," + INSERT INTO data_100008 + SELECT i, 'my test data ' || i + FROM generate_series(-1,1)i;"); + +my $output = $node_coordinator->safe_psql('postgres',"SELECT * FROM pg_logical_slot_get_changes('cdc_replication_slot', NULL, NULL);"); +print($output); + +my $change_string_expected = '[0,"my test data 0"]'; +if ($output =~ /$change_string_expected/) { + $result = 1; +} else { + $result = 0; +} + +is($result, 1, 'CDC create_distributed_table - wal2json test'); +$node_coordinator->safe_psql('postgres',"SELECT pg_drop_replication_slot('cdc_replication_slot');"); + +done_testing();