Test columnar recovery (#4485)

DESCRIPTION: Add tests to verify crash recovery for columnar tables

Based on the Postgres TAP tooling we add a new test suite to the array of test suites for citus. It is modelled after `src/test/recovery` in the postgres project and takes the same place in our repository. It uses the perl modules defined in the postgres project to control the postgres nodes.

The test we add here focus on crash recovery. Our follower tests should cover the streaming replication behaviour.

It is hooked to our CI for both postgres 12 and postgres 13. We omit the recovery tests for postgres 11 as we do not have support for the columnar table access method.

.circleci/config.yml

@@ -310,6 +310,71 @@ jobs:
           flags: 'test_<< parameters.pg_major >>,<< parameters.make >>'
           when: always
 
+  tap-test-citus:
+    description: Runs tap tests for citus
+    parameters:
+      pg_major:
+        description: "postgres major version"
+        type: integer
+      image:
+        description: 'docker image to use as for the tests'
+        type: string
+        default: citus/exttester
+      image_tag:
+        description: 'docker image tag to use'
+        type: string
+      suite:
+        description: 'name of the tap test suite to run'
+        type: string
+      make:
+        description: "make target"
+        type: string
+        default: installcheck
+    docker:
+      - image: '<< parameters.image >>:<< parameters.image_tag >>'
+    working_directory: /home/circleci/project
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - run:
+          name: 'Install Extension'
+          command: |
+            tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-${PG_MAJOR}.tar" --directory /
+      - run:
+          name: 'Configure'
+          command: |
+            chown -R circleci .
+            gosu circleci ./configure
+      - run:
+          name: 'Enable core dumps'
+          command: |
+            ulimit -c unlimited
+      - run:
+          name: 'Run Test'
+          command: |
+            gosu circleci make -C src/test/<< parameters.suite >> << parameters.make >>
+          no_output_timeout: 2m
+      - run:
+          name: 'Copy coredumps'
+          command: |
+            mkdir -p /tmp/core_dumps
+            if ls core.* 1> /dev/null 2>&1; then
+                cp core.* /tmp/core_dumps
+            fi
+          when: on_fail
+      - store_artifacts:
+          name: 'Save tap logs'
+          path: /home/circleci/project/src/test/recovery/tmp_check/log
+          when: on_fail
+      - store_artifacts:
+          name: 'Save core dumps'
+          path: /tmp/core_dumps
+          when: on_fail
+      - codecov/upload:
+          flags: 'test_<< parameters.pg_major >>,tap_<< parameters.suite >>_<< parameters.make >>'
+          when: always
+
   check-merge-to-enterprise:
     docker:
       - image: citus/extbuilder:13.0
@@ -481,6 +546,12 @@ workflows:
           image_tag: '12.4'
           make: check-columnar-isolation
           requires: [build-12]
+      - tap-test-citus:
+          name: 'test_12_tap-recovery'
+          pg_major: 12
+          image_tag: '12.4'
+          suite: recovery
+          requires: [build-12]
       - test-citus:
           name: 'test-12_check-failure'
           pg_major: 12
@@ -543,6 +614,12 @@ workflows:
           image_tag: '13.0'
           make: check-columnar-isolation
           requires: [build-13]
+      - tap-test-citus:
+          name: 'test_13_tap-recovery'
+          pg_major: 13
+          image_tag: '13.0'
+          suite: recovery
+          requires: [build-13]
       - test-citus:
           name: 'test-13_check-failure'
           pg_major: 13

src/test/recovery/.gitignore

@@ -0,0 +1,2 @@
+# Generated by test suite
+/tmp_check/

src/test/recovery/Makefile

@@ -0,0 +1,39 @@
+#-------------------------------------------------------------------------
+#
+# Makefile for src/test/recovery
+#
+# Losely based on the makefile found in postgres' src/test/recovery.
+# We need to define our own invocation of prove to pass the correct path
+# to pg_regress and include citus in the shared preload libraries.
+#
+#-------------------------------------------------------------------------
+
+subdir = src/test/recovery
+top_builddir = ../../..
+include $(top_builddir)/Makefile.global
+
+# copied from pgxs/Makefile.global to use postgres' abs build dir for pg_regress
+ifeq ($(enable_tap_tests),yes)
+
+define citus_prove_installcheck
+rm -rf '$(CURDIR)'/tmp_check
+$(MKDIR_P) '$(CURDIR)'/tmp_check
+cd $(srcdir) && \
+TESTDIR='$(CURDIR)' \
+PATH="$(bindir):$$PATH" \
+PGPORT='6$(DEF_PGPORT)' \
+top_builddir='$(CURDIR)/$(top_builddir)' \
+PG_REGRESS='$(pgxsdir)/src/test/regress/pg_regress' \
+TEMP_CONFIG='$(CURDIR)'/postgresql.conf \
+$(PROVE) $(PG_PROVE_FLAGS) $(PROVE_FLAGS) $(if $(PROVE_TESTS),$(PROVE_TESTS),t/*.pl)
+endef
+
+else
+citus_prove_installcheck = @echo "TAP tests not enabled when postgres was compiled"
+endif
+
+installcheck:
+	$(citus_prove_installcheck)
+
+clean distclean maintainer-clean:
+	rm -rf tmp_check

src/test/recovery/postgresql.conf

@@ -0,0 +1 @@
+shared_preload_libraries=citus

src/test/recovery/t/001_columnar_crash_recovery.pl

@@ -0,0 +1,98 @@
+# Minimal test testing streaming replication
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 6;
+
+# Initialize single node
+my $node_one = get_new_node('node_one');
+$node_one->init();
+$node_one->start;
+
+# initialize the citus extension
+$node_one->safe_psql('postgres', "CREATE EXTENSION citus;");
+
+# create columnar table and insert simple data to verify the data survives a crash
+$node_one->safe_psql('postgres', "
+BEGIN;
+CREATE TABLE t1 (a int, b text) USING columnar;
+INSERT INTO t1 SELECT a, 'hello world ' || a FROM generate_series(1,1002) AS a;
+COMMIT;
+");
+
+# simulate crash
+$node_one->stop('immediate');
+$node_one->start;
+
+my $result = $node_one->safe_psql('postgres', "SELECT count(*) FROM t1;");
+print "node one count: $result\n";
+is($result, qq(1002), 'columnar recovered data from before crash');
+
+
+# truncate the table to verify the truncation survives a crash
+$node_one->safe_psql('postgres', "
+TRUNCATE t1;
+");
+
+# simulate crash
+$node_one->stop('immediate');
+$node_one->start;
+
+$result = $node_one->safe_psql('postgres', "SELECT count(*) FROM t1;");
+print "node one count: $result\n";
+is($result, qq(0), 'columnar recovered truncation');
+
+# test crashing while having an open transaction
+$node_one->safe_psql('postgres', "
+BEGIN;
+INSERT INTO t1 SELECT a, 'hello world ' || a FROM generate_series(1,1003) AS a;
+");
+
+# simulate crash
+$node_one->stop('immediate');
+$node_one->start;
+
+$result = $node_one->safe_psql('postgres', "SELECT count(*) FROM t1;");
+print "node one count: $result\n";
+is($result, qq(0), 'columnar crash during uncommitted transaction');
+
+# test crashing while having a prepared transaction
+$node_one->safe_psql('postgres', "
+BEGIN;
+INSERT INTO t1 SELECT a, 'hello world ' || a FROM generate_series(1,1004) AS a;
+PREPARE TRANSACTION 'prepared_xact_crash';
+");
+
+# simulate crash
+$node_one->stop('immediate');
+$node_one->start;
+
+$result = $node_one->safe_psql('postgres', "SELECT count(*) FROM t1;");
+print "node one count: $result\n";
+is($result, qq(0), 'columnar crash during prepared transaction (before commit)');
+
+$node_one->safe_psql('postgres', "
+COMMIT PREPARED 'prepared_xact_crash';
+");
+
+$result = $node_one->safe_psql('postgres', "SELECT count(*) FROM t1;");
+print "node one count: $result\n";
+is($result, qq(1004), 'columnar crash during prepared transaction (after commit)');
+
+# test crash recovery with copied data
+$node_one->safe_psql('postgres', "
+\\copy t1 FROM stdin delimiter ','
+1,a
+2,b
+3,c
+\\.
+");
+
+# simulate crash
+$node_one->stop('immediate');
+$node_one->start;
+
+$result = $node_one->safe_psql('postgres', "SELECT count(*) FROM t1;");
+print "node one count: $result\n";
+is($result, qq(1007), 'columnar crash after copy command');