diff --git a/.circleci/config.yml b/.circleci/config.yml index a286b7ba7..d8c3e81cf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -372,7 +372,7 @@ jobs: when: on_fail - store_artifacts: name: 'Save tap logs' - path: /home/circleci/project/src/test/recovery/tmp_check/log + path: /home/circleci/project/src/test/<< parameters.suite >>/tmp_check/log when: on_fail - store_artifacts: name: 'Save core dumps' @@ -505,6 +505,12 @@ workflows: image_tag: '12.6' suite: recovery requires: [build-12] + - tap-test-citus: + name: 'test-12_tap-columnar-freezing' + pg_major: 12 + image_tag: '12.6' + suite: columnar_freezing + requires: [build-12] - test-citus: name: 'test-12_check-failure' pg_major: 12 @@ -573,6 +579,12 @@ workflows: image_tag: '13.2' suite: recovery requires: [build-13] + - tap-test-citus: + name: 'test-13_tap-columnar-freezing' + pg_major: 13 + image_tag: '13.2' + suite: columnar_freezing + requires: [build-13] - test-citus: name: 'test-13_check-failure' pg_major: 13 diff --git a/src/backend/columnar/columnar_tableam.c b/src/backend/columnar/columnar_tableam.c index 086b565eb..578322089 100644 --- a/src/backend/columnar/columnar_tableam.c +++ b/src/backend/columnar/columnar_tableam.c @@ -674,6 +674,27 @@ NeededColumnsList(TupleDesc tupdesc, Bitmapset *attr_needed) } +/* + * ColumnarTableTupleCount returns the number of tuples that columnar + * table with relationId has by using stripe metadata. + */ +static uint64 +ColumnarTableTupleCount(Relation relation) +{ + List *stripeList = StripesForRelfilenode(relation->rd_node); + uint64 tupleCount = 0; + + ListCell *lc = NULL; + foreach(lc, stripeList) + { + StripeMetadata *stripe = lfirst(lc); + tupleCount += stripe->rowCount; + } + + return tupleCount; +} + + /* * columnar_vacuum_rel implements VACUUM without FULL option. */ @@ -690,6 +711,9 @@ columnar_vacuum_rel(Relation rel, VacuumParams *params, return; } + pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, + RelationGetRelid(rel)); + int elevel = (params->options & VACOPT_VERBOSE) ? INFO : DEBUG2; /* this should have been resolved by vacuum.c until now */ @@ -705,6 +729,52 @@ columnar_vacuum_rel(Relation rel, VacuumParams *params, { TruncateColumnar(rel, elevel); } + + RelationOpenSmgr(rel); + BlockNumber new_rel_pages = smgrnblocks(rel->rd_smgr, MAIN_FORKNUM); + + /* get the number of indexes */ + List *indexList = RelationGetIndexList(rel); + int nindexes = list_length(indexList); + + TransactionId oldestXmin; + TransactionId freezeLimit; + MultiXactId multiXactCutoff; + + /* initialize xids */ + TransactionId xidFullScanLimit; + MultiXactId mxactFullScanLimit; + vacuum_set_xid_limits(rel, + params->freeze_min_age, + params->freeze_table_age, + params->multixact_freeze_min_age, + params->multixact_freeze_table_age, + &oldestXmin, &freezeLimit, &xidFullScanLimit, + &multiXactCutoff, &mxactFullScanLimit); + + Assert(TransactionIdPrecedesOrEquals(freezeLimit, oldestXmin)); + + /* + * Columnar storage doesn't hold any transaction IDs, so we can always + * just advance to the most aggressive value. + */ + TransactionId newRelFrozenXid = oldestXmin; + MultiXactId newRelminMxid = multiXactCutoff; + + double new_live_tuples = ColumnarTableTupleCount(rel); + + /* all visible pages are always 0 */ + BlockNumber new_rel_allvisible = 0; + + vac_update_relstats(rel, new_rel_pages, new_live_tuples, + new_rel_allvisible, nindexes > 0, + newRelFrozenXid, newRelminMxid, false); + + pgstat_report_vacuum(RelationGetRelid(rel), + rel->rd_rel->relisshared, + Max(new_live_tuples, 0), + 0); + pgstat_progress_end_command(); } diff --git a/src/test/columnar_freezing/Makefile b/src/test/columnar_freezing/Makefile new file mode 100644 index 000000000..565e204fd --- /dev/null +++ b/src/test/columnar_freezing/Makefile @@ -0,0 +1,37 @@ +#------------------------------------------------------------------------- +# +# Makefile for src/test/columnar_freezing +# +# Test that columnar freezing works. +# +#------------------------------------------------------------------------- + +subdir = src/test/columnar_freezing +top_builddir = ../../.. +include $(top_builddir)/Makefile.global + +# copied from pgxs/Makefile.global to use postgres' abs build dir for pg_regress +ifeq ($(enable_tap_tests),yes) + +define citus_prove_installcheck +rm -rf '$(CURDIR)'/tmp_check +$(MKDIR_P) '$(CURDIR)'/tmp_check +cd $(srcdir) && \ +TESTDIR='$(CURDIR)' \ +PATH="$(bindir):$$PATH" \ +PGPORT='6$(DEF_PGPORT)' \ +top_builddir='$(CURDIR)/$(top_builddir)' \ +PG_REGRESS='$(pgxsdir)/src/test/regress/pg_regress' \ +TEMP_CONFIG='$(CURDIR)'/postgresql.conf \ +$(PROVE) $(PG_PROVE_FLAGS) $(PROVE_FLAGS) $(if $(PROVE_TESTS),$(PROVE_TESTS),t/*.pl) +endef + +else +citus_prove_installcheck = @echo "TAP tests not enabled when postgres was compiled" +endif + +installcheck: + $(citus_prove_installcheck) + +clean distclean maintainer-clean: + rm -rf tmp_check diff --git a/src/test/columnar_freezing/postgresql.conf b/src/test/columnar_freezing/postgresql.conf new file mode 100644 index 000000000..39521cc33 --- /dev/null +++ b/src/test/columnar_freezing/postgresql.conf @@ -0,0 +1,7 @@ +shared_preload_libraries=citus +shared_preload_libraries='citus' +vacuum_freeze_min_age = 50000 +vacuum_freeze_table_age = 50000 +synchronous_commit = off +fsync = off + diff --git a/src/test/columnar_freezing/t/001_columnar_freezing.pl b/src/test/columnar_freezing/t/001_columnar_freezing.pl new file mode 100644 index 000000000..1985da2a5 --- /dev/null +++ b/src/test/columnar_freezing/t/001_columnar_freezing.pl @@ -0,0 +1,52 @@ +# Minimal test testing streaming replication +use strict; +use warnings; +use PostgresNode; +use TestLib; +use Test::More tests => 2; + +# Initialize single node +my $node_one = get_new_node('node_one'); +$node_one->init(); +$node_one->start; + +# initialize the citus extension +$node_one->safe_psql('postgres', "CREATE EXTENSION citus;"); + +# create columnar table and insert simple data to verify the data survives a crash +$node_one->safe_psql('postgres', " +CREATE TABLE test_row(i int); +INSERT INTO test_row VALUES (1); +CREATE TABLE test_columnar_freeze(i int) USING columnar WITH(autovacuum_enabled=false); +INSERT INTO test_columnar_freeze VALUES (1); +"); + +my $ten_thousand_updates = ""; + +foreach (1..10000) { + $ten_thousand_updates .= "UPDATE test_row SET i = i + 1;\n"; +} + +# 70K updates +foreach (1..7) { + $node_one->safe_psql('postgres', $ten_thousand_updates); +} + +my $result = $node_one->safe_psql('postgres', " +select age(relfrozenxid) < 70000 as was_frozen + from pg_class where relname='test_columnar_freeze'; +"); +print "node one count: $result\n"; +is($result, qq(f), 'columnar table was not frozen'); + +$node_one->safe_psql('postgres', 'VACUUM FREEZE test_columnar_freeze;'); + +$result = $node_one->safe_psql('postgres', " +select age(relfrozenxid) < 70000 as was_frozen + from pg_class where relname='test_columnar_freeze'; +"); +print "node one count: $result\n"; +is($result, qq(t), 'columnar table was frozen'); + +$node_one->stop('fast'); +