From 8cee2b092b168ede9b2e510e09200347b5155c7a Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Thu, 19 Nov 2020 17:21:07 -0800 Subject: [PATCH] remove columnar FDW code --- src/backend/columnar/cstore_fdw.c | 2199 ----------------- src/backend/columnar/mod.c | 6 - .../columnar/sql/columnar--9.5-1--10.0-1.sql | 28 - .../downgrades/columnar--10.0-1--9.5-1.sql | 9 - src/include/columnar/cstore_fdw.h | 35 - src/test/regress/Makefile | 4 - src/test/regress/columnar_am_schedule | 1 - .../regress/columnar_fdw_isolation_schedule | 1 - src/test/regress/columnar_fdw_schedule | 14 - src/test/regress/expected/am_functions.out | 18 - src/test/regress/expected/am_truncate_0.out | 6 - src/test/regress/expected/fdw_alter.out | 174 -- src/test/regress/expected/fdw_analyze.out | 19 - src/test/regress/expected/fdw_clean.out | 10 - src/test/regress/expected/fdw_drop.out | 58 - src/test/regress/expected/fdw_functions.out | 18 - src/test/regress/expected/fdw_insert.out | 88 - src/test/regress/expected/fdw_query.out | 105 - src/test/regress/expected/fdw_rollback.out | 77 - src/test/regress/expected/fdw_truncate.out | 265 -- src/test/regress/expected/fdw_truncate_0.out | 262 -- src/test/regress/expected/multi_extension.out | 8 +- .../regress/expected/multi_extension_0.out | 8 +- .../expected/upgrade_list_citus_objects.out | 8 +- .../expected/upgrade_list_citus_objects_0.out | 8 +- .../regress/input/fdw_block_filtering.source | 69 - src/test/regress/input/fdw_copyto.source | 17 - src/test/regress/input/fdw_create.source | 39 - src/test/regress/input/fdw_data_types.source | 68 - src/test/regress/input/fdw_load.source | 44 - .../regress/output/fdw_block_filtering.source | 116 - src/test/regress/output/fdw_copyto.source | 23 - src/test/regress/output/fdw_create.source | 42 - src/test/regress/output/fdw_data_types.source | 78 - src/test/regress/output/fdw_load.source | 39 - src/test/regress/sql/am_functions.sql | 20 - src/test/regress/sql/fdw_alter.sql | 85 - src/test/regress/sql/fdw_analyze.sql | 11 - src/test/regress/sql/fdw_clean.sql | 10 - src/test/regress/sql/fdw_drop.sql | 56 - src/test/regress/sql/fdw_functions.sql | 20 - src/test/regress/sql/fdw_insert.sql | 56 - src/test/regress/sql/fdw_query.sql | 34 - src/test/regress/sql/fdw_rollback.sql | 41 - src/test/regress/sql/fdw_truncate.sql | 135 - 45 files changed, 4 insertions(+), 4428 deletions(-) delete mode 100644 src/backend/columnar/cstore_fdw.c delete mode 100644 src/include/columnar/cstore_fdw.h delete mode 100644 src/test/regress/columnar_fdw_isolation_schedule delete mode 100644 src/test/regress/columnar_fdw_schedule delete mode 100644 src/test/regress/expected/am_functions.out delete mode 100644 src/test/regress/expected/fdw_alter.out delete mode 100644 src/test/regress/expected/fdw_analyze.out delete mode 100644 src/test/regress/expected/fdw_clean.out delete mode 100644 src/test/regress/expected/fdw_drop.out delete mode 100644 src/test/regress/expected/fdw_functions.out delete mode 100644 src/test/regress/expected/fdw_insert.out delete mode 100644 src/test/regress/expected/fdw_query.out delete mode 100644 src/test/regress/expected/fdw_rollback.out delete mode 100644 src/test/regress/expected/fdw_truncate.out delete mode 100644 src/test/regress/expected/fdw_truncate_0.out delete mode 100644 src/test/regress/input/fdw_block_filtering.source delete mode 100644 src/test/regress/input/fdw_copyto.source delete mode 100644 src/test/regress/input/fdw_create.source delete mode 100644 src/test/regress/input/fdw_data_types.source delete mode 100644 src/test/regress/input/fdw_load.source delete mode 100644 src/test/regress/output/fdw_block_filtering.source delete mode 100644 src/test/regress/output/fdw_copyto.source delete mode 100644 src/test/regress/output/fdw_create.source delete mode 100644 src/test/regress/output/fdw_data_types.source delete mode 100644 src/test/regress/output/fdw_load.source delete mode 100644 src/test/regress/sql/am_functions.sql delete mode 100644 src/test/regress/sql/fdw_alter.sql delete mode 100644 src/test/regress/sql/fdw_analyze.sql delete mode 100644 src/test/regress/sql/fdw_clean.sql delete mode 100644 src/test/regress/sql/fdw_drop.sql delete mode 100644 src/test/regress/sql/fdw_functions.sql delete mode 100644 src/test/regress/sql/fdw_insert.sql delete mode 100644 src/test/regress/sql/fdw_query.sql delete mode 100644 src/test/regress/sql/fdw_rollback.sql delete mode 100644 src/test/regress/sql/fdw_truncate.sql diff --git a/src/backend/columnar/cstore_fdw.c b/src/backend/columnar/cstore_fdw.c deleted file mode 100644 index 634a76f5d..000000000 --- a/src/backend/columnar/cstore_fdw.c +++ /dev/null @@ -1,2199 +0,0 @@ -/*------------------------------------------------------------------------- - * - * cstore_fdw.c - * - * This file contains the function definitions for scanning, analyzing, and - * copying into cstore_fdw foreign tables. Note that this file uses the API - * provided by cstore_reader and cstore_writer for reading and writing cstore - * files. - * - * Copyright (c) 2016, Citus Data, Inc. - * - * $Id$ - * - *------------------------------------------------------------------------- - */ - -#include "postgres.h" - -#include - -#include "access/heapam.h" -#include "access/reloptions.h" -#if PG_VERSION_NUM >= 130000 -#include "access/heaptoast.h" -#else -#include "access/tuptoaster.h" -#endif -#include "access/xact.h" -#include "catalog/catalog.h" -#include "catalog/indexing.h" -#include "catalog/namespace.h" -#include "catalog/objectaccess.h" -#include "catalog/pg_foreign_table.h" -#include "catalog/pg_namespace.h" -#include "catalog/storage.h" -#include "commands/copy.h" -#include "commands/dbcommands.h" -#include "commands/defrem.h" -#include "commands/event_trigger.h" -#include "commands/explain.h" -#include "commands/extension.h" -#include "commands/vacuum.h" -#include "foreign/fdwapi.h" -#include "foreign/foreign.h" -#include "miscadmin.h" -#include "nodes/makefuncs.h" -#if PG_VERSION_NUM < 120000 -#include "optimizer/cost.h" -#endif -#include "optimizer/pathnode.h" -#include "optimizer/planmain.h" -#include "optimizer/restrictinfo.h" -#if PG_VERSION_NUM >= 120000 -#include "access/heapam.h" -#include "optimizer/optimizer.h" -#else -#include "optimizer/var.h" -#endif -#include "parser/parser.h" -#include "parser/parse_coerce.h" -#include "parser/parse_type.h" -#include "storage/lmgr.h" -#include "storage/smgr.h" -#include "tcop/utility.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/lsyscache.h" -#if PG_VERSION_NUM < 120000 -#include "utils/rel.h" -#endif -#if PG_VERSION_NUM >= 120000 -#include "utils/snapmgr.h" -#else -#include "utils/tqual.h" -#endif -#include "utils/syscache.h" - -#include "columnar/cstore.h" -#include "columnar/cstore_fdw.h" -#include "columnar/cstore_version_compat.h" -#include "distributed/citus_safe_lib.h" - -/* table containing information about how to partition distributed tables */ -#define CITUS_EXTENSION_NAME "citus" -#define CITUS_PARTITION_TABLE_NAME "pg_dist_partition" - -/* human-readable names for addressing columns of the pg_dist_partition table */ -#define ATTR_NUM_PARTITION_RELATION_ID 1 -#define ATTR_NUM_PARTITION_TYPE 2 -#define ATTR_NUM_PARTITION_KEY 3 - -/* - * CStoreValidOption keeps an option name and a context. When an option is passed - * into cstore_fdw objects (server and foreign table), we compare this option's - * name and context against those of valid options. - */ -typedef struct CStoreValidOption -{ - const char *optionName; - Oid optionContextId; -} CStoreValidOption; - -#define COMPRESSION_STRING_DELIMITED_LIST "none, pglz" - -/* Array of options that are valid for cstore_fdw */ -static const uint32 ValidOptionCount = 3; -static const CStoreValidOption ValidOptionArray[] = -{ - /* foreign table options */ - { OPTION_NAME_COMPRESSION_TYPE, ForeignTableRelationId }, - { OPTION_NAME_STRIPE_ROW_COUNT, ForeignTableRelationId }, - { OPTION_NAME_BLOCK_ROW_COUNT, ForeignTableRelationId } -}; - -static object_access_hook_type prevObjectAccessHook = NULL; - -/* local functions forward declarations */ -#if PG_VERSION_NUM >= 130000 -static void CStoreProcessUtility(PlannedStmt *plannedStatement, const char *queryString, - ProcessUtilityContext context, - ParamListInfo paramListInfo, - QueryEnvironment *queryEnvironment, - DestReceiver *destReceiver, - QueryCompletion *queryCompletion); -#elif PG_VERSION_NUM >= 100000 -static void CStoreProcessUtility(PlannedStmt *plannedStatement, const char *queryString, - ProcessUtilityContext context, - ParamListInfo paramListInfo, - QueryEnvironment *queryEnvironment, - DestReceiver *destReceiver, char *completionTag); -#else -static void CStoreProcessUtility(Node *parseTree, const char *queryString, - ProcessUtilityContext context, - ParamListInfo paramListInfo, - DestReceiver *destReceiver, char *completionTag); -#endif -static bool CopyCStoreTableStatement(CopyStmt *copyStatement); -static void CheckSuperuserPrivilegesForCopy(const CopyStmt *copyStatement); -static void CStoreProcessCopyCommand(CopyStmt *copyStatement, const char *queryString, - char *completionTag); -static uint64 CopyIntoCStoreTable(const CopyStmt *copyStatement, - const char *queryString); -static uint64 CopyOutCStoreTable(CopyStmt *copyStatement, const char *queryString); -static void CStoreProcessAlterTableCommand(AlterTableStmt *alterStatement); -static List * FindCStoreTables(List *tableList); -static List * OpenRelationsForTruncate(List *cstoreTableList); -static void FdwNewRelFileNode(Relation relation); -static void TruncateCStoreTables(List *cstoreRelationList); -static bool IsCStoreFdwTable(Oid relationId); -static bool IsCStoreServer(ForeignServer *server); -static bool DistributedTable(Oid relationId); -static bool DistributedWorkerCopy(CopyStmt *copyStatement); -static StringInfo OptionNamesString(Oid currentContextId); -static HeapTuple GetSlotHeapTuple(TupleTableSlot *tts); -static CStoreOptions * CStoreGetOptions(Oid foreignTableId); -static char * CStoreGetOptionValue(Oid foreignTableId, const char *optionName); -static void ValidateForeignTableOptions(char *compressionTypeString, - char *stripeRowCountString, - char *blockRowCountString); -static void CStoreGetForeignRelSize(PlannerInfo *root, RelOptInfo *baserel, - Oid foreignTableId); -static void CStoreGetForeignPaths(PlannerInfo *root, RelOptInfo *baserel, - Oid foreignTableId); -#if PG_VERSION_NUM >= 90500 -static ForeignScan * CStoreGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, - Oid foreignTableId, ForeignPath *bestPath, - List *targetList, List *scanClauses, - Plan *outerPlan); -#else -static ForeignScan * CStoreGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, - Oid foreignTableId, ForeignPath *bestPath, - List *targetList, List *scanClauses); -#endif -static double TupleCountEstimate(Relation relation, RelOptInfo *baserel); -static BlockNumber PageCount(Relation relation); -static List * ColumnList(RelOptInfo *baserel, Oid foreignTableId); -static void CStoreExplainForeignScan(ForeignScanState *scanState, - ExplainState *explainState); -static void CStoreBeginForeignScan(ForeignScanState *scanState, int executorFlags); -static TupleTableSlot * CStoreIterateForeignScan(ForeignScanState *scanState); -static void CStoreEndForeignScan(ForeignScanState *scanState); -static void CStoreReScanForeignScan(ForeignScanState *scanState); -static bool CStoreAnalyzeForeignTable(Relation relation, - AcquireSampleRowsFunc *acquireSampleRowsFunc, - BlockNumber *totalPageCount); -static int CStoreAcquireSampleRows(Relation relation, int logLevel, - HeapTuple *sampleRows, int targetRowCount, - double *totalRowCount, double *totalDeadRowCount); -static List * CStorePlanForeignModify(PlannerInfo *plannerInfo, ModifyTable *plan, - Index resultRelation, int subplanIndex); -static void CStoreBeginForeignModify(ModifyTableState *modifyTableState, - ResultRelInfo *relationInfo, List *fdwPrivate, - int subplanIndex, int executorflags); -static void CStoreBeginForeignInsert(ModifyTableState *modifyTableState, - ResultRelInfo *relationInfo); -static TupleTableSlot * CStoreExecForeignInsert(EState *executorState, - ResultRelInfo *relationInfo, - TupleTableSlot *tupleSlot, - TupleTableSlot *planSlot); -static void CStoreEndForeignModify(EState *executorState, ResultRelInfo *relationInfo); -static void CStoreEndForeignInsert(EState *executorState, ResultRelInfo *relationInfo); -#if PG_VERSION_NUM >= 90600 -static bool CStoreIsForeignScanParallelSafe(PlannerInfo *root, RelOptInfo *rel, - RangeTblEntry *rte); -#endif -static void cstore_fdw_initrel(Relation rel); -static Relation cstore_fdw_open(Oid relationId, LOCKMODE lockmode); -static Relation cstore_fdw_openrv(RangeVar *relation, LOCKMODE lockmode); -static void CStoreFdwObjectAccessHook(ObjectAccessType access, Oid classId, Oid objectId, - int subId, - void *arg); - -PG_FUNCTION_INFO_V1(cstore_ddl_event_end_trigger); -PG_FUNCTION_INFO_V1(cstore_table_size); -PG_FUNCTION_INFO_V1(cstore_fdw_handler); -PG_FUNCTION_INFO_V1(cstore_fdw_validator); - - -/* saved hook value in case of unload */ -static ProcessUtility_hook_type PreviousProcessUtilityHook = NULL; - - -/* - * Called when the module is loaded. In this function we save the - * previous utility hook, and then install our hook to pre-intercept calls to - * the copy command. - */ -void -cstore_fdw_init() -{ - PreviousProcessUtilityHook = (ProcessUtility_hook != NULL) ? - ProcessUtility_hook : standard_ProcessUtility; - ProcessUtility_hook = CStoreProcessUtility; - prevObjectAccessHook = object_access_hook; - object_access_hook = CStoreFdwObjectAccessHook; -} - - -/* - * Called when the module is unloaded. This function uninstalls the - * extension's hooks. - */ -void -cstore_fdw_finish() -{ - ProcessUtility_hook = PreviousProcessUtilityHook; -} - - -/* - * cstore_ddl_event_end_trigger is the event trigger function which is called on - * ddl_command_end event. This function creates required directories after the - * CREATE SERVER statement and valid data and footer files after the CREATE FOREIGN - * TABLE statement. - */ -Datum -cstore_ddl_event_end_trigger(PG_FUNCTION_ARGS) -{ - /* error if event trigger manager did not call this function */ - if (!CALLED_AS_EVENT_TRIGGER(fcinfo)) - { - ereport(ERROR, (errmsg("trigger not fired by event trigger manager"))); - } - - EventTriggerData *triggerData = (EventTriggerData *) fcinfo->context; - Node *parseTree = triggerData->parsetree; - - if (nodeTag(parseTree) == T_CreateForeignTableStmt) - { - CreateForeignTableStmt *createStatement = (CreateForeignTableStmt *) parseTree; - char *serverName = createStatement->servername; - - bool missingOK = false; - ForeignServer *server = GetForeignServerByName(serverName, missingOK); - if (IsCStoreServer(server)) - { - Oid relationId = RangeVarGetRelid(createStatement->base.relation, - AccessShareLock, false); - Relation relation = cstore_fdw_open(relationId, AccessExclusiveLock); - CStoreOptions *options = CStoreGetOptions(relationId); - InitCStoreDataFileMetadata(relation->rd_node.relNode, options->blockRowCount, - options->stripeRowCount, options->compressionType); - heap_close(relation, AccessExclusiveLock); - } - } - - PG_RETURN_NULL(); -} - - -/* - * CStoreProcessUtility is the hook for handling utility commands. This function - * customizes the behaviour of "COPY cstore_table" and "DROP FOREIGN TABLE - * cstore_table" commands. For all other utility statements, the function calls - * the previous utility hook or the standard utility command via macro - * CALL_PREVIOUS_UTILITY. - */ -#if PG_VERSION_NUM >= 130000 -static void -CStoreProcessUtility(PlannedStmt *plannedStatement, const char *queryString, - ProcessUtilityContext context, - ParamListInfo paramListInfo, - QueryEnvironment *queryEnvironment, - DestReceiver *destReceiver, QueryCompletion *queryCompletion) -#elif PG_VERSION_NUM >= 100000 -static void -CStoreProcessUtility(PlannedStmt * plannedStatement, const char * queryString, - ProcessUtilityContext context, - ParamListInfo paramListInfo, - QueryEnvironment * queryEnvironment, - DestReceiver * destReceiver, char * completionTag) -#else -static void -CStoreProcessUtility(Node * parseTree, const char * queryString, - ProcessUtilityContext context, - ParamListInfo paramListInfo, - DestReceiver * destReceiver, char * completionTag) -#endif -{ -#if PG_VERSION_NUM >= 130000 - char *completionTag = NULL; -#endif -#if PG_VERSION_NUM >= 100000 - Node *parseTree = plannedStatement->utilityStmt; -#endif - - if (nodeTag(parseTree) == T_CopyStmt) - { - CopyStmt *copyStatement = (CopyStmt *) parseTree; - - if (CopyCStoreTableStatement(copyStatement)) - { - CStoreProcessCopyCommand(copyStatement, queryString, completionTag); - } - else - { - CALL_PREVIOUS_UTILITY(); - } - } - else if (nodeTag(parseTree) == T_TruncateStmt) - { - TruncateStmt *truncateStatement = (TruncateStmt *) parseTree; - List *allTablesList = truncateStatement->relations; - List *cstoreTablesList = FindCStoreTables(allTablesList); - List *otherTablesList = list_difference(allTablesList, cstoreTablesList); - List *cstoreRelationList = OpenRelationsForTruncate(cstoreTablesList); - ListCell *cstoreRelationCell = NULL; - - if (otherTablesList != NIL) - { - truncateStatement->relations = otherTablesList; - - CALL_PREVIOUS_UTILITY(); - - /* restore the former relation list. Our - * replacement could be freed but still needed - * in a cached plan. A truncate can be cached - * if run from a pl/pgSQL function */ - truncateStatement->relations = allTablesList; - } - - TruncateCStoreTables(cstoreRelationList); - - foreach(cstoreRelationCell, cstoreRelationList) - { - Relation relation = (Relation) lfirst(cstoreRelationCell); - heap_close(relation, AccessExclusiveLock); - } - } - else if (nodeTag(parseTree) == T_AlterTableStmt) - { - AlterTableStmt *alterTable = (AlterTableStmt *) parseTree; - CStoreProcessAlterTableCommand(alterTable); - CALL_PREVIOUS_UTILITY(); - } - else if (nodeTag(parseTree) == T_DropdbStmt) - { - /* let postgres handle error checking and dropping of the database */ - CALL_PREVIOUS_UTILITY(); - } - - /* handle other utility statements */ - else - { - CALL_PREVIOUS_UTILITY(); - } -} - - -/* - * CopyCStoreTableStatement check whether the COPY statement is a "COPY cstore_table FROM - * ..." or "COPY cstore_table TO ...." statement. If it is then the function returns - * true. The function returns false otherwise. - */ -static bool -CopyCStoreTableStatement(CopyStmt *copyStatement) -{ - bool copyCStoreTableStatement = false; - - if (copyStatement->relation != NULL) - { - Oid relationId = RangeVarGetRelid(copyStatement->relation, - AccessShareLock, true); - bool cstoreTable = IsCStoreFdwTable(relationId); - if (cstoreTable) - { - bool distributedTable = DistributedTable(relationId); - bool distributedCopy = DistributedWorkerCopy(copyStatement); - - if (distributedTable || distributedCopy) - { - /* let COPY on distributed tables fall through to Citus */ - copyCStoreTableStatement = false; - } - else - { - copyCStoreTableStatement = true; - } - } - } - - return copyCStoreTableStatement; -} - - -/* - * CheckSuperuserPrivilegesForCopy checks if superuser privilege is required by - * copy operation and reports error if user does not have superuser rights. - */ -static void -CheckSuperuserPrivilegesForCopy(const CopyStmt *copyStatement) -{ - /* - * We disallow copy from file or program except to superusers. These checks - * are based on the checks in DoCopy() function of copy.c. - */ - if (copyStatement->filename != NULL && !superuser()) - { - if (copyStatement->is_program) - { - ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("must be superuser to COPY to or from a program"), - errhint("Anyone can COPY to stdout or from stdin. " - "psql's \\copy command also works for anyone."))); - } - else - { - ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("must be superuser to COPY to or from a file"), - errhint("Anyone can COPY to stdout or from stdin. " - "psql's \\copy command also works for anyone."))); - } - } -} - - -/* - * CStoreProcessCopyCommand handles COPY FROM/TO ... statements. - * It determines the copy direction and forwards execution to appropriate function. - */ -static void -CStoreProcessCopyCommand(CopyStmt *copyStatement, const char *queryString, - char *completionTag) -{ - uint64 processedCount = 0; - - if (copyStatement->is_from) - { - processedCount = CopyIntoCStoreTable(copyStatement, queryString); - } - else - { - processedCount = CopyOutCStoreTable(copyStatement, queryString); - } - - if (completionTag != NULL) - { - SafeSnprintf(completionTag, COMPLETION_TAG_BUFSIZE, "COPY " UINT64_FORMAT, - processedCount); - } -} - - -/* - * CopyIntoCStoreTable handles a "COPY cstore_table FROM" statement. This - * function uses the COPY command's functions to read and parse rows from - * the data source specified in the COPY statement. The function then writes - * each row to the file specified in the cstore foreign table options. Finally, - * the function returns the number of copied rows. - */ -static uint64 -CopyIntoCStoreTable(const CopyStmt *copyStatement, const char *queryString) -{ - uint64 processedRowCount = 0; - Relation relation = NULL; - CopyState copyState = NULL; - bool nextRowFound = true; - TableWriteState *writeState = NULL; - - /* Only superuser can copy from or to local file */ - CheckSuperuserPrivilegesForCopy(copyStatement); - - Assert(copyStatement->relation != NULL); - - /* - * Open and lock the relation. We acquire RowExclusiveLock to allow - * concurrent reads and writes. - */ - relation = cstore_fdw_openrv(copyStatement->relation, RowExclusiveLock); - Oid relationId = RelationGetRelid(relation); - - /* allocate column values and nulls arrays */ - TupleDesc tupleDescriptor = RelationGetDescr(relation); - uint32 columnCount = tupleDescriptor->natts; - Datum *columnValues = palloc0(columnCount * sizeof(Datum)); - bool *columnNulls = palloc0(columnCount * sizeof(bool)); - - CStoreOptions *cstoreOptions = CStoreGetOptions(relationId); - - /* - * We create a new memory context called tuple context, and read and write - * each row's values within this memory context. After each read and write, - * we reset the memory context. That way, we immediately release memory - * allocated for each row, and don't bloat memory usage with large input - * files. - */ - MemoryContext tupleContext = AllocSetContextCreate(CurrentMemoryContext, - "CStore COPY Row Memory Context", - ALLOCSET_DEFAULT_SIZES); - - /* init state to read from COPY data source */ -#if (PG_VERSION_NUM >= 100000) - { - ParseState *pstate = make_parsestate(NULL); - pstate->p_sourcetext = queryString; - - copyState = BeginCopyFrom(pstate, relation, copyStatement->filename, - copyStatement->is_program, - NULL, - copyStatement->attlist, - copyStatement->options); - free_parsestate(pstate); - } -#else - copyState = BeginCopyFrom(relation, copyStatement->filename, - copyStatement->is_program, - copyStatement->attlist, - copyStatement->options); -#endif - - /* init state to write to the cstore file */ - writeState = CStoreBeginWrite(relation->rd_node, - cstoreOptions->compressionType, - cstoreOptions->stripeRowCount, - cstoreOptions->blockRowCount, - tupleDescriptor); - - while (nextRowFound) - { - /* read the next row in tupleContext */ - MemoryContext oldContext = MemoryContextSwitchTo(tupleContext); -#if PG_VERSION_NUM >= 120000 - nextRowFound = NextCopyFrom(copyState, NULL, columnValues, columnNulls); -#else - nextRowFound = NextCopyFrom(copyState, NULL, columnValues, columnNulls, NULL); -#endif - MemoryContextSwitchTo(oldContext); - - /* write the row to the cstore file */ - if (nextRowFound) - { - CStoreWriteRow(writeState, columnValues, columnNulls); - processedRowCount++; - } - - MemoryContextReset(tupleContext); - - CHECK_FOR_INTERRUPTS(); - } - - /* end read/write sessions and close the relation */ - EndCopyFrom(copyState); - CStoreEndWrite(writeState); - heap_close(relation, RowExclusiveLock); - - return processedRowCount; -} - - -/* - * CopyFromCStoreTable handles a "COPY cstore_table TO ..." statement. Statement - * is converted to "COPY (SELECT * FROM cstore_table) TO ..." and forwarded to - * postgres native COPY handler. Function returns number of files copied to external - * stream. Copying selected columns from cstore table is not currently supported. - */ -static uint64 -CopyOutCStoreTable(CopyStmt *copyStatement, const char *queryString) -{ - uint64 processedCount = 0; - - StringInfo newQuerySubstring = makeStringInfo(); - - if (copyStatement->attlist != NIL) - { - ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("copy column list is not supported"), - errhint("use 'copy (select from ) to " - "...' instead"))); - } - - RangeVar *relation = copyStatement->relation; - char *qualifiedName = quote_qualified_identifier(relation->schemaname, - relation->relname); - appendStringInfo(newQuerySubstring, "select * from %s", qualifiedName); - List *queryList = raw_parser(newQuerySubstring->data); - - /* take the first parse tree */ - Node *rawQuery = linitial(queryList); - - /* - * Set the relation field to NULL so that COPY command works on - * query field instead. - */ - copyStatement->relation = NULL; - -#if (PG_VERSION_NUM >= 100000) - - /* - * raw_parser returns list of RawStmt* in PG 10+ we need to - * extract actual query from it. - */ - { - ParseState *pstate = make_parsestate(NULL); - RawStmt *rawStatement = (RawStmt *) rawQuery; - - pstate->p_sourcetext = newQuerySubstring->data; - copyStatement->query = rawStatement->stmt; - - DoCopy(pstate, copyStatement, -1, -1, &processedCount); - free_parsestate(pstate); - } -#else - copyStatement->query = rawQuery; - - DoCopy(copyStatement, queryString, &processedCount); -#endif - - return processedCount; -} - - -/* - * CStoreProcessAlterTableCommand checks if given alter table statement is - * compatible with underlying data structure. Currently it only checks alter - * column type. The function errors out if current column type can not be safely - * converted to requested column type. This check is more restrictive than - * PostgreSQL's because we can not change existing data. - */ -static void -CStoreProcessAlterTableCommand(AlterTableStmt *alterStatement) -{ - ObjectType objectType = alterStatement->relkind; - RangeVar *relationRangeVar = alterStatement->relation; - List *commandList = alterStatement->cmds; - ListCell *commandCell = NULL; - - /* we are only interested in foreign table changes */ - if (objectType != OBJECT_TABLE && objectType != OBJECT_FOREIGN_TABLE) - { - return; - } - - Oid relationId = RangeVarGetRelid(relationRangeVar, AccessShareLock, true); - if (!IsCStoreFdwTable(relationId)) - { - return; - } - - foreach(commandCell, commandList) - { - AlterTableCmd *alterCommand = (AlterTableCmd *) lfirst(commandCell); - if (alterCommand->subtype == AT_AlterColumnType) - { - char *columnName = alterCommand->name; - ColumnDef *columnDef = (ColumnDef *) alterCommand->def; - Oid targetTypeId = typenameTypeId(NULL, columnDef->typeName); - char *typeName = TypeNameToString(columnDef->typeName); - AttrNumber attributeNumber = get_attnum(relationId, columnName); - - if (attributeNumber <= 0) - { - /* let standard utility handle this */ - continue; - } - - Oid currentTypeId = get_atttype(relationId, attributeNumber); - - /* - * We are only interested in implicit coersion type compatibility. - * Erroring out here to prevent further processing. - */ - if (!can_coerce_type(1, ¤tTypeId, &targetTypeId, COERCION_IMPLICIT)) - { - ereport(ERROR, (errmsg("Column %s cannot be cast automatically to " - "type %s", columnName, typeName))); - } - } - } -} - - -/* FindCStoreTables returns list of CStore tables from given table list */ -static List * -FindCStoreTables(List *tableList) -{ - List *cstoreTableList = NIL; - ListCell *relationCell = NULL; - foreach(relationCell, tableList) - { - RangeVar *rangeVar = (RangeVar *) lfirst(relationCell); - Oid relationId = RangeVarGetRelid(rangeVar, NoLock, true); - if (IsCStoreFdwTable(relationId) && !DistributedTable(relationId)) - { - cstoreTableList = lappend(cstoreTableList, rangeVar); - } - } - - return cstoreTableList; -} - - -/* - * OpenRelationsForTruncate opens and locks relations for tables to be truncated. - * - * It also performs a permission checks to see if the user has truncate privilege - * on tables. - */ -static List * -OpenRelationsForTruncate(List *cstoreTableList) -{ - ListCell *relationCell = NULL; - List *relationIdList = NIL; - List *relationList = NIL; - foreach(relationCell, cstoreTableList) - { - RangeVar *rangeVar = (RangeVar *) lfirst(relationCell); - Relation relation = cstore_fdw_openrv(rangeVar, AccessExclusiveLock); - Oid relationId = relation->rd_id; - AclResult aclresult = pg_class_aclcheck(relationId, GetUserId(), - ACL_TRUNCATE); - if (aclresult != ACLCHECK_OK) - { - aclcheck_error(aclresult, ACLCHECK_OBJECT_TABLE, get_rel_name(relationId)); - } - - /* check if this relation is repeated */ - if (list_member_oid(relationIdList, relationId)) - { - heap_close(relation, AccessExclusiveLock); - } - - /* - * Type of cstore tables can change since we checked last time, - * since we didn't hold a lock when checking. - */ - else if (!IsCStoreFdwTable(relationId)) - { - ereport(ERROR, (errmsg("relation \"%s\" not columnar anymore", - RelationGetRelationName(relation)), - errhint("try repeating the truncate command"))); - } - else - { - relationIdList = lappend_oid(relationIdList, relationId); - relationList = lappend(relationList, relation); - } - } - - return relationList; -} - - -/* TruncateCStoreTable truncates given cstore tables */ -static void -TruncateCStoreTables(List *cstoreRelationList) -{ - ListCell *relationCell = NULL; - foreach(relationCell, cstoreRelationList) - { - Relation relation = (Relation) lfirst(relationCell); - Oid relationId = relation->rd_id; - CStoreOptions *options = CStoreGetOptions(relationId); - - Assert(IsCStoreFdwTable(relationId)); - - FdwNewRelFileNode(relation); - InitCStoreDataFileMetadata(relation->rd_node.relNode, options->blockRowCount, - options->stripeRowCount, options->compressionType); - } -} - - -/* - * Version 11 and earlier already assign a relfilenode for foreign - * tables. Version 12 and later do not, so we need to create one manually. - */ -static void -FdwNewRelFileNode(Relation relation) -{ - Relation pg_class = heap_open(RelationRelationId, RowExclusiveLock); - - HeapTuple tuple = SearchSysCacheCopy1(RELOID, - ObjectIdGetDatum(RelationGetRelid(relation))); - if (!HeapTupleIsValid(tuple)) - { - elog(ERROR, "could not find tuple for relation %u", - RelationGetRelid(relation)); - } - Form_pg_class classform = (Form_pg_class) GETSTRUCT(tuple); - - if (true) - { - char persistence = relation->rd_rel->relpersistence; - Oid tablespace; - - /* - * Upgrade to AccessExclusiveLock, and hold until the end of the - * transaction. This shouldn't happen during a read, but it's hard to - * prove that because it happens lazily. - */ - Relation tmprel = heap_open(relation->rd_id, AccessExclusiveLock); - heap_close(tmprel, NoLock); - - if (OidIsValid(relation->rd_rel->relfilenode)) - { - RelationDropStorage(relation); - DeleteDataFileMetadataRowIfExists(relation->rd_rel->relfilenode); - } - - if (OidIsValid(relation->rd_rel->reltablespace)) - { - tablespace = relation->rd_rel->reltablespace; - } - else - { - tablespace = MyDatabaseTableSpace; - } - - Oid filenode = GetNewRelFileNode(tablespace, NULL, persistence); - - classform->relfilenode = filenode; - classform->relpages = 0; /* it's empty until further notice */ - classform->reltuples = 0; - classform->relallvisible = 0; - classform->relfrozenxid = InvalidTransactionId; - classform->relminmxid = InvalidTransactionId; - - CatalogTupleUpdate(pg_class, &tuple->t_self, tuple); - CommandCounterIncrement(); - - relation->rd_node.spcNode = tablespace; - relation->rd_node.dbNode = MyDatabaseId; - relation->rd_node.relNode = filenode; - } - - heap_freetuple(tuple); - heap_close(pg_class, RowExclusiveLock); -} - - -static void -FdwCreateStorage(Relation relation) -{ - Assert(OidIsValid(relation->rd_rel->relfilenode)); - RelationOpenSmgr(relation); - if (!smgrexists(relation->rd_smgr, MAIN_FORKNUM)) - { -#if PG_VERSION_NUM >= 120000 - SMgrRelation srel = RelationCreateStorage(relation->rd_node, - relation->rd_rel->relpersistence); - smgrclose(srel); -#else - RelationCreateStorage(relation->rd_node, - relation->rd_rel->relpersistence); -#endif - } -} - - -/* - * IsCStoreFdwTable checks if the given table name belongs to a foreign columnar store - * table. If it does, the function returns true. Otherwise, it returns false. - */ -bool -IsCStoreFdwTable(Oid relationId) -{ - bool cstoreTable = false; - - if (relationId == InvalidOid) - { - return false; - } - - char relationKind = get_rel_relkind(relationId); - if (relationKind == RELKIND_FOREIGN_TABLE) - { - ForeignTable *foreignTable = GetForeignTable(relationId); - ForeignServer *server = GetForeignServer(foreignTable->serverid); - if (IsCStoreServer(server)) - { - cstoreTable = true; - } - } - - return cstoreTable; -} - - -/* - * IsCStoreServer checks if the given foreign server belongs to cstore_fdw. If it - * does, the function returns true. Otherwise, it returns false. - */ -static bool -IsCStoreServer(ForeignServer *server) -{ - ForeignDataWrapper *foreignDataWrapper = GetForeignDataWrapper(server->fdwid); - bool cstoreServer = false; - - char *foreignWrapperName = foreignDataWrapper->fdwname; - if (strncmp(foreignWrapperName, CSTORE_FDW_NAME, NAMEDATALEN) == 0) - { - cstoreServer = true; - } - - return cstoreServer; -} - - -/* - * DistributedTable checks if the given relationId is the OID of a distributed table, - * which may also be a cstore_fdw table, but in that case COPY should be handled by - * Citus. - */ -static bool -DistributedTable(Oid relationId) -{ - const int scanKeyCount = 1; - ScanKeyData scanKey[1]; - - bool missingOK = true; - Oid extensionOid = get_extension_oid(CITUS_EXTENSION_NAME, missingOK); - if (extensionOid == InvalidOid) - { - /* if the citus extension isn't created, no tables are distributed */ - return false; - } - - Oid partitionOid = get_relname_relid(CITUS_PARTITION_TABLE_NAME, - PG_CATALOG_NAMESPACE); - if (partitionOid == InvalidOid) - { - /* the pg_dist_partition table does not exist */ - return false; - } - - Relation heapRelation = heap_open(partitionOid, AccessShareLock); - - ScanKeyInit(&scanKey[0], ATTR_NUM_PARTITION_RELATION_ID, InvalidStrategy, - F_OIDEQ, ObjectIdGetDatum(relationId)); - - TableScanDesc scanDesc = table_beginscan(heapRelation, SnapshotSelf, scanKeyCount, - scanKey); - - HeapTuple heapTuple = heap_getnext(scanDesc, ForwardScanDirection); - - bool distributedTable = HeapTupleIsValid(heapTuple); - - table_endscan(scanDesc); - relation_close(heapRelation, AccessShareLock); - - return distributedTable; -} - - -/* - * DistributedWorkerCopy returns whether the Citus-specific master_host option is - * present in the COPY options. - */ -static bool -DistributedWorkerCopy(CopyStmt *copyStatement) -{ - ListCell *optionCell = NULL; - foreach(optionCell, copyStatement->options) - { - DefElem *defel = (DefElem *) lfirst(optionCell); - if (strncmp(defel->defname, "master_host", NAMEDATALEN) == 0) - { - return true; - } - } - - return false; -} - - -/* - * cstore_table_size returns the total on-disk size of a cstore table in bytes. - * The result includes the sizes of data file and footer file. - */ -Datum -cstore_table_size(PG_FUNCTION_ARGS) -{ - Oid relationId = PG_GETARG_OID(0); - bool cstoreTable = IsCStoreFdwTable(relationId); - - if (!cstoreTable) - { - ereport(ERROR, (errmsg("relation is not a cstore table"))); - } - - Relation relation = cstore_fdw_open(relationId, AccessShareLock); - RelationOpenSmgr(relation); - BlockNumber nblocks = smgrnblocks(relation->rd_smgr, MAIN_FORKNUM); - heap_close(relation, AccessShareLock); - PG_RETURN_INT64(nblocks * BLCKSZ); -} - - -/* - * cstore_fdw_handler creates and returns a struct with pointers to foreign - * table callback functions. - */ -Datum -cstore_fdw_handler(PG_FUNCTION_ARGS) -{ - FdwRoutine *fdwRoutine = makeNode(FdwRoutine); - - fdwRoutine->GetForeignRelSize = CStoreGetForeignRelSize; - fdwRoutine->GetForeignPaths = CStoreGetForeignPaths; - fdwRoutine->GetForeignPlan = CStoreGetForeignPlan; - fdwRoutine->ExplainForeignScan = CStoreExplainForeignScan; - fdwRoutine->BeginForeignScan = CStoreBeginForeignScan; - fdwRoutine->IterateForeignScan = CStoreIterateForeignScan; - fdwRoutine->ReScanForeignScan = CStoreReScanForeignScan; - fdwRoutine->EndForeignScan = CStoreEndForeignScan; - fdwRoutine->AnalyzeForeignTable = CStoreAnalyzeForeignTable; - fdwRoutine->PlanForeignModify = CStorePlanForeignModify; - fdwRoutine->BeginForeignModify = CStoreBeginForeignModify; - fdwRoutine->ExecForeignInsert = CStoreExecForeignInsert; - fdwRoutine->EndForeignModify = CStoreEndForeignModify; - -#if PG_VERSION_NUM >= 110000 - fdwRoutine->BeginForeignInsert = CStoreBeginForeignInsert; - fdwRoutine->EndForeignInsert = CStoreEndForeignInsert; -#endif - -#if PG_VERSION_NUM >= 90600 - fdwRoutine->IsForeignScanParallelSafe = CStoreIsForeignScanParallelSafe; -#endif - - PG_RETURN_POINTER(fdwRoutine); -} - - -/* - * cstore_fdw_validator validates options given to one of the following commands: - * foreign data wrapper, server, user mapping, or foreign table. This function - * errors out if the given option name or its value is considered invalid. - */ -Datum -cstore_fdw_validator(PG_FUNCTION_ARGS) -{ - Datum optionArray = PG_GETARG_DATUM(0); - Oid optionContextId = PG_GETARG_OID(1); - List *optionList = untransformRelOptions(optionArray); - ListCell *optionCell = NULL; - char *compressionTypeString = NULL; - char *stripeRowCountString = NULL; - char *blockRowCountString = NULL; - - foreach(optionCell, optionList) - { - DefElem *optionDef = (DefElem *) lfirst(optionCell); - char *optionName = optionDef->defname; - bool optionValid = false; - - int32 optionIndex = 0; - for (optionIndex = 0; optionIndex < ValidOptionCount; optionIndex++) - { - const CStoreValidOption *validOption = &(ValidOptionArray[optionIndex]); - - if ((optionContextId == validOption->optionContextId) && - (strncmp(optionName, validOption->optionName, NAMEDATALEN) == 0)) - { - optionValid = true; - break; - } - } - - /* if invalid option, display an informative error message */ - if (!optionValid) - { - StringInfo optionNamesString = OptionNamesString(optionContextId); - - ereport(ERROR, (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), - errmsg("invalid option \"%s\"", optionName), - errhint("Valid options in this context are: %s", - optionNamesString->data))); - } - - if (strncmp(optionName, OPTION_NAME_COMPRESSION_TYPE, NAMEDATALEN) == 0) - { - compressionTypeString = defGetString(optionDef); - } - else if (strncmp(optionName, OPTION_NAME_STRIPE_ROW_COUNT, NAMEDATALEN) == 0) - { - stripeRowCountString = defGetString(optionDef); - } - else if (strncmp(optionName, OPTION_NAME_BLOCK_ROW_COUNT, NAMEDATALEN) == 0) - { - blockRowCountString = defGetString(optionDef); - } - } - - if (optionContextId == ForeignTableRelationId) - { - ValidateForeignTableOptions(compressionTypeString, - stripeRowCountString, blockRowCountString); - } - - PG_RETURN_VOID(); -} - - -/* - * OptionNamesString finds all options that are valid for the current context, - * and concatenates these option names in a comma separated string. The function - * is unchanged from mongo_fdw. - */ -static StringInfo -OptionNamesString(Oid currentContextId) -{ - StringInfo optionNamesString = makeStringInfo(); - bool firstOptionAppended = false; - - int32 optionIndex = 0; - for (optionIndex = 0; optionIndex < ValidOptionCount; optionIndex++) - { - const CStoreValidOption *validOption = &(ValidOptionArray[optionIndex]); - - /* if option belongs to current context, append option name */ - if (currentContextId == validOption->optionContextId) - { - if (firstOptionAppended) - { - appendStringInfoString(optionNamesString, ", "); - } - - appendStringInfoString(optionNamesString, validOption->optionName); - firstOptionAppended = true; - } - } - - return optionNamesString; -} - - -/* - * GetSlotHeapTuple abstracts getting HeapTuple from TupleTableSlot between versions - */ -static HeapTuple -GetSlotHeapTuple(TupleTableSlot *tts) -{ -#if PG_VERSION_NUM >= 120000 - return tts->tts_ops->copy_heap_tuple(tts); -#else - return tts->tts_tuple; -#endif -} - - -/* - * CStoreGetOptions returns the option values to be used when reading and writing - * the cstore file. To resolve these values, the function checks options for the - * foreign table, and if not present, falls back to default values. This function - * errors out if given option values are considered invalid. - */ -static CStoreOptions * -CStoreGetOptions(Oid foreignTableId) -{ - CompressionType compressionType = cstore_compression; - int32 stripeRowCount = cstore_stripe_row_count; - int32 blockRowCount = cstore_block_row_count; - - char *compressionTypeString = CStoreGetOptionValue(foreignTableId, - OPTION_NAME_COMPRESSION_TYPE); - char *stripeRowCountString = CStoreGetOptionValue(foreignTableId, - OPTION_NAME_STRIPE_ROW_COUNT); - char *blockRowCountString = CStoreGetOptionValue(foreignTableId, - OPTION_NAME_BLOCK_ROW_COUNT); - - ValidateForeignTableOptions(compressionTypeString, - stripeRowCountString, blockRowCountString); - - /* parse provided options */ - if (compressionTypeString != NULL) - { - compressionType = ParseCompressionType(compressionTypeString); - } - if (stripeRowCountString != NULL) - { - stripeRowCount = pg_atoi(stripeRowCountString, sizeof(int32), 0); - } - if (blockRowCountString != NULL) - { - blockRowCount = pg_atoi(blockRowCountString, sizeof(int32), 0); - } - - CStoreOptions *cstoreOptions = palloc0(sizeof(CStoreOptions)); - cstoreOptions->compressionType = compressionType; - cstoreOptions->stripeRowCount = stripeRowCount; - cstoreOptions->blockRowCount = blockRowCount; - - return cstoreOptions; -} - - -/* - * CStoreGetOptionValue walks over foreign table and foreign server options, and - * looks for the option with the given name. If found, the function returns the - * option's value. This function is unchanged from mongo_fdw. - */ -static char * -CStoreGetOptionValue(Oid foreignTableId, const char *optionName) -{ - List *optionList = NIL; - ListCell *optionCell = NULL; - char *optionValue = NULL; - - ForeignTable *foreignTable = GetForeignTable(foreignTableId); - ForeignServer *foreignServer = GetForeignServer(foreignTable->serverid); - - optionList = list_concat(optionList, foreignTable->options); - optionList = list_concat(optionList, foreignServer->options); - - foreach(optionCell, optionList) - { - DefElem *optionDef = (DefElem *) lfirst(optionCell); - char *optionDefName = optionDef->defname; - - if (strncmp(optionDefName, optionName, NAMEDATALEN) == 0) - { - optionValue = defGetString(optionDef); - break; - } - } - - return optionValue; -} - - -/* - * ValidateForeignTableOptions verifies if given options are valid cstore_fdw - * foreign table options. This function errors out if given option value is - * considered invalid. - */ -static void -ValidateForeignTableOptions(char *compressionTypeString, - char *stripeRowCountString, char *blockRowCountString) -{ - /* check if the provided compression type is valid */ - if (compressionTypeString != NULL) - { - CompressionType compressionType = ParseCompressionType(compressionTypeString); - if (compressionType == COMPRESSION_TYPE_INVALID) - { - ereport(ERROR, (errmsg("invalid compression type"), - errhint("Valid options are: %s", - COMPRESSION_STRING_DELIMITED_LIST))); - } - } - - /* check if the provided stripe row count has correct format and range */ - if (stripeRowCountString != NULL) - { - /* pg_atoi() errors out if the given string is not a valid 32-bit integer */ - int32 stripeRowCount = pg_atoi(stripeRowCountString, sizeof(int32), 0); - if (stripeRowCount < STRIPE_ROW_COUNT_MINIMUM || - stripeRowCount > STRIPE_ROW_COUNT_MAXIMUM) - { - ereport(ERROR, (errmsg("invalid stripe row count"), - errhint("Stripe row count must be an integer between " - "%d and %d", STRIPE_ROW_COUNT_MINIMUM, - STRIPE_ROW_COUNT_MAXIMUM))); - } - } - - /* check if the provided block row count has correct format and range */ - if (blockRowCountString != NULL) - { - /* pg_atoi() errors out if the given string is not a valid 32-bit integer */ - int32 blockRowCount = pg_atoi(blockRowCountString, sizeof(int32), 0); - if (blockRowCount < BLOCK_ROW_COUNT_MINIMUM || - blockRowCount > BLOCK_ROW_COUNT_MAXIMUM) - { - ereport(ERROR, (errmsg("invalid block row count"), - errhint("Block row count must be an integer between " - "%d and %d", BLOCK_ROW_COUNT_MINIMUM, - BLOCK_ROW_COUNT_MAXIMUM))); - } - } -} - - -/* - * CStoreGetForeignRelSize obtains relation size estimates for a foreign table and - * puts its estimate for row count into baserel->rows. - */ -static void -CStoreGetForeignRelSize(PlannerInfo *root, RelOptInfo *baserel, Oid foreignTableId) -{ - Relation relation = cstore_fdw_open(foreignTableId, AccessShareLock); - double tupleCountEstimate = TupleCountEstimate(relation, baserel); - double rowSelectivity = clauselist_selectivity(root, baserel->baserestrictinfo, - 0, JOIN_INNER, NULL); - - double outputRowCount = clamp_row_est(tupleCountEstimate * rowSelectivity); - baserel->rows = outputRowCount; - heap_close(relation, AccessShareLock); -} - - -/* - * CStoreGetForeignPaths creates possible access paths for a scan on the foreign - * table. We currently have one possible access path. This path filters out row - * blocks that are refuted by where clauses, and only returns values for the - * projected columns. - */ -static void -CStoreGetForeignPaths(PlannerInfo *root, RelOptInfo *baserel, Oid foreignTableId) -{ - Path *foreignScanPath = NULL; - Relation relation = cstore_fdw_open(foreignTableId, AccessShareLock); - - /* - * We skip reading columns that are not in query. Here we assume that all - * columns in relation have the same width, and estimate the number pages - * that will be read by query. - * - * Ideally, we should also take into account the row blocks that will be - * suppressed. But for that we need to know which columns are used for - * sorting. If we wrongly assume that we are sorted by a specific column - * and underestimate the page count, planner may choose nested loop join - * in a place it shouldn't be used. Choosing merge join or hash join is - * usually safer than nested loop join, so we take the more conservative - * approach and assume all rows in the columnar store file will be read. - * We intend to fix this in later version by improving the row sampling - * algorithm and using the correlation statistics to detect which columns - * are in stored in sorted order. - */ - List *queryColumnList = ColumnList(baserel, foreignTableId); - uint32 queryColumnCount = list_length(queryColumnList); - BlockNumber relationPageCount = PageCount(relation); - uint32 relationColumnCount = RelationGetNumberOfAttributes(relation); - - double queryColumnRatio = (double) queryColumnCount / relationColumnCount; - double queryPageCount = relationPageCount * queryColumnRatio; - double totalDiskAccessCost = seq_page_cost * queryPageCount; - - double tupleCountEstimate = TupleCountEstimate(relation, baserel); - - /* - * We estimate costs almost the same way as cost_seqscan(), thus assuming - * that I/O costs are equivalent to a regular table file of the same size. - */ - double filterCostPerTuple = baserel->baserestrictcost.per_tuple; - double cpuCostPerTuple = cpu_tuple_cost + filterCostPerTuple; - double totalCpuCost = cpuCostPerTuple * tupleCountEstimate; - - double startupCost = baserel->baserestrictcost.startup; - double totalCost = startupCost + totalCpuCost + totalDiskAccessCost; - - /* create a foreign path node and add it as the only possible path */ -#if PG_VERSION_NUM >= 90600 - foreignScanPath = (Path *) create_foreignscan_path(root, baserel, - NULL, /* path target */ - baserel->rows, - startupCost, totalCost, - NIL, /* no known ordering */ - NULL, /* not parameterized */ - NULL, /* no outer path */ - NIL); /* no fdw_private */ - -#elif PG_VERSION_NUM >= 90500 - foreignScanPath = (Path *) create_foreignscan_path(root, baserel, baserel->rows, - startupCost, totalCost, - NIL, /* no known ordering */ - NULL, /* not parameterized */ - NULL, /* no outer path */ - NIL); /* no fdw_private */ -#else - foreignScanPath = (Path *) create_foreignscan_path(root, baserel, baserel->rows, - startupCost, totalCost, - NIL, /* no known ordering */ - NULL, /* not parameterized */ - NIL); /* no fdw_private */ -#endif - - add_path(baserel, foreignScanPath); - heap_close(relation, AccessShareLock); -} - - -/* - * CStoreGetForeignPlan creates a ForeignScan plan node for scanning the foreign - * table. We also add the query column list to scan nodes private list, because - * we need it later for skipping over unused columns in the query. - */ -#if PG_VERSION_NUM >= 90500 -static ForeignScan * -CStoreGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreignTableId, - ForeignPath *bestPath, List *targetList, List *scanClauses, - Plan *outerPlan) -#else -static ForeignScan * -CStoreGetForeignPlan(PlannerInfo * root, RelOptInfo * baserel, Oid foreignTableId, - ForeignPath * bestPath, List * targetList, List * scanClauses) -#endif -{ - ForeignScan *foreignScan = NULL; - - /* - * Although we skip row blocks that are refuted by the WHERE clause, but - * we have no native ability to evaluate restriction clauses and make sure - * that all non-related rows are filtered out. So we just put all of the - * scanClauses into the plan node's qual list for the executor to check. - */ - scanClauses = extract_actual_clauses(scanClauses, - false); /* extract regular clauses */ - - /* - * As an optimization, we only read columns that are present in the query. - * To find these columns, we need baserel. We don't have access to baserel - * in executor's callback functions, so we get the column list here and put - * it into foreign scan node's private list. - */ - List *columnList = ColumnList(baserel, foreignTableId); - List *foreignPrivateList = list_make1(columnList); - - /* create the foreign scan node */ -#if PG_VERSION_NUM >= 90500 - foreignScan = make_foreignscan(targetList, scanClauses, baserel->relid, - NIL, /* no expressions to evaluate */ - foreignPrivateList, - NIL, - NIL, - NULL); /* no outer path */ -#else - foreignScan = make_foreignscan(targetList, scanClauses, baserel->relid, - NIL, /* no expressions to evaluate */ - foreignPrivateList); -#endif - - return foreignScan; -} - - -/* - * TupleCountEstimate estimates the number of base relation tuples in the given - * file. - */ -static double -TupleCountEstimate(Relation relation, RelOptInfo *baserel) -{ - double tupleCountEstimate = 0.0; - - /* check if the user executed Analyze on this foreign table before */ - if (baserel->pages > 0) - { - /* - * We have number of pages and number of tuples from pg_class (from a - * previous ANALYZE), so compute a tuples-per-page estimate and scale - * that by the current file size. - */ - double tupleDensity = baserel->tuples / (double) baserel->pages; - BlockNumber pageCount = PageCount(relation); - - tupleCountEstimate = clamp_row_est(tupleDensity * (double) pageCount); - } - else - { - tupleCountEstimate = (double) CStoreTableRowCount(relation); - } - - return tupleCountEstimate; -} - - -/* PageCount calculates and returns the number of pages in a file. */ -static BlockNumber -PageCount(Relation relation) -{ - RelationOpenSmgr(relation); - BlockNumber nblocks = smgrnblocks(relation->rd_smgr, MAIN_FORKNUM); - - return (nblocks > 0) ? nblocks : 1; -} - - -/* - * ColumnList takes in the planner's information about this foreign table. The - * function then finds all columns needed for query execution, including those - * used in projections, joins, and filter clauses, de-duplicates these columns, - * and returns them in a new list. This function is taken from mongo_fdw with - * slight modifications. - */ -static List * -ColumnList(RelOptInfo *baserel, Oid foreignTableId) -{ - List *columnList = NIL; - List *neededColumnList = NIL; - AttrNumber columnIndex = 1; - AttrNumber columnCount = baserel->max_attr; -#if PG_VERSION_NUM >= 90600 - List *targetColumnList = baserel->reltarget->exprs; -#else - List *targetColumnList = baserel->reltargetlist; -#endif - ListCell *targetColumnCell = NULL; - List *restrictInfoList = baserel->baserestrictinfo; - ListCell *restrictInfoCell = NULL; - const AttrNumber wholeRow = 0; - Relation relation = cstore_fdw_open(foreignTableId, AccessShareLock); - TupleDesc tupleDescriptor = RelationGetDescr(relation); - - /* first add the columns used in joins and projections */ - foreach(targetColumnCell, targetColumnList) - { - List *targetVarList = NIL; - Node *targetExpr = (Node *) lfirst(targetColumnCell); - -#if PG_VERSION_NUM >= 90600 - targetVarList = pull_var_clause(targetExpr, - PVC_RECURSE_AGGREGATES | - PVC_RECURSE_PLACEHOLDERS); -#else - targetVarList = pull_var_clause(targetExpr, - PVC_RECURSE_AGGREGATES, - PVC_RECURSE_PLACEHOLDERS); -#endif - - neededColumnList = list_union(neededColumnList, targetVarList); - } - - /* then walk over all restriction clauses, and pull up any used columns */ - foreach(restrictInfoCell, restrictInfoList) - { - RestrictInfo *restrictInfo = (RestrictInfo *) lfirst(restrictInfoCell); - Node *restrictClause = (Node *) restrictInfo->clause; - List *clauseColumnList = NIL; - - /* recursively pull up any columns used in the restriction clause */ -#if PG_VERSION_NUM >= 90600 - clauseColumnList = pull_var_clause(restrictClause, - PVC_RECURSE_AGGREGATES | - PVC_RECURSE_PLACEHOLDERS); -#else - clauseColumnList = pull_var_clause(restrictClause, - PVC_RECURSE_AGGREGATES, - PVC_RECURSE_PLACEHOLDERS); -#endif - - neededColumnList = list_union(neededColumnList, clauseColumnList); - } - - /* walk over all column definitions, and de-duplicate column list */ - for (columnIndex = 1; columnIndex <= columnCount; columnIndex++) - { - ListCell *neededColumnCell = NULL; - Var *column = NULL; - Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, columnIndex - 1); - - if (attributeForm->attisdropped) - { - continue; - } - - /* look for this column in the needed column list */ - foreach(neededColumnCell, neededColumnList) - { - Var *neededColumn = (Var *) lfirst(neededColumnCell); - if (neededColumn->varattno == columnIndex) - { - column = neededColumn; - break; - } - else if (neededColumn->varattno == wholeRow) - { - Index tableId = neededColumn->varno; - - column = makeVar(tableId, columnIndex, attributeForm->atttypid, - attributeForm->atttypmod, attributeForm->attcollation, - 0); - break; - } - } - - if (column != NULL) - { - columnList = lappend(columnList, column); - } - } - - heap_close(relation, AccessShareLock); - - return columnList; -} - - -/* CStoreExplainForeignScan produces extra output for the Explain command. */ -static void -CStoreExplainForeignScan(ForeignScanState *scanState, ExplainState *explainState) -{ - Relation relation = scanState->ss.ss_currentRelation; - - cstore_fdw_initrel(relation); - - /* supress file size if we're not showing cost details */ - if (explainState->costs) - { - RelationOpenSmgr(relation); - long nblocks = smgrnblocks(relation->rd_smgr, MAIN_FORKNUM); - ExplainPropertyLong("CStore File Size", (long) (nblocks * BLCKSZ), - explainState); - } -} - - -/* CStoreBeginForeignScan starts reading the underlying cstore file. */ -static void -CStoreBeginForeignScan(ForeignScanState *scanState, int executorFlags) -{ - Relation currentRelation = scanState->ss.ss_currentRelation; - TupleDesc tupleDescriptor = RelationGetDescr(currentRelation); - - cstore_fdw_initrel(currentRelation); - - /* if Explain with no Analyze, do nothing */ - if (executorFlags & EXEC_FLAG_EXPLAIN_ONLY) - { - return; - } - - Oid foreignTableId = RelationGetRelid(scanState->ss.ss_currentRelation); - - ForeignScan *foreignScan = (ForeignScan *) scanState->ss.ps.plan; - List *foreignPrivateList = (List *) foreignScan->fdw_private; - List *whereClauseList = foreignScan->scan.plan.qual; - - List *columnList = (List *) linitial(foreignPrivateList); - Relation relation = cstore_fdw_open(foreignTableId, AccessShareLock); - TableReadState *readState = CStoreBeginRead(relation, tupleDescriptor, columnList, - whereClauseList); - - scanState->fdw_state = (void *) readState; -} - - -/* - * CStoreIterateForeignScan reads the next record from the cstore file, converts - * it to a Postgres tuple, and stores the converted tuple into the ScanTupleSlot - * as a virtual tuple. - */ -static TupleTableSlot * -CStoreIterateForeignScan(ForeignScanState *scanState) -{ - TableReadState *readState = (TableReadState *) scanState->fdw_state; - TupleTableSlot *tupleSlot = scanState->ss.ss_ScanTupleSlot; - - TupleDesc tupleDescriptor = tupleSlot->tts_tupleDescriptor; - Datum *columnValues = tupleSlot->tts_values; - bool *columnNulls = tupleSlot->tts_isnull; - uint32 columnCount = tupleDescriptor->natts; - - /* initialize all values for this row to null */ - memset(columnValues, 0, columnCount * sizeof(Datum)); - memset(columnNulls, true, columnCount * sizeof(bool)); - - ExecClearTuple(tupleSlot); - - bool nextRowFound = CStoreReadNextRow(readState, columnValues, columnNulls); - if (nextRowFound) - { - ExecStoreVirtualTuple(tupleSlot); - } - - return tupleSlot; -} - - -/* CStoreEndForeignScan finishes scanning the foreign table. */ -static void -CStoreEndForeignScan(ForeignScanState *scanState) -{ - TableReadState *readState = (TableReadState *) scanState->fdw_state; - if (readState != NULL) - { - heap_close(readState->relation, AccessShareLock); - CStoreEndRead(readState); - } -} - - -/* CStoreReScanForeignScan rescans the foreign table. */ -static void -CStoreReScanForeignScan(ForeignScanState *scanState) -{ - CStoreEndForeignScan(scanState); - CStoreBeginForeignScan(scanState, 0); -} - - -/* - * CStoreAnalyzeForeignTable sets the total page count and the function pointer - * used to acquire a random sample of rows from the foreign file. - */ -static bool -CStoreAnalyzeForeignTable(Relation relation, - AcquireSampleRowsFunc *acquireSampleRowsFunc, - BlockNumber *totalPageCount) -{ - cstore_fdw_initrel(relation); - RelationOpenSmgr(relation); - (*totalPageCount) = smgrnblocks(relation->rd_smgr, MAIN_FORKNUM); - (*acquireSampleRowsFunc) = CStoreAcquireSampleRows; - - return true; -} - - -/* - * CStoreAcquireSampleRows acquires a random sample of rows from the foreign - * table. Selected rows are returned in the caller allocated sampleRows array, - * which must have at least target row count entries. The actual number of rows - * selected is returned as the function result. We also count the number of rows - * in the collection and return it in total row count. We also always set dead - * row count to zero. - * - * Note that the returned list of rows does not always follow their actual order - * in the cstore file. Therefore, correlation estimates derived later could be - * inaccurate, but that's OK. We currently don't use correlation estimates (the - * planner only pays attention to correlation for index scans). - */ -static int -CStoreAcquireSampleRows(Relation relation, int logLevel, - HeapTuple *sampleRows, int targetRowCount, - double *totalRowCount, double *totalDeadRowCount) -{ - int sampleRowCount = 0; - double rowCount = 0.0; - double rowCountToSkip = -1; /* -1 means not set yet */ - double selectionState = 0; - MemoryContext oldContext = CurrentMemoryContext; - MemoryContext tupleContext = NULL; - TupleTableSlot *scanTupleSlot = NULL; - List *columnList = NIL; - ForeignScanState *scanState = NULL; - char *relationName = NULL; - int executorFlags = 0; - uint32 columnIndex = 0; - - TupleDesc tupleDescriptor = RelationGetDescr(relation); - uint32 columnCount = tupleDescriptor->natts; - - cstore_fdw_initrel(relation); - - /* create list of columns of the relation */ - for (columnIndex = 0; columnIndex < columnCount; columnIndex++) - { - Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, columnIndex); - const Index tableId = 1; - - if (!attributeForm->attisdropped) - { - Var *column = makeVar(tableId, columnIndex + 1, attributeForm->atttypid, - attributeForm->atttypmod, attributeForm->attcollation, - 0); - columnList = lappend(columnList, column); - } - } - - /* setup foreign scan plan node */ - List *foreignPrivateList = list_make1(columnList); - ForeignScan *foreignScan = makeNode(ForeignScan); - foreignScan->fdw_private = foreignPrivateList; - - /* set up tuple slot */ - Datum *columnValues = palloc0(columnCount * sizeof(Datum)); - bool *columnNulls = palloc0(columnCount * sizeof(bool)); -#if PG_VERSION_NUM >= 120000 - scanTupleSlot = MakeTupleTableSlot(NULL, &TTSOpsVirtual); -#elif PG_VERSION_NUM >= 110000 - scanTupleSlot = MakeTupleTableSlot(NULL); -#else - scanTupleSlot = MakeTupleTableSlot(); -#endif - scanTupleSlot->tts_tupleDescriptor = tupleDescriptor; - scanTupleSlot->tts_values = columnValues; - scanTupleSlot->tts_isnull = columnNulls; - - /* setup scan state */ - scanState = makeNode(ForeignScanState); - scanState->ss.ss_currentRelation = relation; - scanState->ss.ps.plan = (Plan *) foreignScan; - scanState->ss.ss_ScanTupleSlot = scanTupleSlot; - - /* - * Use per-tuple memory context to prevent leak of memory used to read and - * parse rows from the file. - */ - tupleContext = AllocSetContextCreate(CurrentMemoryContext, - "cstore_fdw temporary context", - ALLOCSET_DEFAULT_SIZES); - - CStoreBeginForeignScan(scanState, executorFlags); - - /* prepare for sampling rows */ - selectionState = anl_init_selection_state(targetRowCount); - - for (;;) - { - /* check for user-requested abort or sleep */ - vacuum_delay_point(); - - memset(columnValues, 0, columnCount * sizeof(Datum)); - memset(columnNulls, true, columnCount * sizeof(bool)); - - MemoryContextReset(tupleContext); - MemoryContextSwitchTo(tupleContext); - - /* read the next record */ - CStoreIterateForeignScan(scanState); - - MemoryContextSwitchTo(oldContext); - - /* if there are no more records to read, break */ - if (TTS_EMPTY(scanTupleSlot)) - { - break; - } - - /* - * The first targetRowCount sample rows are simply copied into the - * reservoir. Then we start replacing tuples in the sample until we - * reach the end of the relation. This algorithm is from Jeff Vitter's - * paper (see more info in commands/analyze.c). - */ - if (sampleRowCount < targetRowCount) - { - sampleRows[sampleRowCount] = heap_form_tuple(tupleDescriptor, columnValues, - columnNulls); - sampleRowCount++; - } - else - { - /* - * t in Vitter's paper is the number of records already processed. - * If we need to compute a new S value, we must use the "not yet - * incremented" value of rowCount as t. - */ - if (rowCountToSkip < 0) - { - rowCountToSkip = anl_get_next_S(rowCount, targetRowCount, - &selectionState); - } - - if (rowCountToSkip <= 0) - { - /* - * Found a suitable tuple, so save it, replacing one old tuple - * at random. - */ - int rowIndex = (int) (targetRowCount * anl_random_fract()); - Assert(rowIndex >= 0); - Assert(rowIndex < targetRowCount); - - heap_freetuple(sampleRows[rowIndex]); - sampleRows[rowIndex] = heap_form_tuple(tupleDescriptor, - columnValues, columnNulls); - } - - rowCountToSkip--; - } - - rowCount++; - } - - /* clean up */ - MemoryContextDelete(tupleContext); - pfree(columnValues); - pfree(columnNulls); - - CStoreEndForeignScan(scanState); - - /* emit some interesting relation info */ - relationName = RelationGetRelationName(relation); - ereport(logLevel, (errmsg("\"%s\": file contains %.0f rows; %d rows in sample", - relationName, rowCount, sampleRowCount))); - - (*totalRowCount) = rowCount; - (*totalDeadRowCount) = 0; - - return sampleRowCount; -} - - -/* - * CStorePlanForeignModify checks if operation is supported. Only insert - * command with subquery (ie insert into
select ...) is supported. - * Other forms of insert, delete, and update commands are not supported. It - * throws an error when the command is not supported. - */ -static List * -CStorePlanForeignModify(PlannerInfo *plannerInfo, ModifyTable *plan, - Index resultRelation, int subplanIndex) -{ - bool operationSupported = false; - - if (plan->operation == CMD_INSERT) - { - ListCell *tableCell = NULL; - - /* - * Only insert operation with select subquery is supported. Other forms - * of insert, update, and delete operations are not supported. - */ - Query *query = plannerInfo->parse; - foreach(tableCell, query->rtable) - { - RangeTblEntry *tableEntry = lfirst(tableCell); - - if (tableEntry->rtekind == RTE_SUBQUERY && - tableEntry->subquery != NULL && - tableEntry->subquery->commandType == CMD_SELECT) - { - operationSupported = true; - break; - } - } - } - - if (!operationSupported) - { - ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("operation is not supported"))); - } - - return NIL; -} - - -/* - * CStoreBeginForeignModify prepares cstore table for a modification. - * Only insert is currently supported. - */ -static void -CStoreBeginForeignModify(ModifyTableState *modifyTableState, - ResultRelInfo *relationInfo, List *fdwPrivate, - int subplanIndex, int executorFlags) -{ - /* if Explain with no Analyze, do nothing */ - if (executorFlags & EXEC_FLAG_EXPLAIN_ONLY) - { - return; - } - - Assert(modifyTableState->operation == CMD_INSERT); - - CStoreBeginForeignInsert(modifyTableState, relationInfo); -} - - -/* - * CStoreBeginForeignInsert prepares a cstore table for an insert or rows - * coming from a COPY. - */ -static void -CStoreBeginForeignInsert(ModifyTableState *modifyTableState, ResultRelInfo *relationInfo) -{ - Oid foreignTableOid = RelationGetRelid(relationInfo->ri_RelationDesc); - Relation relation = cstore_fdw_open(foreignTableOid, RowExclusiveLock); - CStoreOptions *cstoreOptions = CStoreGetOptions(foreignTableOid); - TupleDesc tupleDescriptor = RelationGetDescr(relationInfo->ri_RelationDesc); - - TableWriteState *writeState = CStoreBeginWrite(relation->rd_node, - cstoreOptions->compressionType, - cstoreOptions->stripeRowCount, - cstoreOptions->blockRowCount, - tupleDescriptor); - - relationInfo->ri_FdwState = (void *) writeState; - - /* keep the lock */ - relation_close(relation, NoLock); -} - - -/* - * CStoreExecForeignInsert inserts a single row to cstore table - * and returns inserted row's data values. - */ -static TupleTableSlot * -CStoreExecForeignInsert(EState *executorState, ResultRelInfo *relationInfo, - TupleTableSlot *tupleSlot, TupleTableSlot *planSlot) -{ - TableWriteState *writeState = (TableWriteState *) relationInfo->ri_FdwState; - - Assert(writeState != NULL); - - HeapTuple heapTuple = GetSlotHeapTuple(tupleSlot); - - if (HeapTupleHasExternal(heapTuple)) - { - /* detoast any toasted attributes */ - HeapTuple newTuple = toast_flatten_tuple(heapTuple, - tupleSlot->tts_tupleDescriptor); - - ExecForceStoreHeapTuple(newTuple, tupleSlot, true); - } - - slot_getallattrs(tupleSlot); - - CStoreWriteRow(writeState, tupleSlot->tts_values, tupleSlot->tts_isnull); - - return tupleSlot; -} - - -/* - * CStoreEndForeignModify ends the current modification. Only insert is currently - * supported. - */ -static void -CStoreEndForeignModify(EState *executorState, ResultRelInfo *relationInfo) -{ - CStoreEndForeignInsert(executorState, relationInfo); -} - - -/* - * CStoreEndForeignInsert ends the current insert or COPY operation. - */ -static void -CStoreEndForeignInsert(EState *executorState, ResultRelInfo *relationInfo) -{ - TableWriteState *writeState = (TableWriteState *) relationInfo->ri_FdwState; - - /* writeState is NULL during Explain queries */ - if (writeState != NULL) - { - CStoreEndWrite(writeState); - } -} - - -#if PG_VERSION_NUM >= 90600 - -/* - * CStoreIsForeignScanParallelSafe always returns true to indicate that - * reading from a cstore_fdw table in a parallel worker is safe. This - * does not enable parallelism for queries on individual cstore_fdw - * tables, but does allow parallel scans of cstore_fdw partitions. - * - * cstore_fdw is parallel-safe because all writes are immediately committed - * to disk and then read from disk. There is no uncommitted state that needs - * to be shared across processes. - */ -static bool -CStoreIsForeignScanParallelSafe(PlannerInfo *root, RelOptInfo *rel, - RangeTblEntry *rte) -{ - return true; -} - - -#endif - -/* - * Versions 12 and later do not initialize rd_node even if the relation has a - * valid relfilenode, so we need to initialize it each time a cstore FDW - * relation is opened. - */ -static void -cstore_fdw_initrel(Relation rel) -{ -#if PG_VERSION_NUM >= 120000 - if (rel->rd_rel->relfilenode == InvalidOid) - { - FdwNewRelFileNode(rel); - } - - /* - * Copied code from RelationInitPhysicalAddr(), which doesn't - * work on foreign tables. - */ - if (OidIsValid(rel->rd_rel->reltablespace)) - { - rel->rd_node.spcNode = rel->rd_rel->reltablespace; - } - else - { - rel->rd_node.spcNode = MyDatabaseTableSpace; - } - - rel->rd_node.dbNode = MyDatabaseId; - rel->rd_node.relNode = rel->rd_rel->relfilenode; -#endif - FdwCreateStorage(rel); -} - - -static Relation -cstore_fdw_open(Oid relationId, LOCKMODE lockmode) -{ - Relation rel = heap_open(relationId, lockmode); - - cstore_fdw_initrel(rel); - - return rel; -} - - -static Relation -cstore_fdw_openrv(RangeVar *relation, LOCKMODE lockmode) -{ - Relation rel = heap_openrv(relation, lockmode); - - cstore_fdw_initrel(rel); - - return rel; -} - - -/* - * Implements object_access_hook. One of the places this is called is just - * before dropping an object, which allows us to clean-up resources for - * cstore tables. - * - * When cleaning up resources, we need to have access to the pg_class record - * for the table so we can indentify the relfilenode belonging to the relation. - * We don't have access to this information in sql_drop event triggers, since - * the relation has already been dropped there. object_access_hook is called - * __before__ dropping tables, so we still have access to the pg_class - * entry here. - * - * Note that the utility hook is called once per __command__, and not for - * every object dropped, and since a drop can cascade to other objects, it - * is difficult to get full set of dropped objects in the utility hook. - * But object_access_hook is called once per dropped object, so it is - * much easier to clean-up all dropped objects here. - */ -static void -CStoreFdwObjectAccessHook(ObjectAccessType access, Oid classId, Oid objectId, - int subId, void *arg) -{ - if (prevObjectAccessHook) - { - prevObjectAccessHook(access, classId, objectId, subId, arg); - } - - /* - * Do nothing if this is not a DROP relation command. - */ - if (access != OAT_DROP || classId != RelationRelationId || OidIsValid(subId)) - { - return; - } - - /* - * Lock relation to prevent it from being dropped and to avoid - * race conditions in the next if block. - */ - LockRelationOid(objectId, AccessShareLock); - - if (IsCStoreFdwTable(objectId)) - { - /* - * Drop both metadata and storage. We need to drop storage here since - * we manage relfilenode for FDW tables in the extension. - */ - Relation rel = cstore_fdw_open(objectId, AccessExclusiveLock); - RelationOpenSmgr(rel); - RelationDropStorage(rel); - DeleteDataFileMetadataRowIfExists(rel->rd_node.relNode); - - /* keep the lock since we did physical changes to the relation */ - relation_close(rel, NoLock); - } -} diff --git a/src/backend/columnar/mod.c b/src/backend/columnar/mod.c index ad4605e24..a98c81021 100644 --- a/src/backend/columnar/mod.c +++ b/src/backend/columnar/mod.c @@ -18,20 +18,16 @@ #include "citus_version.h" #include "columnar/cstore.h" -#include "columnar/cstore_fdw.h" #include "columnar/mod.h" #ifdef HAS_TABLEAM #include "columnar/cstore_tableam.h" #endif - void columnar_init(void) { cstore_init(); - cstore_fdw_init(); - #ifdef HAS_TABLEAM cstore_tableam_init(); #endif @@ -41,8 +37,6 @@ columnar_init(void) void columnar_fini(void) { - cstore_fdw_finish(); - #if HAS_TABLEAM cstore_tableam_finish(); #endif diff --git a/src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql b/src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql index e1b6b0f54..c3a34f5af 100644 --- a/src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql +++ b/src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql @@ -3,34 +3,6 @@ CREATE SCHEMA cstore; SET search_path TO cstore; -CREATE FUNCTION cstore_fdw_handler() -RETURNS fdw_handler -AS 'MODULE_PATHNAME' -LANGUAGE C STRICT; - -CREATE FUNCTION cstore_fdw_validator(text[], oid) -RETURNS void -AS 'MODULE_PATHNAME' -LANGUAGE C STRICT; - -CREATE FOREIGN DATA WRAPPER cstore_fdw -HANDLER cstore_fdw_handler -VALIDATOR cstore_fdw_validator; - -CREATE FUNCTION cstore_ddl_event_end_trigger() -RETURNS event_trigger -AS 'MODULE_PATHNAME' -LANGUAGE C STRICT; - -CREATE EVENT TRIGGER cstore_ddl_event_end -ON ddl_command_end -EXECUTE PROCEDURE cstore_ddl_event_end_trigger(); - -CREATE FUNCTION pg_catalog.cstore_table_size(relation regclass) -RETURNS bigint -AS 'MODULE_PATHNAME' -LANGUAGE C STRICT; - CREATE TABLE cstore_data_files ( relfilenode oid NOT NULL, block_row_count int NOT NULL, diff --git a/src/backend/columnar/sql/downgrades/columnar--10.0-1--9.5-1.sql b/src/backend/columnar/sql/downgrades/columnar--10.0-1--9.5-1.sql index 5b6a9cbd7..74742d8ea 100644 --- a/src/backend/columnar/sql/downgrades/columnar--10.0-1--9.5-1.sql +++ b/src/backend/columnar/sql/downgrades/columnar--10.0-1--9.5-1.sql @@ -32,15 +32,6 @@ DROP TABLE cstore_skipnodes; DROP TABLE cstore_stripes; DROP TABLE cstore_data_files; -DROP FUNCTION pg_catalog.cstore_table_size(relation regclass); - -DROP EVENT TRIGGER cstore_ddl_event_end; -DROP FUNCTION cstore_ddl_event_end_trigger(); - -DROP FOREIGN DATA WRAPPER cstore_fdw; -DROP FUNCTION cstore_fdw_validator(text[], oid); -DROP FUNCTION cstore_fdw_handler(); - DROP FUNCTION citus_internal.cstore_ensure_objects_exist(); RESET search_path; diff --git a/src/include/columnar/cstore_fdw.h b/src/include/columnar/cstore_fdw.h deleted file mode 100644 index 1c8170ae8..000000000 --- a/src/include/columnar/cstore_fdw.h +++ /dev/null @@ -1,35 +0,0 @@ -/*------------------------------------------------------------------------- - * - * cstore_fdw.h - * - * Type and function declarations for CStore foreign data wrapper. - * - * Copyright (c) 2016, Citus Data, Inc. - * - * $Id$ - * - *------------------------------------------------------------------------- - */ - -#ifndef CSTORE_FDW_H -#define CSTORE_FDW_H - -#include "postgres.h" - -#include "fmgr.h" - -void cstore_fdw_init(void); -void cstore_fdw_finish(void); - -/* event trigger function declarations */ -extern Datum cstore_ddl_event_end_trigger(PG_FUNCTION_ARGS); - -/* Function declarations for utility UDFs */ -extern Datum cstore_table_size(PG_FUNCTION_ARGS); -extern Datum cstore_clean_table_resources(PG_FUNCTION_ARGS); - -/* Function declarations for foreign data wrapper */ -extern Datum cstore_fdw_handler(PG_FUNCTION_ARGS); -extern Datum cstore_fdw_validator(PG_FUNCTION_ARGS); - -#endif /* CSTORE_FDW_H */ diff --git a/src/test/regress/Makefile b/src/test/regress/Makefile index 178bb6356..9b5a38fcb 100644 --- a/src/test/regress/Makefile +++ b/src/test/regress/Makefile @@ -163,10 +163,6 @@ check-follower-cluster: all COLUMNAR_SCHEDULES = COLUMNAR_ISOLATION_SCHEDULES = -# even though we always add the fdw schedules, keep them separate from the declaration -# above for easy removabl when fdw support is removed -COLUMNAR_SCHEDULES += columnar_fdw_schedule -COLUMNAR_ISOLATION_SCHEDULES += columnar_fdw_isolation_schedule ifeq ($(HAS_TABLEAM),yes) COLUMNAR_SCHEDULES += columnar_am_schedule COLUMNAR_ISOLATION_SCHEDULES += columnar_am_isolation_schedule diff --git a/src/test/regress/columnar_am_schedule b/src/test/regress/columnar_am_schedule index cd52d702b..1519da29f 100644 --- a/src/test/regress/columnar_am_schedule +++ b/src/test/regress/columnar_am_schedule @@ -3,7 +3,6 @@ test: am_load test: am_query test: am_analyze test: am_data_types -test: am_functions test: am_drop test: am_insert test: am_copyto diff --git a/src/test/regress/columnar_fdw_isolation_schedule b/src/test/regress/columnar_fdw_isolation_schedule deleted file mode 100644 index 4b9bae8a3..000000000 --- a/src/test/regress/columnar_fdw_isolation_schedule +++ /dev/null @@ -1 +0,0 @@ -# just an empty file now, please remove when we have a test diff --git a/src/test/regress/columnar_fdw_schedule b/src/test/regress/columnar_fdw_schedule deleted file mode 100644 index 6998f3bf5..000000000 --- a/src/test/regress/columnar_fdw_schedule +++ /dev/null @@ -1,14 +0,0 @@ -test: fdw_create -test: fdw_load -test: fdw_query -test: fdw_analyze -test: fdw_data_types -test: fdw_functions -test: fdw_block_filtering -test: fdw_drop -test: fdw_insert -test: fdw_copyto -test: fdw_alter -test: fdw_rollback -test: fdw_truncate -test: fdw_clean diff --git a/src/test/regress/expected/am_functions.out b/src/test/regress/expected/am_functions.out deleted file mode 100644 index a4a8e9c90..000000000 --- a/src/test/regress/expected/am_functions.out +++ /dev/null @@ -1,18 +0,0 @@ --- --- Test utility functions for cstore_fdw tables. --- -CREATE TABLE empty_table (a int) USING columnar; -CREATE TABLE table_with_data (a int) USING columnar; -CREATE TABLE non_cstore_table (a int); -COPY table_with_data FROM STDIN; -SELECT pg_relation_size('empty_table') < pg_relation_size('table_with_data'); - ?column? ---------------------------------------------------------------------- - t -(1 row) - -SELECT cstore_table_size('non_cstore_table'); -ERROR: relation is not a cstore table -DROP TABLE empty_table; -DROP TABLE table_with_data; -DROP TABLE non_cstore_table; diff --git a/src/test/regress/expected/am_truncate_0.out b/src/test/regress/expected/am_truncate_0.out index 073e8f042..ffe2e6825 100644 --- a/src/test/regress/expected/am_truncate_0.out +++ b/src/test/regress/expected/am_truncate_0.out @@ -69,12 +69,6 @@ SELECT count(*) FROM cstore_truncate_test_compressed; 0 (1 row) -SELECT cstore_table_size('cstore_truncate_test_compressed'); - cstore_table_size ---------------------------------------------------------------------- - 26 -(1 row) - -- make sure data files still present SELECT count(*) FROM ( SELECT pg_ls_dir('cstore_fdw/' || databaseoid ) FROM ( diff --git a/src/test/regress/expected/fdw_alter.out b/src/test/regress/expected/fdw_alter.out deleted file mode 100644 index 7e69a4178..000000000 --- a/src/test/regress/expected/fdw_alter.out +++ /dev/null @@ -1,174 +0,0 @@ --- --- Testing ALTER TABLE on cstore_fdw tables. --- -CREATE FOREIGN TABLE test_alter_table (a int, b int, c int) SERVER cstore_server; -WITH sample_data AS (VALUES - (1, 2, 3), - (4, 5, 6), - (7, 8, 9) -) -INSERT INTO test_alter_table SELECT * FROM sample_data; --- drop a column -ALTER FOREIGN TABLE test_alter_table DROP COLUMN a; --- test analyze -ANALYZE test_alter_table; --- verify select queries run as expected -SELECT * FROM test_alter_table; - b | c ---------------------------------------------------------------------- - 2 | 3 - 5 | 6 - 8 | 9 -(3 rows) - -SELECT a FROM test_alter_table; -ERROR: column "a" does not exist -SELECT b FROM test_alter_table; - b ---------------------------------------------------------------------- - 2 - 5 - 8 -(3 rows) - --- verify insert runs as expected -INSERT INTO test_alter_table (SELECT 3, 5, 8); -ERROR: INSERT has more expressions than target columns -INSERT INTO test_alter_table (SELECT 5, 8); --- add a column with no defaults -ALTER FOREIGN TABLE test_alter_table ADD COLUMN d int; -SELECT * FROM test_alter_table; - b | c | d ---------------------------------------------------------------------- - 2 | 3 | - 5 | 6 | - 8 | 9 | - 5 | 8 | -(4 rows) - -INSERT INTO test_alter_table (SELECT 3, 5, 8); -SELECT * FROM test_alter_table; - b | c | d ---------------------------------------------------------------------- - 2 | 3 | - 5 | 6 | - 8 | 9 | - 5 | 8 | - 3 | 5 | 8 -(5 rows) - --- add a fixed-length column with default value -ALTER FOREIGN TABLE test_alter_table ADD COLUMN e int default 3; -SELECT * from test_alter_table; - b | c | d | e ---------------------------------------------------------------------- - 2 | 3 | | 3 - 5 | 6 | | 3 - 8 | 9 | | 3 - 5 | 8 | | 3 - 3 | 5 | 8 | 3 -(5 rows) - -INSERT INTO test_alter_table (SELECT 1, 2, 4, 8); -SELECT * from test_alter_table; - b | c | d | e ---------------------------------------------------------------------- - 2 | 3 | | 3 - 5 | 6 | | 3 - 8 | 9 | | 3 - 5 | 8 | | 3 - 3 | 5 | 8 | 3 - 1 | 2 | 4 | 8 -(6 rows) - --- add a variable-length column with default value -ALTER FOREIGN TABLE test_alter_table ADD COLUMN f text DEFAULT 'TEXT ME'; -SELECT * from test_alter_table; - b | c | d | e | f ---------------------------------------------------------------------- - 2 | 3 | | 3 | TEXT ME - 5 | 6 | | 3 | TEXT ME - 8 | 9 | | 3 | TEXT ME - 5 | 8 | | 3 | TEXT ME - 3 | 5 | 8 | 3 | TEXT ME - 1 | 2 | 4 | 8 | TEXT ME -(6 rows) - -INSERT INTO test_alter_table (SELECT 1, 2, 4, 8, 'ABCDEF'); -SELECT * from test_alter_table; - b | c | d | e | f ---------------------------------------------------------------------- - 2 | 3 | | 3 | TEXT ME - 5 | 6 | | 3 | TEXT ME - 8 | 9 | | 3 | TEXT ME - 5 | 8 | | 3 | TEXT ME - 3 | 5 | 8 | 3 | TEXT ME - 1 | 2 | 4 | 8 | TEXT ME - 1 | 2 | 4 | 8 | ABCDEF -(7 rows) - --- drop couple of columns -ALTER FOREIGN TABLE test_alter_table DROP COLUMN c; -ALTER FOREIGN TABLE test_alter_table DROP COLUMN e; -ANALYZE test_alter_table; -SELECT * from test_alter_table; - b | d | f ---------------------------------------------------------------------- - 2 | | TEXT ME - 5 | | TEXT ME - 8 | | TEXT ME - 5 | | TEXT ME - 3 | 8 | TEXT ME - 1 | 4 | TEXT ME - 1 | 4 | ABCDEF -(7 rows) - -SELECT count(*) from test_alter_table; - count ---------------------------------------------------------------------- - 7 -(1 row) - -SELECT count(t.*) from test_alter_table t; - count ---------------------------------------------------------------------- - 7 -(1 row) - --- unsupported default values -ALTER FOREIGN TABLE test_alter_table ADD COLUMN g boolean DEFAULT isfinite(current_date); -ALTER FOREIGN TABLE test_alter_table ADD COLUMN h DATE DEFAULT current_date; -SELECT * FROM test_alter_table; -ERROR: unsupported default value for column "g" -HINT: Expression is either mutable or does not evaluate to constant value -ALTER FOREIGN TABLE test_alter_table ALTER COLUMN g DROP DEFAULT; -SELECT * FROM test_alter_table; -ERROR: unsupported default value for column "h" -HINT: Expression is either mutable or does not evaluate to constant value -ALTER FOREIGN TABLE test_alter_table ALTER COLUMN h DROP DEFAULT; -ANALYZE test_alter_table; -SELECT * FROM test_alter_table; - b | d | f | g | h ---------------------------------------------------------------------- - 2 | | TEXT ME | | - 5 | | TEXT ME | | - 8 | | TEXT ME | | - 5 | | TEXT ME | | - 3 | 8 | TEXT ME | | - 1 | 4 | TEXT ME | | - 1 | 4 | ABCDEF | | -(7 rows) - --- unsupported type change -ALTER FOREIGN TABLE test_alter_table ADD COLUMN i int; -ALTER FOREIGN TABLE test_alter_table ADD COLUMN j float; -ALTER FOREIGN TABLE test_alter_table ADD COLUMN k text; --- this is valid type change -ALTER FOREIGN TABLE test_alter_table ALTER COLUMN i TYPE float; --- this is not valid -ALTER FOREIGN TABLE test_alter_table ALTER COLUMN j TYPE int; -ERROR: Column j cannot be cast automatically to type pg_catalog.int4 --- text / varchar conversion is valid both ways -ALTER FOREIGN TABLE test_alter_table ALTER COLUMN k TYPE varchar(20); -ALTER FOREIGN TABLE test_alter_table ALTER COLUMN k TYPE text; -DROP FOREIGN TABLE test_alter_table; diff --git a/src/test/regress/expected/fdw_analyze.out b/src/test/regress/expected/fdw_analyze.out deleted file mode 100644 index 654391597..000000000 --- a/src/test/regress/expected/fdw_analyze.out +++ /dev/null @@ -1,19 +0,0 @@ --- --- Test the ANALYZE command for cstore_fdw tables. --- --- ANALYZE uncompressed table -ANALYZE contestant; -SELECT count(*) FROM pg_stats WHERE tablename='contestant'; - count ---------------------------------------------------------------------- - 6 -(1 row) - --- ANALYZE compressed table -ANALYZE contestant_compressed; -SELECT count(*) FROM pg_stats WHERE tablename='contestant_compressed'; - count ---------------------------------------------------------------------- - 6 -(1 row) - diff --git a/src/test/regress/expected/fdw_clean.out b/src/test/regress/expected/fdw_clean.out deleted file mode 100644 index ecd4d67a1..000000000 --- a/src/test/regress/expected/fdw_clean.out +++ /dev/null @@ -1,10 +0,0 @@ -DROP FOREIGN TABLE collation_block_filtering_test; -DROP FOREIGN TABLE test_block_filtering; -DROP FOREIGN TABLE test_null_values; -DROP FOREIGN TABLE test_other_types; -DROP FOREIGN TABLE test_range_types; -DROP FOREIGN TABLE test_enum_and_composite_types; -DROP TYPE composite_type; -DROP TYPE enum_type; -DROP FOREIGN TABLE test_datetime_types; -DROP FOREIGN TABLE test_array_types; diff --git a/src/test/regress/expected/fdw_drop.out b/src/test/regress/expected/fdw_drop.out deleted file mode 100644 index def073b20..000000000 --- a/src/test/regress/expected/fdw_drop.out +++ /dev/null @@ -1,58 +0,0 @@ --- --- Tests the different DROP commands for cstore_fdw tables. --- --- DROP FOREIGN TABL --- DROP SCHEMA --- DROP EXTENSION --- DROP DATABASE --- --- Note that travis does not create --- cstore_fdw extension in default database (postgres). This has caused --- different behavior between travis tests and local tests. Thus --- 'postgres' directory is excluded from comparison to have the same result. --- store postgres database oid -SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset -SELECT count(*) AS cstore_data_files_before_drop FROM cstore.cstore_data_files \gset --- DROP cstore_fdw tables -DROP FOREIGN TABLE contestant; -DROP FOREIGN TABLE contestant_compressed; --- make sure DROP deletes metadata -SELECT :cstore_data_files_before_drop - count(*) FROM cstore.cstore_data_files; - ?column? ---------------------------------------------------------------------- - 2 -(1 row) - --- Create a cstore_fdw table under a schema and drop it. -CREATE SCHEMA test_schema; -CREATE FOREIGN TABLE test_schema.test_table(data int) SERVER cstore_server; -SELECT count(*) AS cstore_data_files_before_drop FROM cstore.cstore_data_files \gset -DROP SCHEMA test_schema CASCADE; -NOTICE: drop cascades to foreign table test_schema.test_table -SELECT :cstore_data_files_before_drop - count(*) FROM cstore.cstore_data_files; - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT current_database() datname \gset -CREATE DATABASE db_to_drop; -NOTICE: Citus partially supports CREATE DATABASE for distributed databases -DETAIL: Citus does not propagate CREATE DATABASE command to workers -HINT: You can manually create a database and its extensions on workers. -\c db_to_drop -CREATE EXTENSION citus; -CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw; -SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset -CREATE FOREIGN TABLE test_table(data int) SERVER cstore_server; -DROP EXTENSION citus CASCADE; -NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to server cstore_server -drop cascades to foreign table test_table --- test database drop -CREATE EXTENSION citus; -CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw; -SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset -CREATE FOREIGN TABLE test_table(data int) SERVER cstore_server; -\c :datname -DROP DATABASE db_to_drop; diff --git a/src/test/regress/expected/fdw_functions.out b/src/test/regress/expected/fdw_functions.out deleted file mode 100644 index cf047f578..000000000 --- a/src/test/regress/expected/fdw_functions.out +++ /dev/null @@ -1,18 +0,0 @@ --- --- Test utility functions for cstore_fdw tables. --- -CREATE FOREIGN TABLE empty_table (a int) SERVER cstore_server; -CREATE FOREIGN TABLE table_with_data (a int) SERVER cstore_server; -CREATE TABLE non_cstore_table (a int); -COPY table_with_data FROM STDIN; -SELECT cstore_table_size('empty_table') < cstore_table_size('table_with_data'); - ?column? ---------------------------------------------------------------------- - t -(1 row) - -SELECT cstore_table_size('non_cstore_table'); -ERROR: relation is not a cstore table -DROP FOREIGN TABLE empty_table; -DROP FOREIGN TABLE table_with_data; -DROP TABLE non_cstore_table; diff --git a/src/test/regress/expected/fdw_insert.out b/src/test/regress/expected/fdw_insert.out deleted file mode 100644 index 1cc94e678..000000000 --- a/src/test/regress/expected/fdw_insert.out +++ /dev/null @@ -1,88 +0,0 @@ --- --- Testing insert on cstore_fdw tables. --- -CREATE FOREIGN TABLE test_insert_command (a int) SERVER cstore_server; --- test single row inserts fail -select count(*) from test_insert_command; - count ---------------------------------------------------------------------- - 0 -(1 row) - -insert into test_insert_command values(1); -ERROR: operation is not supported -select count(*) from test_insert_command; - count ---------------------------------------------------------------------- - 0 -(1 row) - -insert into test_insert_command default values; -ERROR: operation is not supported -select count(*) from test_insert_command; - count ---------------------------------------------------------------------- - 0 -(1 row) - --- test inserting from another table succeed -CREATE TABLE test_insert_command_data (a int); -select count(*) from test_insert_command_data; - count ---------------------------------------------------------------------- - 0 -(1 row) - -insert into test_insert_command_data values(1); -select count(*) from test_insert_command_data; - count ---------------------------------------------------------------------- - 1 -(1 row) - -insert into test_insert_command select * from test_insert_command_data; -select count(*) from test_insert_command; - count ---------------------------------------------------------------------- - 1 -(1 row) - -drop table test_insert_command_data; -drop foreign table test_insert_command; --- test long attribute value insertion --- create sufficiently long text so that data is stored in toast -CREATE TABLE test_long_text AS -SELECT a as int_val, string_agg(random()::text, '') as text_val -FROM generate_series(1, 10) a, generate_series(1, 1000) b -GROUP BY a ORDER BY a; --- store hash values of text for later comparison -CREATE TABLE test_long_text_hash AS -SELECT int_val, md5(text_val) AS hash -FROM test_long_text; -CREATE FOREIGN TABLE test_cstore_long_text(int_val int, text_val text) -SERVER cstore_server; --- store long text in cstore table -INSERT INTO test_cstore_long_text SELECT * FROM test_long_text; --- drop source table to remove original text from toast -DROP TABLE test_long_text; --- check if text data is still available in cstore table --- by comparing previously stored hash. -SELECT a.int_val -FROM test_long_text_hash a, test_cstore_long_text c -WHERE a.int_val = c.int_val AND a.hash = md5(c.text_val); - int_val ---------------------------------------------------------------------- - 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 - 10 -(10 rows) - -DROP TABLE test_long_text_hash; -DROP FOREIGN TABLE test_cstore_long_text; diff --git a/src/test/regress/expected/fdw_query.out b/src/test/regress/expected/fdw_query.out deleted file mode 100644 index a83b18451..000000000 --- a/src/test/regress/expected/fdw_query.out +++ /dev/null @@ -1,105 +0,0 @@ --- --- Test querying cstore_fdw tables. --- --- Settings to make the result deterministic -SET datestyle = "ISO, YMD"; --- Query uncompressed data -SELECT count(*) FROM contestant; - count ---------------------------------------------------------------------- - 8 -(1 row) - -SELECT avg(rating), stddev_samp(rating) FROM contestant; - avg | stddev_samp ---------------------------------------------------------------------- - 2344.3750000000000000 | 433.746119785032 -(1 row) - -SELECT country, avg(rating) FROM contestant WHERE rating > 2200 - GROUP BY country ORDER BY country; - country | avg ---------------------------------------------------------------------- - XA | 2203.0000000000000000 - XB | 2610.5000000000000000 - XC | 2236.0000000000000000 - XD | 3090.0000000000000000 -(4 rows) - -SELECT * FROM contestant ORDER BY handle; - handle | birthdate | rating | percentile | country | achievements ---------------------------------------------------------------------- - a | 1990-01-10 | 2090 | 97.1 | XA | {a} - b | 1990-11-01 | 2203 | 98.1 | XA | {a,b} - c | 1988-11-01 | 2907 | 99.4 | XB | {w,y} - d | 1985-05-05 | 2314 | 98.3 | XB | {} - e | 1995-05-05 | 2236 | 98.2 | XC | {a} - f | 1983-04-02 | 3090 | 99.6 | XD | {a,b,c,y} - g | 1991-12-13 | 1803 | 85.1 | XD | {a,c} - h | 1987-10-26 | 2112 | 95.4 | XD | {w,a} -(8 rows) - --- Query compressed data -SELECT count(*) FROM contestant_compressed; - count ---------------------------------------------------------------------- - 8 -(1 row) - -SELECT avg(rating), stddev_samp(rating) FROM contestant_compressed; - avg | stddev_samp ---------------------------------------------------------------------- - 2344.3750000000000000 | 433.746119785032 -(1 row) - -SELECT country, avg(rating) FROM contestant_compressed WHERE rating > 2200 - GROUP BY country ORDER BY country; - country | avg ---------------------------------------------------------------------- - XA | 2203.0000000000000000 - XB | 2610.5000000000000000 - XC | 2236.0000000000000000 - XD | 3090.0000000000000000 -(4 rows) - -SELECT * FROM contestant_compressed ORDER BY handle; - handle | birthdate | rating | percentile | country | achievements ---------------------------------------------------------------------- - a | 1990-01-10 | 2090 | 97.1 | XA | {a} - b | 1990-11-01 | 2203 | 98.1 | XA | {a,b} - c | 1988-11-01 | 2907 | 99.4 | XB | {w,y} - d | 1985-05-05 | 2314 | 98.3 | XB | {} - e | 1995-05-05 | 2236 | 98.2 | XC | {a} - f | 1983-04-02 | 3090 | 99.6 | XD | {a,b,c,y} - g | 1991-12-13 | 1803 | 85.1 | XD | {a,c} - h | 1987-10-26 | 2112 | 95.4 | XD | {w,a} -(8 rows) - --- Verify that we handle whole-row references correctly -SELECT to_json(v) FROM contestant v ORDER BY rating LIMIT 1; - to_json ---------------------------------------------------------------------- - {"handle":"g","birthdate":"1991-12-13","rating":1803,"percentile":85.1,"country":"XD ","achievements":["a","c"]} -(1 row) - --- Test variables used in expressions -CREATE FOREIGN TABLE union_first (a int, b int) SERVER cstore_server; -CREATE FOREIGN TABLE union_second (a int, b int) SERVER cstore_server; -INSERT INTO union_first SELECT a, a FROM generate_series(1, 5) a; -INSERT INTO union_second SELECT a, a FROM generate_series(11, 15) a; -(SELECT a*1, b FROM union_first) union all (SELECT a*1, b FROM union_second); - ?column? | b ---------------------------------------------------------------------- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 - 11 | 11 - 12 | 12 - 13 | 13 - 14 | 14 - 15 | 15 -(10 rows) - -DROP FOREIGN TABLE union_first, union_second; diff --git a/src/test/regress/expected/fdw_rollback.out b/src/test/regress/expected/fdw_rollback.out deleted file mode 100644 index c70b2a49e..000000000 --- a/src/test/regress/expected/fdw_rollback.out +++ /dev/null @@ -1,77 +0,0 @@ --- --- Testing we handle rollbacks properly --- -CREATE FOREIGN TABLE t(a int, b int) SERVER cstore_server; -BEGIN; -INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i; -ROLLBACK; -SELECT count(*) FROM t; - count ---------------------------------------------------------------------- - 0 -(1 row) - --- check stripe metadata also have been rolled-back -SELECT count(*) FROM cstore.cstore_stripes a, pg_class b -WHERE a.relfilenode = b.relfilenode AND b.relname = 't'; - count ---------------------------------------------------------------------- - 0 -(1 row) - -INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i; -SELECT count(*) FROM t; - count ---------------------------------------------------------------------- - 10 -(1 row) - -SELECT count(*) FROM cstore.cstore_stripes a, pg_class b -WHERE a.relfilenode = b.relfilenode AND b.relname = 't'; - count ---------------------------------------------------------------------- - 1 -(1 row) - --- savepoint rollback -BEGIN; -SAVEPOINT s0; -INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i; -SAVEPOINT s1; -INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i; -SELECT count(*) FROM t; - count ---------------------------------------------------------------------- - 30 -(1 row) - -ROLLBACK TO SAVEPOINT s1; -SELECT count(*) FROM t; - count ---------------------------------------------------------------------- - 20 -(1 row) - -ROLLBACK TO SAVEPOINT s0; -SELECT count(*) FROM t; - count ---------------------------------------------------------------------- - 10 -(1 row) - -INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i; -COMMIT; -SELECT count(*) FROM t; - count ---------------------------------------------------------------------- - 20 -(1 row) - -SELECT count(*) FROM cstore.cstore_stripes a, pg_class b -WHERE a.relfilenode = b.relfilenode AND b.relname = 't'; - count ---------------------------------------------------------------------- - 2 -(1 row) - -DROP FOREIGN TABLE t; diff --git a/src/test/regress/expected/fdw_truncate.out b/src/test/regress/expected/fdw_truncate.out deleted file mode 100644 index a2c95da01..000000000 --- a/src/test/regress/expected/fdw_truncate.out +++ /dev/null @@ -1,265 +0,0 @@ --- --- Test the TRUNCATE TABLE command for cstore_fdw tables. --- --- print whether we're using version > 10 to make version-specific tests clear -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten; - version_above_ten ---------------------------------------------------------------------- - t -(1 row) - --- CREATE a cstore_fdw table, fill with some data -- -CREATE FOREIGN TABLE cstore_truncate_test (a int, b int) SERVER cstore_server; -CREATE FOREIGN TABLE cstore_truncate_test_second (a int, b int) SERVER cstore_server; -CREATE FOREIGN TABLE cstore_truncate_test_compressed (a int, b int) SERVER cstore_server OPTIONS (compression 'pglz'); -CREATE TABLE cstore_truncate_test_regular (a int, b int); -SELECT count(*) AS cstore_data_files_before_truncate FROM cstore.cstore_data_files \gset -INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a; -INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a; -INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a; --- query rows -SELECT * FROM cstore_truncate_test; - a | b ---------------------------------------------------------------------- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 6 - 7 | 7 - 8 | 8 - 9 | 9 - 10 | 10 -(10 rows) - -TRUNCATE TABLE cstore_truncate_test; -SELECT * FROM cstore_truncate_test; - a | b ---------------------------------------------------------------------- -(0 rows) - -SELECT COUNT(*) from cstore_truncate_test; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT count(*) FROM cstore_truncate_test_compressed; - count ---------------------------------------------------------------------- - 20 -(1 row) - -TRUNCATE TABLE cstore_truncate_test_compressed; -SELECT count(*) FROM cstore_truncate_test_compressed; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT cstore_table_size('cstore_truncate_test_compressed'); - cstore_table_size ---------------------------------------------------------------------- - 0 -(1 row) - -INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a; -INSERT INTO cstore_truncate_test_regular select a, a from generate_series(10, 20) a; -INSERT INTO cstore_truncate_test_second select a, a from generate_series(20, 30) a; -SELECT * from cstore_truncate_test; - a | b ---------------------------------------------------------------------- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 6 - 7 | 7 - 8 | 8 - 9 | 9 - 10 | 10 -(10 rows) - -SELECT * from cstore_truncate_test_second; - a | b ---------------------------------------------------------------------- - 20 | 20 - 21 | 21 - 22 | 22 - 23 | 23 - 24 | 24 - 25 | 25 - 26 | 26 - 27 | 27 - 28 | 28 - 29 | 29 - 30 | 30 -(11 rows) - -SELECT * from cstore_truncate_test_regular; - a | b ---------------------------------------------------------------------- - 10 | 10 - 11 | 11 - 12 | 12 - 13 | 13 - 14 | 14 - 15 | 15 - 16 | 16 - 17 | 17 - 18 | 18 - 19 | 19 - 20 | 20 -(11 rows) - --- make sure multi truncate works --- notice that the same table might be repeated -TRUNCATE TABLE cstore_truncate_test, - cstore_truncate_test_regular, - cstore_truncate_test_second, - cstore_truncate_test; -SELECT * from cstore_truncate_test; - a | b ---------------------------------------------------------------------- -(0 rows) - -SELECT * from cstore_truncate_test_second; - a | b ---------------------------------------------------------------------- -(0 rows) - -SELECT * from cstore_truncate_test_regular; - a | b ---------------------------------------------------------------------- -(0 rows) - --- test if truncate on empty table works -TRUNCATE TABLE cstore_truncate_test; -SELECT * from cstore_truncate_test; - a | b ---------------------------------------------------------------------- -(0 rows) - --- make sure TRUNATE deletes metadata for old relfilenode -SELECT :cstore_data_files_before_truncate - count(*) FROM cstore.cstore_data_files; - ?column? ---------------------------------------------------------------------- - 0 -(1 row) - --- test if truncation in the same transaction that created the table works properly -BEGIN; -CREATE FOREIGN TABLE cstore_same_transaction_truncate(a int) SERVER cstore_server; -INSERT INTO cstore_same_transaction_truncate SELECT * FROM generate_series(1, 100); -TRUNCATE cstore_same_transaction_truncate; -INSERT INTO cstore_same_transaction_truncate SELECT * FROM generate_series(20, 23); -COMMIT; --- should output "1" for the newly created relation -SELECT count(*) - :cstore_data_files_before_truncate FROM cstore.cstore_data_files; - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT * FROM cstore_same_transaction_truncate; - a ---------------------------------------------------------------------- - 20 - 21 - 22 - 23 -(4 rows) - -DROP FOREIGN TABLE cstore_same_transaction_truncate; --- test if a cached truncate from a pl/pgsql function works -CREATE FUNCTION cstore_truncate_test_regular_func() RETURNS void AS $$ -BEGIN - INSERT INTO cstore_truncate_test_regular select a, a from generate_series(1, 10) a; - TRUNCATE TABLE cstore_truncate_test_regular; -END;$$ -LANGUAGE plpgsql; -SELECT cstore_truncate_test_regular_func(); - cstore_truncate_test_regular_func ---------------------------------------------------------------------- - -(1 row) - --- the cached plans are used stating from the second call -SELECT cstore_truncate_test_regular_func(); - cstore_truncate_test_regular_func ---------------------------------------------------------------------- - -(1 row) - -DROP FUNCTION cstore_truncate_test_regular_func(); -DROP FOREIGN TABLE cstore_truncate_test, cstore_truncate_test_second; -DROP TABLE cstore_truncate_test_regular; -DROP FOREIGN TABLE cstore_truncate_test_compressed; --- test truncate with schema -CREATE SCHEMA truncate_schema; -CREATE FOREIGN TABLE truncate_schema.truncate_tbl (id int) SERVER cstore_server OPTIONS(compression 'pglz'); -INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100); -SELECT COUNT(*) FROM truncate_schema.truncate_tbl; - count ---------------------------------------------------------------------- - 100 -(1 row) - -TRUNCATE TABLE truncate_schema.truncate_tbl; -SELECT COUNT(*) FROM truncate_schema.truncate_tbl; - count ---------------------------------------------------------------------- - 0 -(1 row) - -INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100); --- create a user that can not truncate -CREATE USER truncate_user; -NOTICE: not propagating CREATE ROLE/USER commands to worker nodes -HINT: Connect to worker nodes directly to manually create all necessary users and roles. -GRANT USAGE ON SCHEMA truncate_schema TO truncate_user; -GRANT SELECT ON TABLE truncate_schema.truncate_tbl TO truncate_user; -REVOKE TRUNCATE ON TABLE truncate_schema.truncate_tbl FROM truncate_user; -SELECT current_user \gset -\c - truncate_user --- verify truncate command fails and check number of rows -SELECT count(*) FROM truncate_schema.truncate_tbl; - count ---------------------------------------------------------------------- - 100 -(1 row) - -TRUNCATE TABLE truncate_schema.truncate_tbl; -ERROR: permission denied for table truncate_tbl -SELECT count(*) FROM truncate_schema.truncate_tbl; - count ---------------------------------------------------------------------- - 100 -(1 row) - --- switch to super user, grant truncate to truncate_user -\c - :current_user -GRANT TRUNCATE ON TABLE truncate_schema.truncate_tbl TO truncate_user; --- verify truncate_user can truncate now -\c - truncate_user -SELECT count(*) FROM truncate_schema.truncate_tbl; - count ---------------------------------------------------------------------- - 100 -(1 row) - -TRUNCATE TABLE truncate_schema.truncate_tbl; -SELECT count(*) FROM truncate_schema.truncate_tbl; - count ---------------------------------------------------------------------- - 0 -(1 row) - -\c - :current_user --- cleanup -DROP SCHEMA truncate_schema CASCADE; -NOTICE: drop cascades to foreign table truncate_schema.truncate_tbl -DROP USER truncate_user; diff --git a/src/test/regress/expected/fdw_truncate_0.out b/src/test/regress/expected/fdw_truncate_0.out deleted file mode 100644 index 073e8f042..000000000 --- a/src/test/regress/expected/fdw_truncate_0.out +++ /dev/null @@ -1,262 +0,0 @@ --- --- Test the TRUNCATE TABLE command for cstore_fdw tables. --- --- print whether we're using version > 10 to make version-specific tests clear -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten; - version_above_ten ---------------------------------------------------------------------- - f -(1 row) - --- Check that files for the automatically managed table exist in the --- cstore_fdw/{databaseoid} directory. -SELECT count(*) FROM ( - SELECT pg_ls_dir('cstore_fdw/' || databaseoid ) FROM ( - SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() - ) AS q1) AS q2; - count ---------------------------------------------------------------------- - 0 -(1 row) - --- CREATE a cstore_fdw table, fill with some data -- -CREATE FOREIGN TABLE cstore_truncate_test (a int, b int) SERVER cstore_server; -CREATE FOREIGN TABLE cstore_truncate_test_second (a int, b int) SERVER cstore_server; -CREATE FOREIGN TABLE cstore_truncate_test_compressed (a int, b int) SERVER cstore_server OPTIONS (compression 'pglz'); -CREATE TABLE cstore_truncate_test_regular (a int, b int); -INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a; -INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a; -INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a; --- query rows -SELECT * FROM cstore_truncate_test; - a | b ---------------------------------------------------------------------- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 6 - 7 | 7 - 8 | 8 - 9 | 9 - 10 | 10 -(10 rows) - -TRUNCATE TABLE cstore_truncate_test; -SELECT * FROM cstore_truncate_test; - a | b ---------------------------------------------------------------------- -(0 rows) - -SELECT COUNT(*) from cstore_truncate_test; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT count(*) FROM cstore_truncate_test_compressed; - count ---------------------------------------------------------------------- - 20 -(1 row) - -TRUNCATE TABLE cstore_truncate_test_compressed; -SELECT count(*) FROM cstore_truncate_test_compressed; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT cstore_table_size('cstore_truncate_test_compressed'); - cstore_table_size ---------------------------------------------------------------------- - 26 -(1 row) - --- make sure data files still present -SELECT count(*) FROM ( - SELECT pg_ls_dir('cstore_fdw/' || databaseoid ) FROM ( - SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() - ) AS q1) AS q2; - count ---------------------------------------------------------------------- - 6 -(1 row) - -INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a; -INSERT INTO cstore_truncate_test_regular select a, a from generate_series(10, 20) a; -INSERT INTO cstore_truncate_test_second select a, a from generate_series(20, 30) a; -SELECT * from cstore_truncate_test; - a | b ---------------------------------------------------------------------- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 6 - 7 | 7 - 8 | 8 - 9 | 9 - 10 | 10 -(10 rows) - -SELECT * from cstore_truncate_test_second; - a | b ---------------------------------------------------------------------- - 20 | 20 - 21 | 21 - 22 | 22 - 23 | 23 - 24 | 24 - 25 | 25 - 26 | 26 - 27 | 27 - 28 | 28 - 29 | 29 - 30 | 30 -(11 rows) - -SELECT * from cstore_truncate_test_regular; - a | b ---------------------------------------------------------------------- - 10 | 10 - 11 | 11 - 12 | 12 - 13 | 13 - 14 | 14 - 15 | 15 - 16 | 16 - 17 | 17 - 18 | 18 - 19 | 19 - 20 | 20 -(11 rows) - --- make sure multi truncate works --- notice that the same table might be repeated -TRUNCATE TABLE cstore_truncate_test, - cstore_truncate_test_regular, - cstore_truncate_test_second, - cstore_truncate_test; -SELECT * from cstore_truncate_test; - a | b ---------------------------------------------------------------------- -(0 rows) - -SELECT * from cstore_truncate_test_second; - a | b ---------------------------------------------------------------------- -(0 rows) - -SELECT * from cstore_truncate_test_regular; - a | b ---------------------------------------------------------------------- -(0 rows) - --- test if truncate on empty table works -TRUNCATE TABLE cstore_truncate_test; -SELECT * from cstore_truncate_test; - a | b ---------------------------------------------------------------------- -(0 rows) - --- test if a cached truncate from a pl/pgsql function works -CREATE FUNCTION cstore_truncate_test_regular_func() RETURNS void AS $$ -BEGIN - INSERT INTO cstore_truncate_test_regular select a, a from generate_series(1, 10) a; - TRUNCATE TABLE cstore_truncate_test_regular; -END;$$ -LANGUAGE plpgsql; -SELECT cstore_truncate_test_regular_func(); - cstore_truncate_test_regular_func ---------------------------------------------------------------------- - -(1 row) - --- the cached plans are used stating from the second call -SELECT cstore_truncate_test_regular_func(); - cstore_truncate_test_regular_func ---------------------------------------------------------------------- - -(1 row) - -DROP FUNCTION cstore_truncate_test_regular_func(); -DROP FOREIGN TABLE cstore_truncate_test, cstore_truncate_test_second; -DROP TABLE cstore_truncate_test_regular; -DROP FOREIGN TABLE cstore_truncate_test_compressed; --- test truncate with schema -CREATE SCHEMA truncate_schema; -CREATE FOREIGN TABLE truncate_schema.truncate_tbl (id int) SERVER cstore_server OPTIONS(compression 'pglz'); -INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100); -SELECT COUNT(*) FROM truncate_schema.truncate_tbl; - count ---------------------------------------------------------------------- - 100 -(1 row) - -TRUNCATE TABLE truncate_schema.truncate_tbl; -SELECT COUNT(*) FROM truncate_schema.truncate_tbl; - count ---------------------------------------------------------------------- - 0 -(1 row) - -INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100); --- create a user that can not truncate -CREATE USER truncate_user; -GRANT USAGE ON SCHEMA truncate_schema TO truncate_user; -GRANT SELECT ON TABLE truncate_schema.truncate_tbl TO truncate_user; -REVOKE TRUNCATE ON TABLE truncate_schema.truncate_tbl FROM truncate_user; -SELECT current_user \gset -\c - truncate_user --- verify truncate command fails and check number of rows -SELECT count(*) FROM truncate_schema.truncate_tbl; - count ---------------------------------------------------------------------- - 100 -(1 row) - -TRUNCATE TABLE truncate_schema.truncate_tbl; -ERROR: permission denied for relation truncate_tbl -SELECT count(*) FROM truncate_schema.truncate_tbl; - count ---------------------------------------------------------------------- - 100 -(1 row) - --- switch to super user, grant truncate to truncate_user -\c - :current_user -GRANT TRUNCATE ON TABLE truncate_schema.truncate_tbl TO truncate_user; --- verify truncate_user can truncate now -\c - truncate_user -SELECT count(*) FROM truncate_schema.truncate_tbl; - count ---------------------------------------------------------------------- - 100 -(1 row) - -TRUNCATE TABLE truncate_schema.truncate_tbl; -SELECT count(*) FROM truncate_schema.truncate_tbl; - count ---------------------------------------------------------------------- - 0 -(1 row) - -\c - :current_user --- cleanup -DROP SCHEMA truncate_schema CASCADE; -NOTICE: drop cascades to foreign table truncate_schema.truncate_tbl -DROP USER truncate_user; --- verify files are removed -SELECT count(*) FROM ( - SELECT pg_ls_dir('cstore_fdw/' || databaseoid ) FROM ( - SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() - ) AS q1) AS q2; - count ---------------------------------------------------------------------- - 0 -(1 row) - diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 67c46eaec..f951de582 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -478,22 +478,16 @@ SELECT * FROM print_extension_changes(); previous_object | current_object --------------------------------------------------------------------- | access method columnar - | event trigger cstore_ddl_event_end - | foreign-data wrapper cstore_fdw | function alter_columnar_table_reset(regclass,boolean,boolean,boolean) | function alter_columnar_table_set(regclass,integer,integer,name) | function citus_internal.cstore_ensure_objects_exist() | function cstore.columnar_handler(internal) - | function cstore.cstore_ddl_event_end_trigger() - | function cstore.cstore_fdw_handler() - | function cstore.cstore_fdw_validator(text[],oid) - | function cstore_table_size(regclass) | schema cstore | table cstore.cstore_data_files | table cstore.cstore_skipnodes | table cstore.cstore_stripes | view cstore.columnar_options -(16 rows) +(10 rows) DROP TABLE prev_objects, extension_diff; -- show running version diff --git a/src/test/regress/expected/multi_extension_0.out b/src/test/regress/expected/multi_extension_0.out index 57811d164..83d9c55e1 100644 --- a/src/test/regress/expected/multi_extension_0.out +++ b/src/test/regress/expected/multi_extension_0.out @@ -477,19 +477,13 @@ ALTER EXTENSION citus UPDATE TO '10.0-1'; SELECT * FROM print_extension_changes(); previous_object | current_object --------------------------------------------------------------------- - | event trigger cstore_ddl_event_end - | foreign-data wrapper cstore_fdw | function citus_internal.cstore_ensure_objects_exist() - | function cstore.cstore_ddl_event_end_trigger() - | function cstore.cstore_fdw_handler() - | function cstore.cstore_fdw_validator(text[],oid) - | function cstore_table_size(regclass) | schema cstore | table cstore.cstore_data_files | table cstore.cstore_skipnodes | table cstore.cstore_stripes | view cstore.columnar_options -(12 rows) +(6 rows) DROP TABLE prev_objects, extension_diff; -- show running version diff --git a/src/test/regress/expected/upgrade_list_citus_objects.out b/src/test/regress/expected/upgrade_list_citus_objects.out index daf4c5648..2ffacffdb 100644 --- a/src/test/regress/expected/upgrade_list_citus_objects.out +++ b/src/test/regress/expected/upgrade_list_citus_objects.out @@ -18,8 +18,6 @@ ORDER BY 1; --------------------------------------------------------------------- access method columnar event trigger citus_cascade_to_partition - event trigger cstore_ddl_event_end - foreign-data wrapper cstore_fdw function alter_columnar_table_reset(regclass,boolean,boolean,boolean) function alter_columnar_table_set(regclass,integer,integer,name) function alter_role_if_exists(text,text) @@ -83,10 +81,6 @@ ORDER BY 1; function create_intermediate_result(text,text) function create_reference_table(regclass) function cstore.columnar_handler(internal) - function cstore.cstore_ddl_event_end_trigger() - function cstore.cstore_fdw_handler() - function cstore.cstore_fdw_validator(text[],oid) - function cstore_table_size(regclass) function distributed_tables_colocated(regclass,regclass) function dump_global_wait_edges() function dump_local_wait_edges() @@ -223,5 +217,5 @@ ORDER BY 1; view citus_worker_stat_activity view cstore.columnar_options view pg_dist_shard_placement -(207 rows) +(201 rows) diff --git a/src/test/regress/expected/upgrade_list_citus_objects_0.out b/src/test/regress/expected/upgrade_list_citus_objects_0.out index 2c6e4a4bb..0cd1f51c1 100644 --- a/src/test/regress/expected/upgrade_list_citus_objects_0.out +++ b/src/test/regress/expected/upgrade_list_citus_objects_0.out @@ -17,8 +17,6 @@ ORDER BY 1; description --------------------------------------------------------------------- event trigger citus_cascade_to_partition - event trigger cstore_ddl_event_end - foreign-data wrapper cstore_fdw function alter_role_if_exists(text,text) function any_value(anyelement) function any_value_agg(anyelement,anyelement) @@ -79,10 +77,6 @@ ORDER BY 1; function create_distributed_table(regclass,text,citus.distribution_type,text) function create_intermediate_result(text,text) function create_reference_table(regclass) - function cstore.cstore_ddl_event_end_trigger() - function cstore.cstore_fdw_handler() - function cstore.cstore_fdw_validator(text[],oid) - function cstore_table_size(regclass) function distributed_tables_colocated(regclass,regclass) function dump_global_wait_edges() function dump_local_wait_edges() @@ -219,5 +213,5 @@ ORDER BY 1; view citus_worker_stat_activity view cstore.columnar_options view pg_dist_shard_placement -(203 rows) +(197 rows) diff --git a/src/test/regress/input/fdw_block_filtering.source b/src/test/regress/input/fdw_block_filtering.source deleted file mode 100644 index dc3170f0d..000000000 --- a/src/test/regress/input/fdw_block_filtering.source +++ /dev/null @@ -1,69 +0,0 @@ --- --- Test block filtering in cstore_fdw using min/max values in stripe skip lists. --- - - --- --- filtered_row_count returns number of rows filtered by the WHERE clause. --- If blocks get filtered by cstore_fdw, less rows are passed to WHERE --- clause, so this function should return a lower number. --- -CREATE OR REPLACE FUNCTION filtered_row_count (query text) RETURNS bigint AS -$$ - DECLARE - result bigint; - rec text; - BEGIN - result := 0; - - FOR rec IN EXECUTE 'EXPLAIN ANALYZE ' || query LOOP - IF rec ~ '^\s+Rows Removed by Filter' then - result := regexp_replace(rec, '[^0-9]*', '', 'g'); - END IF; - END LOOP; - - RETURN result; - END; -$$ LANGUAGE PLPGSQL; - - --- Create and load data -CREATE FOREIGN TABLE test_block_filtering (a int) - SERVER cstore_server - OPTIONS(block_row_count '1000', stripe_row_count '2000'); - -COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV; - - --- Verify that filtered_row_count is less than 1000 for the following queries -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering'); -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200'); -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 200'); -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 9900'); -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 9900'); -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0'); - - --- Verify that filtered_row_count is less than 2000 for the following queries -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 1 AND 10'); -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010'); -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN -10 AND 0'); - - --- Load data for second time and verify that filtered_row_count is exactly twice as before -COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV; -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200'); -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0'); -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010'); - - --- Verify that we are fine with collations which use a different alphabet order -CREATE FOREIGN TABLE collation_block_filtering_test(A text collate "da_DK") - SERVER cstore_server; -COPY collation_block_filtering_test FROM STDIN; -A -Å -B -\. - -SELECT * FROM collation_block_filtering_test WHERE A > 'B'; diff --git a/src/test/regress/input/fdw_copyto.source b/src/test/regress/input/fdw_copyto.source deleted file mode 100644 index a4b753a8d..000000000 --- a/src/test/regress/input/fdw_copyto.source +++ /dev/null @@ -1,17 +0,0 @@ --- --- Test copying data from cstore_fdw tables. --- -CREATE FOREIGN TABLE test_contestant(handle TEXT, birthdate DATE, rating INT, - percentile FLOAT, country CHAR(3), achievements TEXT[]) - SERVER cstore_server; - --- load table data from file -COPY test_contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; - --- export using COPY table TO ... -COPY test_contestant TO STDOUT; - --- export using COPY (SELECT * FROM table) TO ... -COPY (select * from test_contestant) TO STDOUT; - -DROP FOREIGN TABLE test_contestant CASCADE; diff --git a/src/test/regress/input/fdw_create.source b/src/test/regress/input/fdw_create.source deleted file mode 100644 index de834ceb0..000000000 --- a/src/test/regress/input/fdw_create.source +++ /dev/null @@ -1,39 +0,0 @@ --- --- Test the CREATE statements related to cstore_fdw. --- - -CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw; - - --- Validator tests -CREATE FOREIGN TABLE test_validator_invalid_option () - SERVER cstore_server - OPTIONS(bad_option_name '1'); -- ERROR - -CREATE FOREIGN TABLE test_validator_invalid_stripe_row_count () - SERVER cstore_server - OPTIONS(stripe_row_count '0'); -- ERROR - -CREATE FOREIGN TABLE test_validator_invalid_block_row_count () - SERVER cstore_server - OPTIONS(block_row_count '0'); -- ERROR - -CREATE FOREIGN TABLE test_validator_invalid_compression_type () - SERVER cstore_server - OPTIONS(compression 'invalid_compression'); -- ERROR - --- Create uncompressed table -CREATE FOREIGN TABLE contestant (handle TEXT, birthdate DATE, rating INT, - percentile FLOAT, country CHAR(3), achievements TEXT[]) - SERVER cstore_server; - - --- Create compressed table with automatically determined file path -CREATE FOREIGN TABLE contestant_compressed (handle TEXT, birthdate DATE, rating INT, - percentile FLOAT, country CHAR(3), achievements TEXT[]) - SERVER cstore_server - OPTIONS(compression 'pglz'); - --- Test that querying an empty table works -ANALYZE contestant; -SELECT count(*) FROM contestant; diff --git a/src/test/regress/input/fdw_data_types.source b/src/test/regress/input/fdw_data_types.source deleted file mode 100644 index ec83c4d8c..000000000 --- a/src/test/regress/input/fdw_data_types.source +++ /dev/null @@ -1,68 +0,0 @@ --- --- Test loading and reading different data types to/from cstore_fdw foreign tables. --- - - --- Settings to make the result deterministic -SET datestyle = "ISO, YMD"; -SET timezone to 'GMT'; -SET intervalstyle TO 'POSTGRES_VERBOSE'; - - --- Test array types -CREATE FOREIGN TABLE test_array_types (int_array int[], bigint_array bigint[], - text_array text[]) SERVER cstore_server; - -COPY test_array_types FROM '@abs_srcdir@/data/array_types.csv' WITH CSV; - -SELECT * FROM test_array_types; - - --- Test date/time types -CREATE FOREIGN TABLE test_datetime_types (timestamp timestamp, - timestamp_with_timezone timestamp with time zone, date date, time time, - interval interval) SERVER cstore_server; - -COPY test_datetime_types FROM '@abs_srcdir@/data/datetime_types.csv' WITH CSV; - -SELECT * FROM test_datetime_types; - - --- Test enum and composite types -CREATE TYPE enum_type AS ENUM ('a', 'b', 'c'); -CREATE TYPE composite_type AS (a int, b text); - -CREATE FOREIGN TABLE test_enum_and_composite_types (enum enum_type, - composite composite_type) SERVER cstore_server; - -COPY test_enum_and_composite_types FROM - '@abs_srcdir@/data/enum_and_composite_types.csv' WITH CSV; - -SELECT * FROM test_enum_and_composite_types; - - --- Test range types -CREATE FOREIGN TABLE test_range_types (int4range int4range, int8range int8range, - numrange numrange, tsrange tsrange) SERVER cstore_server; - -COPY test_range_types FROM '@abs_srcdir@/data/range_types.csv' WITH CSV; - -SELECT * FROM test_range_types; - - --- Test other types -CREATE FOREIGN TABLE test_other_types (bool boolean, bytea bytea, money money, - inet inet, bitstring bit varying(5), uuid uuid, json json) SERVER cstore_server; - -COPY test_other_types FROM '@abs_srcdir@/data/other_types.csv' WITH CSV; - -SELECT * FROM test_other_types; - - --- Test null values -CREATE FOREIGN TABLE test_null_values (a int, b int[], c composite_type) - SERVER cstore_server; - -COPY test_null_values FROM '@abs_srcdir@/data/null_values.csv' WITH CSV; - -SELECT * FROM test_null_values; diff --git a/src/test/regress/input/fdw_load.source b/src/test/regress/input/fdw_load.source deleted file mode 100644 index 0913acde7..000000000 --- a/src/test/regress/input/fdw_load.source +++ /dev/null @@ -1,44 +0,0 @@ --- --- Test loading data into cstore_fdw tables. --- - --- COPY with incorrect delimiter -COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' - WITH DELIMITER '|'; -- ERROR - --- COPY with invalid program -COPY contestant FROM PROGRAM 'invalid_program' WITH CSV; -- ERROR - --- COPY into uncompressed table from file -COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; - --- COPY into uncompressed table from program -COPY contestant FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' WITH CSV; - --- COPY into compressed table -COPY contestant_compressed FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; - --- COPY into uncompressed table from program -COPY contestant_compressed FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' - WITH CSV; - --- Test column list -CREATE FOREIGN TABLE famous_constants (id int, name text, value real) - SERVER cstore_server; -COPY famous_constants (value, name, id) FROM STDIN WITH CSV; -3.141,pi,1 -2.718,e,2 -0.577,gamma,3 -5.291e-11,bohr radius,4 -\. - -COPY famous_constants (name, value) FROM STDIN WITH CSV; -avagadro,6.022e23 -electron mass,9.109e-31 -proton mass,1.672e-27 -speed of light,2.997e8 -\. - -SELECT * FROM famous_constants ORDER BY id, name; - -DROP FOREIGN TABLE famous_constants; diff --git a/src/test/regress/output/fdw_block_filtering.source b/src/test/regress/output/fdw_block_filtering.source deleted file mode 100644 index 2f664a78a..000000000 --- a/src/test/regress/output/fdw_block_filtering.source +++ /dev/null @@ -1,116 +0,0 @@ --- --- Test block filtering in cstore_fdw using min/max values in stripe skip lists. --- --- --- filtered_row_count returns number of rows filtered by the WHERE clause. --- If blocks get filtered by cstore_fdw, less rows are passed to WHERE --- clause, so this function should return a lower number. --- -CREATE OR REPLACE FUNCTION filtered_row_count (query text) RETURNS bigint AS -$$ - DECLARE - result bigint; - rec text; - BEGIN - result := 0; - - FOR rec IN EXECUTE 'EXPLAIN ANALYZE ' || query LOOP - IF rec ~ '^\s+Rows Removed by Filter' then - result := regexp_replace(rec, '[^0-9]*', '', 'g'); - END IF; - END LOOP; - - RETURN result; - END; -$$ LANGUAGE PLPGSQL; --- Create and load data -CREATE FOREIGN TABLE test_block_filtering (a int) - SERVER cstore_server - OPTIONS(block_row_count '1000', stripe_row_count '2000'); -COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV; --- Verify that filtered_row_count is less than 1000 for the following queries -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering'); - filtered_row_count --------------------- - 0 -(1 row) - -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200'); - filtered_row_count --------------------- - 801 -(1 row) - -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 200'); - filtered_row_count --------------------- - 200 -(1 row) - -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 9900'); - filtered_row_count --------------------- - 101 -(1 row) - -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 9900'); - filtered_row_count --------------------- - 900 -(1 row) - -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0'); - filtered_row_count --------------------- - 0 -(1 row) - --- Verify that filtered_row_count is less than 2000 for the following queries -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 1 AND 10'); - filtered_row_count --------------------- - 990 -(1 row) - -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010'); - filtered_row_count --------------------- - 1979 -(1 row) - -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN -10 AND 0'); - filtered_row_count --------------------- - 0 -(1 row) - --- Load data for second time and verify that filtered_row_count is exactly twice as before -COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV; -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200'); - filtered_row_count --------------------- - 1602 -(1 row) - -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0'); - filtered_row_count --------------------- - 0 -(1 row) - -SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010'); - filtered_row_count --------------------- - 3958 -(1 row) - --- Verify that we are fine with collations which use a different alphabet order -CREATE FOREIGN TABLE collation_block_filtering_test(A text collate "da_DK") - SERVER cstore_server; -COPY collation_block_filtering_test FROM STDIN; -SELECT * FROM collation_block_filtering_test WHERE A > 'B'; - a ---- - Å -(1 row) - diff --git a/src/test/regress/output/fdw_copyto.source b/src/test/regress/output/fdw_copyto.source deleted file mode 100644 index a8d841f18..000000000 --- a/src/test/regress/output/fdw_copyto.source +++ /dev/null @@ -1,23 +0,0 @@ --- --- Test copying data from cstore_fdw tables. --- -CREATE FOREIGN TABLE test_contestant(handle TEXT, birthdate DATE, rating INT, - percentile FLOAT, country CHAR(3), achievements TEXT[]) - SERVER cstore_server; --- load table data from file -COPY test_contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; --- export using COPY table TO ... -COPY test_contestant TO STDOUT; -a 01-10-1990 2090 97.1 XA {a} -b 11-01-1990 2203 98.1 XA {a,b} -c 11-01-1988 2907 99.4 XB {w,y} -d 05-05-1985 2314 98.3 XB {} -e 05-05-1995 2236 98.2 XC {a} --- export using COPY (SELECT * FROM table) TO ... -COPY (select * from test_contestant) TO STDOUT; -a 01-10-1990 2090 97.1 XA {a} -b 11-01-1990 2203 98.1 XA {a,b} -c 11-01-1988 2907 99.4 XB {w,y} -d 05-05-1985 2314 98.3 XB {} -e 05-05-1995 2236 98.2 XC {a} -DROP FOREIGN TABLE test_contestant CASCADE; diff --git a/src/test/regress/output/fdw_create.source b/src/test/regress/output/fdw_create.source deleted file mode 100644 index 41f17fdd8..000000000 --- a/src/test/regress/output/fdw_create.source +++ /dev/null @@ -1,42 +0,0 @@ --- --- Test the CREATE statements related to cstore_fdw. --- -CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw; --- Validator tests -CREATE FOREIGN TABLE test_validator_invalid_option () - SERVER cstore_server - OPTIONS(bad_option_name '1'); -- ERROR -ERROR: invalid option "bad_option_name" -HINT: Valid options in this context are: compression, stripe_row_count, block_row_count -CREATE FOREIGN TABLE test_validator_invalid_stripe_row_count () - SERVER cstore_server - OPTIONS(stripe_row_count '0'); -- ERROR -ERROR: invalid stripe row count -HINT: Stripe row count must be an integer between 1000 and 10000000 -CREATE FOREIGN TABLE test_validator_invalid_block_row_count () - SERVER cstore_server - OPTIONS(block_row_count '0'); -- ERROR -ERROR: invalid block row count -HINT: Block row count must be an integer between 1000 and 100000 -CREATE FOREIGN TABLE test_validator_invalid_compression_type () - SERVER cstore_server - OPTIONS(compression 'invalid_compression'); -- ERROR -ERROR: invalid compression type -HINT: Valid options are: none, pglz --- Create uncompressed table -CREATE FOREIGN TABLE contestant (handle TEXT, birthdate DATE, rating INT, - percentile FLOAT, country CHAR(3), achievements TEXT[]) - SERVER cstore_server; --- Create compressed table with automatically determined file path -CREATE FOREIGN TABLE contestant_compressed (handle TEXT, birthdate DATE, rating INT, - percentile FLOAT, country CHAR(3), achievements TEXT[]) - SERVER cstore_server - OPTIONS(compression 'pglz'); --- Test that querying an empty table works -ANALYZE contestant; -SELECT count(*) FROM contestant; - count -------- - 0 -(1 row) - diff --git a/src/test/regress/output/fdw_data_types.source b/src/test/regress/output/fdw_data_types.source deleted file mode 100644 index 23fdcfa29..000000000 --- a/src/test/regress/output/fdw_data_types.source +++ /dev/null @@ -1,78 +0,0 @@ --- --- Test loading and reading different data types to/from cstore_fdw foreign tables. --- --- Settings to make the result deterministic -SET datestyle = "ISO, YMD"; -SET timezone to 'GMT'; -SET intervalstyle TO 'POSTGRES_VERBOSE'; --- Test array types -CREATE FOREIGN TABLE test_array_types (int_array int[], bigint_array bigint[], - text_array text[]) SERVER cstore_server; -COPY test_array_types FROM '@abs_srcdir@/data/array_types.csv' WITH CSV; -SELECT * FROM test_array_types; - int_array | bigint_array | text_array ---------------------------+--------------------------------------------+------------ - {1,2,3} | {1,2,3} | {a,b,c} - {} | {} | {} - {-2147483648,2147483647} | {-9223372036854775808,9223372036854775807} | {""} -(3 rows) - --- Test date/time types -CREATE FOREIGN TABLE test_datetime_types (timestamp timestamp, - timestamp_with_timezone timestamp with time zone, date date, time time, - interval interval) SERVER cstore_server; -COPY test_datetime_types FROM '@abs_srcdir@/data/datetime_types.csv' WITH CSV; -SELECT * FROM test_datetime_types; - timestamp | timestamp_with_timezone | date | time | interval ----------------------+-------------------------+------------+----------+----------- - 2000-01-02 04:05:06 | 1999-01-08 12:05:06+00 | 2000-01-02 | 04:05:06 | @ 4 hours - 1970-01-01 00:00:00 | infinity | -infinity | 00:00:00 | @ 0 -(2 rows) - --- Test enum and composite types -CREATE TYPE enum_type AS ENUM ('a', 'b', 'c'); -CREATE TYPE composite_type AS (a int, b text); -CREATE FOREIGN TABLE test_enum_and_composite_types (enum enum_type, - composite composite_type) SERVER cstore_server; -COPY test_enum_and_composite_types FROM - '@abs_srcdir@/data/enum_and_composite_types.csv' WITH CSV; -SELECT * FROM test_enum_and_composite_types; - enum | composite -------+----------- - a | (2,b) - b | (3,c) -(2 rows) - --- Test range types -CREATE FOREIGN TABLE test_range_types (int4range int4range, int8range int8range, - numrange numrange, tsrange tsrange) SERVER cstore_server; -COPY test_range_types FROM '@abs_srcdir@/data/range_types.csv' WITH CSV; -SELECT * FROM test_range_types; - int4range | int8range | numrange | tsrange ------------+-----------+----------+----------------------------------------------- - [1,3) | [1,3) | [1,3) | ["2000-01-02 00:30:00","2010-02-03 12:30:00") - empty | [1,) | (,) | empty -(2 rows) - --- Test other types -CREATE FOREIGN TABLE test_other_types (bool boolean, bytea bytea, money money, - inet inet, bitstring bit varying(5), uuid uuid, json json) SERVER cstore_server; -COPY test_other_types FROM '@abs_srcdir@/data/other_types.csv' WITH CSV; -SELECT * FROM test_other_types; - bool | bytea | money | inet | bitstring | uuid | json -------+------------+-------+-------------+-----------+--------------------------------------+------------------ - f | \xdeadbeef | $1.00 | 192.168.1.2 | 10101 | a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 | {"key": "value"} - t | \xcdb0 | $1.50 | 127.0.0.1 | | a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 | [] -(2 rows) - --- Test null values -CREATE FOREIGN TABLE test_null_values (a int, b int[], c composite_type) - SERVER cstore_server; -COPY test_null_values FROM '@abs_srcdir@/data/null_values.csv' WITH CSV; -SELECT * FROM test_null_values; - a | b | c ----+--------+----- - | {NULL} | (,) - | | -(2 rows) - diff --git a/src/test/regress/output/fdw_load.source b/src/test/regress/output/fdw_load.source deleted file mode 100644 index c76f203eb..000000000 --- a/src/test/regress/output/fdw_load.source +++ /dev/null @@ -1,39 +0,0 @@ --- --- Test loading data into cstore_fdw tables. --- --- COPY with incorrect delimiter -COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' - WITH DELIMITER '|'; -- ERROR -ERROR: missing data for column "birthdate" --- COPY with invalid program -COPY contestant FROM PROGRAM 'invalid_program' WITH CSV; -- ERROR -ERROR: program "invalid_program" failed -DETAIL: command not found --- COPY into uncompressed table from file -COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; --- COPY into uncompressed table from program -COPY contestant FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' WITH CSV; --- COPY into compressed table -COPY contestant_compressed FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; --- COPY into uncompressed table from program -COPY contestant_compressed FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' - WITH CSV; --- Test column list -CREATE FOREIGN TABLE famous_constants (id int, name text, value real) - SERVER cstore_server; -COPY famous_constants (value, name, id) FROM STDIN WITH CSV; -COPY famous_constants (name, value) FROM STDIN WITH CSV; -SELECT * FROM famous_constants ORDER BY id, name; - id | name | value -----+----------------+----------- - 1 | pi | 3.141 - 2 | e | 2.718 - 3 | gamma | 0.577 - 4 | bohr radius | 5.291e-11 - | avagadro | 6.022e+23 - | electron mass | 9.109e-31 - | proton mass | 1.672e-27 - | speed of light | 2.997e+08 -(8 rows) - -DROP FOREIGN TABLE famous_constants; diff --git a/src/test/regress/sql/am_functions.sql b/src/test/regress/sql/am_functions.sql deleted file mode 100644 index 14e4c6f2b..000000000 --- a/src/test/regress/sql/am_functions.sql +++ /dev/null @@ -1,20 +0,0 @@ --- --- Test utility functions for cstore_fdw tables. --- - -CREATE TABLE empty_table (a int) USING columnar; -CREATE TABLE table_with_data (a int) USING columnar; -CREATE TABLE non_cstore_table (a int); - -COPY table_with_data FROM STDIN; -1 -2 -3 -\. - -SELECT pg_relation_size('empty_table') < pg_relation_size('table_with_data'); -SELECT cstore_table_size('non_cstore_table'); - -DROP TABLE empty_table; -DROP TABLE table_with_data; -DROP TABLE non_cstore_table; diff --git a/src/test/regress/sql/fdw_alter.sql b/src/test/regress/sql/fdw_alter.sql deleted file mode 100644 index 5ba3beb34..000000000 --- a/src/test/regress/sql/fdw_alter.sql +++ /dev/null @@ -1,85 +0,0 @@ --- --- Testing ALTER TABLE on cstore_fdw tables. --- - -CREATE FOREIGN TABLE test_alter_table (a int, b int, c int) SERVER cstore_server; - -WITH sample_data AS (VALUES - (1, 2, 3), - (4, 5, 6), - (7, 8, 9) -) -INSERT INTO test_alter_table SELECT * FROM sample_data; - --- drop a column -ALTER FOREIGN TABLE test_alter_table DROP COLUMN a; - --- test analyze -ANALYZE test_alter_table; - --- verify select queries run as expected -SELECT * FROM test_alter_table; -SELECT a FROM test_alter_table; -SELECT b FROM test_alter_table; - --- verify insert runs as expected -INSERT INTO test_alter_table (SELECT 3, 5, 8); -INSERT INTO test_alter_table (SELECT 5, 8); - - --- add a column with no defaults -ALTER FOREIGN TABLE test_alter_table ADD COLUMN d int; -SELECT * FROM test_alter_table; -INSERT INTO test_alter_table (SELECT 3, 5, 8); -SELECT * FROM test_alter_table; - - --- add a fixed-length column with default value -ALTER FOREIGN TABLE test_alter_table ADD COLUMN e int default 3; -SELECT * from test_alter_table; -INSERT INTO test_alter_table (SELECT 1, 2, 4, 8); -SELECT * from test_alter_table; - - --- add a variable-length column with default value -ALTER FOREIGN TABLE test_alter_table ADD COLUMN f text DEFAULT 'TEXT ME'; -SELECT * from test_alter_table; -INSERT INTO test_alter_table (SELECT 1, 2, 4, 8, 'ABCDEF'); -SELECT * from test_alter_table; - - --- drop couple of columns -ALTER FOREIGN TABLE test_alter_table DROP COLUMN c; -ALTER FOREIGN TABLE test_alter_table DROP COLUMN e; -ANALYZE test_alter_table; -SELECT * from test_alter_table; -SELECT count(*) from test_alter_table; -SELECT count(t.*) from test_alter_table t; - - --- unsupported default values -ALTER FOREIGN TABLE test_alter_table ADD COLUMN g boolean DEFAULT isfinite(current_date); -ALTER FOREIGN TABLE test_alter_table ADD COLUMN h DATE DEFAULT current_date; -SELECT * FROM test_alter_table; -ALTER FOREIGN TABLE test_alter_table ALTER COLUMN g DROP DEFAULT; -SELECT * FROM test_alter_table; -ALTER FOREIGN TABLE test_alter_table ALTER COLUMN h DROP DEFAULT; -ANALYZE test_alter_table; -SELECT * FROM test_alter_table; - --- unsupported type change -ALTER FOREIGN TABLE test_alter_table ADD COLUMN i int; -ALTER FOREIGN TABLE test_alter_table ADD COLUMN j float; -ALTER FOREIGN TABLE test_alter_table ADD COLUMN k text; - --- this is valid type change -ALTER FOREIGN TABLE test_alter_table ALTER COLUMN i TYPE float; - --- this is not valid -ALTER FOREIGN TABLE test_alter_table ALTER COLUMN j TYPE int; - --- text / varchar conversion is valid both ways -ALTER FOREIGN TABLE test_alter_table ALTER COLUMN k TYPE varchar(20); -ALTER FOREIGN TABLE test_alter_table ALTER COLUMN k TYPE text; - -DROP FOREIGN TABLE test_alter_table; diff --git a/src/test/regress/sql/fdw_analyze.sql b/src/test/regress/sql/fdw_analyze.sql deleted file mode 100644 index 4476454a6..000000000 --- a/src/test/regress/sql/fdw_analyze.sql +++ /dev/null @@ -1,11 +0,0 @@ --- --- Test the ANALYZE command for cstore_fdw tables. --- - --- ANALYZE uncompressed table -ANALYZE contestant; -SELECT count(*) FROM pg_stats WHERE tablename='contestant'; - --- ANALYZE compressed table -ANALYZE contestant_compressed; -SELECT count(*) FROM pg_stats WHERE tablename='contestant_compressed'; diff --git a/src/test/regress/sql/fdw_clean.sql b/src/test/regress/sql/fdw_clean.sql deleted file mode 100644 index ecd4d67a1..000000000 --- a/src/test/regress/sql/fdw_clean.sql +++ /dev/null @@ -1,10 +0,0 @@ -DROP FOREIGN TABLE collation_block_filtering_test; -DROP FOREIGN TABLE test_block_filtering; -DROP FOREIGN TABLE test_null_values; -DROP FOREIGN TABLE test_other_types; -DROP FOREIGN TABLE test_range_types; -DROP FOREIGN TABLE test_enum_and_composite_types; -DROP TYPE composite_type; -DROP TYPE enum_type; -DROP FOREIGN TABLE test_datetime_types; -DROP FOREIGN TABLE test_array_types; diff --git a/src/test/regress/sql/fdw_drop.sql b/src/test/regress/sql/fdw_drop.sql deleted file mode 100644 index 3246ed016..000000000 --- a/src/test/regress/sql/fdw_drop.sql +++ /dev/null @@ -1,56 +0,0 @@ --- --- Tests the different DROP commands for cstore_fdw tables. --- --- DROP FOREIGN TABL --- DROP SCHEMA --- DROP EXTENSION --- DROP DATABASE --- - --- Note that travis does not create --- cstore_fdw extension in default database (postgres). This has caused --- different behavior between travis tests and local tests. Thus --- 'postgres' directory is excluded from comparison to have the same result. - --- store postgres database oid -SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset - -SELECT count(*) AS cstore_data_files_before_drop FROM cstore.cstore_data_files \gset - --- DROP cstore_fdw tables -DROP FOREIGN TABLE contestant; -DROP FOREIGN TABLE contestant_compressed; - --- make sure DROP deletes metadata -SELECT :cstore_data_files_before_drop - count(*) FROM cstore.cstore_data_files; - --- Create a cstore_fdw table under a schema and drop it. -CREATE SCHEMA test_schema; -CREATE FOREIGN TABLE test_schema.test_table(data int) SERVER cstore_server; - -SELECT count(*) AS cstore_data_files_before_drop FROM cstore.cstore_data_files \gset -DROP SCHEMA test_schema CASCADE; -SELECT :cstore_data_files_before_drop - count(*) FROM cstore.cstore_data_files; - -SELECT current_database() datname \gset - -CREATE DATABASE db_to_drop; -\c db_to_drop -CREATE EXTENSION citus; -CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw; -SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset - -CREATE FOREIGN TABLE test_table(data int) SERVER cstore_server; - -DROP EXTENSION citus CASCADE; - --- test database drop -CREATE EXTENSION citus; -CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw; -SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset - -CREATE FOREIGN TABLE test_table(data int) SERVER cstore_server; - -\c :datname - -DROP DATABASE db_to_drop; diff --git a/src/test/regress/sql/fdw_functions.sql b/src/test/regress/sql/fdw_functions.sql deleted file mode 100644 index ed7e260b3..000000000 --- a/src/test/regress/sql/fdw_functions.sql +++ /dev/null @@ -1,20 +0,0 @@ --- --- Test utility functions for cstore_fdw tables. --- - -CREATE FOREIGN TABLE empty_table (a int) SERVER cstore_server; -CREATE FOREIGN TABLE table_with_data (a int) SERVER cstore_server; -CREATE TABLE non_cstore_table (a int); - -COPY table_with_data FROM STDIN; -1 -2 -3 -\. - -SELECT cstore_table_size('empty_table') < cstore_table_size('table_with_data'); -SELECT cstore_table_size('non_cstore_table'); - -DROP FOREIGN TABLE empty_table; -DROP FOREIGN TABLE table_with_data; -DROP TABLE non_cstore_table; diff --git a/src/test/regress/sql/fdw_insert.sql b/src/test/regress/sql/fdw_insert.sql deleted file mode 100644 index 7a6b075ce..000000000 --- a/src/test/regress/sql/fdw_insert.sql +++ /dev/null @@ -1,56 +0,0 @@ --- --- Testing insert on cstore_fdw tables. --- - -CREATE FOREIGN TABLE test_insert_command (a int) SERVER cstore_server; - --- test single row inserts fail -select count(*) from test_insert_command; -insert into test_insert_command values(1); -select count(*) from test_insert_command; - -insert into test_insert_command default values; -select count(*) from test_insert_command; - --- test inserting from another table succeed -CREATE TABLE test_insert_command_data (a int); - -select count(*) from test_insert_command_data; -insert into test_insert_command_data values(1); -select count(*) from test_insert_command_data; - -insert into test_insert_command select * from test_insert_command_data; -select count(*) from test_insert_command; - -drop table test_insert_command_data; -drop foreign table test_insert_command; - --- test long attribute value insertion --- create sufficiently long text so that data is stored in toast -CREATE TABLE test_long_text AS -SELECT a as int_val, string_agg(random()::text, '') as text_val -FROM generate_series(1, 10) a, generate_series(1, 1000) b -GROUP BY a ORDER BY a; - --- store hash values of text for later comparison -CREATE TABLE test_long_text_hash AS -SELECT int_val, md5(text_val) AS hash -FROM test_long_text; - -CREATE FOREIGN TABLE test_cstore_long_text(int_val int, text_val text) -SERVER cstore_server; - --- store long text in cstore table -INSERT INTO test_cstore_long_text SELECT * FROM test_long_text; - --- drop source table to remove original text from toast -DROP TABLE test_long_text; - --- check if text data is still available in cstore table --- by comparing previously stored hash. -SELECT a.int_val -FROM test_long_text_hash a, test_cstore_long_text c -WHERE a.int_val = c.int_val AND a.hash = md5(c.text_val); - -DROP TABLE test_long_text_hash; -DROP FOREIGN TABLE test_cstore_long_text; diff --git a/src/test/regress/sql/fdw_query.sql b/src/test/regress/sql/fdw_query.sql deleted file mode 100644 index 87743e7bd..000000000 --- a/src/test/regress/sql/fdw_query.sql +++ /dev/null @@ -1,34 +0,0 @@ --- --- Test querying cstore_fdw tables. --- - --- Settings to make the result deterministic -SET datestyle = "ISO, YMD"; - --- Query uncompressed data -SELECT count(*) FROM contestant; -SELECT avg(rating), stddev_samp(rating) FROM contestant; -SELECT country, avg(rating) FROM contestant WHERE rating > 2200 - GROUP BY country ORDER BY country; -SELECT * FROM contestant ORDER BY handle; - --- Query compressed data -SELECT count(*) FROM contestant_compressed; -SELECT avg(rating), stddev_samp(rating) FROM contestant_compressed; -SELECT country, avg(rating) FROM contestant_compressed WHERE rating > 2200 - GROUP BY country ORDER BY country; -SELECT * FROM contestant_compressed ORDER BY handle; - --- Verify that we handle whole-row references correctly -SELECT to_json(v) FROM contestant v ORDER BY rating LIMIT 1; - --- Test variables used in expressions -CREATE FOREIGN TABLE union_first (a int, b int) SERVER cstore_server; -CREATE FOREIGN TABLE union_second (a int, b int) SERVER cstore_server; - -INSERT INTO union_first SELECT a, a FROM generate_series(1, 5) a; -INSERT INTO union_second SELECT a, a FROM generate_series(11, 15) a; - -(SELECT a*1, b FROM union_first) union all (SELECT a*1, b FROM union_second); - -DROP FOREIGN TABLE union_first, union_second; diff --git a/src/test/regress/sql/fdw_rollback.sql b/src/test/regress/sql/fdw_rollback.sql deleted file mode 100644 index 804868ac9..000000000 --- a/src/test/regress/sql/fdw_rollback.sql +++ /dev/null @@ -1,41 +0,0 @@ --- --- Testing we handle rollbacks properly --- - -CREATE FOREIGN TABLE t(a int, b int) SERVER cstore_server; - -BEGIN; -INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i; -ROLLBACK; -SELECT count(*) FROM t; - --- check stripe metadata also have been rolled-back -SELECT count(*) FROM cstore.cstore_stripes a, pg_class b -WHERE a.relfilenode = b.relfilenode AND b.relname = 't'; - -INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i; -SELECT count(*) FROM t; - -SELECT count(*) FROM cstore.cstore_stripes a, pg_class b -WHERE a.relfilenode = b.relfilenode AND b.relname = 't'; - --- savepoint rollback -BEGIN; -SAVEPOINT s0; -INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i; -SAVEPOINT s1; -INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i; -SELECT count(*) FROM t; -ROLLBACK TO SAVEPOINT s1; -SELECT count(*) FROM t; -ROLLBACK TO SAVEPOINT s0; -SELECT count(*) FROM t; -INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i; -COMMIT; - -SELECT count(*) FROM t; - -SELECT count(*) FROM cstore.cstore_stripes a, pg_class b -WHERE a.relfilenode = b.relfilenode AND b.relname = 't'; - -DROP FOREIGN TABLE t; diff --git a/src/test/regress/sql/fdw_truncate.sql b/src/test/regress/sql/fdw_truncate.sql deleted file mode 100644 index f9e3b15fc..000000000 --- a/src/test/regress/sql/fdw_truncate.sql +++ /dev/null @@ -1,135 +0,0 @@ --- --- Test the TRUNCATE TABLE command for cstore_fdw tables. --- - --- print whether we're using version > 10 to make version-specific tests clear -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten; - --- CREATE a cstore_fdw table, fill with some data -- -CREATE FOREIGN TABLE cstore_truncate_test (a int, b int) SERVER cstore_server; -CREATE FOREIGN TABLE cstore_truncate_test_second (a int, b int) SERVER cstore_server; -CREATE FOREIGN TABLE cstore_truncate_test_compressed (a int, b int) SERVER cstore_server OPTIONS (compression 'pglz'); -CREATE TABLE cstore_truncate_test_regular (a int, b int); - -SELECT count(*) AS cstore_data_files_before_truncate FROM cstore.cstore_data_files \gset - -INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a; - -INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a; -INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a; - --- query rows -SELECT * FROM cstore_truncate_test; - -TRUNCATE TABLE cstore_truncate_test; - -SELECT * FROM cstore_truncate_test; - -SELECT COUNT(*) from cstore_truncate_test; - -SELECT count(*) FROM cstore_truncate_test_compressed; -TRUNCATE TABLE cstore_truncate_test_compressed; -SELECT count(*) FROM cstore_truncate_test_compressed; - -SELECT cstore_table_size('cstore_truncate_test_compressed'); - -INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a; -INSERT INTO cstore_truncate_test_regular select a, a from generate_series(10, 20) a; -INSERT INTO cstore_truncate_test_second select a, a from generate_series(20, 30) a; - -SELECT * from cstore_truncate_test; - -SELECT * from cstore_truncate_test_second; - -SELECT * from cstore_truncate_test_regular; - --- make sure multi truncate works --- notice that the same table might be repeated -TRUNCATE TABLE cstore_truncate_test, - cstore_truncate_test_regular, - cstore_truncate_test_second, - cstore_truncate_test; - -SELECT * from cstore_truncate_test; -SELECT * from cstore_truncate_test_second; -SELECT * from cstore_truncate_test_regular; - --- test if truncate on empty table works -TRUNCATE TABLE cstore_truncate_test; -SELECT * from cstore_truncate_test; - --- make sure TRUNATE deletes metadata for old relfilenode -SELECT :cstore_data_files_before_truncate - count(*) FROM cstore.cstore_data_files; - --- test if truncation in the same transaction that created the table works properly -BEGIN; -CREATE FOREIGN TABLE cstore_same_transaction_truncate(a int) SERVER cstore_server; -INSERT INTO cstore_same_transaction_truncate SELECT * FROM generate_series(1, 100); -TRUNCATE cstore_same_transaction_truncate; -INSERT INTO cstore_same_transaction_truncate SELECT * FROM generate_series(20, 23); -COMMIT; - --- should output "1" for the newly created relation -SELECT count(*) - :cstore_data_files_before_truncate FROM cstore.cstore_data_files; -SELECT * FROM cstore_same_transaction_truncate; - -DROP FOREIGN TABLE cstore_same_transaction_truncate; - --- test if a cached truncate from a pl/pgsql function works -CREATE FUNCTION cstore_truncate_test_regular_func() RETURNS void AS $$ -BEGIN - INSERT INTO cstore_truncate_test_regular select a, a from generate_series(1, 10) a; - TRUNCATE TABLE cstore_truncate_test_regular; -END;$$ -LANGUAGE plpgsql; - -SELECT cstore_truncate_test_regular_func(); --- the cached plans are used stating from the second call -SELECT cstore_truncate_test_regular_func(); -DROP FUNCTION cstore_truncate_test_regular_func(); - -DROP FOREIGN TABLE cstore_truncate_test, cstore_truncate_test_second; -DROP TABLE cstore_truncate_test_regular; -DROP FOREIGN TABLE cstore_truncate_test_compressed; - --- test truncate with schema -CREATE SCHEMA truncate_schema; -CREATE FOREIGN TABLE truncate_schema.truncate_tbl (id int) SERVER cstore_server OPTIONS(compression 'pglz'); -INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100); -SELECT COUNT(*) FROM truncate_schema.truncate_tbl; - -TRUNCATE TABLE truncate_schema.truncate_tbl; -SELECT COUNT(*) FROM truncate_schema.truncate_tbl; - -INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100); - --- create a user that can not truncate -CREATE USER truncate_user; -GRANT USAGE ON SCHEMA truncate_schema TO truncate_user; -GRANT SELECT ON TABLE truncate_schema.truncate_tbl TO truncate_user; -REVOKE TRUNCATE ON TABLE truncate_schema.truncate_tbl FROM truncate_user; - -SELECT current_user \gset - -\c - truncate_user --- verify truncate command fails and check number of rows -SELECT count(*) FROM truncate_schema.truncate_tbl; -TRUNCATE TABLE truncate_schema.truncate_tbl; -SELECT count(*) FROM truncate_schema.truncate_tbl; - --- switch to super user, grant truncate to truncate_user -\c - :current_user -GRANT TRUNCATE ON TABLE truncate_schema.truncate_tbl TO truncate_user; - --- verify truncate_user can truncate now -\c - truncate_user -SELECT count(*) FROM truncate_schema.truncate_tbl; -TRUNCATE TABLE truncate_schema.truncate_tbl; -SELECT count(*) FROM truncate_schema.truncate_tbl; - -\c - :current_user - --- cleanup -DROP SCHEMA truncate_schema CASCADE; -DROP USER truncate_user;