Use client side \copy when accessing test files

pull/6138/head
Hanefi Onaldi 2022-08-09 14:16:46 +03:00
parent a58523f1d8
commit 6ef96ac560
No known key found for this signature in database
GPG Key ID: F18CDB10BA0DFDC7
28 changed files with 388 additions and 196 deletions

View File

@ -6,7 +6,8 @@ CREATE TABLE test_contestant(handle TEXT, birthdate DATE, rating INT,
USING columnar; USING columnar;
-- load table data from file -- load table data from file
\set contestants_1_csv_file :abs_srcdir '/data/contestants.1.csv' \set contestants_1_csv_file :abs_srcdir '/data/contestants.1.csv'
COPY test_contestant FROM :'contestants_1_csv_file' WITH CSV; \set client_side_copy_command '\\copy test_contestant FROM ' :'contestants_1_csv_file' ' WITH CSV;'
:client_side_copy_command
-- export using COPY table TO ... -- export using COPY table TO ...
COPY test_contestant TO STDOUT; COPY test_contestant TO STDOUT;
a 01-10-1990 2090 97.1 XA {a} a 01-10-1990 2090 97.1 XA {a}

View File

@ -9,7 +9,8 @@ SET intervalstyle TO 'POSTGRES_VERBOSE';
CREATE TABLE test_array_types (int_array int[], bigint_array bigint[], CREATE TABLE test_array_types (int_array int[], bigint_array bigint[],
text_array text[]) USING columnar; text_array text[]) USING columnar;
\set array_types_csv_file :abs_srcdir '/data/array_types.csv' \set array_types_csv_file :abs_srcdir '/data/array_types.csv'
COPY test_array_types FROM :'array_types_csv_file' WITH CSV; \set client_side_copy_command '\\copy test_array_types FROM ' :'array_types_csv_file' ' WITH CSV;'
:client_side_copy_command
SELECT * FROM test_array_types; SELECT * FROM test_array_types;
int_array | bigint_array | text_array int_array | bigint_array | text_array
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -23,7 +24,8 @@ CREATE TABLE test_datetime_types (timestamp timestamp,
timestamp_with_timezone timestamp with time zone, date date, time time, timestamp_with_timezone timestamp with time zone, date date, time time,
interval interval) USING columnar; interval interval) USING columnar;
\set datetime_types_csv_file :abs_srcdir '/data/datetime_types.csv' \set datetime_types_csv_file :abs_srcdir '/data/datetime_types.csv'
COPY test_datetime_types FROM :'datetime_types_csv_file' WITH CSV; \set client_side_copy_command '\\copy test_datetime_types FROM ' :'datetime_types_csv_file' ' WITH CSV;'
:client_side_copy_command
SELECT * FROM test_datetime_types; SELECT * FROM test_datetime_types;
timestamp | timestamp_with_timezone | date | time | interval timestamp | timestamp_with_timezone | date | time | interval
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -50,7 +52,8 @@ SELECT * FROM test_enum_and_composite_types;
CREATE TABLE test_range_types (int4range int4range, int8range int8range, CREATE TABLE test_range_types (int4range int4range, int8range int8range,
numrange numrange, tsrange tsrange) USING columnar; numrange numrange, tsrange tsrange) USING columnar;
\set range_types_csv_file :abs_srcdir '/data/range_types.csv' \set range_types_csv_file :abs_srcdir '/data/range_types.csv'
COPY test_range_types FROM :'range_types_csv_file' WITH CSV; \set client_side_copy_command '\\copy test_range_types FROM ' :'range_types_csv_file' ' WITH CSV;'
:client_side_copy_command
SELECT * FROM test_range_types; SELECT * FROM test_range_types;
int4range | int8range | numrange | tsrange int4range | int8range | numrange | tsrange
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -62,7 +65,8 @@ SELECT * FROM test_range_types;
CREATE TABLE test_other_types (bool boolean, bytea bytea, money money, CREATE TABLE test_other_types (bool boolean, bytea bytea, money money,
inet inet, bitstring bit varying(5), uuid uuid, json json) USING columnar; inet inet, bitstring bit varying(5), uuid uuid, json json) USING columnar;
\set other_types_csv_file :abs_srcdir '/data/other_types.csv' \set other_types_csv_file :abs_srcdir '/data/other_types.csv'
COPY test_other_types FROM :'other_types_csv_file' WITH CSV; \set client_side_copy_command '\\copy test_other_types FROM ' :'other_types_csv_file' ' WITH CSV;'
:client_side_copy_command
SELECT * FROM test_other_types; SELECT * FROM test_other_types;
bool | bytea | money | inet | bitstring | uuid | json bool | bytea | money | inet | bitstring | uuid | json
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -74,7 +78,8 @@ SELECT * FROM test_other_types;
CREATE TABLE test_null_values (a int, b int[], c composite_type) CREATE TABLE test_null_values (a int, b int[], c composite_type)
USING columnar; USING columnar;
\set null_values_csv_file :abs_srcdir '/data/null_values.csv' \set null_values_csv_file :abs_srcdir '/data/null_values.csv'
COPY test_null_values FROM :'null_values_csv_file' WITH CSV; \set client_side_copy_command '\\copy test_null_values FROM ' :'null_values_csv_file' ' WITH CSV;'
:client_side_copy_command
SELECT * FROM test_null_values; SELECT * FROM test_null_values;
a | b | c a | b | c
--------------------------------------------------------------------- ---------------------------------------------------------------------

View File

@ -3,8 +3,8 @@
-- --
-- COPY with incorrect delimiter -- COPY with incorrect delimiter
\set contestants_1_csv_file :abs_srcdir '/data/contestants.1.csv' \set contestants_1_csv_file :abs_srcdir '/data/contestants.1.csv'
COPY contestant FROM :'contestants_1_csv_file' \set client_side_copy_command '\\copy contestant FROM ' :'contestants_1_csv_file' ' WITH DELIMITER '''|''';'
WITH DELIMITER '|'; -- ERROR :client_side_copy_command -- ERROR
ERROR: missing data for column "birthdate" ERROR: missing data for column "birthdate"
CONTEXT: COPY contestant, line 1: "a,1990-01-10,2090,97.1,XA ,{a}" CONTEXT: COPY contestant, line 1: "a,1990-01-10,2090,97.1,XA ,{a}"
-- COPY with invalid program -- COPY with invalid program
@ -12,7 +12,8 @@ COPY contestant FROM PROGRAM 'invalid_program' WITH CSV; -- ERROR
ERROR: program "invalid_program" failed ERROR: program "invalid_program" failed
DETAIL: command not found DETAIL: command not found
-- COPY into uncompressed table from file -- COPY into uncompressed table from file
COPY contestant FROM :'contestants_1_csv_file' WITH CSV; \set client_side_copy_command '\\copy contestant FROM ' :'contestants_1_csv_file' ' WITH CSV;'
:client_side_copy_command
-- COPY into uncompressed table from program -- COPY into uncompressed table from program
\set cat_contestants_2_csv_file 'cat ' :abs_srcdir '/data/contestants.2.csv' \set cat_contestants_2_csv_file 'cat ' :abs_srcdir '/data/contestants.2.csv'
COPY contestant FROM PROGRAM :'cat_contestants_2_csv_file' WITH CSV; COPY contestant FROM PROGRAM :'cat_contestants_2_csv_file' WITH CSV;
@ -25,7 +26,8 @@ select
(1 row) (1 row)
-- COPY into compressed table -- COPY into compressed table
COPY contestant_compressed FROM :'contestants_1_csv_file' WITH CSV; \set client_side_copy_command '\\copy contestant_compressed FROM ' :'contestants_1_csv_file' ' WITH CSV;'
:client_side_copy_command
-- COPY into uncompressed table from program -- COPY into uncompressed table from program
COPY contestant_compressed FROM PROGRAM :'cat_contestants_2_csv_file' COPY contestant_compressed FROM PROGRAM :'cat_contestants_2_csv_file'
WITH CSV; WITH CSV;

View File

@ -36,8 +36,10 @@ UPDATE pg_dist_shard SET shardminvalue = 8997, shardmaxvalue = 14947
WHERE shardid = :new_shard_id; WHERE shardid = :new_shard_id;
\set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data' \set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data'
\set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data' \set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data'
COPY lineitem_range FROM :'lineitem_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy lineitem_range FROM ' :'lineitem_1_data_file' ' with delimiter '''|''';'
COPY lineitem_range FROM :'lineitem_2_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy lineitem_range FROM ' :'lineitem_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
-- Run aggregate(distinct) on partition column for range partitioned table -- Run aggregate(distinct) on partition column for range partitioned table
SELECT count(distinct l_orderkey) FROM lineitem_range; SELECT count(distinct l_orderkey) FROM lineitem_range;
count count
@ -161,8 +163,10 @@ SELECT create_distributed_table('lineitem_hash', 'l_orderkey', 'hash');
(1 row) (1 row)
COPY lineitem_hash FROM :'lineitem_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy lineitem_hash FROM ' :'lineitem_1_data_file' ' with delimiter '''|''';'
COPY lineitem_hash FROM :'lineitem_2_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy lineitem_hash FROM ' :'lineitem_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
-- aggregate(distinct) on partition column is allowed -- aggregate(distinct) on partition column is allowed
SELECT count(distinct l_orderkey) FROM lineitem_hash; SELECT count(distinct l_orderkey) FROM lineitem_hash;
count count

View File

@ -23,8 +23,10 @@ SELECT create_distributed_table('events_table', 'user_id');
\set users_table_data_file :abs_srcdir '/data/users_table.data' \set users_table_data_file :abs_srcdir '/data/users_table.data'
\set events_table_data_file :abs_srcdir '/data/events_table.data' \set events_table_data_file :abs_srcdir '/data/events_table.data'
COPY users_table FROM :'users_table_data_file' WITH CSV; \set client_side_copy_command '\\copy users_table FROM ' :'users_table_data_file' ' WITH CSV;'
COPY events_table FROM :'events_table_data_file' WITH CSV; :client_side_copy_command
\set client_side_copy_command '\\copy events_table FROM ' :'events_table_data_file' ' WITH CSV;'
:client_side_copy_command
SET citus.shard_count = 96; SET citus.shard_count = 96;
CREATE SCHEMA subquery_and_ctes; CREATE SCHEMA subquery_and_ctes;
SET search_path TO subquery_and_ctes; SET search_path TO subquery_and_ctes;
@ -42,8 +44,10 @@ SELECT create_distributed_table('events_table', 'user_id');
(1 row) (1 row)
COPY users_table FROM :'users_table_data_file' WITH CSV; \set client_side_copy_command '\\copy users_table FROM ' :'users_table_data_file' ' WITH CSV;'
COPY events_table FROM :'events_table_data_file' WITH CSV; :client_side_copy_command
\set client_side_copy_command '\\copy events_table FROM ' :'events_table_data_file' ' WITH CSV;'
:client_side_copy_command
SET citus.shard_count TO DEFAULT; SET citus.shard_count TO DEFAULT;
SET search_path TO DEFAULT; SET search_path TO DEFAULT;
CREATE TABLE users_table (user_id int, time timestamp, value_1 int, value_2 int, value_3 float, value_4 bigint); CREATE TABLE users_table (user_id int, time timestamp, value_1 int, value_2 int, value_3 float, value_4 bigint);
@ -112,8 +116,10 @@ INSERT INTO users_ref_test_table VALUES(3,'User_3',47);
INSERT INTO users_ref_test_table VALUES(4,'User_4',48); INSERT INTO users_ref_test_table VALUES(4,'User_4',48);
INSERT INTO users_ref_test_table VALUES(5,'User_5',49); INSERT INTO users_ref_test_table VALUES(5,'User_5',49);
INSERT INTO users_ref_test_table VALUES(6,'User_6',50); INSERT INTO users_ref_test_table VALUES(6,'User_6',50);
COPY users_table FROM :'users_table_data_file' WITH CSV; \set client_side_copy_command '\\copy users_table FROM ' :'users_table_data_file' ' WITH CSV;'
COPY events_table FROM :'events_table_data_file' WITH CSV; :client_side_copy_command
\set client_side_copy_command '\\copy events_table FROM ' :'events_table_data_file' ' WITH CSV;'
:client_side_copy_command
-- create indexes for -- create indexes for
CREATE INDEX is_index1 ON users_table(user_id); CREATE INDEX is_index1 ON users_table(user_id);
CREATE INDEX is_index2 ON events_table(user_id); CREATE INDEX is_index2 ON events_table(user_id);

View File

@ -331,10 +331,14 @@ SELECT master_create_empty_shard('orders_subquery') AS new_shard_id
UPDATE pg_dist_shard SET shardminvalue = 8997, shardmaxvalue = 14947 UPDATE pg_dist_shard SET shardminvalue = 8997, shardmaxvalue = 14947
WHERE shardid = :new_shard_id; WHERE shardid = :new_shard_id;
\set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data' \set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data'
COPY lineitem_subquery FROM :'lineitem_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy lineitem_subquery FROM ' :'lineitem_1_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data' \set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data'
COPY lineitem_subquery FROM :'lineitem_2_data_file' with delimiter '|'; \set client_side_copy_command '\\copy lineitem_subquery FROM ' :'lineitem_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set orders_1_data_file :abs_srcdir '/data/orders.1.data' \set orders_1_data_file :abs_srcdir '/data/orders.1.data'
COPY orders_subquery FROM :'orders_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy orders_subquery FROM ' :'orders_1_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set orders_2_data_file :abs_srcdir '/data/orders.2.data' \set orders_2_data_file :abs_srcdir '/data/orders.2.data'
COPY orders_subquery FROM :'orders_2_data_file' with delimiter '|'; \set client_side_copy_command '\\copy orders_subquery FROM ' :'orders_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command

View File

@ -31,8 +31,10 @@ SELECT create_distributed_table('lineitem_hash', 'l_orderkey', 'hash');
\set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data' \set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data'
\set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data' \set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data'
COPY lineitem_hash FROM :'lineitem_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy lineitem_hash FROM ' :'lineitem_1_data_file' ' with delimiter '''|''';'
COPY lineitem_hash FROM :'lineitem_2_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy lineitem_hash FROM ' :'lineitem_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
ANALYZE lineitem_hash; ANALYZE lineitem_hash;
-- count(distinct) is supported on top level query if there -- count(distinct) is supported on top level query if there
-- is a grouping on the partition key -- is a grouping on the partition key

View File

@ -106,7 +106,8 @@ SELECT count(*) FROM customer_copy_hash WHERE c_custkey = 9;
(1 row) (1 row)
-- Test server-side copy from file -- Test server-side copy from file
COPY customer_copy_hash FROM :'customer2datafile' WITH (DELIMITER '|'); \set client_side_copy_command '\\copy customer_copy_hash FROM ' :'customer2datafile' ' WITH (DELIMITER '''|''');'
:client_side_copy_command
-- Confirm that data was copied -- Confirm that data was copied
SELECT count(*) FROM customer_copy_hash; SELECT count(*) FROM customer_copy_hash;
count count
@ -208,7 +209,8 @@ SELECT master_create_distributed_table('customer_copy_range', 'c_custkey', 'rang
(1 row) (1 row)
-- Test COPY into empty range-partitioned table -- Test COPY into empty range-partitioned table
COPY customer_copy_range FROM :'customer1datafile' WITH (DELIMITER '|'); \set client_side_copy_command '\\copy customer_copy_range FROM ' :'customer1datafile' ' WITH (DELIMITER '''|''');'
:client_side_copy_command
ERROR: could not find any shards into which to copy ERROR: could not find any shards into which to copy
DETAIL: No shards exist for distributed table "customer_copy_range". DETAIL: No shards exist for distributed table "customer_copy_range".
SELECT master_create_empty_shard('customer_copy_range') AS new_shard_id SELECT master_create_empty_shard('customer_copy_range') AS new_shard_id
@ -220,7 +222,8 @@ SELECT master_create_empty_shard('customer_copy_range') AS new_shard_id
UPDATE pg_dist_shard SET shardminvalue = 501, shardmaxvalue = 1000 UPDATE pg_dist_shard SET shardminvalue = 501, shardmaxvalue = 1000
WHERE shardid = :new_shard_id; WHERE shardid = :new_shard_id;
-- Test copy into range-partitioned table -- Test copy into range-partitioned table
COPY customer_copy_range FROM :'customer1datafile' WITH (DELIMITER '|'); \set client_side_copy_command '\\copy customer_copy_range FROM ' :'customer1datafile' ' WITH (DELIMITER '''|''');'
:client_side_copy_command
-- Check whether data went into the right shard (maybe) -- Check whether data went into the right shard (maybe)
SELECT min(c_custkey), max(c_custkey), avg(c_custkey), count(*) SELECT min(c_custkey), max(c_custkey), avg(c_custkey), count(*)
FROM customer_copy_range WHERE c_custkey <= 500; FROM customer_copy_range WHERE c_custkey <= 500;
@ -360,7 +363,8 @@ SELECT create_distributed_table('lineitem_copy_append', 'l_orderkey', 'append');
BEGIN; BEGIN;
SELECT master_create_empty_shard('lineitem_copy_append') AS shardid \gset SELECT master_create_empty_shard('lineitem_copy_append') AS shardid \gset
COPY lineitem_copy_append FROM :'lineitem1datafile' with (delimiter '|', append_to_shard :shardid); \set client_side_copy_command '\\copy lineitem_copy_append FROM ' :'lineitem1datafile' ' with (delimiter '''|''', append_to_shard ' :shardid ');'
:client_side_copy_command
END; END;
SELECT count(*) FROM pg_dist_shard WHERE logicalrelid = 'lineitem_copy_append'::regclass; SELECT count(*) FROM pg_dist_shard WHERE logicalrelid = 'lineitem_copy_append'::regclass;
count count
@ -369,9 +373,11 @@ SELECT count(*) FROM pg_dist_shard WHERE logicalrelid = 'lineitem_copy_append'::
(1 row) (1 row)
-- trigger some errors on the append_to_shard option -- trigger some errors on the append_to_shard option
COPY lineitem_copy_append FROM :'lineitem1datafile' with (delimiter '|', append_to_shard xxxxx); \set client_side_copy_command '\\copy lineitem_copy_append FROM ' :'lineitem1datafile' ' with (delimiter '''|''', append_to_shard xxxxx);'
:client_side_copy_command
ERROR: could not find valid entry for shard xxxxx ERROR: could not find valid entry for shard xxxxx
COPY lineitem_copy_append FROM :'lineitem1datafile' with (delimiter '|', append_to_shard xxxxx); \set client_side_copy_command '\\copy lineitem_copy_append FROM ' :'lineitem1datafile' ' with (delimiter '''|''', append_to_shard xxxxx);'
:client_side_copy_command
ERROR: shard xxxxx does not belong to table lineitem_copy_append ERROR: shard xxxxx does not belong to table lineitem_copy_append
-- Test schema support on append partitioned tables -- Test schema support on append partitioned tables
CREATE SCHEMA append; CREATE SCHEMA append;
@ -393,8 +399,10 @@ SELECT create_distributed_table('append.customer_copy', 'c_custkey', 'append');
SELECT master_create_empty_shard('append.customer_copy') AS shardid1 \gset SELECT master_create_empty_shard('append.customer_copy') AS shardid1 \gset
SELECT master_create_empty_shard('append.customer_copy') AS shardid2 \gset SELECT master_create_empty_shard('append.customer_copy') AS shardid2 \gset
-- Test copy from the master node -- Test copy from the master node
COPY append.customer_copy FROM :'customer1datafile' with (delimiter '|', append_to_shard :shardid1); \set client_side_copy_command '\\copy append.customer_copy FROM ' :'customer1datafile' ' with (delimiter '''|''', append_to_shard ' :shardid1 ');'
COPY append.customer_copy FROM :'customer2datafile' with (delimiter '|', append_to_shard :shardid2); :client_side_copy_command
\set client_side_copy_command '\\copy append.customer_copy FROM ' :'customer2datafile' ' with (delimiter '''|''', append_to_shard ' :shardid2 ');'
:client_side_copy_command
-- Test the content of the table -- Test the content of the table
SELECT min(c_custkey), max(c_custkey), avg(c_acctbal), count(*) FROM append.customer_copy; SELECT min(c_custkey), max(c_custkey), avg(c_acctbal), count(*) FROM append.customer_copy;
min | max | avg | count min | max | avg | count

View File

@ -3,22 +3,35 @@
-- --
\set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data' \set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data'
\set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data' \set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data'
COPY lineitem FROM :'lineitem_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy lineitem FROM ' :'lineitem_1_data_file' ' with delimiter '''|''';'
COPY lineitem FROM :'lineitem_2_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy lineitem FROM ' :'lineitem_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set orders_1_data_file :abs_srcdir '/data/orders.1.data' \set orders_1_data_file :abs_srcdir '/data/orders.1.data'
\set orders_2_data_file :abs_srcdir '/data/orders.2.data' \set orders_2_data_file :abs_srcdir '/data/orders.2.data'
COPY orders FROM :'orders_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy orders FROM ' :'orders_1_data_file' ' with delimiter '''|''';'
COPY orders FROM :'orders_2_data_file' with delimiter '|'; :client_side_copy_command
COPY orders_reference FROM :'orders_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy orders FROM ' :'orders_2_data_file' ' with delimiter '''|''';'
COPY orders_reference FROM :'orders_2_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy orders_reference FROM ' :'orders_1_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy orders_reference FROM ' :'orders_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set customer_1_data_file :abs_srcdir '/data/customer.1.data' \set customer_1_data_file :abs_srcdir '/data/customer.1.data'
\set nation_data_file :abs_srcdir '/data/nation.data' \set nation_data_file :abs_srcdir '/data/nation.data'
\set part_data_file :abs_srcdir '/data/part.data' \set part_data_file :abs_srcdir '/data/part.data'
\set supplier_data_file :abs_srcdir '/data/supplier.data' \set supplier_data_file :abs_srcdir '/data/supplier.data'
COPY customer FROM :'customer_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy customer FROM ' :'customer_1_data_file' ' with delimiter '''|''';'
COPY customer_append FROM :'customer_1_data_file' with (delimiter '|', append_to_shard xxxxx); :client_side_copy_command
COPY nation FROM :'nation_data_file' with delimiter '|'; \set client_side_copy_command '\\copy customer_append FROM ' :'customer_1_data_file' ' with (delimiter '''|''', append_to_shard xxxxx);'
COPY part FROM :'part_data_file' with delimiter '|'; :client_side_copy_command
COPY part_append FROM :'part_data_file' with (delimiter '|', append_to_shard xxxxx); \set client_side_copy_command '\\copy nation FROM ' :'nation_data_file' ' with delimiter '''|''';'
COPY supplier FROM :'supplier_data_file' with delimiter '|'; :client_side_copy_command
COPY supplier_single_shard FROM :'supplier_data_file' with delimiter '|'; \set client_side_copy_command '\\copy part FROM ' :'part_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy part_append FROM ' :'part_data_file' ' with (delimiter '''|''', append_to_shard xxxxx);'
:client_side_copy_command
\set client_side_copy_command '\\copy supplier FROM ' :'supplier_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy supplier_single_shard FROM ' :'supplier_data_file' ' with delimiter '''|''';'
:client_side_copy_command

View File

@ -2,7 +2,11 @@
\set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data' \set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data'
\set orders_1_data_file :abs_srcdir '/data/orders.1.data' \set orders_1_data_file :abs_srcdir '/data/orders.1.data'
\set orders_2_data_file :abs_srcdir '/data/orders.2.data' \set orders_2_data_file :abs_srcdir '/data/orders.2.data'
COPY lineitem_hash_part FROM :'lineitem_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy lineitem_hash_part FROM ' :'lineitem_1_data_file' ' with delimiter '''|''';'
COPY lineitem_hash_part FROM :'lineitem_2_data_file' with delimiter '|'; :client_side_copy_command
COPY orders_hash_part FROM :'orders_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy lineitem_hash_part FROM ' :'lineitem_2_data_file' ' with delimiter '''|''';'
COPY orders_hash_part FROM :'orders_2_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy orders_hash_part FROM ' :'orders_1_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy orders_hash_part FROM ' :'orders_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command

View File

@ -8,9 +8,12 @@ SET citus.next_shard_id TO 280000;
\set customer_2_data_file :abs_srcdir '/data/customer.2.data' \set customer_2_data_file :abs_srcdir '/data/customer.2.data'
\set customer_3_data_file :abs_srcdir '/data/customer.3.data' \set customer_3_data_file :abs_srcdir '/data/customer.3.data'
\set part_more_data_file :abs_srcdir '/data/part.more.data' \set part_more_data_file :abs_srcdir '/data/part.more.data'
COPY customer FROM :'customer_2_data_file' with delimiter '|'; \set client_side_copy_command '\\copy customer FROM ' :'customer_2_data_file' ' with delimiter '''|''';'
COPY customer FROM :'customer_3_data_file' with delimiter '|'; :client_side_copy_command
COPY part FROM :'part_more_data_file' with delimiter '|'; \set client_side_copy_command '\\copy customer FROM ' :'customer_3_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy part FROM ' :'part_more_data_file' ' with delimiter '''|''';'
:client_side_copy_command
SELECT master_create_empty_shard('customer_append') AS shardid1 \gset SELECT master_create_empty_shard('customer_append') AS shardid1 \gset
SELECT master_create_empty_shard('customer_append') AS shardid2 \gset SELECT master_create_empty_shard('customer_append') AS shardid2 \gset
copy customer_append FROM :'customer_2_data_file' with (delimiter '|', append_to_shard :shardid1); copy customer_append FROM :'customer_2_data_file' with (delimiter '|', append_to_shard :shardid1);

View File

@ -2,12 +2,17 @@
-- MULTI_MX_COPY_DATA -- MULTI_MX_COPY_DATA
-- --
\set nation_data_file :abs_srcdir '/data/nation.data' \set nation_data_file :abs_srcdir '/data/nation.data'
COPY nation_hash FROM :'nation_data_file' with delimiter '|'; \set client_side_copy_command '\\copy nation_hash FROM ' :'nation_data_file' ' with delimiter '''|''';'
:client_side_copy_command
SET search_path TO citus_mx_test_schema; SET search_path TO citus_mx_test_schema;
COPY nation_hash FROM :'nation_data_file' with delimiter '|'; \set client_side_copy_command '\\copy nation_hash FROM ' :'nation_data_file' ' with delimiter '''|''';'
COPY citus_mx_test_schema_join_1.nation_hash FROM :'nation_data_file' with delimiter '|'; :client_side_copy_command
COPY citus_mx_test_schema_join_1.nation_hash_2 FROM :'nation_data_file' with delimiter '|'; \set client_side_copy_command '\\copy citus_mx_test_schema_join_1.nation_hash FROM ' :'nation_data_file' ' with delimiter '''|''';'
COPY citus_mx_test_schema_join_2.nation_hash FROM :'nation_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy citus_mx_test_schema_join_1.nation_hash_2 FROM ' :'nation_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy citus_mx_test_schema_join_2.nation_hash FROM ' :'nation_data_file' ' with delimiter '''|''';'
:client_side_copy_command
SET citus.shard_replication_factor TO 2; SET citus.shard_replication_factor TO 2;
CREATE TABLE citus_mx_test_schema.nation_hash_replicated AS SELECT * FROM citus_mx_test_schema.nation_hash; CREATE TABLE citus_mx_test_schema.nation_hash_replicated AS SELECT * FROM citus_mx_test_schema.nation_hash;
SELECT create_distributed_table('citus_mx_test_schema.nation_hash_replicated', 'n_nationkey'); SELECT create_distributed_table('citus_mx_test_schema.nation_hash_replicated', 'n_nationkey');
@ -20,24 +25,31 @@ HINT: To remove the local data, run: SELECT truncate_local_data_after_distribut
(1 row) (1 row)
COPY nation_hash_replicated FROM :'nation_data_file' with delimiter '|'; \set client_side_copy_command '\\copy nation_hash_replicated FROM ' :'nation_data_file' ' with delimiter '''|''';'
:client_side_copy_command
-- now try loading data from worker node -- now try loading data from worker node
\c - - - :worker_1_port \c - - - :worker_1_port
SET search_path TO public; SET search_path TO public;
\set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data' \set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data'
\set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data' \set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data'
COPY lineitem_mx FROM :'lineitem_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy lineitem_mx FROM ' :'lineitem_1_data_file' ' with delimiter '''|''';'
COPY lineitem_mx FROM :'lineitem_2_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy lineitem_mx FROM ' :'lineitem_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set nation_data_file :abs_srcdir '/data/nation.data' \set nation_data_file :abs_srcdir '/data/nation.data'
COPY citus_mx_test_schema.nation_hash_replicated FROM :'nation_data_file' with delimiter '|'; \set client_side_copy_command '\\copy citus_mx_test_schema.nation_hash_replicated FROM ' :'nation_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\c - - - :worker_2_port \c - - - :worker_2_port
-- and use second worker as well -- and use second worker as well
\set orders_1_data_file :abs_srcdir '/data/orders.1.data' \set orders_1_data_file :abs_srcdir '/data/orders.1.data'
\set orders_2_data_file :abs_srcdir '/data/orders.2.data' \set orders_2_data_file :abs_srcdir '/data/orders.2.data'
\set nation_data_file :abs_srcdir '/data/nation.data' \set nation_data_file :abs_srcdir '/data/nation.data'
COPY orders_mx FROM :'orders_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy orders_mx FROM ' :'orders_1_data_file' ' with delimiter '''|''';'
COPY orders_mx FROM :'orders_2_data_file' with delimiter '|'; :client_side_copy_command
COPY citus_mx_test_schema.nation_hash_replicated FROM :'nation_data_file' with delimiter '|'; \set client_side_copy_command '\\copy orders_mx FROM ' :'orders_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy citus_mx_test_schema.nation_hash_replicated FROM ' :'nation_data_file' ' with delimiter '''|''';'
:client_side_copy_command
-- get ready for the next test -- get ready for the next test
TRUNCATE orders_mx; TRUNCATE orders_mx;
\c - - - :worker_2_port \c - - - :worker_2_port
@ -64,7 +76,8 @@ show citus.local_shared_pool_size;
\set orders_1_data_file :abs_srcdir '/data/orders.1.data' \set orders_1_data_file :abs_srcdir '/data/orders.1.data'
\set orders_2_data_file :abs_srcdir '/data/orders.2.data' \set orders_2_data_file :abs_srcdir '/data/orders.2.data'
COPY orders_mx FROM :'orders_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy orders_mx FROM ' :'orders_1_data_file' ' with delimiter '''|''';'
:client_side_copy_command
NOTICE: executing the copy locally for shard xxxxx NOTICE: executing the copy locally for shard xxxxx
CONTEXT: COPY orders_mx, line 3: "3|1234|F|205654.30|1993-10-14|5-LOW|Clerk#000000955|0|sly final accounts boost. carefully regular id..." CONTEXT: COPY orders_mx, line 3: "3|1234|F|205654.30|1993-10-14|5-LOW|Clerk#000000955|0|sly final accounts boost. carefully regular id..."
NOTICE: executing the copy locally for shard xxxxx NOTICE: executing the copy locally for shard xxxxx
@ -81,7 +94,8 @@ NOTICE: executing the copy locally for shard xxxxx
CONTEXT: COPY orders_mx, line 25: "97|211|F|100572.55|1993-01-29|3-MEDIUM|Clerk#000000547|0|hang blithely along the regular accounts. f..." CONTEXT: COPY orders_mx, line 25: "97|211|F|100572.55|1993-01-29|3-MEDIUM|Clerk#000000547|0|hang blithely along the regular accounts. f..."
NOTICE: executing the copy locally for shard xxxxx NOTICE: executing the copy locally for shard xxxxx
CONTEXT: COPY orders_mx, line 38: "134|62|F|208201.46|1992-05-01|4-NOT SPECIFIED|Clerk#000000711|0|lar theodolites boos" CONTEXT: COPY orders_mx, line 38: "134|62|F|208201.46|1992-05-01|4-NOT SPECIFIED|Clerk#000000711|0|lar theodolites boos"
COPY orders_mx FROM :'orders_2_data_file' with delimiter '|'; \set client_side_copy_command '\\copy orders_mx FROM ' :'orders_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
NOTICE: executing the copy locally for shard xxxxx NOTICE: executing the copy locally for shard xxxxx
CONTEXT: COPY orders_mx, line 2: "8998|80|F|147264.16|1993-01-04|5-LOW|Clerk#000000733|0| fluffily pending sauternes cajo" CONTEXT: COPY orders_mx, line 2: "8998|80|F|147264.16|1993-01-04|5-LOW|Clerk#000000733|0| fluffily pending sauternes cajo"
NOTICE: executing the copy locally for shard xxxxx NOTICE: executing the copy locally for shard xxxxx
@ -99,7 +113,8 @@ CONTEXT: COPY orders_mx, line 43: "9159|1135|O|99594.61|1995-07-26|1-URGENT|Cle
NOTICE: executing the copy locally for shard xxxxx NOTICE: executing the copy locally for shard xxxxx
CONTEXT: COPY orders_mx, line 69: "9281|904|F|173278.28|1992-02-24|1-URGENT|Clerk#000000530|0|eep furiously according to the requests; ..." CONTEXT: COPY orders_mx, line 69: "9281|904|F|173278.28|1992-02-24|1-URGENT|Clerk#000000530|0|eep furiously according to the requests; ..."
\set nation_data_file :abs_srcdir '/data/nation.data' \set nation_data_file :abs_srcdir '/data/nation.data'
COPY citus_mx_test_schema.nation_hash_replicated FROM :'nation_data_file' with delimiter '|'; \set client_side_copy_command '\\copy citus_mx_test_schema.nation_hash_replicated FROM ' :'nation_data_file' ' with delimiter '''|''';'
:client_side_copy_command
NOTICE: executing the copy locally for shard xxxxx NOTICE: executing the copy locally for shard xxxxx
CONTEXT: COPY nation_hash_replicated, line 1: "0|ALGERIA|0| haggle. carefully final deposits detect slyly agai" CONTEXT: COPY nation_hash_replicated, line 1: "0|ALGERIA|0| haggle. carefully final deposits detect slyly agai"
NOTICE: executing the copy locally for shard xxxxx NOTICE: executing the copy locally for shard xxxxx
@ -138,7 +153,11 @@ SET search_path TO public;
\set nation_data_file :abs_srcdir '/data/nation.data' \set nation_data_file :abs_srcdir '/data/nation.data'
\set part_data_file :abs_srcdir '/data/part.data' \set part_data_file :abs_srcdir '/data/part.data'
\set supplier_data_file :abs_srcdir '/data/supplier.data' \set supplier_data_file :abs_srcdir '/data/supplier.data'
COPY customer_mx FROM :'customer_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy customer_mx FROM ' :'customer_1_data_file' ' with delimiter '''|''';'
COPY nation_mx FROM :'nation_data_file' with delimiter '|'; :client_side_copy_command
COPY part_mx FROM :'part_data_file' with delimiter '|'; \set client_side_copy_command '\\copy nation_mx FROM ' :'nation_data_file' ' with delimiter '''|''';'
COPY supplier_mx FROM :'supplier_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy part_mx FROM ' :'part_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy supplier_mx FROM ' :'supplier_data_file' ' with delimiter '''|''';'
:client_side_copy_command

View File

@ -89,10 +89,14 @@ SELECT create_reference_table('multi_outer_join_third_reference');
\set customer_1_10_data :abs_srcdir '/data/customer-1-10.data' \set customer_1_10_data :abs_srcdir '/data/customer-1-10.data'
\set customer_11_20_data :abs_srcdir '/data/customer-11-20.data' \set customer_11_20_data :abs_srcdir '/data/customer-11-20.data'
\set customer_1_15_data :abs_srcdir '/data/customer-1-15.data' \set customer_1_15_data :abs_srcdir '/data/customer-1-15.data'
COPY multi_outer_join_left FROM :'customer_1_10_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_left FROM ' :'customer_1_10_data' ' with delimiter '''|''';'
COPY multi_outer_join_left FROM :'customer_11_20_data' with delimiter '|'; :client_side_copy_command
COPY multi_outer_join_right FROM :'customer_1_15_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_left FROM ' :'customer_11_20_data' ' with delimiter '''|''';'
COPY multi_outer_join_right_reference FROM :'customer_1_15_data' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_right FROM ' :'customer_1_15_data' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_right_reference FROM ' :'customer_1_15_data' ' with delimiter '''|''';'
:client_side_copy_command
-- Make sure we do not crash if one table has no shards -- Make sure we do not crash if one table has no shards
SELECT SELECT
min(l_custkey), max(l_custkey) min(l_custkey), max(l_custkey)
@ -114,8 +118,10 @@ FROM
-- Third table is a single shard table with all data -- Third table is a single shard table with all data
\set customer_1_30_data :abs_srcdir '/data/customer-1-30.data' \set customer_1_30_data :abs_srcdir '/data/customer-1-30.data'
COPY multi_outer_join_third FROM :'customer_1_30_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_third FROM ' :'customer_1_30_data' ' with delimiter '''|''';'
COPY multi_outer_join_third_reference FROM :'customer_1_30_data' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_third_reference FROM ' :'customer_1_30_data' ' with delimiter '''|''';'
:client_side_copy_command
-- Regular outer join should return results for all rows -- Regular outer join should return results for all rows
SELECT SELECT
min(l_custkey), max(l_custkey) min(l_custkey), max(l_custkey)
@ -228,7 +234,8 @@ FROM
-- Turn the right table into a large table -- Turn the right table into a large table
\set customer_21_30_data :abs_srcdir '/data/customer-21-30.data' \set customer_21_30_data :abs_srcdir '/data/customer-21-30.data'
COPY multi_outer_join_right FROM :'customer_21_30_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_right FROM ' :'customer_21_30_data' ' with delimiter '''|''';'
:client_side_copy_command
-- Shards do not have 1-1 matching. We should error here. -- Shards do not have 1-1 matching. We should error here.
SELECT SELECT
min(l_custkey), max(l_custkey) min(l_custkey), max(l_custkey)
@ -244,11 +251,15 @@ TRUNCATE multi_outer_join_left;
TRUNCATE multi_outer_join_right; TRUNCATE multi_outer_join_right;
-- reload shards with 1-1 matching -- reload shards with 1-1 matching
\set customer_subset_11_20_data :abs_srcdir '/data/customer-subset-11-20.data' \set customer_subset_11_20_data :abs_srcdir '/data/customer-subset-11-20.data'
COPY multi_outer_join_left FROM :'customer_subset_11_20_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_left FROM ' :'customer_subset_11_20_data' ' with delimiter '''|''';'
COPY multi_outer_join_left FROM :'customer_21_30_data' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_left FROM ' :'customer_21_30_data' ' with delimiter '''|''';'
:client_side_copy_command
\set customer_subset_21_30_data :abs_srcdir '/data/customer-subset-21-30.data' \set customer_subset_21_30_data :abs_srcdir '/data/customer-subset-21-30.data'
COPY multi_outer_join_right FROM :'customer_11_20_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_right FROM ' :'customer_11_20_data' ' with delimiter '''|''';'
COPY multi_outer_join_right FROM :'customer_subset_21_30_data' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_right FROM ' :'customer_subset_21_30_data' ' with delimiter '''|''';'
:client_side_copy_command
-- multi_outer_join_third is a single shard table -- multi_outer_join_third is a single shard table
-- Regular left join should work as expected -- Regular left join should work as expected
SELECT SELECT
@ -803,7 +814,8 @@ LIMIT 20;
-- Add a shard to the left table that overlaps with multiple shards in the right -- Add a shard to the left table that overlaps with multiple shards in the right
\set customer_1_data_file :abs_srcdir '/data/customer.1.data' \set customer_1_data_file :abs_srcdir '/data/customer.1.data'
COPY multi_outer_join_left FROM :'customer_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_left FROM ' :'customer_1_data_file' ' with delimiter '''|''';'
:client_side_copy_command
-- All outer joins should error out -- All outer joins should error out
SELECT SELECT
min(l_custkey), max(l_custkey) min(l_custkey), max(l_custkey)

View File

@ -83,12 +83,16 @@ FROM
-- Left table is a large table -- Left table is a large table
\set customer_1_10_data :abs_srcdir '/data/customer-1-10.data' \set customer_1_10_data :abs_srcdir '/data/customer-1-10.data'
\set customer_11_20_data :abs_srcdir '/data/customer-11-20.data' \set customer_11_20_data :abs_srcdir '/data/customer-11-20.data'
COPY multi_outer_join_left_hash FROM :'customer_1_10_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_left_hash FROM ' :'customer_1_10_data' ' with delimiter '''|''';'
COPY multi_outer_join_left_hash FROM :'customer_11_20_data' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_left_hash FROM ' :'customer_11_20_data' ' with delimiter '''|''';'
:client_side_copy_command
-- Right table is a small table -- Right table is a small table
\set customer_1_15_data :abs_srcdir '/data/customer-1-15.data' \set customer_1_15_data :abs_srcdir '/data/customer-1-15.data'
COPY multi_outer_join_right_reference FROM :'customer_1_15_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_right_reference FROM ' :'customer_1_15_data' ' with delimiter '''|''';'
COPY multi_outer_join_right_hash FROM :'customer_1_15_data' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_right_hash FROM ' :'customer_1_15_data' ' with delimiter '''|''';'
:client_side_copy_command
-- Make sure we do not crash if one table has data -- Make sure we do not crash if one table has data
SELECT SELECT
min(l_custkey), max(l_custkey) min(l_custkey), max(l_custkey)
@ -110,8 +114,10 @@ FROM
-- Third table is a single shard table with all data -- Third table is a single shard table with all data
\set customer_1_30_data :abs_srcdir '/data/customer-1-30.data' \set customer_1_30_data :abs_srcdir '/data/customer-1-30.data'
COPY multi_outer_join_third_reference FROM :'customer_1_30_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_third_reference FROM ' :'customer_1_30_data' ' with delimiter '''|''';'
COPY multi_outer_join_right_hash FROM :'customer_1_30_data' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_right_hash FROM ' :'customer_1_30_data' ' with delimiter '''|''';'
:client_side_copy_command
-- Regular outer join should return results for all rows -- Regular outer join should return results for all rows
SELECT SELECT
min(l_custkey), max(l_custkey) min(l_custkey), max(l_custkey)
@ -221,7 +227,8 @@ FROM
-- load some more data -- load some more data
\set customer_21_30_data :abs_srcdir '/data/customer-21-30.data' \set customer_21_30_data :abs_srcdir '/data/customer-21-30.data'
COPY multi_outer_join_right_reference FROM :'customer_21_30_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_right_reference FROM ' :'customer_21_30_data' ' with delimiter '''|''';'
:client_side_copy_command
-- Update shards so that they do not have 1-1 matching, triggering an error. -- Update shards so that they do not have 1-1 matching, triggering an error.
UPDATE pg_dist_shard SET shardminvalue = '2147483646' WHERE shardid = 1260006; UPDATE pg_dist_shard SET shardminvalue = '2147483646' WHERE shardid = 1260006;
UPDATE pg_dist_shard SET shardmaxvalue = '2147483647' WHERE shardid = 1260006; UPDATE pg_dist_shard SET shardmaxvalue = '2147483647' WHERE shardid = 1260006;
@ -235,12 +242,18 @@ UPDATE pg_dist_shard SET shardmaxvalue = '-1073741825' WHERE shardid = 1260006;
-- empty tables -- empty tables
TRUNCATE multi_outer_join_left_hash, multi_outer_join_right_hash, multi_outer_join_right_reference; TRUNCATE multi_outer_join_left_hash, multi_outer_join_right_hash, multi_outer_join_right_reference;
-- reload shards with 1-1 matching -- reload shards with 1-1 matching
COPY multi_outer_join_left_hash FROM :'customer_1_15_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_left_hash FROM ' :'customer_1_15_data' ' with delimiter '''|''';'
COPY multi_outer_join_left_hash FROM :'customer_21_30_data' with delimiter '|'; :client_side_copy_command
COPY multi_outer_join_right_reference FROM :'customer_11_20_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_left_hash FROM ' :'customer_21_30_data' ' with delimiter '''|''';'
COPY multi_outer_join_right_reference FROM :'customer_21_30_data' with delimiter '|'; :client_side_copy_command
COPY multi_outer_join_right_hash FROM :'customer_11_20_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_right_reference FROM ' :'customer_11_20_data' ' with delimiter '''|''';'
COPY multi_outer_join_right_hash FROM :'customer_21_30_data' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_right_reference FROM ' :'customer_21_30_data' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_right_hash FROM ' :'customer_11_20_data' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_right_hash FROM ' :'customer_21_30_data' ' with delimiter '''|''';'
:client_side_copy_command
-- multi_outer_join_third_reference is a single shard table -- multi_outer_join_third_reference is a single shard table
-- Regular left join should work as expected -- Regular left join should work as expected
SELECT SELECT

View File

@ -7,7 +7,8 @@ CREATE TABLE test_contestant(handle TEXT, birthdate DATE, rating INT,
-- load table data from file -- load table data from file
\set contestants_1_csv_file :abs_srcdir '/data/contestants.1.csv' \set contestants_1_csv_file :abs_srcdir '/data/contestants.1.csv'
COPY test_contestant FROM :'contestants_1_csv_file' WITH CSV; \set client_side_copy_command '\\copy test_contestant FROM ' :'contestants_1_csv_file' ' WITH CSV;'
:client_side_copy_command
-- export using COPY table TO ... -- export using COPY table TO ...
COPY test_contestant TO STDOUT; COPY test_contestant TO STDOUT;

View File

@ -14,7 +14,8 @@ CREATE TABLE test_array_types (int_array int[], bigint_array bigint[],
text_array text[]) USING columnar; text_array text[]) USING columnar;
\set array_types_csv_file :abs_srcdir '/data/array_types.csv' \set array_types_csv_file :abs_srcdir '/data/array_types.csv'
COPY test_array_types FROM :'array_types_csv_file' WITH CSV; \set client_side_copy_command '\\copy test_array_types FROM ' :'array_types_csv_file' ' WITH CSV;'
:client_side_copy_command
SELECT * FROM test_array_types; SELECT * FROM test_array_types;
@ -25,7 +26,8 @@ CREATE TABLE test_datetime_types (timestamp timestamp,
interval interval) USING columnar; interval interval) USING columnar;
\set datetime_types_csv_file :abs_srcdir '/data/datetime_types.csv' \set datetime_types_csv_file :abs_srcdir '/data/datetime_types.csv'
COPY test_datetime_types FROM :'datetime_types_csv_file' WITH CSV; \set client_side_copy_command '\\copy test_datetime_types FROM ' :'datetime_types_csv_file' ' WITH CSV;'
:client_side_copy_command
SELECT * FROM test_datetime_types; SELECT * FROM test_datetime_types;
@ -49,7 +51,8 @@ CREATE TABLE test_range_types (int4range int4range, int8range int8range,
numrange numrange, tsrange tsrange) USING columnar; numrange numrange, tsrange tsrange) USING columnar;
\set range_types_csv_file :abs_srcdir '/data/range_types.csv' \set range_types_csv_file :abs_srcdir '/data/range_types.csv'
COPY test_range_types FROM :'range_types_csv_file' WITH CSV; \set client_side_copy_command '\\copy test_range_types FROM ' :'range_types_csv_file' ' WITH CSV;'
:client_side_copy_command
SELECT * FROM test_range_types; SELECT * FROM test_range_types;
@ -59,7 +62,8 @@ CREATE TABLE test_other_types (bool boolean, bytea bytea, money money,
inet inet, bitstring bit varying(5), uuid uuid, json json) USING columnar; inet inet, bitstring bit varying(5), uuid uuid, json json) USING columnar;
\set other_types_csv_file :abs_srcdir '/data/other_types.csv' \set other_types_csv_file :abs_srcdir '/data/other_types.csv'
COPY test_other_types FROM :'other_types_csv_file' WITH CSV; \set client_side_copy_command '\\copy test_other_types FROM ' :'other_types_csv_file' ' WITH CSV;'
:client_side_copy_command
SELECT * FROM test_other_types; SELECT * FROM test_other_types;
@ -69,7 +73,8 @@ CREATE TABLE test_null_values (a int, b int[], c composite_type)
USING columnar; USING columnar;
\set null_values_csv_file :abs_srcdir '/data/null_values.csv' \set null_values_csv_file :abs_srcdir '/data/null_values.csv'
COPY test_null_values FROM :'null_values_csv_file' WITH CSV; \set client_side_copy_command '\\copy test_null_values FROM ' :'null_values_csv_file' ' WITH CSV;'
:client_side_copy_command
SELECT * FROM test_null_values; SELECT * FROM test_null_values;

View File

@ -4,14 +4,15 @@
-- COPY with incorrect delimiter -- COPY with incorrect delimiter
\set contestants_1_csv_file :abs_srcdir '/data/contestants.1.csv' \set contestants_1_csv_file :abs_srcdir '/data/contestants.1.csv'
COPY contestant FROM :'contestants_1_csv_file' \set client_side_copy_command '\\copy contestant FROM ' :'contestants_1_csv_file' ' WITH DELIMITER '''|''';'
WITH DELIMITER '|'; -- ERROR :client_side_copy_command -- ERROR
-- COPY with invalid program -- COPY with invalid program
COPY contestant FROM PROGRAM 'invalid_program' WITH CSV; -- ERROR COPY contestant FROM PROGRAM 'invalid_program' WITH CSV; -- ERROR
-- COPY into uncompressed table from file -- COPY into uncompressed table from file
COPY contestant FROM :'contestants_1_csv_file' WITH CSV; \set client_side_copy_command '\\copy contestant FROM ' :'contestants_1_csv_file' ' WITH CSV;'
:client_side_copy_command
-- COPY into uncompressed table from program -- COPY into uncompressed table from program
\set cat_contestants_2_csv_file 'cat ' :abs_srcdir '/data/contestants.2.csv' \set cat_contestants_2_csv_file 'cat ' :abs_srcdir '/data/contestants.2.csv'
@ -22,7 +23,8 @@ select
from columnar_test_helpers.columnar_storage_info('contestant'); from columnar_test_helpers.columnar_storage_info('contestant');
-- COPY into compressed table -- COPY into compressed table
COPY contestant_compressed FROM :'contestants_1_csv_file' WITH CSV; \set client_side_copy_command '\\copy contestant_compressed FROM ' :'contestants_1_csv_file' ' WITH CSV;'
:client_side_copy_command
-- COPY into uncompressed table from program -- COPY into uncompressed table from program
COPY contestant_compressed FROM PROGRAM :'cat_contestants_2_csv_file' COPY contestant_compressed FROM PROGRAM :'cat_contestants_2_csv_file'

View File

@ -36,8 +36,10 @@ WHERE shardid = :new_shard_id;
\set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data' \set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data'
\set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data' \set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data'
COPY lineitem_range FROM :'lineitem_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy lineitem_range FROM ' :'lineitem_1_data_file' ' with delimiter '''|''';'
COPY lineitem_range FROM :'lineitem_2_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy lineitem_range FROM ' :'lineitem_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
-- Run aggregate(distinct) on partition column for range partitioned table -- Run aggregate(distinct) on partition column for range partitioned table
@ -95,8 +97,10 @@ CREATE TABLE lineitem_hash (
SET citus.shard_replication_factor TO 1; SET citus.shard_replication_factor TO 1;
SELECT create_distributed_table('lineitem_hash', 'l_orderkey', 'hash'); SELECT create_distributed_table('lineitem_hash', 'l_orderkey', 'hash');
COPY lineitem_hash FROM :'lineitem_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy lineitem_hash FROM ' :'lineitem_1_data_file' ' with delimiter '''|''';'
COPY lineitem_hash FROM :'lineitem_2_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy lineitem_hash FROM ' :'lineitem_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
-- aggregate(distinct) on partition column is allowed -- aggregate(distinct) on partition column is allowed
SELECT count(distinct l_orderkey) FROM lineitem_hash; SELECT count(distinct l_orderkey) FROM lineitem_hash;

View File

@ -17,8 +17,10 @@ SELECT create_distributed_table('events_table', 'user_id');
\set users_table_data_file :abs_srcdir '/data/users_table.data' \set users_table_data_file :abs_srcdir '/data/users_table.data'
\set events_table_data_file :abs_srcdir '/data/events_table.data' \set events_table_data_file :abs_srcdir '/data/events_table.data'
COPY users_table FROM :'users_table_data_file' WITH CSV; \set client_side_copy_command '\\copy users_table FROM ' :'users_table_data_file' ' WITH CSV;'
COPY events_table FROM :'events_table_data_file' WITH CSV; :client_side_copy_command
\set client_side_copy_command '\\copy events_table FROM ' :'events_table_data_file' ' WITH CSV;'
:client_side_copy_command
SET citus.shard_count = 96; SET citus.shard_count = 96;
CREATE SCHEMA subquery_and_ctes; CREATE SCHEMA subquery_and_ctes;
@ -30,8 +32,10 @@ SELECT create_distributed_table('users_table', 'user_id');
CREATE TABLE events_table (user_id int, time timestamp, event_type int, value_2 int, value_3 float, value_4 bigint); CREATE TABLE events_table (user_id int, time timestamp, event_type int, value_2 int, value_3 float, value_4 bigint);
SELECT create_distributed_table('events_table', 'user_id'); SELECT create_distributed_table('events_table', 'user_id');
COPY users_table FROM :'users_table_data_file' WITH CSV; \set client_side_copy_command '\\copy users_table FROM ' :'users_table_data_file' ' WITH CSV;'
COPY events_table FROM :'events_table_data_file' WITH CSV; :client_side_copy_command
\set client_side_copy_command '\\copy events_table FROM ' :'events_table_data_file' ' WITH CSV;'
:client_side_copy_command
SET citus.shard_count TO DEFAULT; SET citus.shard_count TO DEFAULT;
SET search_path TO DEFAULT; SET search_path TO DEFAULT;
@ -70,8 +74,10 @@ INSERT INTO users_ref_test_table VALUES(4,'User_4',48);
INSERT INTO users_ref_test_table VALUES(5,'User_5',49); INSERT INTO users_ref_test_table VALUES(5,'User_5',49);
INSERT INTO users_ref_test_table VALUES(6,'User_6',50); INSERT INTO users_ref_test_table VALUES(6,'User_6',50);
COPY users_table FROM :'users_table_data_file' WITH CSV; \set client_side_copy_command '\\copy users_table FROM ' :'users_table_data_file' ' WITH CSV;'
COPY events_table FROM :'events_table_data_file' WITH CSV; :client_side_copy_command
\set client_side_copy_command '\\copy events_table FROM ' :'events_table_data_file' ' WITH CSV;'
:client_side_copy_command
-- create indexes for -- create indexes for
CREATE INDEX is_index1 ON users_table(user_id); CREATE INDEX is_index1 ON users_table(user_id);

View File

@ -291,11 +291,15 @@ UPDATE pg_dist_shard SET shardminvalue = 8997, shardmaxvalue = 14947
WHERE shardid = :new_shard_id; WHERE shardid = :new_shard_id;
\set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data' \set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data'
COPY lineitem_subquery FROM :'lineitem_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy lineitem_subquery FROM ' :'lineitem_1_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data' \set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data'
COPY lineitem_subquery FROM :'lineitem_2_data_file' with delimiter '|'; \set client_side_copy_command '\\copy lineitem_subquery FROM ' :'lineitem_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set orders_1_data_file :abs_srcdir '/data/orders.1.data' \set orders_1_data_file :abs_srcdir '/data/orders.1.data'
COPY orders_subquery FROM :'orders_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy orders_subquery FROM ' :'orders_1_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set orders_2_data_file :abs_srcdir '/data/orders.2.data' \set orders_2_data_file :abs_srcdir '/data/orders.2.data'
COPY orders_subquery FROM :'orders_2_data_file' with delimiter '|'; \set client_side_copy_command '\\copy orders_subquery FROM ' :'orders_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command

View File

@ -31,8 +31,10 @@ SELECT create_distributed_table('lineitem_hash', 'l_orderkey', 'hash');
\set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data' \set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data'
\set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data' \set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data'
COPY lineitem_hash FROM :'lineitem_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy lineitem_hash FROM ' :'lineitem_1_data_file' ' with delimiter '''|''';'
COPY lineitem_hash FROM :'lineitem_2_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy lineitem_hash FROM ' :'lineitem_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
ANALYZE lineitem_hash; ANALYZE lineitem_hash;

View File

@ -102,7 +102,8 @@ WITH (DELIMITER ' ');
SELECT count(*) FROM customer_copy_hash WHERE c_custkey = 9; SELECT count(*) FROM customer_copy_hash WHERE c_custkey = 9;
-- Test server-side copy from file -- Test server-side copy from file
COPY customer_copy_hash FROM :'customer2datafile' WITH (DELIMITER '|'); \set client_side_copy_command '\\copy customer_copy_hash FROM ' :'customer2datafile' ' WITH (DELIMITER '''|''');'
:client_side_copy_command
-- Confirm that data was copied -- Confirm that data was copied
SELECT count(*) FROM customer_copy_hash; SELECT count(*) FROM customer_copy_hash;
@ -184,7 +185,8 @@ CREATE TABLE customer_copy_range (
SELECT master_create_distributed_table('customer_copy_range', 'c_custkey', 'range'); SELECT master_create_distributed_table('customer_copy_range', 'c_custkey', 'range');
-- Test COPY into empty range-partitioned table -- Test COPY into empty range-partitioned table
COPY customer_copy_range FROM :'customer1datafile' WITH (DELIMITER '|'); \set client_side_copy_command '\\copy customer_copy_range FROM ' :'customer1datafile' ' WITH (DELIMITER '''|''');'
:client_side_copy_command
SELECT master_create_empty_shard('customer_copy_range') AS new_shard_id SELECT master_create_empty_shard('customer_copy_range') AS new_shard_id
\gset \gset
@ -197,7 +199,8 @@ UPDATE pg_dist_shard SET shardminvalue = 501, shardmaxvalue = 1000
WHERE shardid = :new_shard_id; WHERE shardid = :new_shard_id;
-- Test copy into range-partitioned table -- Test copy into range-partitioned table
COPY customer_copy_range FROM :'customer1datafile' WITH (DELIMITER '|'); \set client_side_copy_command '\\copy customer_copy_range FROM ' :'customer1datafile' ' WITH (DELIMITER '''|''');'
:client_side_copy_command
-- Check whether data went into the right shard (maybe) -- Check whether data went into the right shard (maybe)
SELECT min(c_custkey), max(c_custkey), avg(c_custkey), count(*) SELECT min(c_custkey), max(c_custkey), avg(c_custkey), count(*)
@ -293,14 +296,17 @@ SELECT create_distributed_table('lineitem_copy_append', 'l_orderkey', 'append');
BEGIN; BEGIN;
SELECT master_create_empty_shard('lineitem_copy_append') AS shardid \gset SELECT master_create_empty_shard('lineitem_copy_append') AS shardid \gset
COPY lineitem_copy_append FROM :'lineitem1datafile' with (delimiter '|', append_to_shard :shardid); \set client_side_copy_command '\\copy lineitem_copy_append FROM ' :'lineitem1datafile' ' with (delimiter '''|''', append_to_shard ' :shardid ');'
:client_side_copy_command
END; END;
SELECT count(*) FROM pg_dist_shard WHERE logicalrelid = 'lineitem_copy_append'::regclass; SELECT count(*) FROM pg_dist_shard WHERE logicalrelid = 'lineitem_copy_append'::regclass;
-- trigger some errors on the append_to_shard option -- trigger some errors on the append_to_shard option
COPY lineitem_copy_append FROM :'lineitem1datafile' with (delimiter '|', append_to_shard 1); \set client_side_copy_command '\\copy lineitem_copy_append FROM ' :'lineitem1datafile' ' with (delimiter '''|''', append_to_shard 1);'
COPY lineitem_copy_append FROM :'lineitem1datafile' with (delimiter '|', append_to_shard 560000); :client_side_copy_command
\set client_side_copy_command '\\copy lineitem_copy_append FROM ' :'lineitem1datafile' ' with (delimiter '''|''', append_to_shard 560000);'
:client_side_copy_command
-- Test schema support on append partitioned tables -- Test schema support on append partitioned tables
CREATE SCHEMA append; CREATE SCHEMA append;
@ -319,8 +325,10 @@ SELECT master_create_empty_shard('append.customer_copy') AS shardid1 \gset
SELECT master_create_empty_shard('append.customer_copy') AS shardid2 \gset SELECT master_create_empty_shard('append.customer_copy') AS shardid2 \gset
-- Test copy from the master node -- Test copy from the master node
COPY append.customer_copy FROM :'customer1datafile' with (delimiter '|', append_to_shard :shardid1); \set client_side_copy_command '\\copy append.customer_copy FROM ' :'customer1datafile' ' with (delimiter '''|''', append_to_shard ' :shardid1 ');'
COPY append.customer_copy FROM :'customer2datafile' with (delimiter '|', append_to_shard :shardid2); :client_side_copy_command
\set client_side_copy_command '\\copy append.customer_copy FROM ' :'customer2datafile' ' with (delimiter '''|''', append_to_shard ' :shardid2 ');'
:client_side_copy_command
-- Test the content of the table -- Test the content of the table
SELECT min(c_custkey), max(c_custkey), avg(c_acctbal), count(*) FROM append.customer_copy; SELECT min(c_custkey), max(c_custkey), avg(c_acctbal), count(*) FROM append.customer_copy;

View File

@ -4,25 +4,38 @@
\set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data' \set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data'
\set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data' \set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data'
COPY lineitem FROM :'lineitem_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy lineitem FROM ' :'lineitem_1_data_file' ' with delimiter '''|''';'
COPY lineitem FROM :'lineitem_2_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy lineitem FROM ' :'lineitem_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set orders_1_data_file :abs_srcdir '/data/orders.1.data' \set orders_1_data_file :abs_srcdir '/data/orders.1.data'
\set orders_2_data_file :abs_srcdir '/data/orders.2.data' \set orders_2_data_file :abs_srcdir '/data/orders.2.data'
COPY orders FROM :'orders_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy orders FROM ' :'orders_1_data_file' ' with delimiter '''|''';'
COPY orders FROM :'orders_2_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy orders FROM ' :'orders_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
COPY orders_reference FROM :'orders_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy orders_reference FROM ' :'orders_1_data_file' ' with delimiter '''|''';'
COPY orders_reference FROM :'orders_2_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy orders_reference FROM ' :'orders_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set customer_1_data_file :abs_srcdir '/data/customer.1.data' \set customer_1_data_file :abs_srcdir '/data/customer.1.data'
\set nation_data_file :abs_srcdir '/data/nation.data' \set nation_data_file :abs_srcdir '/data/nation.data'
\set part_data_file :abs_srcdir '/data/part.data' \set part_data_file :abs_srcdir '/data/part.data'
\set supplier_data_file :abs_srcdir '/data/supplier.data' \set supplier_data_file :abs_srcdir '/data/supplier.data'
COPY customer FROM :'customer_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy customer FROM ' :'customer_1_data_file' ' with delimiter '''|''';'
COPY customer_append FROM :'customer_1_data_file' with (delimiter '|', append_to_shard 360006); :client_side_copy_command
COPY nation FROM :'nation_data_file' with delimiter '|'; \set client_side_copy_command '\\copy customer_append FROM ' :'customer_1_data_file' ' with (delimiter '''|''', append_to_shard 360006);'
COPY part FROM :'part_data_file' with delimiter '|'; :client_side_copy_command
COPY part_append FROM :'part_data_file' with (delimiter '|', append_to_shard 360009); \set client_side_copy_command '\\copy nation FROM ' :'nation_data_file' ' with delimiter '''|''';'
COPY supplier FROM :'supplier_data_file' with delimiter '|'; :client_side_copy_command
COPY supplier_single_shard FROM :'supplier_data_file' with delimiter '|'; \set client_side_copy_command '\\copy part FROM ' :'part_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy part_append FROM ' :'part_data_file' ' with (delimiter '''|''', append_to_shard 360009);'
:client_side_copy_command
\set client_side_copy_command '\\copy supplier FROM ' :'supplier_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy supplier_single_shard FROM ' :'supplier_data_file' ' with delimiter '''|''';'
:client_side_copy_command

View File

@ -2,7 +2,11 @@
\set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data' \set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data'
\set orders_1_data_file :abs_srcdir '/data/orders.1.data' \set orders_1_data_file :abs_srcdir '/data/orders.1.data'
\set orders_2_data_file :abs_srcdir '/data/orders.2.data' \set orders_2_data_file :abs_srcdir '/data/orders.2.data'
COPY lineitem_hash_part FROM :'lineitem_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy lineitem_hash_part FROM ' :'lineitem_1_data_file' ' with delimiter '''|''';'
COPY lineitem_hash_part FROM :'lineitem_2_data_file' with delimiter '|'; :client_side_copy_command
COPY orders_hash_part FROM :'orders_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy lineitem_hash_part FROM ' :'lineitem_2_data_file' ' with delimiter '''|''';'
COPY orders_hash_part FROM :'orders_2_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy orders_hash_part FROM ' :'orders_1_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy orders_hash_part FROM ' :'orders_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command

View File

@ -13,9 +13,12 @@ SET citus.next_shard_id TO 280000;
\set customer_2_data_file :abs_srcdir '/data/customer.2.data' \set customer_2_data_file :abs_srcdir '/data/customer.2.data'
\set customer_3_data_file :abs_srcdir '/data/customer.3.data' \set customer_3_data_file :abs_srcdir '/data/customer.3.data'
\set part_more_data_file :abs_srcdir '/data/part.more.data' \set part_more_data_file :abs_srcdir '/data/part.more.data'
COPY customer FROM :'customer_2_data_file' with delimiter '|'; \set client_side_copy_command '\\copy customer FROM ' :'customer_2_data_file' ' with delimiter '''|''';'
COPY customer FROM :'customer_3_data_file' with delimiter '|'; :client_side_copy_command
COPY part FROM :'part_more_data_file' with delimiter '|'; \set client_side_copy_command '\\copy customer FROM ' :'customer_3_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy part FROM ' :'part_more_data_file' ' with delimiter '''|''';'
:client_side_copy_command
SELECT master_create_empty_shard('customer_append') AS shardid1 \gset SELECT master_create_empty_shard('customer_append') AS shardid1 \gset
SELECT master_create_empty_shard('customer_append') AS shardid2 \gset SELECT master_create_empty_shard('customer_append') AS shardid2 \gset

View File

@ -3,18 +3,24 @@
-- --
\set nation_data_file :abs_srcdir '/data/nation.data' \set nation_data_file :abs_srcdir '/data/nation.data'
COPY nation_hash FROM :'nation_data_file' with delimiter '|'; \set client_side_copy_command '\\copy nation_hash FROM ' :'nation_data_file' ' with delimiter '''|''';'
:client_side_copy_command
SET search_path TO citus_mx_test_schema; SET search_path TO citus_mx_test_schema;
COPY nation_hash FROM :'nation_data_file' with delimiter '|'; \set client_side_copy_command '\\copy nation_hash FROM ' :'nation_data_file' ' with delimiter '''|''';'
COPY citus_mx_test_schema_join_1.nation_hash FROM :'nation_data_file' with delimiter '|'; :client_side_copy_command
COPY citus_mx_test_schema_join_1.nation_hash_2 FROM :'nation_data_file' with delimiter '|'; \set client_side_copy_command '\\copy citus_mx_test_schema_join_1.nation_hash FROM ' :'nation_data_file' ' with delimiter '''|''';'
COPY citus_mx_test_schema_join_2.nation_hash FROM :'nation_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy citus_mx_test_schema_join_1.nation_hash_2 FROM ' :'nation_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy citus_mx_test_schema_join_2.nation_hash FROM ' :'nation_data_file' ' with delimiter '''|''';'
:client_side_copy_command
SET citus.shard_replication_factor TO 2; SET citus.shard_replication_factor TO 2;
CREATE TABLE citus_mx_test_schema.nation_hash_replicated AS SELECT * FROM citus_mx_test_schema.nation_hash; CREATE TABLE citus_mx_test_schema.nation_hash_replicated AS SELECT * FROM citus_mx_test_schema.nation_hash;
SELECT create_distributed_table('citus_mx_test_schema.nation_hash_replicated', 'n_nationkey'); SELECT create_distributed_table('citus_mx_test_schema.nation_hash_replicated', 'n_nationkey');
COPY nation_hash_replicated FROM :'nation_data_file' with delimiter '|'; \set client_side_copy_command '\\copy nation_hash_replicated FROM ' :'nation_data_file' ' with delimiter '''|''';'
:client_side_copy_command
-- now try loading data from worker node -- now try loading data from worker node
\c - - - :worker_1_port \c - - - :worker_1_port
@ -22,20 +28,26 @@ SET search_path TO public;
\set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data' \set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data'
\set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data' \set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data'
COPY lineitem_mx FROM :'lineitem_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy lineitem_mx FROM ' :'lineitem_1_data_file' ' with delimiter '''|''';'
COPY lineitem_mx FROM :'lineitem_2_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy lineitem_mx FROM ' :'lineitem_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set nation_data_file :abs_srcdir '/data/nation.data' \set nation_data_file :abs_srcdir '/data/nation.data'
COPY citus_mx_test_schema.nation_hash_replicated FROM :'nation_data_file' with delimiter '|'; \set client_side_copy_command '\\copy citus_mx_test_schema.nation_hash_replicated FROM ' :'nation_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\c - - - :worker_2_port \c - - - :worker_2_port
-- and use second worker as well -- and use second worker as well
\set orders_1_data_file :abs_srcdir '/data/orders.1.data' \set orders_1_data_file :abs_srcdir '/data/orders.1.data'
\set orders_2_data_file :abs_srcdir '/data/orders.2.data' \set orders_2_data_file :abs_srcdir '/data/orders.2.data'
\set nation_data_file :abs_srcdir '/data/nation.data' \set nation_data_file :abs_srcdir '/data/nation.data'
COPY orders_mx FROM :'orders_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy orders_mx FROM ' :'orders_1_data_file' ' with delimiter '''|''';'
COPY orders_mx FROM :'orders_2_data_file' with delimiter '|'; :client_side_copy_command
COPY citus_mx_test_schema.nation_hash_replicated FROM :'nation_data_file' with delimiter '|'; \set client_side_copy_command '\\copy orders_mx FROM ' :'orders_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy citus_mx_test_schema.nation_hash_replicated FROM ' :'nation_data_file' ' with delimiter '''|''';'
:client_side_copy_command
-- get ready for the next test -- get ready for the next test
TRUNCATE orders_mx; TRUNCATE orders_mx;
@ -50,11 +62,14 @@ show citus.local_shared_pool_size;
\set orders_1_data_file :abs_srcdir '/data/orders.1.data' \set orders_1_data_file :abs_srcdir '/data/orders.1.data'
\set orders_2_data_file :abs_srcdir '/data/orders.2.data' \set orders_2_data_file :abs_srcdir '/data/orders.2.data'
COPY orders_mx FROM :'orders_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy orders_mx FROM ' :'orders_1_data_file' ' with delimiter '''|''';'
COPY orders_mx FROM :'orders_2_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy orders_mx FROM ' :'orders_2_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set nation_data_file :abs_srcdir '/data/nation.data' \set nation_data_file :abs_srcdir '/data/nation.data'
COPY citus_mx_test_schema.nation_hash_replicated FROM :'nation_data_file' with delimiter '|'; \set client_side_copy_command '\\copy citus_mx_test_schema.nation_hash_replicated FROM ' :'nation_data_file' ' with delimiter '''|''';'
:client_side_copy_command
-- set it back -- set it back
ALTER SYSTEM RESET citus.local_shared_pool_size; ALTER SYSTEM RESET citus.local_shared_pool_size;
@ -73,7 +88,11 @@ SET search_path TO public;
\set nation_data_file :abs_srcdir '/data/nation.data' \set nation_data_file :abs_srcdir '/data/nation.data'
\set part_data_file :abs_srcdir '/data/part.data' \set part_data_file :abs_srcdir '/data/part.data'
\set supplier_data_file :abs_srcdir '/data/supplier.data' \set supplier_data_file :abs_srcdir '/data/supplier.data'
COPY customer_mx FROM :'customer_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy customer_mx FROM ' :'customer_1_data_file' ' with delimiter '''|''';'
COPY nation_mx FROM :'nation_data_file' with delimiter '|'; :client_side_copy_command
COPY part_mx FROM :'part_data_file' with delimiter '|'; \set client_side_copy_command '\\copy nation_mx FROM ' :'nation_data_file' ' with delimiter '''|''';'
COPY supplier_mx FROM :'supplier_data_file' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy part_mx FROM ' :'part_data_file' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy supplier_mx FROM ' :'supplier_data_file' ' with delimiter '''|''';'
:client_side_copy_command

View File

@ -72,10 +72,14 @@ SELECT create_reference_table('multi_outer_join_third_reference');
\set customer_1_10_data :abs_srcdir '/data/customer-1-10.data' \set customer_1_10_data :abs_srcdir '/data/customer-1-10.data'
\set customer_11_20_data :abs_srcdir '/data/customer-11-20.data' \set customer_11_20_data :abs_srcdir '/data/customer-11-20.data'
\set customer_1_15_data :abs_srcdir '/data/customer-1-15.data' \set customer_1_15_data :abs_srcdir '/data/customer-1-15.data'
COPY multi_outer_join_left FROM :'customer_1_10_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_left FROM ' :'customer_1_10_data' ' with delimiter '''|''';'
COPY multi_outer_join_left FROM :'customer_11_20_data' with delimiter '|'; :client_side_copy_command
COPY multi_outer_join_right FROM :'customer_1_15_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_left FROM ' :'customer_11_20_data' ' with delimiter '''|''';'
COPY multi_outer_join_right_reference FROM :'customer_1_15_data' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_right FROM ' :'customer_1_15_data' ' with delimiter '''|''';'
:client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_right_reference FROM ' :'customer_1_15_data' ' with delimiter '''|''';'
:client_side_copy_command
-- Make sure we do not crash if one table has no shards -- Make sure we do not crash if one table has no shards
SELECT SELECT
@ -90,8 +94,10 @@ FROM
-- Third table is a single shard table with all data -- Third table is a single shard table with all data
\set customer_1_30_data :abs_srcdir '/data/customer-1-30.data' \set customer_1_30_data :abs_srcdir '/data/customer-1-30.data'
COPY multi_outer_join_third FROM :'customer_1_30_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_third FROM ' :'customer_1_30_data' ' with delimiter '''|''';'
COPY multi_outer_join_third_reference FROM :'customer_1_30_data' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_third_reference FROM ' :'customer_1_30_data' ' with delimiter '''|''';'
:client_side_copy_command
-- Regular outer join should return results for all rows -- Regular outer join should return results for all rows
SELECT SELECT
@ -171,7 +177,8 @@ FROM
-- Turn the right table into a large table -- Turn the right table into a large table
\set customer_21_30_data :abs_srcdir '/data/customer-21-30.data' \set customer_21_30_data :abs_srcdir '/data/customer-21-30.data'
COPY multi_outer_join_right FROM :'customer_21_30_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_right FROM ' :'customer_21_30_data' ' with delimiter '''|''';'
:client_side_copy_command
-- Shards do not have 1-1 matching. We should error here. -- Shards do not have 1-1 matching. We should error here.
@ -186,12 +193,16 @@ TRUNCATE multi_outer_join_right;
-- reload shards with 1-1 matching -- reload shards with 1-1 matching
\set customer_subset_11_20_data :abs_srcdir '/data/customer-subset-11-20.data' \set customer_subset_11_20_data :abs_srcdir '/data/customer-subset-11-20.data'
COPY multi_outer_join_left FROM :'customer_subset_11_20_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_left FROM ' :'customer_subset_11_20_data' ' with delimiter '''|''';'
COPY multi_outer_join_left FROM :'customer_21_30_data' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_left FROM ' :'customer_21_30_data' ' with delimiter '''|''';'
:client_side_copy_command
\set customer_subset_21_30_data :abs_srcdir '/data/customer-subset-21-30.data' \set customer_subset_21_30_data :abs_srcdir '/data/customer-subset-21-30.data'
COPY multi_outer_join_right FROM :'customer_11_20_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_right FROM ' :'customer_11_20_data' ' with delimiter '''|''';'
COPY multi_outer_join_right FROM :'customer_subset_21_30_data' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_right FROM ' :'customer_subset_21_30_data' ' with delimiter '''|''';'
:client_side_copy_command
-- multi_outer_join_third is a single shard table -- multi_outer_join_third is a single shard table
-- Regular left join should work as expected -- Regular left join should work as expected
@ -454,7 +465,8 @@ LIMIT 20;
-- Add a shard to the left table that overlaps with multiple shards in the right -- Add a shard to the left table that overlaps with multiple shards in the right
\set customer_1_data_file :abs_srcdir '/data/customer.1.data' \set customer_1_data_file :abs_srcdir '/data/customer.1.data'
COPY multi_outer_join_left FROM :'customer_1_data_file' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_left FROM ' :'customer_1_data_file' ' with delimiter '''|''';'
:client_side_copy_command
-- All outer joins should error out -- All outer joins should error out

View File

@ -66,13 +66,17 @@ FROM
-- Left table is a large table -- Left table is a large table
\set customer_1_10_data :abs_srcdir '/data/customer-1-10.data' \set customer_1_10_data :abs_srcdir '/data/customer-1-10.data'
\set customer_11_20_data :abs_srcdir '/data/customer-11-20.data' \set customer_11_20_data :abs_srcdir '/data/customer-11-20.data'
COPY multi_outer_join_left_hash FROM :'customer_1_10_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_left_hash FROM ' :'customer_1_10_data' ' with delimiter '''|''';'
COPY multi_outer_join_left_hash FROM :'customer_11_20_data' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_left_hash FROM ' :'customer_11_20_data' ' with delimiter '''|''';'
:client_side_copy_command
-- Right table is a small table -- Right table is a small table
\set customer_1_15_data :abs_srcdir '/data/customer-1-15.data' \set customer_1_15_data :abs_srcdir '/data/customer-1-15.data'
COPY multi_outer_join_right_reference FROM :'customer_1_15_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_right_reference FROM ' :'customer_1_15_data' ' with delimiter '''|''';'
COPY multi_outer_join_right_hash FROM :'customer_1_15_data' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_right_hash FROM ' :'customer_1_15_data' ' with delimiter '''|''';'
:client_side_copy_command
-- Make sure we do not crash if one table has data -- Make sure we do not crash if one table has data
SELECT SELECT
@ -87,8 +91,10 @@ FROM
-- Third table is a single shard table with all data -- Third table is a single shard table with all data
\set customer_1_30_data :abs_srcdir '/data/customer-1-30.data' \set customer_1_30_data :abs_srcdir '/data/customer-1-30.data'
COPY multi_outer_join_third_reference FROM :'customer_1_30_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_third_reference FROM ' :'customer_1_30_data' ' with delimiter '''|''';'
COPY multi_outer_join_right_hash FROM :'customer_1_30_data' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_right_hash FROM ' :'customer_1_30_data' ' with delimiter '''|''';'
:client_side_copy_command
-- Regular outer join should return results for all rows -- Regular outer join should return results for all rows
@ -169,7 +175,8 @@ FROM
-- load some more data -- load some more data
\set customer_21_30_data :abs_srcdir '/data/customer-21-30.data' \set customer_21_30_data :abs_srcdir '/data/customer-21-30.data'
COPY multi_outer_join_right_reference FROM :'customer_21_30_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_right_reference FROM ' :'customer_21_30_data' ' with delimiter '''|''';'
:client_side_copy_command
-- Update shards so that they do not have 1-1 matching, triggering an error. -- Update shards so that they do not have 1-1 matching, triggering an error.
UPDATE pg_dist_shard SET shardminvalue = '2147483646' WHERE shardid = 1260006; UPDATE pg_dist_shard SET shardminvalue = '2147483646' WHERE shardid = 1260006;
@ -185,14 +192,20 @@ UPDATE pg_dist_shard SET shardmaxvalue = '-1073741825' WHERE shardid = 1260006;
TRUNCATE multi_outer_join_left_hash, multi_outer_join_right_hash, multi_outer_join_right_reference; TRUNCATE multi_outer_join_left_hash, multi_outer_join_right_hash, multi_outer_join_right_reference;
-- reload shards with 1-1 matching -- reload shards with 1-1 matching
COPY multi_outer_join_left_hash FROM :'customer_1_15_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_left_hash FROM ' :'customer_1_15_data' ' with delimiter '''|''';'
COPY multi_outer_join_left_hash FROM :'customer_21_30_data' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_left_hash FROM ' :'customer_21_30_data' ' with delimiter '''|''';'
:client_side_copy_command
COPY multi_outer_join_right_reference FROM :'customer_11_20_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_right_reference FROM ' :'customer_11_20_data' ' with delimiter '''|''';'
COPY multi_outer_join_right_reference FROM :'customer_21_30_data' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_right_reference FROM ' :'customer_21_30_data' ' with delimiter '''|''';'
:client_side_copy_command
COPY multi_outer_join_right_hash FROM :'customer_11_20_data' with delimiter '|'; \set client_side_copy_command '\\copy multi_outer_join_right_hash FROM ' :'customer_11_20_data' ' with delimiter '''|''';'
COPY multi_outer_join_right_hash FROM :'customer_21_30_data' with delimiter '|'; :client_side_copy_command
\set client_side_copy_command '\\copy multi_outer_join_right_hash FROM ' :'customer_21_30_data' ' with delimiter '''|''';'
:client_side_copy_command
-- multi_outer_join_third_reference is a single shard table -- multi_outer_join_third_reference is a single shard table