external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1294 from citusdata/fix_test_outputs_for_valgrind 

Prepare for valgrind automation
pull/1938/head
Burak Yücesoy 2017-04-21 05:51:14 -08:00 committed by GitHub
parent 384f32b191 a35d0cd8af
commit e291887227
7 changed files with 90 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/test/regress/Makefile
View File

@ -51,6 +51,7 @@ check-multi: all tempinstall-main
check-multi-vg: all tempinstall-main check-multi-vg: all tempinstall-main
$(pg_regress_multi_check) --load-extension=citus --valgrind \ $(pg_regress_multi_check) --load-extension=citus --valgrind \
--pg_ctl-timeout=360 --connection-timeout=500000 --valgrind-path=valgrind --valgrind-log-file=$(VALGRIND_LOG_FILE) \
-- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule $(EXTRA_TESTS) -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule $(EXTRA_TESTS)
check-isolation: all tempinstall-main check-isolation: all tempinstall-main
@ -70,7 +71,6 @@ check-multi-task-tracker-extra: all tempinstall-main
--server-option=citus.large_table_shard_count=1 \ --server-option=citus.large_table_shard_count=1 \
-- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_task_tracker_extra_schedule $(EXTRA_TESTS) -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_task_tracker_extra_schedule $(EXTRA_TESTS)
check-multi-binary: all tempinstall-main check-multi-binary: all tempinstall-main
$(pg_regress_multi_check) --load-extension=citus \ $(pg_regress_multi_check) --load-extension=citus \
--server-option=citus.binary_worker_copy_format=on \ --server-option=citus.binary_worker_copy_format=on \

46
src/test/regress/expected/multi_explain.out
View File

@ -34,6 +34,9 @@ BEGIN
RETURN result; RETURN result;
END; END;
$BODY$ LANGUAGE plpgsql; $BODY$ LANGUAGE plpgsql;
-- VACUMM related tables to ensure test outputs are stable
VACUUM ANALYZE lineitem;
VACUUM ANALYZE orders;
-- Test Text format -- Test Text format
EXPLAIN (COSTS FALSE, FORMAT TEXT) EXPLAIN (COSTS FALSE, FORMAT TEXT)
SELECT l_quantity, count(*) count_quantity FROM lineitem SELECT l_quantity, count(*) count_quantity FROM lineitem
@ -287,12 +290,13 @@ Limit
-> Limit -> Limit
-> Sort -> Sort
Sort Key: lineitem.l_quantity Sort Key: lineitem.l_quantity
-> Hash Join -> Merge Join
Hash Cond: (lineitem.l_orderkey = orders.o_orderkey) Merge Cond: (orders.o_orderkey = lineitem.l_orderkey)
-> Seq Scan on lineitem_290001 lineitem -> Index Scan using orders_pkey_290008 on orders_290008 orders
Filter: (l_quantity < 5.0) -> Sort
-> Hash Sort Key: lineitem.l_orderkey
-> Seq Scan on orders_290008 orders -> Seq Scan on lineitem_290001 lineitem
Filter: (l_quantity < 5.0)
-- Test insert -- Test insert
EXPLAIN (COSTS FALSE) EXPLAIN (COSTS FALSE)
INSERT INTO lineitem VALUES(1,0); INSERT INTO lineitem VALUES(1,0);
@ -314,11 +318,9 @@ Custom Scan (Citus Router)
-> Task -> Task
Node: host=localhost port=57638 dbname=regression Node: host=localhost port=57638 dbname=regression
-> Update on lineitem_290000 -> Update on lineitem_290000
-> Bitmap Heap Scan on lineitem_290000 -> Index Scan using lineitem_pkey_290000 on lineitem_290000
Recheck Cond: (l_orderkey = 1) Index Cond: (l_orderkey = 1)
Filter: (l_partkey = 0) Filter: (l_partkey = 0)
-> Bitmap Index Scan on lineitem_pkey_290000
Index Cond: (l_orderkey = 1)
-- Test delete -- Test delete
EXPLAIN (COSTS FALSE) EXPLAIN (COSTS FALSE)
DELETE FROM lineitem DELETE FROM lineitem
@ -329,11 +331,9 @@ Custom Scan (Citus Router)
-> Task -> Task
Node: host=localhost port=57638 dbname=regression Node: host=localhost port=57638 dbname=regression
-> Delete on lineitem_290000 -> Delete on lineitem_290000
-> Bitmap Heap Scan on lineitem_290000 -> Index Scan using lineitem_pkey_290000 on lineitem_290000
Recheck Cond: (l_orderkey = 1) Index Cond: (l_orderkey = 1)
Filter: (l_partkey = 0) Filter: (l_partkey = 0)
-> Bitmap Index Scan on lineitem_pkey_290000
Index Cond: (l_orderkey = 1)
-- Test single-shard SELECT -- Test single-shard SELECT
EXPLAIN (COSTS FALSE) EXPLAIN (COSTS FALSE)
SELECT l_quantity FROM lineitem WHERE l_orderkey = 5; SELECT l_quantity FROM lineitem WHERE l_orderkey = 5;
@ -342,10 +342,8 @@ Custom Scan (Citus Router)
Tasks Shown: All Tasks Shown: All
-> Task -> Task
Node: host=localhost port=57637 dbname=regression Node: host=localhost port=57637 dbname=regression
-> Bitmap Heap Scan on lineitem_290000 lineitem -> Index Scan using lineitem_pkey_290000 on lineitem_290000 lineitem
Recheck Cond: (l_orderkey = 5) Index Cond: (l_orderkey = 5)
-> Bitmap Index Scan on lineitem_pkey_290000
Index Cond: (l_orderkey = 5)
SELECT true AS valid FROM explain_xml($$ SELECT true AS valid FROM explain_xml($$
SELECT l_quantity FROM lineitem WHERE l_orderkey = 5$$); SELECT l_quantity FROM lineitem WHERE l_orderkey = 5$$);
t t
@ -646,10 +644,8 @@ Custom Scan (Citus Router) (cost=0.00..0.00 rows=0 width=0)
Tasks Shown: All Tasks Shown: All
-> Task -> Task
Node: host=localhost port=57637 dbname=regression Node: host=localhost port=57637 dbname=regression
-> Bitmap Heap Scan on lineitem_290000 lineitem (cost=4.30..13.44 rows=3 width=18) -> Index Scan using lineitem_pkey_290000 on lineitem_290000 lineitem (cost=0.28..11.83 rows=3 width=5)
Recheck Cond: (l_orderkey = 5) Index Cond: (l_orderkey = 5)
-> Bitmap Index Scan on lineitem_pkey_290000 (cost=0.00..4.30 rows=3 width=0)
Index Cond: (l_orderkey = 5)
PREPARE real_time_executor_query AS PREPARE real_time_executor_query AS
SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030; SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030;
EXPLAIN (COSTS FALSE) EXECUTE real_time_executor_query; EXPLAIN (COSTS FALSE) EXECUTE real_time_executor_query;
@ -671,7 +667,5 @@ Custom Scan (Citus Router) (cost=0.00..0.00 rows=0 width=0)
Tasks Shown: All Tasks Shown: All
-> Task -> Task
Node: host=localhost port=57637 dbname=regression Node: host=localhost port=57637 dbname=regression
-> Bitmap Heap Scan on lineitem_290000 lineitem (cost=4.30..13.44 rows=3 width=18) -> Index Scan using lineitem_pkey_290000 on lineitem_290000 lineitem (cost=0.28..11.83 rows=3 width=5)
Recheck Cond: (l_orderkey = 5) Index Cond: (l_orderkey = 5)
-> Bitmap Index Scan on lineitem_pkey_290000 (cost=0.00..4.30 rows=3 width=0)
Index Cond: (l_orderkey = 5)

46
src/test/regress/expected/multi_explain_0.out
View File

@ -34,6 +34,9 @@ BEGIN
RETURN result; RETURN result;
END; END;
$BODY$ LANGUAGE plpgsql; $BODY$ LANGUAGE plpgsql;
-- VACUMM related tables to ensure test outputs are stable
VACUUM ANALYZE lineitem;
VACUUM ANALYZE orders;
-- Test Text format -- Test Text format
EXPLAIN (COSTS FALSE, FORMAT TEXT) EXPLAIN (COSTS FALSE, FORMAT TEXT)
SELECT l_quantity, count(*) count_quantity FROM lineitem SELECT l_quantity, count(*) count_quantity FROM lineitem
@ -266,12 +269,13 @@ Limit
-> Limit -> Limit
-> Sort -> Sort
Sort Key: lineitem.l_quantity Sort Key: lineitem.l_quantity
-> Hash Join -> Merge Join
Hash Cond: (lineitem.l_orderkey = orders.o_orderkey) Merge Cond: (orders.o_orderkey = lineitem.l_orderkey)
-> Seq Scan on lineitem_290001 lineitem -> Index Scan using orders_pkey_290008 on orders_290008 orders
Filter: (l_quantity < 5.0) -> Sort
-> Hash Sort Key: lineitem.l_orderkey
-> Seq Scan on orders_290008 orders -> Seq Scan on lineitem_290001 lineitem
Filter: (l_quantity < 5.0)
-- Test insert -- Test insert
EXPLAIN (COSTS FALSE) EXPLAIN (COSTS FALSE)
INSERT INTO lineitem VALUES(1,0); INSERT INTO lineitem VALUES(1,0);
@ -293,11 +297,9 @@ Custom Scan (Citus Router)
-> Task -> Task
Node: host=localhost port=57638 dbname=regression Node: host=localhost port=57638 dbname=regression
-> Update on lineitem_290000 -> Update on lineitem_290000
-> Bitmap Heap Scan on lineitem_290000 -> Index Scan using lineitem_pkey_290000 on lineitem_290000
Recheck Cond: (l_orderkey = 1) Index Cond: (l_orderkey = 1)
Filter: (l_partkey = 0) Filter: (l_partkey = 0)
-> Bitmap Index Scan on lineitem_pkey_290000
Index Cond: (l_orderkey = 1)
-- Test delete -- Test delete
EXPLAIN (COSTS FALSE) EXPLAIN (COSTS FALSE)
DELETE FROM lineitem DELETE FROM lineitem
@ -308,11 +310,9 @@ Custom Scan (Citus Router)
-> Task -> Task
Node: host=localhost port=57638 dbname=regression Node: host=localhost port=57638 dbname=regression
-> Delete on lineitem_290000 -> Delete on lineitem_290000
-> Bitmap Heap Scan on lineitem_290000 -> Index Scan using lineitem_pkey_290000 on lineitem_290000
Recheck Cond: (l_orderkey = 1) Index Cond: (l_orderkey = 1)
Filter: (l_partkey = 0) Filter: (l_partkey = 0)
-> Bitmap Index Scan on lineitem_pkey_290000
Index Cond: (l_orderkey = 1)
-- Test single-shard SELECT -- Test single-shard SELECT
EXPLAIN (COSTS FALSE) EXPLAIN (COSTS FALSE)
SELECT l_quantity FROM lineitem WHERE l_orderkey = 5; SELECT l_quantity FROM lineitem WHERE l_orderkey = 5;
@ -321,10 +321,8 @@ Custom Scan (Citus Router)
Tasks Shown: All Tasks Shown: All
-> Task -> Task
Node: host=localhost port=57637 dbname=regression Node: host=localhost port=57637 dbname=regression
-> Bitmap Heap Scan on lineitem_290000 lineitem -> Index Scan using lineitem_pkey_290000 on lineitem_290000 lineitem
Recheck Cond: (l_orderkey = 5) Index Cond: (l_orderkey = 5)
-> Bitmap Index Scan on lineitem_pkey_290000
Index Cond: (l_orderkey = 5)
SELECT true AS valid FROM explain_xml($$ SELECT true AS valid FROM explain_xml($$
SELECT l_quantity FROM lineitem WHERE l_orderkey = 5$$); SELECT l_quantity FROM lineitem WHERE l_orderkey = 5$$);
t t
@ -617,10 +615,8 @@ Custom Scan (Citus Router) (cost=0.00..0.00 rows=0 width=0)
Tasks Shown: All Tasks Shown: All
-> Task -> Task
Node: host=localhost port=57637 dbname=regression Node: host=localhost port=57637 dbname=regression
-> Bitmap Heap Scan on lineitem_290000 lineitem (cost=4.30..13.44 rows=3 width=18) -> Index Scan using lineitem_pkey_290000 on lineitem_290000 lineitem (cost=0.28..11.83 rows=3 width=5)
Recheck Cond: (l_orderkey = 5) Index Cond: (l_orderkey = 5)
-> Bitmap Index Scan on lineitem_pkey_290000 (cost=0.00..4.30 rows=3 width=0)
Index Cond: (l_orderkey = 5)
PREPARE real_time_executor_query AS PREPARE real_time_executor_query AS
SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030; SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030;
EXPLAIN (COSTS FALSE) EXECUTE real_time_executor_query; EXPLAIN (COSTS FALSE) EXECUTE real_time_executor_query;
@ -642,7 +638,5 @@ Custom Scan (Citus Router) (cost=0.00..0.00 rows=0 width=0)
Tasks Shown: All Tasks Shown: All
-> Task -> Task
Node: host=localhost port=57637 dbname=regression Node: host=localhost port=57637 dbname=regression
-> Bitmap Heap Scan on lineitem_290000 lineitem (cost=4.30..13.44 rows=3 width=18) -> Index Scan using lineitem_pkey_290000 on lineitem_290000 lineitem (cost=0.28..11.83 rows=3 width=5)
Recheck Cond: (l_orderkey = 5) Index Cond: (l_orderkey = 5)
-> Bitmap Index Scan on lineitem_pkey_290000 (cost=0.00..4.30 rows=3 width=0)
Index Cond: (l_orderkey = 5)

3
src/test/regress/input/multi_outer_join_reference.source
View File

@ -447,7 +447,8 @@ SELECT
t_custkey, r_custkey t_custkey, r_custkey
FROM FROM
multi_outer_join_right_reference FULL JOIN multi_outer_join_right_reference FULL JOIN
multi_outer_join_third_reference ON (t_custkey = r_custkey); multi_outer_join_third_reference ON (t_custkey = r_custkey)
ORDER BY 1;
-- DROP unused tables to clean up workspace -- DROP unused tables to clean up workspace
DROP TABLE multi_outer_join_left_hash; DROP TABLE multi_outer_join_left_hash;

23
src/test/regress/output/multi_outer_join_reference.source
View File

@ -798,9 +798,20 @@ SELECT
t_custkey, r_custkey t_custkey, r_custkey
FROM FROM
multi_outer_join_right_reference FULL JOIN multi_outer_join_right_reference FULL JOIN
multi_outer_join_third_reference ON (t_custkey = r_custkey); multi_outer_join_third_reference ON (t_custkey = r_custkey)
ORDER BY 1;
t_custkey | r_custkey t_custkey | r_custkey
-----------+----------- -----------+-----------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 | 11 11 | 11
12 | 12 12 | 12
13 | 13 13 | 13
@ -821,16 +832,6 @@ FROM
28 | 28 28 | 28
29 | 29 29 | 29
30 | 30 30 | 30
10 |
2 |
5 |
8 |
6 |
4 |
1 |
3 |
9 |
7 |
(30 rows) (30 rows)
-- DROP unused tables to clean up workspace -- DROP unused tables to clean up workspace

47
src/test/regress/pg_regress_multi.pl
View File

@ -26,16 +26,20 @@ sub Usage()
print " pg_regress_multi [MULTI OPTIONS] -- [PG REGRESS OPTS]\n"; print " pg_regress_multi [MULTI OPTIONS] -- [PG REGRESS OPTS]\n";
print "\n"; print "\n";
print "Multi Options:\n"; print "Multi Options:\n";
print " --isolationtester Run isolationtester tests instead of plain tests\n"; print " --isolationtester Run isolationtester tests instead of plain tests\n";
print " --vanillatest Run postgres tests with citus loaded as shared preload library\n"; print " --vanillatest Run postgres tests with citus loaded as shared preload library\n";
print " --bindir Path to postgres binary directory\n"; print " --bindir Path to postgres binary directory\n";
print " --libdir Path to postgres library directory\n"; print " --libdir Path to postgres library directory\n";
print " --postgres-builddir Path to postgres build directory\n"; print " --postgres-builddir Path to postgres build directory\n";
print " --postgres-srcdir Path to postgres build directory\n"; print " --postgres-srcdir Path to postgres build directory\n";
print " --pgxsdir Path to the PGXS directory\n"; print " --pgxsdir Path to the PGXS directory\n";
print " --load-extension Extensions to install in all nodes\n"; print " --load-extension Extensions to install in all nodes\n";
print " --server-option Config option to pass to the server\n"; print " --server-option Config option to pass to the server\n";
print " --valgrind Run server via valgrind\n"; print " --valgrind Run server via valgrind\n";
print " --valgrind-path Path to the valgrind executable\n";
print " --valgrind-log-file Path to the write valgrind logs\n";
print " --pg_ctl-timeout Timeout for pg_ctl\n";
print " --connection-timeout Timeout for connecting to worker nodes\n";
exit 1; exit 1;
} }
@ -56,6 +60,10 @@ my %fdwServers = ();
my %functions = (); my %functions = ();
my %operators = (); my %operators = ();
my $valgrind = 0; my $valgrind = 0;
my $valgrindPath = "valgrind";
my $valgrindLogFile = "valgrind_test_log.txt";
my $pgCtlTimeout = undef;
my $connectionTimeout = 5000;
my $serversAreShutdown = "TRUE"; my $serversAreShutdown = "TRUE";
@ -71,6 +79,10 @@ GetOptions(
'load-extension=s' => \@extensions, 'load-extension=s' => \@extensions,
'server-option=s' => \@userPgOptions, 'server-option=s' => \@userPgOptions,
'valgrind' => \$valgrind, 'valgrind' => \$valgrind,
'valgrind-path=s' => \$valgrindPath,
'valgrind-log-file=s' => \$valgrindLogFile,
'pg_ctl-timeout=s' => \$pgCtlTimeout,
'connection-timeout=s' => \$connectionTimeout,
'help' => sub { Usage() }); 'help' => sub { Usage() });
# Update environment to include [DY]LD_LIBRARY_PATH/LIBDIR/etc - # Update environment to include [DY]LD_LIBRARY_PATH/LIBDIR/etc -
@ -135,10 +147,12 @@ are present.
MESSAGE MESSAGE
} }
# valgrind starts slow, need to increase timeout # If pgCtlTimeout is defined, we will set related environment variable.
if ($valgrind) # This is generally used with valgrind because valgrind starts slow and we
# need to increase timeout.
if (defined $pgCtlTimeout)
{ {
$ENV{PGCTLTIMEOUT} = '360'; $ENV{PGCTLTIMEOUT} = "$pgCtlTimeout";
} }
# We don't want valgrind to run pg_ctl itself, as that'd trigger a lot # We don't want valgrind to run pg_ctl itself, as that'd trigger a lot
@ -163,13 +177,13 @@ sub replace_postgres
or die "Could not create postgres wrapper at $bindir/postgres"; or die "Could not create postgres wrapper at $bindir/postgres";
print $fh <<"END"; print $fh <<"END";
#!/bin/bash #!/bin/bash
exec valgrind \\ exec $valgrindPath \\
--quiet \\ --quiet \\
--suppressions=${postgresSrcdir}/src/tools/valgrind.supp \\ --suppressions=${postgresSrcdir}/src/tools/valgrind.supp \\
--trace-children=yes --track-origins=yes --read-var-info=yes \\ --trace-children=yes --track-origins=yes --read-var-info=no \\
--leak-check=no \\ --leak-check=no \\
--error-exitcode=128 \\
--error-markers=VALGRINDERROR-BEGIN,VALGRINDERROR-END \\ --error-markers=VALGRINDERROR-BEGIN,VALGRINDERROR-END \\
--log-file=$valgrindLogFile \\
$bindir/postgres.orig \\ $bindir/postgres.orig \\
"\$@" "\$@"
END END
@ -220,6 +234,7 @@ push(@pgOptions, '-c', "citus.expire_cached_shards=on");
push(@pgOptions, '-c', "citus.task_tracker_delay=10ms"); push(@pgOptions, '-c', "citus.task_tracker_delay=10ms");
push(@pgOptions, '-c', "citus.remote_task_check_interval=1ms"); push(@pgOptions, '-c', "citus.remote_task_check_interval=1ms");
push(@pgOptions, '-c', "citus.shard_replication_factor=2"); push(@pgOptions, '-c', "citus.shard_replication_factor=2");
push(@pgOptions, '-c', "citus.node_connection_timeout=${connectionTimeout}");
# Add externally added options last, so they overwrite the default ones above # Add externally added options last, so they overwrite the default ones above
for my $option (@userPgOptions) for my $option (@userPgOptions)

4
src/test/regress/sql/multi_explain.sql
View File

@ -36,6 +36,10 @@ BEGIN
END; END;
$BODY$ LANGUAGE plpgsql; $BODY$ LANGUAGE plpgsql;
-- VACUMM related tables to ensure test outputs are stable
VACUUM ANALYZE lineitem;
VACUUM ANALYZE orders;
-- Test Text format -- Test Text format
EXPLAIN (COSTS FALSE, FORMAT TEXT) EXPLAIN (COSTS FALSE, FORMAT TEXT)
SELECT l_quantity, count(*) count_quantity FROM lineitem SELECT l_quantity, count(*) count_quantity FROM lineitem