From 49255213d4b576fdfdce9af97eb97d461dbbacbf Mon Sep 17 00:00:00 2001 From: Brian Cloutier Date: Tue, 27 Mar 2018 15:17:20 -0700 Subject: [PATCH] Configure appveyor to run regression tests - Add install.pl to instal .sql files on Windows - Remove a hack to PGDLLIMPORT some variables - Add citus_version.o to the Makefile - Fix pg_regress_multi's PATH generation on Windows - Output regression.diffs when the tests fail - Fix permissions in data directory, make sure postgres can play with it --- Makefile | 4 +- appveyor.yml | 20 ++++-- citus.control | 6 ++ src/backend/distributed/install.pl | 96 ++++++++++++++++++++++++++++ src/test/regress/pg_regress_multi.pl | 25 ++++++-- 5 files changed, 134 insertions(+), 17 deletions(-) create mode 100644 citus.control create mode 100755 src/backend/distributed/install.pl diff --git a/Makefile b/Makefile index 29daf7796..8e0d4fea2 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,6 @@ # Citus toplevel Makefile EXTENSION = citus -DATA = citus--7.0-15.sql - MODULE_big = citus @@ -81,6 +79,7 @@ OBJS = src/backend/distributed/shared_library_init.o \ src/backend/distributed/utils/citus_outfuncs.o \ src/backend/distributed/utils/citus_readfuncs.o \ src/backend/distributed/utils/citus_ruleutils.o \ + src/backend/distributed/utils/citus_version.o \ src/backend/distributed/utils/colocation_utils.o \ src/backend/distributed/utils/distribution_column.o \ src/backend/distributed/utils/errormessage.o \ @@ -108,7 +107,6 @@ OBJS = src/backend/distributed/shared_library_init.o \ $(WIN32RES) EXTENSION= -DATA= MODULE_big= OBJS= diff --git a/appveyor.yml b/appveyor.yml index b587d26b2..409e617a9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -49,11 +49,6 @@ install: # make postgres think we're a contrib module - git clone -b REL_10_STABLE https://github.com/postgres/postgres C:\projects\postgres - git -C C:\projects\postgres apply C:\projects\citus\windows\Mkvcbuild.pm.patch - # temporary hack: mark enable_hashagg PGDLLIMPORT - - git -C C:\projects\postgres cherry-pick 935dee9ad5a8d12f4d3b772a6e6c99d245e5ad44 & exit 0 - # an even bigger hack, this file has a merge conflict so skip it - - git -C C:\projects\postgres reset src\include\optimizer\cost.h - - git -C C:\projects\postgres checkout src\include\optimizer\cost.h # only build Citus, don't build the other contrib modules - git -C C:\projects\postgres apply C:\projects\citus\windows\Mkvcbuild-minimize.patch - rmdir /s /q C:\projects\postgres\contrib @@ -78,6 +73,17 @@ build_script: #---------------------------------# # tests configuration # #---------------------------------# +before_test: + - ps: cd C:\projects\postgres\src\tools\msvc + - install C:\projects\pgsql + - cd C:\projects\postgres\contrib\citus\src\backend\distributed + - copy citus--*.sql C:\projects\pgsql\share\extension\ + - perl install.pl C:\projects\pgsql -# disable automatic tests -test: off +test_script: + - ps: cd C:\projects\postgres\contrib\citus\src\test\regress + - perl pg_regress_multi.pl --bindir=C:\projects\pgsql\bin --load-extension=citus -- --schedule=multi_schedule + +on_failure: + - ps: cd C:\projects\postgres\contrib\citus\src\test\regress + - type regression.diffs diff --git a/citus.control b/citus.control new file mode 100644 index 000000000..5493b8234 --- /dev/null +++ b/citus.control @@ -0,0 +1,6 @@ +# Citus extension +comment = 'Citus distributed database' +default_version = '7.4-1' +module_pathname = '$libdir/citus' +relocatable = false +schema = pg_catalog diff --git a/src/backend/distributed/install.pl b/src/backend/distributed/install.pl new file mode 100755 index 000000000..63dcdaab0 --- /dev/null +++ b/src/backend/distributed/install.pl @@ -0,0 +1,96 @@ +#!/usr/bin/env perl + +# Generates all the .sql files and puts them into the requested directory + +use strict; +use warnings; + +use File::Copy "copy"; +use File::Spec::Functions "catdir"; + +my $EXTENSION = "citus"; + +my $argcount = @ARGV; +die "need a single argument saying where to install the sql files\n" if ($argcount != 1); + +my $prefix = $ARGV[0]; +die "install destination must exist!\n" unless (-e $prefix); +my $dest = catdir($prefix, "share/postgresql/extension"); +unless (-e $dest) +{ + $dest = catdir($prefix, "share/extension"); +} +die "install destination must be a postgres installation!\n" unless (-e $dest); + +# 1) check that we're installing into the real postgres installation + +print "installing into $prefix\n"; + +# 2) parse Makefile's EXTVERSIONS to get the list of version pairs + +die "could not find Makefile" unless (-e 'Makefile'); +my $lines; +{ + open(MAKEFILE, "Makefile") or die "cannot open Makefile\n"; + local $/ = undef; # remove the record separator just for this scope + $lines = ; # read all the lines at once into a single variable + close(MAKEFILE); +} + +my $versions; +if ($lines =~ /^EXTVERSIONS = ((?:[0-9. \t-]+(?:\\\n)?)*)/ms) +{ + $versions = $1; +} +else +{ + die "could not find EXTVERSIONS\n"; +} + +$versions =~ s/\\\n//g; # remove all newlines +$versions =~ s/\t/ /g; # also all tabs +$versions =~ s/\s+/ /g; # and, finally, all doubled whitespace +print "found list of versions: $versions\n"; + +# 3) generate all the sql files + +my @versions = split(' ', $versions); +my @files = ("$EXTENSION--5.0.sql"); + +die "could not find $EXTENSION.sql" unless (-e "$EXTENSION.sql"); +copy("$EXTENSION.sql", "$EXTENSION--5.0.sql"); + +while (scalar @versions > 1) +{ + my $version = $versions[0]; + my $next = $versions[1]; + + my $thisFile = "$EXTENSION--$version.sql"; + my $updateFile = "$EXTENSION--$version--$next.sql"; + my $nextFile = "$EXTENSION--$next.sql"; + + print "creating $nextFile\n"; + + copy($thisFile, $nextFile); + + open(NEXT, ">>", $nextFile) or die "could not open $nextFile"; + open(UPDATE, "<", $updateFile) or die "could not open $updateFile"; + while(my $line = ){ + print NEXT $line; + } + close(UPDATE); + close(NEXT); + + push @files, $nextFile; + push @files, $updateFile; + shift @versions; +} + +print "copying .sql files into $dest\n"; + +# 4) copy the sql files into the right place! +while(my $file = shift(@files)) +{ + my $fullDest = catdir($dest, $file); + copy($file, $fullDest); +} diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index e510c1acc..af1b2d2cc 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -107,7 +107,11 @@ if (defined $libdir) $ENV{LD_LIBRARY_PATH} = "$libdir:".($ENV{LD_LIBRARY_PATH} || ''); $ENV{DYLD_LIBRARY_PATH} = "$libdir:".($ENV{DYLD_LIBRARY_PATH} || ''); $ENV{LIBPATH} = "$libdir:".($ENV{LIBPATH} || ''); - $ENV{PATH} = "$libdir:".($ENV{PATH} || ''); + if ($usingWindows) { + $ENV{PATH} = "$libdir;".($ENV{PATH} || ''); + } else { + $ENV{PATH} = "$libdir:".($ENV{PATH} || ''); + } } # Put $bindir to the end of PATH. We want to prefer system binaries by @@ -115,10 +119,13 @@ if (defined $libdir) # want to find binaries if they're not in PATH. if (defined $bindir) { - $ENV{PATH} = ($ENV{PATH} || '').":$bindir"; + if ($usingWindows) { + $ENV{PATH} = ($ENV{PATH} || '').";$bindir"; + } else { + $ENV{PATH} = ($ENV{PATH} || '').":$bindir"; + } } - # Most people are used to unified diffs these days, rather than the # context diffs pg_regress defaults to. Change default to avoid # everyone having to (re-)learn how to change that setting. Also add @@ -381,23 +388,21 @@ if ($usingWindows) { print $fh "--variable=dev_null=\"/nul\" "; print $fh "--variable=temp_dir=\"%TEMP%\" "; - print $fh "--variable=psql=\"".catfile($bindir, "psql")."\" "; } else { print $fh "--variable=dev_null=\"/dev/null\" "; print $fh "--variable=temp_dir=\"/tmp/\" "; - print $fh "--variable=psql=\"psql\" "; } if ($usingWindows) { - print $fh "%*\n"; # pass on the commandline arguments + print $fh " %*\n"; # pass on the commandline arguments } else { - print $fh "\"\$@\"\n"; # pass on the commandline arguments + print $fh " \"\$@\"\n"; # pass on the commandline arguments } close $fh; @@ -448,6 +453,12 @@ else } } +if ($usingWindows) +{ + # takeown returns a failing result code no matter what happens, so skip the check + system("takeown /f data"); + system("icacls data /grant \%userdomain\%\\\%username\%:F"); +} # Routine to shutdown servers at failure/exit sub ShutdownServers()