mirror of https://github.com/citusdata/citus.git
Merge 2f349ce456 into 31911d8297
eaydingol
GitHub
commit
c8f28631b2
|
|
@ -132,127 +132,60 @@ jobs:
|
|||
./build-${{ env.PG_MAJOR }}/*
|
||||
./install-${{ env.PG_MAJOR }}.tar
|
||||
test-citus:
|
||||
name: PG${{ fromJson(matrix.pg_version).major }} - ${{ matrix.make }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
suite:
|
||||
- regress
|
||||
image_name:
|
||||
- ${{ needs.params.outputs.test_image_name }}
|
||||
pg_version:
|
||||
- ${{ needs.params.outputs.pg15_version }}
|
||||
- ${{ needs.params.outputs.pg16_version }}
|
||||
- ${{ needs.params.outputs.pg17_version }}
|
||||
make:
|
||||
- check-split
|
||||
- check-multi
|
||||
- check-multi-1
|
||||
- check-multi-mx
|
||||
- check-vanilla
|
||||
- check-isolation
|
||||
- check-operations
|
||||
- check-follower-cluster
|
||||
- check-add-backup-node
|
||||
- check-columnar
|
||||
- check-columnar-isolation
|
||||
- check-enterprise
|
||||
- check-enterprise-isolation
|
||||
- check-enterprise-isolation-logicalrep-1
|
||||
- check-enterprise-isolation-logicalrep-2
|
||||
- check-enterprise-isolation-logicalrep-3
|
||||
include:
|
||||
- make: check-failure
|
||||
pg_version: ${{ needs.params.outputs.pg15_version }}
|
||||
suite: regress
|
||||
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
||||
- make: check-failure
|
||||
pg_version: ${{ needs.params.outputs.pg16_version }}
|
||||
suite: regress
|
||||
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
||||
- make: check-failure
|
||||
pg_version: ${{ needs.params.outputs.pg17_version }}
|
||||
suite: regress
|
||||
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
||||
- make: check-enterprise-failure
|
||||
pg_version: ${{ needs.params.outputs.pg15_version }}
|
||||
suite: regress
|
||||
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
||||
- make: check-enterprise-failure
|
||||
pg_version: ${{ needs.params.outputs.pg16_version }}
|
||||
suite: regress
|
||||
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
||||
- make: check-enterprise-failure
|
||||
pg_version: ${{ needs.params.outputs.pg17_version }}
|
||||
suite: regress
|
||||
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
||||
- make: check-pytest
|
||||
pg_version: ${{ needs.params.outputs.pg15_version }}
|
||||
suite: regress
|
||||
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
||||
- make: check-pytest
|
||||
pg_version: ${{ needs.params.outputs.pg16_version }}
|
||||
suite: regress
|
||||
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
||||
- make: check-pytest
|
||||
pg_version: ${{ needs.params.outputs.pg17_version }}
|
||||
suite: regress
|
||||
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
||||
- make: installcheck
|
||||
suite: cdc
|
||||
image_name: ${{ needs.params.outputs.test_image_name }}
|
||||
pg_version: ${{ needs.params.outputs.pg15_version }}
|
||||
- make: installcheck
|
||||
suite: cdc
|
||||
image_name: ${{ needs.params.outputs.test_image_name }}
|
||||
pg_version: ${{ needs.params.outputs.pg16_version }}
|
||||
- make: installcheck
|
||||
suite: cdc
|
||||
image_name: ${{ needs.params.outputs.test_image_name }}
|
||||
pg_version: ${{ needs.params.outputs.pg17_version }}
|
||||
- make: check-query-generator
|
||||
pg_version: ${{ needs.params.outputs.pg15_version }}
|
||||
suite: regress
|
||||
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
||||
- make: check-query-generator
|
||||
pg_version: ${{ needs.params.outputs.pg16_version }}
|
||||
suite: regress
|
||||
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
||||
- make: check-query-generator
|
||||
pg_version: ${{ needs.params.outputs.pg17_version }}
|
||||
suite: regress
|
||||
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}"
|
||||
options: >-
|
||||
--user root
|
||||
--dns=8.8.8.8
|
||||
--cap-add=SYS_NICE
|
||||
--security-opt seccomp=unconfined
|
||||
# Due to Github creates a default network for each job, we need to use
|
||||
# --dns= to have similar DNS settings as our other CI systems or local
|
||||
# machines. Otherwise, we may see different results.
|
||||
# and grant caps so PG18's NUMA introspection (pg_shmem_allocations_numa -> move_pages)
|
||||
# doesn't fail with EPERM in CI.
|
||||
name: Test Citus
|
||||
uses: ./.github/workflows/run_tests.yml
|
||||
needs:
|
||||
- params
|
||||
- build
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: "./.github/actions/setup_extension"
|
||||
- name: Run Test
|
||||
run: gosu circleci make -C src/test/${{ matrix.suite }} ${{ matrix.make }}
|
||||
timeout-minutes: 20
|
||||
- uses: "./.github/actions/save_logs_and_results"
|
||||
if: always()
|
||||
with:
|
||||
folder: ${{ fromJson(matrix.pg_version).major }}_${{ matrix.make }}
|
||||
- uses: "./.github/actions/upload_coverage"
|
||||
if: always()
|
||||
with:
|
||||
flags: ${{ env.PG_MAJOR }}_${{ matrix.suite }}_${{ matrix.make }}
|
||||
codecov_token: ${{ secrets.CODECOV_TOKEN }}
|
||||
- params
|
||||
- build
|
||||
with:
|
||||
pg_versions: >
|
||||
[
|
||||
${{ needs.params.outputs.pg15_version }},
|
||||
${{ needs.params.outputs.pg16_version }},
|
||||
${{ needs.params.outputs.pg17_version }}
|
||||
]
|
||||
make_targets: '["check-split", "check-multi", "check-multi-1", "check-multi-mx", "check-vanilla", "check-isolation", "check-operations", "check-follower-cluster", "check-add-backup-node", "check-columnar", "check-columnar-isolation", "check-enterprise", "check-enterprise-isolation", "check-enterprise-isolation-logicalrep-1", "check-enterprise-isolation-logicalrep-2", "check-enterprise-isolation-logicalrep-3"]'
|
||||
image_suffix: ${{ needs.params.outputs.image_suffix }}
|
||||
image_name: ${{ needs.params.outputs.test_image_name }}
|
||||
secrets:
|
||||
codecov_token: ${{ secrets.CODECOV_TOKEN }}
|
||||
test-citus-failure:
|
||||
name: Test Citus Failure
|
||||
uses: ./.github/workflows/run_tests.yml
|
||||
needs:
|
||||
- params
|
||||
- build
|
||||
with:
|
||||
pg_versions: >
|
||||
[
|
||||
${{ needs.params.outputs.pg15_version }},
|
||||
${{ needs.params.outputs.pg16_version }},
|
||||
${{ needs.params.outputs.pg17_version }}
|
||||
]
|
||||
make_targets: '["check-failure", "check-enterprise-failure", "check-pytest", "check-query-generator"]'
|
||||
image_suffix: ${{ needs.params.outputs.image_suffix }}
|
||||
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
||||
secrets:
|
||||
codecov_token: ${{ secrets.CODECOV_TOKEN }}
|
||||
test-citus-cdc:
|
||||
name: Test Citus CDC
|
||||
uses: ./.github/workflows/run_tests.yml
|
||||
needs:
|
||||
- params
|
||||
- build
|
||||
with:
|
||||
pg_versions: >
|
||||
[
|
||||
${{ needs.params.outputs.pg15_version }},
|
||||
${{ needs.params.outputs.pg16_version }},
|
||||
${{ needs.params.outputs.pg17_version }}
|
||||
]
|
||||
make_targets: '["installcheck"]'
|
||||
image_suffix: ${{ needs.params.outputs.image_suffix }}
|
||||
image_name: ${{ needs.params.outputs.test_image_name }}
|
||||
suite: cdc
|
||||
secrets:
|
||||
codecov_token: ${{ secrets.CODECOV_TOKEN }}
|
||||
test-arbitrary-configs:
|
||||
name: PG${{ fromJson(matrix.pg_version).major }} - check-arbitrary-configs-${{ matrix.parallel }}
|
||||
runs-on: ["self-hosted", "1ES.Pool=1es-gha-citusdata-pool"]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,59 @@
|
|||
name: Run Tests
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
pg_versions:
|
||||
required: true
|
||||
type: string
|
||||
make_targets:
|
||||
required: true
|
||||
type: string
|
||||
image_suffix:
|
||||
required: true
|
||||
type: string
|
||||
image_name:
|
||||
required: true
|
||||
type: string
|
||||
suite:
|
||||
required: false
|
||||
type: string
|
||||
default: "regress"
|
||||
citus_version:
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
secrets:
|
||||
codecov_token:
|
||||
required: false
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: PG${{ matrix.pg_version.major }} - ${{ matrix.make }}${{ inputs.citus_version && format(' - {0}', inputs.citus_version) || '' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
pg_version: ${{ fromJson(inputs.pg_versions) }}
|
||||
make: ${{ fromJson(inputs.make_targets) }}
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: "${{ inputs.image_name }}:${{ matrix.pg_version.full }}${{ inputs.image_suffix }}"
|
||||
options: >-
|
||||
--user root
|
||||
--dns=8.8.8.8
|
||||
--cap-add=SYS_NICE
|
||||
--security-opt seccomp=unconfined
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: "./.github/actions/setup_extension"
|
||||
- name: Run Test
|
||||
run: CITUSVERSION=${{ inputs.citus_version }} gosu circleci make -C src/test/${{ inputs.suite }} ${{ matrix.make }}
|
||||
timeout-minutes: 20
|
||||
- uses: "./.github/actions/save_logs_and_results"
|
||||
if: always()
|
||||
with:
|
||||
folder: ${{ matrix.pg_version.major }}_${{ matrix.make }}${{ inputs.citus_version && format('_{0}', inputs.citus_version) || '' }}
|
||||
- uses: "./.github/actions/upload_coverage"
|
||||
if: always()
|
||||
with:
|
||||
flags: ${{ env.PG_MAJOR }}_${{ inputs.suite }}_${{ matrix.make }}
|
||||
codecov_token: ${{ secrets.codecov_token }}
|
||||
|
|
@ -10,6 +10,13 @@ ifndef MAJORVERSION
|
|||
MAJORVERSION := $(basename $(VERSION))
|
||||
endif
|
||||
|
||||
ifndef CITUSVERSION
|
||||
CITUSVERSION := ""
|
||||
endif
|
||||
|
||||
ifndef CITUSLIBDIR
|
||||
CITUSLIBDIR := ""
|
||||
endif
|
||||
# Add ./bin to $PATH so we use our custom diff instead of the default diff tool.
|
||||
# We do this to be able to mask shard Ids, placement Ids, node ports, etc.
|
||||
MAKEFILE_DIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
|
||||
|
|
@ -28,7 +35,7 @@ export PGISOLATIONTIMEOUT = 60
|
|||
## Citus regression support
|
||||
##
|
||||
MULTI_INSTALLDIR=$(CURDIR)/tmp_check/install
|
||||
pg_regress_multi_check = $(PERL) $(citus_abs_srcdir)/pg_regress_multi.pl --pgxsdir="$(pgxsdir)" --bindir="$(bindir)" --libdir="$(libdir)" --majorversion="$(MAJORVERSION)" --postgres-builddir="$(postgres_abs_builddir)" --postgres-srcdir="$(postgres_abs_srcdir)" --citus_abs_srcdir="$(citus_abs_srcdir)"
|
||||
pg_regress_multi_check = $(PERL) $(citus_abs_srcdir)/pg_regress_multi.pl --pgxsdir="$(pgxsdir)" --bindir="$(bindir)" --libdir="$(libdir)" --majorversion="$(MAJORVERSION)" --postgres-builddir="$(postgres_abs_builddir)" --postgres-srcdir="$(postgres_abs_srcdir)" --citus_abs_srcdir="$(citus_abs_srcdir)" --citus-version=$(CITUSVERSION) --citus-libdir="$(CITUSLIBDIR)"
|
||||
MULTI_REGRESS_OPTS = --inputdir=$(citus_abs_srcdir) $(pg_regress_locale_flags) --launcher="$(citus_abs_srcdir)/log_test_times"
|
||||
|
||||
pg_upgrade_check = $(citus_abs_srcdir)/citus_tests/upgrade/pg_upgrade_test.py
|
||||
|
|
@ -157,6 +164,7 @@ check-multi: all
|
|||
check-enterprise: all
|
||||
$(pg_regress_multi_check) --load-extension=citus \
|
||||
-- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/enterprise_schedule $(EXTRA_TESTS)
|
||||
|
||||
check-multi-1: all
|
||||
$(pg_regress_multi_check) --load-extension=citus \
|
||||
-- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_1_schedule $(EXTRA_TESTS)
|
||||
|
|
|
|||
|
|
@ -50,6 +50,8 @@ sub Usage()
|
|||
print " --connection-timeout Timeout for connecting to worker nodes\n";
|
||||
print " --mitmproxy Start a mitmproxy for one of the workers\n";
|
||||
print " --worker-count Number of workers in Citus cluster (default: 2)\n";
|
||||
print " --citus-version Citus version being tested (used for during extension create)\n";
|
||||
print " --citus-libdir Absolute path to alternative Citus library directory\n";
|
||||
exit 1;
|
||||
}
|
||||
|
||||
|
|
@ -87,6 +89,8 @@ my $conninfo = "";
|
|||
my $publicWorker1Host = "localhost";
|
||||
my $publicWorker2Host = "localhost";
|
||||
my $workerCount = 2;
|
||||
my $citusversion = "";
|
||||
my $citusLibdir = "";
|
||||
|
||||
my $serversAreShutdown = "TRUE";
|
||||
my $usingWindows = 0;
|
||||
|
|
@ -121,6 +125,8 @@ GetOptions(
|
|||
'worker-1-public-hostname=s' => \$publicWorker1Host,
|
||||
'worker-2-public-hostname=s' => \$publicWorker2Host,
|
||||
'worker-count=i' => \$workerCount,
|
||||
'citus-version=s' => \$citusversion,
|
||||
'citus-libdir=s' => \$citusLibdir,
|
||||
'help' => sub { Usage() });
|
||||
|
||||
my $fixopen = "$bindir/postgres.fixopen";
|
||||
|
|
@ -307,6 +313,40 @@ sub generate_hba
|
|||
close $fh;
|
||||
}
|
||||
|
||||
sub setup_symlink
|
||||
{
|
||||
my ($originalfile, $targetfile) = @_;
|
||||
|
||||
# Only proceed if not on Windows and both files are defined and non-empty
|
||||
return if $usingWindows;
|
||||
return unless (defined $originalfile && $originalfile ne "");
|
||||
return unless (defined $targetfile && $targetfile ne "");
|
||||
|
||||
-e $targetfile or die "Target is not found at $targetfile";
|
||||
|
||||
my $backup = $originalfile . ".bak";
|
||||
rename($originalfile, $backup) or die "Failed to copy $originalfile to $backup: $!";
|
||||
|
||||
symlink($targetfile, $originalfile) or die "Failed to create symlink $originalfile -> $targetfile: $!";
|
||||
}
|
||||
|
||||
sub restore_original
|
||||
{
|
||||
my ($originalfile) = @_;
|
||||
|
||||
# Only proceed if not on Windows and file is defined and non-empty
|
||||
return if $usingWindows;
|
||||
return unless (defined $originalfile && $originalfile ne "");
|
||||
|
||||
my $backup = $originalfile . ".bak";
|
||||
|
||||
# Return silently if backup doesn't exist
|
||||
return unless -e $backup;
|
||||
|
||||
unlink($originalfile) or die "Failed to remove symlink at $originalfile: $!";
|
||||
rename($backup, $originalfile) or die "Failed to restore original file from $backup to $originalfile: $!";
|
||||
}
|
||||
|
||||
# always want to call initdb under normal postgres, so revert from a
|
||||
# partial run, even if we're now not using valgrind.
|
||||
revert_replace_postgres();
|
||||
|
|
@ -590,6 +630,12 @@ if($isolationtester)
|
|||
push(@pgOptions, "citus.background_task_queue_interval=-1");
|
||||
}
|
||||
|
||||
if($citusversion || $citusLibdir)
|
||||
{
|
||||
push(@pgOptions, "citus.enable_version_checks=off");
|
||||
push(@pgOptions, "columnar.enable_version_checks=off");
|
||||
}
|
||||
|
||||
# Add externally added options last, so they overwrite the default ones above
|
||||
for my $option (@userPgOptions)
|
||||
{
|
||||
|
|
@ -774,6 +820,9 @@ if (!$conninfo)
|
|||
# Routine to shutdown servers at failure/exit
|
||||
sub ShutdownServers()
|
||||
{
|
||||
restore_original(catfile($libdir, "citus.so"));
|
||||
restore_original(catfile($libdir, "citus_columnar.so"));
|
||||
|
||||
if (!$conninfo && $serversAreShutdown eq "FALSE")
|
||||
{
|
||||
system(catfile("$bindir", "pg_ctl"),
|
||||
|
|
@ -899,6 +948,13 @@ if ($followercluster && $backupnodetest == 0)
|
|||
$synchronousReplication = "-c synchronous_standby_names='FIRST 1 (*)' -c synchronous_commit=remote_apply";
|
||||
}
|
||||
|
||||
# Ensure citus.so points to alternative library if provided
|
||||
if ($citusLibdir)
|
||||
{
|
||||
setup_symlink(catfile($libdir, "citus.so"), catfile($citusLibdir, "citus.so"));
|
||||
setup_symlink(catfile($libdir, "citus_columnar.so"), catfile($citusLibdir, "citus_columnar.so"));
|
||||
}
|
||||
|
||||
# Start servers
|
||||
if (!$conninfo)
|
||||
{
|
||||
|
|
@ -970,6 +1026,13 @@ if ($followercluster)
|
|||
}
|
||||
}
|
||||
|
||||
# Restore original citus.so if we modified it
|
||||
if ($citusLibdir)
|
||||
{
|
||||
restore_original(catfile($libdir, "citus.so"));
|
||||
restore_original(catfile($libdir, "citus_columnar.so"));
|
||||
}
|
||||
|
||||
###
|
||||
# Create database, extensions, types, functions and fdws on the workers,
|
||||
# pg_regress won't know to create them for us.
|
||||
|
|
@ -990,10 +1053,20 @@ if (!$conninfo)
|
|||
|
||||
for my $extension (@extensions)
|
||||
{
|
||||
system(catfile($bindir, "psql"),
|
||||
('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression",
|
||||
'-c', "CREATE EXTENSION IF NOT EXISTS $extension;")) == 0
|
||||
or die "Could not create extension $extension on worker port $port.";
|
||||
if ($extension eq "citus" && $citusversion ne "")
|
||||
{
|
||||
system(catfile($bindir, "psql"),
|
||||
('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression",
|
||||
'-c', "CREATE EXTENSION IF NOT EXISTS $extension VERSION '$citusversion';")) == 0
|
||||
or die "Could not create extension $extension on worker port $port.";
|
||||
}
|
||||
else
|
||||
{
|
||||
system(catfile($bindir, "psql"),
|
||||
('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression",
|
||||
'-c', "CREATE EXTENSION IF NOT EXISTS $extension;")) == 0
|
||||
or die "Could not create extension $extension on worker port $port.";
|
||||
}
|
||||
}
|
||||
|
||||
foreach my $function (keys %functions)
|
||||
|
|
|
|||
Loading…
Reference in New Issue