Merge branch 'main' into tenant-schema-isolation

.github/workflows/build_and_test.yml

@@ -27,9 +27,9 @@ jobs:
       style_checker_image_name: "citus/stylechecker"
       style_checker_tools_version: "0.8.18"
       image_suffix: "-v9d71045"
-      pg14_version: "14.9"
+      pg14_version: '{ "major": "14", "full": "14.9" }'
-      pg15_version: "15.4"
+      pg15_version: '{ "major": "15", "full": "15.4" }'
-      pg16_version: "16.0"
+      pg16_version: '{ "major": "16", "full": "16.0" }'
       upgrade_pg_versions: "14.9-15.4-16.0"
     steps:
       # Since GHA jobs needs at least one step we use a noop step here.
@@ -93,7 +93,7 @@ jobs:
       run: ci/check_migration_files.sh
   build:
     needs: params
-    name: Build for PG ${{ matrix.pg_version}}
+    name: Build for PG${{ fromJson(matrix.pg_version).major }}
     strategy:
       fail-fast: false
       matrix:
@@ -107,7 +107,7 @@ jobs:
           - ${{ needs.params.outputs.pg16_version }}
     runs-on: ubuntu-20.04
     container:
-      image: "${{ matrix.image_name }}:${{ matrix.pg_version  }}${{ matrix.image_suffix }}"
+      image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}"
       options: --user root
     steps:
     - uses: actions/checkout@v3.5.0
@@ -124,7 +124,7 @@ jobs:
           ./build-${{ env.PG_MAJOR }}/*
           ./install-${{ env.PG_MAJOR }}.tar
   test-citus:
-    name: PG${{ matrix.pg_version }} - ${{ matrix.make }}
+    name: PG${{ fromJson(matrix.pg_version).major }} - ${{ matrix.make }}
     strategy:
       fail-fast: false
       matrix:
@@ -211,7 +211,7 @@ jobs:
             image_name: ${{ needs.params.outputs.fail_test_image_name }}
     runs-on: ubuntu-20.04
     container:
-      image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ needs.params.outputs.image_suffix }}"
+      image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}"
       options: --user root --dns=8.8.8.8
       # Due to Github creates a default network for each job, we need to use
       # --dns= to have similar DNS settings as our other CI systems or local
@@ -228,17 +228,17 @@ jobs:
     - uses: "./.github/actions/save_logs_and_results"
       if: always()
       with:
-        folder: ${{ matrix.pg_version }}_${{ matrix.make }}
+        folder: ${{ fromJson(matrix.pg_version).major }}_${{ matrix.make }}
     - uses: "./.github/actions/upload_coverage"
       if: always()
       with:
         flags: ${{ env.PG_MAJOR }}_${{ matrix.suite }}_${{ matrix.make }}
         codecov_token: ${{ secrets.CODECOV_TOKEN }}
   test-arbitrary-configs:
-    name: PG${{ matrix.pg_version }} - check-arbitrary-configs-${{ matrix.parallel }}
+    name: PG${{ fromJson(matrix.pg_version).major }} - check-arbitrary-configs-${{ matrix.parallel }}
     runs-on: ["self-hosted", "1ES.Pool=1es-gha-citusdata-pool"]
     container:
-      image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ needs.params.outputs.image_suffix }}"
+      image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}"
       options: --user root
     needs:
       - params
@@ -333,10 +333,10 @@ jobs:
         flags: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade
         codecov_token: ${{ secrets.CODECOV_TOKEN }}
   test-citus-upgrade:
-    name: PG${{ needs.params.outputs.pg14_version }} - check-citus-upgrade
+    name: PG${{ fromJson(needs.params.outputs.pg14_version).major }} - check-citus-upgrade
     runs-on: ubuntu-20.04
     container:
-      image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ needs.params.outputs.pg14_version }}${{ needs.params.outputs.image_suffix }}"
+      image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ fromJson(needs.params.outputs.pg14_version).full }}${{ needs.params.outputs.image_suffix }}"
       options: --user root
     needs:
     - params
@@ -383,7 +383,7 @@ jobs:
       CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
     runs-on: ubuntu-20.04
     container:
-      image: ${{ needs.params.outputs.test_image_name }}:${{ needs.params.outputs.pg16_version }}${{ needs.params.outputs.image_suffix }}
+      image: ${{ needs.params.outputs.test_image_name }}:${{ fromJson(needs.params.outputs.pg16_version).full }}${{ needs.params.outputs.image_suffix }}
     needs:
       - params
       - test-citus
@@ -478,7 +478,7 @@ jobs:
     name: Test flakyness
     runs-on: ubuntu-20.04
     container:
-      image: ${{ needs.params.outputs.fail_test_image_name }}:${{ needs.params.outputs.pg16_version }}${{ needs.params.outputs.image_suffix }}
+      image: ${{ needs.params.outputs.fail_test_image_name }}:${{ fromJson(needs.params.outputs.pg16_version).full }}${{ needs.params.outputs.image_suffix }}
       options: --user root
     env:
       runs: 8

.github/workflows/packaging-test-pipelines.yml

@@ -24,9 +24,11 @@ jobs:
       - name: Get Postgres Versions
         id: get-postgres-versions
         run: |
-          # Postgres versions are stored in .github/workflows/build_and_test.yml file in "pg[pg-version]_version"
+          set -euxo pipefail
-          # format. Below command extracts the versions and get the unique values.
+          # Postgres versions are stored in .github/workflows/build_and_test.yml
-          pg_versions=$(cat .github/workflows/build_and_test.yml | grep -oE 'pg[0-9]+_version: "[0-9.]+"' | sed -E 's/pg([0-9]+)_version: "([0-9.]+)"/\1/g' | sort | uniq | tr '\n', ',')
+          # file in json strings with major and full keys.
+          # Below command extracts the versions and get the unique values.
+          pg_versions=$(cat .github/workflows/build_and_test.yml | grep -oE '"major": "[0-9]+", "full": "[0-9.]+"' | sed -E 's/"major": "([0-9]+)", "full": "([0-9.]+)"/\1/g' | sort | uniq | tr '\n', ',')
           pg_versions_array="[ ${pg_versions} ]"
           echo "Supported PG Versions: ${pg_versions_array}"
           # Below line is needed to set the output variable to be used in the next job

ci/check_all_ci_scripts_are_run.sh

@@ -14,8 +14,8 @@ ci_scripts=$(
     grep -v -E '^(ci_helpers.sh|fix_style.sh)$'
 )
 for script in $ci_scripts; do
-    if ! grep "\\bci/$script\\b" .circleci/config.yml > /dev/null; then
+    if ! grep "\\bci/$script\\b" -r .github > /dev/null; then
-        echo "ERROR: CI script with name \"$script\" is not actually used in .circleci/config.yml"
+        echo "ERROR: CI script with name \"$script\" is not actually used in .github folder"
         exit 1
     fi
     if ! grep "^## \`$script\`\$" ci/README.md > /dev/null; then

ci/check_enterprise_merge.sh

@@ -1,96 +0,0 @@
-#!/bin/bash
-
-# Testing this script locally requires you to set the following environment
-# variables:
-# CIRCLE_BRANCH, GIT_USERNAME and GIT_TOKEN
-
-# fail if trying to reference a variable that is not set.
-set -u
-# exit immediately if a command fails
-set -e
-# Fail on pipe failures
-set -o pipefail
-
-PR_BRANCH="${CIRCLE_BRANCH}"
-ENTERPRISE_REMOTE="https://${GIT_USERNAME}:${GIT_TOKEN}@github.com/citusdata/citus-enterprise"
-
-# shellcheck disable=SC1091
-source ci/ci_helpers.sh
-
-# List executed commands. This is done so debugging this script is easier when
-# it fails. It's explicitly done after git remote add so username and password
-# are not shown in CI output (even though it's also filtered out by CircleCI)
-set -x
-
-check_compile () {
-    echo "INFO: checking if merged code can be compiled"
-    ./configure --without-libcurl
-    make -j10
-}
-
-# Clone current git repo (which should be community) to a temporary working
-# directory and go there
-GIT_DIR_ROOT="$(git rev-parse --show-toplevel)"
-TMP_GIT_DIR="$(mktemp --directory -t citus-merge-check.XXXXXXXXX)"
-git clone "$GIT_DIR_ROOT" "$TMP_GIT_DIR"
-cd "$TMP_GIT_DIR"
-
-# Fails in CI without this
-git config user.email "citus-bot@microsoft.com"
-git config user.name "citus bot"
-
-# Disable "set -x" temporarily, because $ENTERPRISE_REMOTE contains passwords
-{ set +x ; } 2> /dev/null
-git remote add enterprise "$ENTERPRISE_REMOTE"
-set -x
-
-git remote set-url --push enterprise no-pushing
-
-# Fetch enterprise-master
-git fetch enterprise enterprise-master
-
-
-git checkout "enterprise/enterprise-master"
-
-if git merge --no-commit "origin/$PR_BRANCH"; then
-    echo "INFO: community PR branch could be merged into enterprise-master"
-    # check that we can compile after the merge
-    if check_compile; then
-        exit 0
-    fi
-
-    echo "WARN: Failed to compile after community PR branch was merged into enterprise"
-fi
-
-# undo partial merge
-git merge --abort
-
-# If we have a conflict on enterprise merge on the master branch, we have a problem.
-# Provide an error message to indicate that enterprise merge is needed to fix this check.
-if [[ $PR_BRANCH = master ]]; then
-    echo "ERROR: Master branch has merge conflicts with enterprise-master."
-    echo "Try re-running this CI job after merging your changes into enterprise-master."
-    exit 1
-fi
-
-if ! git fetch enterprise "$PR_BRANCH" ; then
-    echo "ERROR: enterprise/$PR_BRANCH was not found and community PR branch could not be merged into enterprise-master"
-    exit 1
-fi
-
-# Show the top commit of the enterprise PR branch to make debugging easier
-git log -n 1 "enterprise/$PR_BRANCH"
-
-# Check that this branch contains the top commit of the current community PR
-# branch. If it does not it means it's not up to date with the current PR, so
-# the enterprise branch should be updated.
-if ! git merge-base --is-ancestor "origin/$PR_BRANCH" "enterprise/$PR_BRANCH" ; then
-    echo "ERROR: enterprise/$PR_BRANCH is not up to date with community PR branch"
-    exit 1
-fi
-
-# Now check if we can merge the enterprise PR into enterprise-master without
-# issues.
-git merge --no-commit "enterprise/$PR_BRANCH"
-# check that we can compile after the merge
-check_compile

src/test/regress/expected/validate_constraint.out

@@ -133,12 +133,6 @@ ORDER BY 1, 2;
  validatable_constraint_8000016 | t
 (10 rows)
 
-DROP TABLE constrained_table;
+SET client_min_messages TO WARNING;
-DROP TABLE referenced_table CASCADE;
-DROP TABLE referencing_table;
 DROP SCHEMA validate_constraint CASCADE;
-NOTICE:  drop cascades to 3 other objects
-DETAIL:  drop cascades to type constraint_validity
-drop cascades to view constraint_validations_in_workers
-drop cascades to view constraint_validations
 SET search_path TO DEFAULT;

src/test/regress/multi_1_schedule

@@ -202,7 +202,8 @@ test: citus_copy_shard_placement
 # multi_utilities cannot be run in parallel with other tests because it checks
 # global locks
 test: multi_utilities
-test: foreign_key_to_reference_table validate_constraint
+test: foreign_key_to_reference_table
+test: validate_constraint
 test: multi_repartition_udt multi_repartitioned_subquery_udf multi_subtransactions
 
 test: multi_modifying_xacts

src/test/regress/multi_schedule_hyperscale

@@ -154,7 +154,8 @@ test: multi_outer_join
 # ---
 test: multi_complex_count_distinct
 test: multi_upsert multi_simple_queries
-test: foreign_key_to_reference_table validate_constraint
+test: foreign_key_to_reference_table
+test: validate_constraint
 
 # ---------
 #  creates hash and range-partitioned tables and performs COPY

src/test/regress/multi_schedule_hyperscale_superuser

@@ -150,7 +150,9 @@ test: multi_outer_join
 test: multi_create_fdw
 test: multi_generate_ddl_commands multi_create_shards multi_prune_shard_list
 test: multi_upsert multi_simple_queries multi_data_types
-test: multi_utilities foreign_key_to_reference_table validate_constraint
+test: multi_utilities
+test: foreign_key_to_reference_table
+test: validate_constraint
 test: multi_repartition_udt multi_repartitioned_subquery_udf
 
 # ---------

src/test/regress/sql/validate_constraint.sql

@@ -116,9 +116,6 @@ SELECT *
 FROM constraint_validations_in_workers
 ORDER BY 1, 2;
 
-DROP TABLE constrained_table;
+SET client_min_messages TO WARNING;
-DROP TABLE referenced_table CASCADE;
-DROP TABLE referencing_table;
-
 DROP SCHEMA validate_constraint CASCADE;
 SET search_path TO DEFAULT;