Bump Citus version to 11.1.7 (#7502)

---------

Co-authored-by: Onur Tirtir <onurcantirtir@gmail.com>

.github/actions/parallelization/action.yml

@@ -0,0 +1,23 @@
+name: 'Parallelization matrix'
+inputs:
+  count:
+    required: false
+    default: 32
+outputs:
+  json:
+    value: ${{ steps.generate_matrix.outputs.json }}
+runs:
+  using: "composite"
+  steps:
+    - name: Generate parallelization matrix
+      id: generate_matrix
+      shell: bash
+      run: |-
+        json_array="{\"include\": ["
+        for ((i = 1; i <= ${{ inputs.count }}; i++)); do
+            json_array+="{\"id\":\"$i\"},"
+        done
+        json_array=${json_array%,}
+        json_array+=" ]}"
+        echo "json=$json_array" >> "$GITHUB_OUTPUT"
+        echo "json=$json_array"

.github/actions/save_logs_and_results/action.yml

@@ -0,0 +1,38 @@
+name: save_logs_and_results
+inputs:
+  folder:
+    required: false
+    default: "log"
+runs:
+  using: composite
+  steps:
+  - uses: actions/upload-artifact@v3.1.1
+    name: Upload logs
+    with:
+      name: ${{ inputs.folder }}
+      if-no-files-found: ignore
+      path: |
+        src/test/**/proxy.output
+        src/test/**/results/
+        src/test/**/tmp_check/master/log
+        src/test/**/tmp_check/worker.57638/log
+        src/test/**/tmp_check/worker.57637/log
+        src/test/**/*.diffs
+        src/test/**/out/ddls.sql
+        src/test/**/out/queries.sql
+        src/test/**/logfile_*
+        /tmp/pg_upgrade_newData_logs
+  - name: Publish regression.diffs
+    run: |-
+      diffs="$(find src/test/regress -name "*.diffs" -exec cat {} \;)"
+      if ! [ -z "$diffs" ]; then
+        echo '```diff' >> $GITHUB_STEP_SUMMARY
+        echo -E "$diffs" >> $GITHUB_STEP_SUMMARY
+        echo '```' >> $GITHUB_STEP_SUMMARY
+        echo -E $diffs
+      fi
+    shell: bash
+  - name: Print stack traces
+    run: "./ci/print_stack_trace.sh"
+    if: failure()
+    shell: bash

.github/actions/setup_extension/action.yml

@@ -0,0 +1,35 @@
+name: setup_extension
+inputs:
+  pg_major:
+    required: false
+  skip_installation:
+    required: false
+    default: false
+    type: boolean
+runs:
+  using: composite
+  steps:
+  - name: Expose $PG_MAJOR to Github Env
+    run: |-
+        if [ -z "${{ inputs.pg_major }}" ]; then
+          echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV
+        else
+          echo "PG_MAJOR=${{ inputs.pg_major }}" >> $GITHUB_ENV
+        fi
+    shell: bash
+  - uses: actions/download-artifact@v3.0.1
+    with:
+      name: build-${{ env.PG_MAJOR }}
+  - name: Install Extension
+    if: ${{ inputs.skip_installation == 'false' }}
+    run: tar xfv "install-$PG_MAJOR.tar" --directory /
+    shell: bash
+  - name: Configure
+    run: |-
+      chown -R circleci .
+      git config --global --add safe.directory ${GITHUB_WORKSPACE}
+      gosu circleci ./configure --without-pg-version-check
+    shell: bash
+  - name: Enable core dumps
+    run: ulimit -c unlimited
+    shell: bash

.github/actions/upload_coverage/action.yml

@@ -0,0 +1,15 @@
+name: coverage
+inputs:
+  flags:
+    required: false
+  codecov_token:
+    required: true
+runs:
+  using: composite
+  steps:
+  - uses: codecov/codecov-action@v3
+    with:
+      flags: ${{ inputs.flags }}
+      token: ${{ inputs.codecov_token }}
+      verbose: true
+      gcov: true

.github/packaging/packaging_ignore.yml

@@ -0,0 +1,3 @@
+base:
+  - ".* warning: ignoring old recipe for target [`']check'"
+  - ".* warning: overriding recipe for target [`']check'"

.github/packaging/validate_build_output.sh

@@ -0,0 +1,20 @@
+package_type=${1}
+
+# Since $HOME is set in GH_Actions as /github/home, pyenv fails to create virtualenvs.
+# For this script, we set $HOME to /root and then set it back to /github/home.
+GITHUB_HOME="${HOME}"
+export HOME="/root"
+
+eval "$(pyenv init -)"
+pyenv versions
+pyenv virtualenv ${PACKAGING_PYTHON_VERSION} packaging_env
+pyenv activate packaging_env
+
+git clone -b v0.8.25 --depth=1  https://github.com/citusdata/tools.git tools
+python3 -m pip install -r tools/packaging_automation/requirements.txt
+python3 -m tools.packaging_automation.validate_build_output --output_file output.log \
+                                                            --ignore_file .github/packaging/packaging_ignore.yml \
+                                                            --package_type ${package_type}
+pyenv deactivate
+# Set $HOME back to /github/home
+export HOME=${GITHUB_HOME}

.github/workflows/build_and_test.yml

@@ -0,0 +1,468 @@
+name: Build & Test
+run-name: Build & Test - ${{ github.event.pull_request.title || github.ref_name }}
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+on:
+  workflow_dispatch:
+    inputs:
+      skip_test_flakyness:
+        required: false
+        default: false
+        type: boolean
+  pull_request:
+    types: [opened, reopened,synchronize]
+jobs:
+  # Since GHA does not interpolate env varibles in matrix context, we need to
+  # define them in a separate job and use them in other jobs.
+  params:
+    runs-on: ubuntu-latest
+    name: Initialize parameters
+    outputs:
+      build_image_name: "citus/extbuilder"
+      test_image_name: "citus/exttester"
+      citusupgrade_image_name: "citus/citusupgradetester"
+      fail_test_image_name: "citus/failtester"
+      pgupgrade_image_name: "citus/pgupgradetester"
+      style_checker_image_name: "citus/stylechecker"
+      style_checker_tools_version: "0.8.18"
+      image_suffix: "-vc4b1573"
+      pg13_version: '{ "major": "13", "full": "13.10" }'
+      pg14_version: '{ "major": "14", "full": "14.7" }'
+      pg15_version: '{ "major": "15", "full": "15.2" }'
+      upgrade_pg_versions: "13.10-14.7-15.2"
+    steps:
+      # Since GHA jobs needs at least one step we use a noop step here.
+      - name: Set up parameters
+        run: echo 'noop'
+  check-sql-snapshots:
+    needs: params
+    runs-on: ubuntu-20.04
+    container:
+      image: ${{ needs.params.outputs.build_image_name }}:latest
+      options: --user root
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - name: Check Snapshots
+      run: |
+        git config --global --add safe.directory ${GITHUB_WORKSPACE}
+        ci/check_sql_snapshots.sh
+  check-style:
+    needs: params
+    runs-on: ubuntu-20.04
+    container:
+      image: ${{ needs.params.outputs.style_checker_image_name }}:${{ needs.params.outputs.style_checker_tools_version }}${{ needs.params.outputs.image_suffix }}
+    steps:
+    - name: Check Snapshots
+      run: |
+        git config --global --add safe.directory ${GITHUB_WORKSPACE}
+    - uses: actions/checkout@v3.5.0
+      with:
+        fetch-depth: 0
+    - name: Check C Style
+      run: citus_indent --check
+    - name: Fix whitespace
+      run: ci/editorconfig.sh && git diff --exit-code
+    - name: Remove useless declarations
+      run: ci/remove_useless_declarations.sh && git diff --cached --exit-code
+    - name: Normalize test output
+      run: ci/normalize_expected.sh && git diff --exit-code
+    - name: Check for C-style comments in migration files
+      run: ci/disallow_c_comments_in_migrations.sh && git diff --exit-code
+    - name: 'Check for comment--cached ns that start with # character in spec files'
+      run: ci/disallow_hash_comments_in_spec_files.sh && git diff --exit-code
+    - name: Check for gitignore entries .for source files
+      run: ci/fix_gitignore.sh && git diff --exit-code
+    - name: Check for lengths of changelog entries
+      run: ci/disallow_long_changelog_entries.sh
+    - name: Check for banned C API usage
+      run: ci/banned.h.sh
+    - name: Check for tests missing in schedules
+      run: ci/check_all_tests_are_run.sh
+    - name: Check if all CI scripts are actually run
+      run: ci/check_all_ci_scripts_are_run.sh
+    - name: Check if all GUCs are sorted alphabetically
+      run: ci/check_gucs_are_alphabetically_sorted.sh
+  build:
+    needs: params
+    name: Build for PG${{ fromJson(matrix.pg_version).major }}
+    strategy:
+      fail-fast: false
+      matrix:
+        image_name:
+          - ${{ needs.params.outputs.build_image_name }}
+        image_suffix:
+          - ${{ needs.params.outputs.image_suffix}}
+        pg_version:
+          - ${{ needs.params.outputs.pg13_version }}
+          - ${{ needs.params.outputs.pg14_version }}
+          - ${{ needs.params.outputs.pg15_version }}
+    runs-on: ubuntu-20.04
+    container:
+      image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}"
+      options: --user root
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - name: Expose $PG_MAJOR to Github Env
+      run: echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV
+      shell: bash
+    - name: Build
+      run: "./ci/build-citus.sh"
+      shell: bash
+    - uses: actions/upload-artifact@v3.1.1
+      with:
+        name: build-${{ env.PG_MAJOR }}
+        path: |-
+          ./build-${{ env.PG_MAJOR }}/*
+          ./install-${{ env.PG_MAJOR }}.tar
+  test-citus:
+    name: PG${{ fromJson(matrix.pg_version).major }} - ${{ matrix.make }}
+    strategy:
+      fail-fast: false
+      matrix:
+        suite:
+          - regress
+        image_name:
+          - ${{ needs.params.outputs.test_image_name }}
+        pg_version:
+          - ${{ needs.params.outputs.pg13_version }}
+          - ${{ needs.params.outputs.pg14_version }}
+          - ${{ needs.params.outputs.pg15_version }}
+        make:
+          - check-split
+          - check-multi
+          - check-multi-1
+          - check-multi-mx
+          - check-vanilla
+          - check-isolation
+          - check-operations
+          - check-follower-cluster
+          - check-columnar
+          - check-columnar-isolation
+          - check-enterprise
+          - check-enterprise-isolation
+          - check-enterprise-isolation-logicalrep-1
+          - check-enterprise-isolation-logicalrep-2
+          - check-enterprise-isolation-logicalrep-3
+        include:
+          - make: check-failure
+            pg_version: ${{ needs.params.outputs.pg13_version }}
+            suite: regress
+            image_name: ${{ needs.params.outputs.fail_test_image_name }}
+          - make: check-failure
+            pg_version: ${{ needs.params.outputs.pg14_version }}
+            suite: regress
+            image_name: ${{ needs.params.outputs.fail_test_image_name }}
+          - make: check-failure
+            pg_version: ${{ needs.params.outputs.pg15_version }}
+            suite: regress
+            image_name: ${{ needs.params.outputs.fail_test_image_name }}
+          - make: check-enterprise-failure
+            pg_version: ${{ needs.params.outputs.pg13_version }}
+            suite: regress
+            image_name: ${{ needs.params.outputs.fail_test_image_name }}
+          - make: check-enterprise-failure
+            pg_version: ${{ needs.params.outputs.pg14_version }}
+            suite: regress
+            image_name: ${{ needs.params.outputs.fail_test_image_name }}
+          - make: check-enterprise-failure
+            pg_version: ${{ needs.params.outputs.pg15_version }}
+            suite: regress
+            image_name: ${{ needs.params.outputs.fail_test_image_name }}
+          - make: installcheck
+            suite: recovery
+            image_name: ${{ needs.params.outputs.test_image_name }}
+            pg_version: ${{ needs.params.outputs.pg13_version }}
+          - make: installcheck
+            suite: recovery
+            image_name: ${{ needs.params.outputs.test_image_name }}
+            pg_version: ${{ needs.params.outputs.pg14_version }}
+          - make: installcheck
+            suite: recovery
+            image_name: ${{ needs.params.outputs.test_image_name }}
+            pg_version: ${{ needs.params.outputs.pg15_version }}
+          - make: installcheck
+            suite: columnar_freezing
+            image_name: ${{ needs.params.outputs.test_image_name }}
+            pg_version: ${{ needs.params.outputs.pg13_version }}
+          - make: installcheck
+            suite: columnar_freezing
+            image_name: ${{ needs.params.outputs.test_image_name }}
+            pg_version: ${{ needs.params.outputs.pg14_version }}
+          - make: installcheck
+            suite: columnar_freezing
+            image_name: ${{ needs.params.outputs.test_image_name }}
+            pg_version: ${{ needs.params.outputs.pg15_version }}
+    runs-on: ubuntu-20.04
+    container:
+      image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}"
+      options: --user root --dns=8.8.8.8
+      # Due to Github creates a default network for each job, we need to use
+      # --dns= to have similar DNS settings as our other CI systems or local
+      # machines. Otherwise, we may see different results.
+    needs:
+    - params
+    - build
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - uses: "./.github/actions/setup_extension"
+    - name: Run Test
+      run: gosu circleci make -C src/test/${{ matrix.suite }} ${{ matrix.make }}
+      timeout-minutes: 20
+    - uses: "./.github/actions/save_logs_and_results"
+      if: always()
+      with:
+        folder: ${{ fromJson(matrix.pg_version).major }}_${{ matrix.make }}
+    - uses: "./.github/actions/upload_coverage"
+      if: always()
+      with:
+        flags: ${{ env.PG_MAJOR }}_${{ matrix.suite }}_${{ matrix.make }}
+        codecov_token: ${{ secrets.CODECOV_TOKEN }}
+  test-arbitrary-configs:
+    name: PG${{ fromJson(matrix.pg_version).major }} - check-arbitrary-configs-${{ matrix.parallel }}
+    runs-on: ["self-hosted", "1ES.Pool=1es-gha-citusdata-pool"]
+    container:
+      image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}"
+      options: --user root
+    needs:
+      - params
+      - build
+    strategy:
+      fail-fast: false
+      matrix:
+        image_name:
+          - ${{ needs.params.outputs.fail_test_image_name }}
+        pg_version:
+          - ${{ needs.params.outputs.pg13_version }}
+          - ${{ needs.params.outputs.pg14_version }}
+          - ${{ needs.params.outputs.pg15_version }}
+        parallel: [0,1,2,3,4,5] # workaround for running 6 parallel jobs
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - uses: "./.github/actions/setup_extension"
+    - name: Test arbitrary configs
+      run: |-
+        # we use parallel jobs to split the tests into 6 parts and run them in parallel
+        # the script below extracts the tests for the current job
+        N=6  # Total number of jobs (see matrix.parallel)
+        X=${{ matrix.parallel }}  # Current job number
+        TESTS=$(src/test/regress/citus_tests/print_test_names.py |
+          tr '\n' ',' | awk -v N="$N" -v X="$X" -F, '{
+            split("", parts)
+            for (i = 1; i <= NF; i++) {
+                parts[i % N] = parts[i % N] $i ","
+            }
+            print substr(parts[X], 1, length(parts[X])-1)
+        }')
+        echo $TESTS
+        gosu circleci \
+          make -C src/test/regress \
+            check-arbitrary-configs parallel=4 CONFIGS=$TESTS
+    - uses: "./.github/actions/save_logs_and_results"
+      if: always()
+    - uses: "./.github/actions/upload_coverage"
+      if: always()
+      with:
+        flags: ${{ env.pg_major }}_upgrade
+        codecov_token: ${{ secrets.CODECOV_TOKEN }}
+  test-pg-upgrade:
+    name: PG${{ matrix.old_pg_major }}-PG${{ matrix.new_pg_major }} - check-pg-upgrade
+    runs-on: ubuntu-20.04
+    container:
+      image: "${{ needs.params.outputs.pgupgrade_image_name }}:${{ needs.params.outputs.upgrade_pg_versions }}${{ needs.params.outputs.image_suffix }}"
+      options: --user root
+    needs:
+    - params
+    - build
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - old_pg_major: 13
+            new_pg_major: 14
+          - old_pg_major: 14
+            new_pg_major: 15
+    env:
+      old_pg_major: ${{ matrix.old_pg_major }}
+      new_pg_major: ${{ matrix.new_pg_major }}
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - uses: "./.github/actions/setup_extension"
+      with:
+        pg_major: "${{ env.old_pg_major }}"
+    - uses: "./.github/actions/setup_extension"
+      with:
+        pg_major: "${{ env.new_pg_major }}"
+    - name: Install and test postgres upgrade
+      run: |-
+        gosu circleci \
+          make -C src/test/regress \
+            check-pg-upgrade \
+            old-bindir=/usr/lib/postgresql/${{ env.old_pg_major }}/bin \
+            new-bindir=/usr/lib/postgresql/${{ env.new_pg_major }}/bin
+    - name: Copy pg_upgrade logs for newData dir
+      run: |-
+        mkdir -p /tmp/pg_upgrade_newData_logs
+        if ls src/test/regress/tmp_upgrade/newData/*.log 1> /dev/null 2>&1; then
+            cp src/test/regress/tmp_upgrade/newData/*.log /tmp/pg_upgrade_newData_logs
+        fi
+      if: failure()
+    - uses: "./.github/actions/save_logs_and_results"
+      if: always()
+    - uses: "./.github/actions/upload_coverage"
+      if: always()
+      with:
+        flags: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade
+        codecov_token: ${{ secrets.CODECOV_TOKEN }}
+  test-citus-upgrade:
+    name: PG${{ fromJson(needs.params.outputs.pg13_version).major }} - check-citus-upgrade
+    runs-on: ubuntu-20.04
+    container:
+      image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ fromJson(needs.params.outputs.pg13_version).full }}${{ needs.params.outputs.image_suffix }}"
+      options: --user root
+    needs:
+    - params
+    - build
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - uses: "./.github/actions/setup_extension"
+      with:
+        skip_installation: true
+    - name: Install and test citus upgrade
+      run: |-
+        # run make check-citus-upgrade for all citus versions
+        # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
+        for citus_version in ${CITUS_VERSIONS}; do \
+          gosu circleci \
+            make -C src/test/regress \
+              check-citus-upgrade \
+              bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
+              citus-old-version=${citus_version} \
+              citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \
+              citus-post-tar=${GITHUB_WORKSPACE}/install-$PG_MAJOR.tar; \
+        done;
+        # run make check-citus-upgrade-mixed for all citus versions
+        # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
+        for citus_version in ${CITUS_VERSIONS}; do \
+          gosu circleci \
+            make -C src/test/regress \
+              check-citus-upgrade-mixed \
+              citus-old-version=${citus_version} \
+              bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
+              citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \
+              citus-post-tar=${GITHUB_WORKSPACE}/install-$PG_MAJOR.tar; \
+        done;
+    - uses: "./.github/actions/save_logs_and_results"
+      if: always()
+    - uses: "./.github/actions/upload_coverage"
+      if: always()
+      with:
+        flags: ${{ env.pg_major }}_upgrade
+        codecov_token: ${{ secrets.CODECOV_TOKEN }}
+  ch_benchmark:
+    name: CH Benchmark
+    if: startsWith(github.ref, 'refs/heads/ch_benchmark/')
+    runs-on: ubuntu-20.04
+    needs:
+    - build
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - uses: azure/login@v1
+      with:
+        creds: ${{ secrets.AZURE_CREDENTIALS }}
+    - name: install dependencies and run ch_benchmark tests
+      uses: azure/CLI@v1
+      with:
+        inlineScript: |
+          cd ./src/test/hammerdb
+          chmod +x run_hammerdb.sh
+          run_hammerdb.sh citusbot_ch_benchmark_rg
+  tpcc_benchmark:
+    name: TPCC Benchmark
+    if: startsWith(github.ref, 'refs/heads/tpcc_benchmark/')
+    runs-on: ubuntu-20.04
+    needs:
+    - build
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - uses: azure/login@v1
+      with:
+        creds: ${{ secrets.AZURE_CREDENTIALS }}
+    - name: install dependencies and run tpcc_benchmark tests
+      uses: azure/CLI@v1
+      with:
+        inlineScript: |
+          cd ./src/test/hammerdb
+          chmod +x run_hammerdb.sh
+          run_hammerdb.sh citusbot_tpcc_benchmark_rg
+  prepare_parallelization_matrix_32:
+    name: Parallel 32
+    if: ${{ needs.test-flakyness-pre.outputs.tests != ''}}
+    needs: test-flakyness-pre
+    runs-on: ubuntu-20.04
+    outputs:
+      json: ${{ steps.parallelization.outputs.json }}
+    steps:
+      - uses: actions/checkout@v3.5.0
+      - uses: "./.github/actions/parallelization"
+        id: parallelization
+        with:
+          count: 32
+  test-flakyness-pre:
+    name: Detect regression tests need to be ran
+    if: ${{ !inputs.skip_test_flakyness }}}
+    runs-on: ubuntu-20.04
+    needs: build
+    outputs:
+      tests: ${{ steps.detect-regression-tests.outputs.tests }}
+    steps:
+    - uses: actions/checkout@v3.5.0
+      with:
+        fetch-depth: 0
+    - name: Detect regression tests need to be ran
+      id: detect-regression-tests
+      run: |-
+        detected_changes=$(git diff origin/release-11.1... --name-only --diff-filter=AM | (grep 'src/test/regress/sql/.*\.sql\|src/test/regress/spec/.*\.spec\|src/test/regress/citus_tests/test/test_.*\.py' || true))
+        tests=${detected_changes}
+        if [ -z "$tests" ]; then
+            echo "No test found."
+        else
+          echo "Detected tests " $tests
+        fi
+
+        echo 'tests<<EOF' >> $GITHUB_OUTPUT
+        echo "$tests" >> "$GITHUB_OUTPUT"
+        echo 'EOF' >> $GITHUB_OUTPUT
+  test-flakyness:
+    if: false
+    name: Test flakyness
+    runs-on: ubuntu-20.04
+    container:
+      image: ${{ needs.params.outputs.fail_test_image_name }}:${{ needs.params.outputs.pg15_version }}${{ needs.params.outputs.image_suffix }}
+      options: --user root
+    env:
+      runs: 8
+    needs:
+    - params
+    - build
+    - test-flakyness-pre
+    - prepare_parallelization_matrix_32
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.prepare_parallelization_matrix_32.outputs.json) }}
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - uses: actions/download-artifact@v3.0.1
+    - uses: "./.github/actions/setup_extension"
+    - name: Run minimal tests
+      run: |-
+        tests="${{ needs.test-flakyness-pre.outputs.tests }}"
+        tests_array=($tests)
+        for test in "${tests_array[@]}"
+        do
+            test_name=$(echo "$test" | sed -r "s/.+\/(.+)\..+/\1/")
+            gosu circleci src/test/regress/citus_tests/run_test.py $test_name --repeat ${{ env.runs }} --use-base-schedule --use-whole-schedule-line
+        done
+      shell: bash
+    - uses: "./.github/actions/save_logs_and_results"
+      if: always()

.github/workflows/flaky_test_debugging.yml

@@ -0,0 +1,79 @@
+name: Flaky test debugging
+run-name: Flaky test debugging - ${{ inputs.flaky_test }} (${{ inputs.flaky_test_runs_per_job }}x${{ inputs.flaky_test_parallel_jobs }})
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+on:
+  workflow_dispatch:
+    inputs:
+      flaky_test:
+        required: true
+        type: string
+        description: Test to run
+      flaky_test_runs_per_job:
+        required: false
+        default: 8
+        type: number
+        description: Number of times to run the test
+      flaky_test_parallel_jobs:
+        required: false
+        default: 32
+        type: number
+        description: Number of parallel jobs to run
+jobs:
+  build:
+    name: Build Citus
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ vars.build_image_name }}:${{ vars.pg15_version  }}${{ vars.image_suffix }}
+      options: --user root
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - name: Configure, Build, and Install
+      run: |
+        echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV
+        ./ci/build-citus.sh
+      shell: bash
+    - uses: actions/upload-artifact@v3.1.1
+      with:
+        name: build-${{ env.PG_MAJOR }}
+        path: |-
+          ./build-${{ env.PG_MAJOR }}/*
+          ./install-${{ env.PG_MAJOR }}.tar
+  prepare_parallelization_matrix:
+    name: Prepare parallelization matrix
+    runs-on: ubuntu-latest
+    outputs:
+      json: ${{ steps.parallelization.outputs.json }}
+    steps:
+      - uses: actions/checkout@v3.5.0
+      - uses: "./.github/actions/parallelization"
+        id: parallelization
+        with:
+          count: ${{ inputs.flaky_test_parallel_jobs }}
+  test_flakyness:
+    name: Test flakyness
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ vars.fail_test_image_name }}:${{ vars.pg15_version  }}${{ vars.image_suffix }}
+      options: --user root
+    needs:
+      [build, prepare_parallelization_matrix]
+    env:
+      test: "${{ inputs.flaky_test }}"
+      runs: "${{ inputs.flaky_test_runs_per_job }}"
+      skip: false
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.prepare_parallelization_matrix.outputs.json) }}
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - uses: "./.github/actions/setup_extension"
+    - name: Run minimal tests
+      run: |-
+          gosu circleci src/test/regress/citus_tests/run_test.py ${{ env.test }} --repeat ${{ env.runs }} --use-base-schedule --use-whole-schedule-line
+      shell: bash
+    - uses: "./.github/actions/save_logs_and_results"
+      if: always()
+      with:
+          folder: ${{ matrix.id }}

.github/workflows/packaging-test-pipelines.yml

@@ -0,0 +1,161 @@
+name: Build tests in packaging images
+
+on:
+  push:
+    branches: "**"
+
+  workflow_dispatch:
+
+jobs:
+
+  get_postgres_versions_from_file:
+    runs-on: ubuntu-latest
+    outputs:
+      pg_versions: ${{ steps.get-postgres-versions.outputs.pg_versions }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      - name: Get Postgres Versions
+        id: get-postgres-versions
+        run: |
+          set -euxo pipefail
+          # Postgres versions are stored in .github/workflows/build_and_test.yml
+          # file in json strings with major and full keys.
+          # Below command extracts the versions and get the unique values.
+          pg_versions=$(cat .github/workflows/build_and_test.yml | grep -oE '"major": "[0-9]+", "full": "[0-9.]+"' | sed -E 's/"major": "([0-9]+)", "full": "([0-9.]+)"/\1/g' | sort | uniq | tr '\n', ',')
+          pg_versions_array="[ ${pg_versions} ]"
+          echo "Supported PG Versions: ${pg_versions_array}"
+          # Below line is needed to set the output variable to be used in the next job
+          echo "pg_versions=${pg_versions_array}" >> $GITHUB_OUTPUT
+        shell: bash
+  rpm_build_tests:
+    name: rpm_build_tests
+    needs: get_postgres_versions_from_file
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # While we use separate images for different Postgres versions in rpm
+        # based distros
+        # For this reason, we need to use a "matrix" to generate names of
+        # rpm images, e.g. citus/packaging:centos-7-pg12
+        packaging_docker_image:
+          - oraclelinux-7
+          - oraclelinux-8
+          - centos-7
+          - almalinux-8
+          - almalinux-9
+        POSTGRES_VERSION: ${{ fromJson(needs.get_postgres_versions_from_file.outputs.pg_versions) }}
+
+    container:
+      image: citus/packaging:${{ matrix.packaging_docker_image }}-pg${{ matrix.POSTGRES_VERSION }}
+      options: --user root
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set Postgres and python parameters for rpm based distros
+        run: |
+          echo "/usr/pgsql-${{ matrix.POSTGRES_VERSION }}/bin" >> $GITHUB_PATH
+          echo "/root/.pyenv/bin:$PATH" >> $GITHUB_PATH
+          echo "PACKAGING_PYTHON_VERSION=3.8.16" >> $GITHUB_ENV
+
+      - name: Configure
+        run: |
+          echo "Current Shell:$0"
+          echo "GCC Version: $(gcc --version)"
+          ./configure 2>&1 | tee output.log
+
+      - name: Make clean
+        run: |
+          make clean
+
+      - name: Make
+        run: |
+          make CFLAGS="-Wno-missing-braces" -sj$(cat /proc/cpuinfo | grep "core id" | wc -l) 2>&1 | tee -a output.log
+
+      - name: Make install
+        run: |
+          make CFLAGS="-Wno-missing-braces" install 2>&1 | tee -a output.log
+
+      - name: Validate output
+        env:
+          POSTGRES_VERSION: ${{ matrix.POSTGRES_VERSION }}
+          PACKAGING_DOCKER_IMAGE: ${{ matrix.packaging_docker_image }}
+        run: |
+          echo "Postgres version: ${POSTGRES_VERSION}"
+
+          ## Install required packages to execute packaging tools for rpm based distros
+          yum install python3-pip python3-devel postgresql-devel -y
+          python3 -m pip install wheel
+
+          ./.github/packaging/validate_build_output.sh "rpm"
+
+  deb_build_tests:
+    name: deb_build_tests
+    needs: get_postgres_versions_from_file
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # On deb based distros, we use the same docker image for
+        # builds based on different Postgres versions because deb
+        # based images include all postgres installations.
+        # For this reason, we have multiple runs --which is 3 today--
+        # for each deb based image and we use POSTGRES_VERSION to set
+        # PG_CONFIG variable in each of those runs.
+        packaging_docker_image:
+          - debian-buster-all
+          - debian-bookworm-all
+          - debian-bullseye-all
+          - ubuntu-focal-all
+          - ubuntu-jammy-all
+
+        POSTGRES_VERSION: ${{ fromJson(needs.get_postgres_versions_from_file.outputs.pg_versions) }}
+
+    container:
+      image: citus/packaging:${{ matrix.packaging_docker_image }}
+      options: --user root
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set pg_config path and python parameters for deb based distros
+        run: |
+          echo "PG_CONFIG=/usr/lib/postgresql/${{ matrix.POSTGRES_VERSION }}/bin/pg_config" >> $GITHUB_ENV
+          echo "/root/.pyenv/bin:$PATH" >> $GITHUB_PATH
+          echo "PACKAGING_PYTHON_VERSION=3.8.16" >> $GITHUB_ENV
+
+      - name: Configure
+        run: |
+          echo "Current Shell:$0"
+          echo "GCC Version: $(gcc --version)"
+          ./configure 2>&1 | tee output.log
+
+      - name: Make clean
+        run: |
+          make clean
+
+      - name: Make
+        run: |
+          make -sj$(cat /proc/cpuinfo | grep "core id" | wc -l) 2>&1 | tee -a output.log
+
+      - name: Make install
+        run: |
+          make install 2>&1 | tee -a output.log
+
+      - name: Validate output
+        env:
+          POSTGRES_VERSION: ${{ matrix.POSTGRES_VERSION }}
+          PACKAGING_DOCKER_IMAGE: ${{ matrix.packaging_docker_image }}
+        run: |
+          echo "Postgres version: ${POSTGRES_VERSION}"
+
+          sudo apt-get update -y || true
+          ## Install required packages to execute packaging tools for deb based distros
+          sudo apt-get purge -y python3-yaml
+          ./.github/packaging/validate_build_output.sh "deb"

CHANGELOG.md

@@ -1,3 +1,233 @@
+### citus v11.1.7 (February 12, 2024) ###
+
+* Fixes memory and memory context leaks in Foreign Constraint Graphs (#7236)
+
+* Fixes a bug related to non-existent objects in DDL commands (#6984)
+
+* Fixes a bug that could cause COPY logic to skip data in case of OOM (#7152)
+
+* Fixes a bug with deleting colocation groups (#6929)
+
+* Fixes incorrect results on fetching scrollable with hold cursors (#7014)
+
+* Fixes the incorrect column count after ALTER TABLE (#7379)
+
+* Improves failure handling of distributed execution (#7090)
+
+* Makes sure to disallow creating a replicated distributed table
+  concurrently (#7219)
+
+* Removes pg_send_cancellation and all references (#7135)
+
+### citus v11.1.6 (April 20, 2023) ###
+
+* Correctly reports shard size in `citus_shards` view (#6748)
+
+* Fixes a bug in shard copy operations (#6721)
+
+* Fixes a bug that breaks pg upgrades if the user has a columnar table (#6624)
+
+* Fixes a bug that causes background rebalancer to fail when a reference table
+  doesn't have a primary key (#6682)
+
+* Fixes a regression in allowed foreign keys on distributed tables (#6550)
+
+* Fixes a use-after-free bug in connection management (#6685)
+
+* Fixes an unexpected foreign table error by disallowing to drop the
+  `table_name` option (#6669)
+
+* Fixes an uninitialized memory access in shard split API (#6845)
+
+* Fixes compilation for PG13.10 and PG14.7 (#6711)
+
+* Fixes crash that happens when trying to replicate a reference table that is
+  actually dropped (#6595)
+
+* Fixes memory leak issue with query results that returns single row (#6724)
+
+* Fixes the modifiers for subscription and role creation (#6603)
+
+* Makes sure to quote all identifiers used for logical replication to prevent
+  potential issues (#6604)
+
+* Makes sure to skip foreign key validations at the end of shard moves (#6640)
+
+### citus v11.1.5 (December 12, 2022) ###
+
+* Fixes two potential dangling pointer issues
+
+* Fixes compilation warning on PG13 + OpenSSL 3.0
+
+### citus v11.1.4 (October 24, 2022) ###
+
+* Fixes an upgrade problem for `worker_fetch_foreign_file` when upgrade path
+  starts from 8.3 up to 11.1
+
+* Fixes an upgrade problem for `worker_repartition_cleanup` when upgrade path
+  starts from 9.1 up to 11.1
+
+### citus v11.1.3 (October 14, 2022) ###
+
+* Adds support for PostgreSQL 15.0
+
+* Fixes a bug in `ALTER EXTENSION citus UPDATE`
+
+* Fixes a bug that causes a crash with empty/null password
+
+* Fixes a bug that causes not retaining trigger enable/disable settings when
+  re-creating them on shards
+
+* Fixes a bug that prevents retaining columnar table options after a
+  table-rewrite
+
+* Raises memory limits in columnar from 256MB to 1GB for reads and writes
+
+### citus v11.1.2 (September 30, 2022) ###
+
+* Adds support for PostgreSQL 15rc1
+
+* Disallows having `ON DELETE/UPDATE SET DEFAULT` actions on columns that
+  default to sequences
+
+* Fixes a bug that might cause inserting incorrect `DEFAULT` values when
+  applying foreign key actions
+
+* Fixes a performance issue related to shard-moves by creating replica
+  identities before copying shards
+
+* Improves logging during shard-splits and resource cleanup
+
+* Makes sure to reuse connections for shard-splits and logical replication
+
+* Makes sure to try dropping replication slots a few more times after a failure
+  at the end of the shard-split
+
+### citus v11.1.1 (September 16, 2022) ###
+
+* Fixes a bug that prevents `create_distributed_table_concurrently()` working
+  on an empty node
+
+### citus v11.1.0 (September 15, 2022) ###
+
+* Adds support for PostgreSQL 15beta4
+
+* Adds ability to run shard rebalancer in the background
+
+* Adds `create_distributed_table_concurrently()` UDF to distribute tables
+  without interrupting the application
+
+* Adds `citus_split_shard_by_split_points()` UDF that allows
+  splitting a shard to specified set of nodes without blocking writes
+  and based on given split points
+
+* Adds support for non-blocking tenant isolation
+
+* Adds support for isolation tenants that use partitioned tables
+  or columnar tables
+
+* Separates columnar table access method into a separate logical extension
+
+* Adds support for online replication in `replicate_reference_tables()`
+
+* Improves performance of blocking shard moves
+
+* Improves non-blocking shard moves with a faster custom copy logic
+
+* Creates all foreign keys quickly at the end of a shard move
+
+* Limits `get_rebalance_progress()` to show shards in moving state
+
+* Makes `citus_move_shard_placement()` idempotent if shard already exists
+  on target node
+
+* Shows `citus_copy_shard_placement()` progress in `get_rebalance_progres()`
+
+* Supports changing CPU priorities for backends and shard moves
+
+* Adds the GUC `citus.allow_unsafe_constraints` to allow unique/exclusion/
+  primary key constraints without distribution column
+
+* Introduces GUC `citus.skip_constraint_validation`
+
+* Introduces `citus_locks` view
+
+* Improves `citus_tables` view by showing local tables added to metadata
+
+* Improves columnar table access method by moving old catalog tables into
+  an internal schema and introduces more secure & informative views based
+  on them
+
+* Adds support for `GRANT/REVOKE` on aggregates
+
+* Adds support for `NULLS NOT DISTINCT` clauses for indexes for PG15+
+
+* Adds support for setting relation options for columnar tables using
+  `ALTER TABLE`
+
+* Adds support for unlogged distributed sequences
+
+* Removes `do_repair` option from `citus_copy_shard_placement()`
+
+* Removes deprecated re-partitioning functions like
+  `worker_hash_partition_table()`
+
+* Drops support for isolation tenants that use replicated tables
+
+* Checks existence of the shards before insert, delete, and update
+
+* Hides tables owned by extensions from `citus_tables` and `citus_shards`
+
+* Propagates `VACUUM` and `ANALYZE` to worker nodes
+
+* Makes non-partitioned table size calculation quicker
+
+* Improves `create_distributed_table()` by creating new colocation entries when
+  using `colocate_with => 'none'`
+
+* Ensures that `SELECT .. FOR UPDATE` opens a transaction block when used in
+  a function call
+
+* Prevents a segfault by disallowing usage of SQL functions referencing to a
+  distributed table
+
+* Prevents creating a new colocation entry when replicating reference tables
+
+* Fixes a bug in query escaping in `undistribute_table()` and
+  `alter_distributed_table()`
+
+* Fixes a bug preventing the usage of `isolate_tenant_to_new_shard()` with text
+  column
+
+* Fixes a bug that may cause `GRANT` to propagate within `CREATE EXTENSION`
+
+* Fixes a bug that causes incorrectly marking `metadatasynced` flag for
+  coordinator
+
+* Fixes a bug that may prevent Citus from creating function in transaction
+  block properly
+
+* Fixes a bug that prevents promoting read-replicas as primaries
+
+* Fixes a bug that prevents setting colocation group of a partitioned
+  distributed table to `none`
+
+* Fixes a bug that prevents using `AUTO` option for `VACUUM (INDEX_CLEANUP)`
+  operation
+
+* Fixes a segfault in `citus_copy_shard_placement()`
+
+* Fixes an issue that can cause logical reference table replication to fail
+
+* Fixes schema name qualification for `RENAME SEQUENCE` statement
+
+* Fixes several small memory leaks
+
+* Fixes the transaction timestamp column of the `get_current_transaction_id()`
+  on coordinator
+
+* Maps any unused parameters to a generic type in prepared statements
+
 ### citus v10.2.8 (August 19, 2022) ###
 
 * Fixes compilation warning caused by latest upgrade script changes

Makefile

@@ -11,7 +11,7 @@ endif
 
 include Makefile.global
 
-all: extension pg_send_cancellation
+all: extension
 
 
 # build columnar only
@@ -40,22 +40,14 @@ clean-full:
 
 install-downgrades:
 	$(MAKE) -C src/backend/distributed/ install-downgrades
-install-all: install-headers install-pg_send_cancellation
+install-all: install-headers
 	$(MAKE) -C src/backend/columnar/ install-all
 	$(MAKE) -C src/backend/distributed/ install-all
 
-# build citus_send_cancellation binary
-pg_send_cancellation:
-	$(MAKE) -C src/bin/pg_send_cancellation/ all
-install-pg_send_cancellation: pg_send_cancellation
-	$(MAKE) -C src/bin/pg_send_cancellation/ install
-clean-pg_send_cancellation:
-	$(MAKE) -C src/bin/pg_send_cancellation/ clean
-.PHONY: pg_send_cancellation install-pg_send_cancellation clean-pg_send_cancellation
 
 # Add to generic targets
-install: install-extension install-headers install-pg_send_cancellation
-clean: clean-extension clean-pg_send_cancellation
+install: install-extension install-headers
+clean: clean-extension
 
 # apply or check style
 reindent:

ci/build-citus.sh

@@ -15,9 +15,6 @@ PG_MAJOR=${PG_MAJOR:?please provide the postgres major version}
 codename=${VERSION#*(}
 codename=${codename%)*}
 
-# get project from argument
-project="${CIRCLE_PROJECT_REPONAME}"
-
 # we'll do everything with absolute paths
 basedir="$(pwd)"
 
@@ -28,7 +25,7 @@ build_ext() {
   pg_major="$1"
 
   builddir="${basedir}/build-${pg_major}"
-  echo "Beginning build of ${project} for PostgreSQL ${pg_major}..." >&2
+  echo "Beginning build for PostgreSQL ${pg_major}..." >&2
 
   # do everything in a subdirectory to avoid clutter in current directory
   mkdir -p "${builddir}" && cd "${builddir}"

ci/check_all_ci_scripts_are_run.sh

@@ -14,8 +14,8 @@ ci_scripts=$(
     grep -v -E '^(ci_helpers.sh|fix_style.sh)$'
 )
 for script in $ci_scripts; do
-    if ! grep "\\bci/$script\\b" .circleci/config.yml > /dev/null; then
-        echo "ERROR: CI script with name \"$script\" is not actually used in .circleci/config.yml"
+    if ! grep "\\bci/$script\\b" -r .github > /dev/null; then
+        echo "ERROR: CI script with name \"$script\" is not actually used in .github folder"
         exit 1
     fi
     if ! grep "^## \`$script\`\$" ci/README.md > /dev/null; then

ci/check_enterprise_merge.sh

@@ -1,96 +0,0 @@
-#!/bin/bash
-
-# Testing this script locally requires you to set the following environment
-# variables:
-# CIRCLE_BRANCH, GIT_USERNAME and GIT_TOKEN
-
-# fail if trying to reference a variable that is not set.
-set -u
-# exit immediately if a command fails
-set -e
-# Fail on pipe failures
-set -o pipefail
-
-PR_BRANCH="${CIRCLE_BRANCH}"
-ENTERPRISE_REMOTE="https://${GIT_USERNAME}:${GIT_TOKEN}@github.com/citusdata/citus-enterprise"
-
-# shellcheck disable=SC1091
-source ci/ci_helpers.sh
-
-# List executed commands. This is done so debugging this script is easier when
-# it fails. It's explicitly done after git remote add so username and password
-# are not shown in CI output (even though it's also filtered out by CircleCI)
-set -x
-
-check_compile () {
-    echo "INFO: checking if merged code can be compiled"
-    ./configure --without-libcurl
-    make -j10
-}
-
-# Clone current git repo (which should be community) to a temporary working
-# directory and go there
-GIT_DIR_ROOT="$(git rev-parse --show-toplevel)"
-TMP_GIT_DIR="$(mktemp --directory -t citus-merge-check.XXXXXXXXX)"
-git clone "$GIT_DIR_ROOT" "$TMP_GIT_DIR"
-cd "$TMP_GIT_DIR"
-
-# Fails in CI without this
-git config user.email "citus-bot@microsoft.com"
-git config user.name "citus bot"
-
-# Disable "set -x" temporarily, because $ENTERPRISE_REMOTE contains passwords
-{ set +x ; } 2> /dev/null
-git remote add enterprise "$ENTERPRISE_REMOTE"
-set -x
-
-git remote set-url --push enterprise no-pushing
-
-# Fetch enterprise-master
-git fetch enterprise enterprise-master
-
-
-git checkout "enterprise/enterprise-master"
-
-if git merge --no-commit "origin/$PR_BRANCH"; then
-    echo "INFO: community PR branch could be merged into enterprise-master"
-    # check that we can compile after the merge
-    if check_compile; then
-        exit 0
-    fi
-
-    echo "WARN: Failed to compile after community PR branch was merged into enterprise"
-fi
-
-# undo partial merge
-git merge --abort
-
-# If we have a conflict on enterprise merge on the master branch, we have a problem.
-# Provide an error message to indicate that enterprise merge is needed to fix this check.
-if [[ $PR_BRANCH = master ]]; then
-    echo "ERROR: Master branch has merge conflicts with enterprise-master."
-    echo "Try re-running this CI job after merging your changes into enterprise-master."
-    exit 1
-fi
-
-if ! git fetch enterprise "$PR_BRANCH" ; then
-    echo "ERROR: enterprise/$PR_BRANCH was not found and community PR branch could not be merged into enterprise-master"
-    exit 1
-fi
-
-# Show the top commit of the enterprise PR branch to make debugging easier
-git log -n 1 "enterprise/$PR_BRANCH"
-
-# Check that this branch contains the top commit of the current community PR
-# branch. If it does not it means it's not up to date with the current PR, so
-# the enterprise branch should be updated.
-if ! git merge-base --is-ancestor "origin/$PR_BRANCH" "enterprise/$PR_BRANCH" ; then
-    echo "ERROR: enterprise/$PR_BRANCH is not up to date with community PR branch"
-    exit 1
-fi
-
-# Now check if we can merge the enterprise PR into enterprise-master without
-# issues.
-git merge --no-commit "enterprise/$PR_BRANCH"
-# check that we can compile after the merge
-check_compile

ci/fix_gitignore.sh

@@ -5,7 +5,8 @@ source ci/ci_helpers.sh
 
 # Remove all the ignored files from git tree, and error out
 # find all ignored files in git tree, and use quotation marks to prevent word splitting on filenames with spaces in them
-ignored_lines_in_git_tree=$(git ls-files --ignored --exclude-standard | sed 's/.*/"&"/')
+# NOTE: Option --cached is needed to avoid a bug in git ls-files command.
+ignored_lines_in_git_tree=$(git ls-files --ignored --cached --exclude-standard | sed 's/.*/"&"/')
 
 if [[ -n $ignored_lines_in_git_tree ]]
 then

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for Citus 11.1devel.
+# Generated by GNU Autoconf 2.69 for Citus 11.1.7.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -579,8 +579,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='Citus'
 PACKAGE_TARNAME='citus'
-PACKAGE_VERSION='11.1devel'
-PACKAGE_STRING='Citus 11.1devel'
+PACKAGE_VERSION='11.1.7'
+PACKAGE_STRING='Citus 11.1.7'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -1262,7 +1262,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures Citus 11.1devel to adapt to many kinds of systems.
+\`configure' configures Citus 11.1.7 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1324,7 +1324,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of Citus 11.1devel:";;
+     short | recursive ) echo "Configuration of Citus 11.1.7:";;
    esac
   cat <<\_ACEOF
 
@@ -1429,7 +1429,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-Citus configure 11.1devel
+Citus configure 11.1.7
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1912,7 +1912,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by Citus $as_me 11.1devel, which was
+It was created by Citus $as_me 11.1.7, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -5393,7 +5393,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by Citus $as_me 11.1devel, which was
+This file was extended by Citus $as_me 11.1.7, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -5455,7 +5455,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-Citus config.status 11.1devel
+Citus config.status 11.1.7
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -5,7 +5,7 @@
 # everyone needing autoconf installed, the resulting files are checked
 # into the SCM.
 
-AC_INIT([Citus], [11.1devel])
+AC_INIT([Citus], [11.1.7])
 AC_COPYRIGHT([Copyright (c) Citus Data, Inc.])
 
 # we'll need sed and awk for some of the version commands

src/backend/columnar/columnar_metadata.c

@@ -60,6 +60,10 @@
 #include "utils/relfilenodemap.h"
 
 #define COLUMNAR_RELOPTION_NAMESPACE "columnar"
+#define SLOW_METADATA_ACCESS_WARNING \
+	"Metadata index %s is not available, this might mean slower read/writes " \
+	"on columnar tables. This is expected during Postgres upgrades and not " \
+	"expected otherwise."
 
 typedef struct
 {
@@ -701,15 +705,23 @@ ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescri
 
 	Oid columnarChunkOid = ColumnarChunkRelationId();
 	Relation columnarChunk = table_open(columnarChunkOid, AccessShareLock);
-	Relation index = index_open(ColumnarChunkIndexRelationId(), AccessShareLock);
 
 	ScanKeyInit(&scanKey[0], Anum_columnar_chunk_storageid,
 				BTEqualStrategyNumber, F_OIDEQ, UInt64GetDatum(storageId));
 	ScanKeyInit(&scanKey[1], Anum_columnar_chunk_stripe,
 				BTEqualStrategyNumber, F_OIDEQ, Int32GetDatum(stripe));
 
-	SysScanDesc scanDescriptor = systable_beginscan_ordered(columnarChunk, index,
-															snapshot, 2, scanKey);
+	Oid indexId = ColumnarChunkIndexRelationId();
+	bool indexOk = OidIsValid(indexId);
+	SysScanDesc scanDescriptor = systable_beginscan(columnarChunk, indexId,
+													indexOk, snapshot, 2, scanKey);
+
+	static bool loggedSlowMetadataAccessWarning = false;
+	if (!indexOk && !loggedSlowMetadataAccessWarning)
+	{
+		ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING, "chunk_pkey")));
+		loggedSlowMetadataAccessWarning = true;
+	}
 
 	StripeSkipList *chunkList = palloc0(sizeof(StripeSkipList));
 	chunkList->chunkCount = chunkCount;
@@ -721,8 +733,7 @@ ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescri
 			palloc0(chunkCount * sizeof(ColumnChunkSkipNode));
 	}
 
-	while (HeapTupleIsValid(heapTuple = systable_getnext_ordered(scanDescriptor,
-																 ForwardScanDirection)))
+	while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor)))
 	{
 		Datum datumArray[Natts_columnar_chunk];
 		bool isNullArray[Natts_columnar_chunk];
@@ -787,8 +798,7 @@ ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescri
 		}
 	}
 
-	systable_endscan_ordered(scanDescriptor);
-	index_close(index, AccessShareLock);
+	systable_endscan(scanDescriptor);
 	table_close(columnarChunk, AccessShareLock);
 
 	chunkList->chunkGroupRowCounts =
@@ -799,9 +809,9 @@ ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescri
 
 
 /*
- * FindStripeByRowNumber returns StripeMetadata for the stripe whose
- * firstRowNumber is greater than given rowNumber. If no such stripe
- * exists, then returns NULL.
+ * FindStripeByRowNumber returns StripeMetadata for the stripe that has the
+ * smallest firstRowNumber among the stripes whose firstRowNumber is grater
+ * than given rowNumber. If no such stripe exists, then returns NULL.
  */
 StripeMetadata *
 FindNextStripeByRowNumber(Relation relation, uint64 rowNumber, Snapshot snapshot)
@@ -891,8 +901,7 @@ StripeGetHighestRowNumber(StripeMetadata *stripeMetadata)
 /*
  * StripeMetadataLookupRowNumber returns StripeMetadata for the stripe whose
  * firstRowNumber is less than or equal to (FIND_LESS_OR_EQUAL), or is
- * greater than (FIND_GREATER) given rowNumber by doing backward index
- * scan on stripe_first_row_number_idx.
+ * greater than (FIND_GREATER) given rowNumber.
  * If no such stripe exists, then returns NULL.
  */
 static StripeMetadata *
@@ -923,31 +932,71 @@ StripeMetadataLookupRowNumber(Relation relation, uint64 rowNumber, Snapshot snap
 	ScanKeyInit(&scanKey[1], Anum_columnar_stripe_first_row_number,
 				strategyNumber, procedure, UInt64GetDatum(rowNumber));
 
-
 	Relation columnarStripes = table_open(ColumnarStripeRelationId(), AccessShareLock);
-	Relation index = index_open(ColumnarStripeFirstRowNumberIndexRelationId(),
-								AccessShareLock);
-	SysScanDesc scanDescriptor = systable_beginscan_ordered(columnarStripes, index,
-															snapshot, 2,
-															scanKey);
 
-	ScanDirection scanDirection = NoMovementScanDirection;
-	if (lookupMode == FIND_LESS_OR_EQUAL)
+	Oid indexId = ColumnarStripeFirstRowNumberIndexRelationId();
+	bool indexOk = OidIsValid(indexId);
+	SysScanDesc scanDescriptor = systable_beginscan(columnarStripes, indexId, indexOk,
+													snapshot, 2, scanKey);
+
+	static bool loggedSlowMetadataAccessWarning = false;
+	if (!indexOk && !loggedSlowMetadataAccessWarning)
 	{
-		scanDirection = BackwardScanDirection;
-	}
-	else if (lookupMode == FIND_GREATER)
-	{
-		scanDirection = ForwardScanDirection;
-	}
-	HeapTuple heapTuple = systable_getnext_ordered(scanDescriptor, scanDirection);
-	if (HeapTupleIsValid(heapTuple))
-	{
-		foundStripeMetadata = BuildStripeMetadata(columnarStripes, heapTuple);
+		ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING,
+								 "stripe_first_row_number_idx")));
+		loggedSlowMetadataAccessWarning = true;
 	}
 
-	systable_endscan_ordered(scanDescriptor);
-	index_close(index, AccessShareLock);
+	if (indexOk)
+	{
+		ScanDirection scanDirection = NoMovementScanDirection;
+		if (lookupMode == FIND_LESS_OR_EQUAL)
+		{
+			scanDirection = BackwardScanDirection;
+		}
+		else if (lookupMode == FIND_GREATER)
+		{
+			scanDirection = ForwardScanDirection;
+		}
+		HeapTuple heapTuple = systable_getnext_ordered(scanDescriptor, scanDirection);
+		if (HeapTupleIsValid(heapTuple))
+		{
+			foundStripeMetadata = BuildStripeMetadata(columnarStripes, heapTuple);
+		}
+	}
+	else
+	{
+		HeapTuple heapTuple = NULL;
+		while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor)))
+		{
+			StripeMetadata *stripe = BuildStripeMetadata(columnarStripes, heapTuple);
+			if (!foundStripeMetadata)
+			{
+				/* first match */
+				foundStripeMetadata = stripe;
+			}
+			else if (lookupMode == FIND_LESS_OR_EQUAL &&
+					 stripe->firstRowNumber > foundStripeMetadata->firstRowNumber)
+			{
+				/*
+				 * Among the stripes with firstRowNumber less-than-or-equal-to given,
+				 * we're looking for the one with the greatest firstRowNumber.
+				 */
+				foundStripeMetadata = stripe;
+			}
+			else if (lookupMode == FIND_GREATER &&
+					 stripe->firstRowNumber < foundStripeMetadata->firstRowNumber)
+			{
+				/*
+				 * Among the stripes with firstRowNumber greater-than given,
+				 * we're looking for the one with the smallest firstRowNumber.
+				 */
+				foundStripeMetadata = stripe;
+			}
+		}
+	}
+
+	systable_endscan(scanDescriptor);
 	table_close(columnarStripes, AccessShareLock);
 
 	return foundStripeMetadata;
@@ -1021,8 +1070,8 @@ CheckStripeMetadataConsistency(StripeMetadata *stripeMetadata)
 
 /*
  * FindStripeWithHighestRowNumber returns StripeMetadata for the stripe that
- * has the row with highest rowNumber by doing backward index scan on
- * stripe_first_row_number_idx. If given relation is empty, then returns NULL.
+ * has the row with highest rowNumber. If given relation is empty, then returns
+ * NULL.
  */
 StripeMetadata *
 FindStripeWithHighestRowNumber(Relation relation, Snapshot snapshot)
@@ -1035,19 +1084,46 @@ FindStripeWithHighestRowNumber(Relation relation, Snapshot snapshot)
 				BTEqualStrategyNumber, F_OIDEQ, Int32GetDatum(storageId));
 
 	Relation columnarStripes = table_open(ColumnarStripeRelationId(), AccessShareLock);
-	Relation index = index_open(ColumnarStripeFirstRowNumberIndexRelationId(),
-								AccessShareLock);
-	SysScanDesc scanDescriptor = systable_beginscan_ordered(columnarStripes, index,
-															snapshot, 1, scanKey);
 
-	HeapTuple heapTuple = systable_getnext_ordered(scanDescriptor, BackwardScanDirection);
-	if (HeapTupleIsValid(heapTuple))
+	Oid indexId = ColumnarStripeFirstRowNumberIndexRelationId();
+	bool indexOk = OidIsValid(indexId);
+	SysScanDesc scanDescriptor = systable_beginscan(columnarStripes, indexId, indexOk,
+													snapshot, 1, scanKey);
+
+	static bool loggedSlowMetadataAccessWarning = false;
+	if (!indexOk && !loggedSlowMetadataAccessWarning)
 	{
-		stripeWithHighestRowNumber = BuildStripeMetadata(columnarStripes, heapTuple);
+		ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING,
+								 "stripe_first_row_number_idx")));
+		loggedSlowMetadataAccessWarning = true;
 	}
 
-	systable_endscan_ordered(scanDescriptor);
-	index_close(index, AccessShareLock);
+	if (indexOk)
+	{
+		/* do one-time fetch using the index */
+		HeapTuple heapTuple = systable_getnext_ordered(scanDescriptor,
+													   BackwardScanDirection);
+		if (HeapTupleIsValid(heapTuple))
+		{
+			stripeWithHighestRowNumber = BuildStripeMetadata(columnarStripes, heapTuple);
+		}
+	}
+	else
+	{
+		HeapTuple heapTuple = NULL;
+		while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor)))
+		{
+			StripeMetadata *stripe = BuildStripeMetadata(columnarStripes, heapTuple);
+			if (!stripeWithHighestRowNumber ||
+				stripe->firstRowNumber > stripeWithHighestRowNumber->firstRowNumber)
+			{
+				/* first or a greater match */
+				stripeWithHighestRowNumber = stripe;
+			}
+		}
+	}
+
+	systable_endscan(scanDescriptor);
 	table_close(columnarStripes, AccessShareLock);
 
 	return stripeWithHighestRowNumber;
@@ -1064,7 +1140,6 @@ ReadChunkGroupRowCounts(uint64 storageId, uint64 stripe, uint32 chunkGroupCount,
 {
 	Oid columnarChunkGroupOid = ColumnarChunkGroupRelationId();
 	Relation columnarChunkGroup = table_open(columnarChunkGroupOid, AccessShareLock);
-	Relation index = index_open(ColumnarChunkGroupIndexRelationId(), AccessShareLock);
 
 	ScanKeyData scanKey[2];
 	ScanKeyInit(&scanKey[0], Anum_columnar_chunkgroup_storageid,
@@ -1072,15 +1147,22 @@ ReadChunkGroupRowCounts(uint64 storageId, uint64 stripe, uint32 chunkGroupCount,
 	ScanKeyInit(&scanKey[1], Anum_columnar_chunkgroup_stripe,
 				BTEqualStrategyNumber, F_OIDEQ, Int32GetDatum(stripe));
 
+	Oid indexId = ColumnarChunkGroupIndexRelationId();
+	bool indexOk = OidIsValid(indexId);
 	SysScanDesc scanDescriptor =
-		systable_beginscan_ordered(columnarChunkGroup, index, snapshot, 2, scanKey);
+		systable_beginscan(columnarChunkGroup, indexId, indexOk, snapshot, 2, scanKey);
+
+	static bool loggedSlowMetadataAccessWarning = false;
+	if (!indexOk && !loggedSlowMetadataAccessWarning)
+	{
+		ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING, "chunk_group_pkey")));
+		loggedSlowMetadataAccessWarning = true;
+	}
 
-	uint32 chunkGroupIndex = 0;
 	HeapTuple heapTuple = NULL;
 	uint32 *chunkGroupRowCounts = palloc0(chunkGroupCount * sizeof(uint32));
 
-	while (HeapTupleIsValid(heapTuple = systable_getnext_ordered(scanDescriptor,
-																 ForwardScanDirection)))
+	while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor)))
 	{
 		Datum datumArray[Natts_columnar_chunkgroup];
 		bool isNullArray[Natts_columnar_chunkgroup];
@@ -1091,24 +1173,16 @@ ReadChunkGroupRowCounts(uint64 storageId, uint64 stripe, uint32 chunkGroupCount,
 
 		uint32 tupleChunkGroupIndex =
 			DatumGetUInt32(datumArray[Anum_columnar_chunkgroup_chunk - 1]);
-		if (chunkGroupIndex >= chunkGroupCount ||
-			tupleChunkGroupIndex != chunkGroupIndex)
+		if (tupleChunkGroupIndex >= chunkGroupCount)
 		{
 			elog(ERROR, "unexpected chunk group");
 		}
 
-		chunkGroupRowCounts[chunkGroupIndex] =
+		chunkGroupRowCounts[tupleChunkGroupIndex] =
 			(uint32) DatumGetUInt64(datumArray[Anum_columnar_chunkgroup_row_count - 1]);
-		chunkGroupIndex++;
 	}
 
-	if (chunkGroupIndex != chunkGroupCount)
-	{
-		elog(ERROR, "unexpected chunk group count");
-	}
-
-	systable_endscan_ordered(scanDescriptor);
-	index_close(index, AccessShareLock);
+	systable_endscan(scanDescriptor);
 	table_close(columnarChunkGroup, AccessShareLock);
 
 	return chunkGroupRowCounts;
@@ -1305,14 +1379,20 @@ UpdateStripeMetadataRow(uint64 storageId, uint64 stripeId, bool *update,
 	Oid columnarStripesOid = ColumnarStripeRelationId();
 
 	Relation columnarStripes = table_open(columnarStripesOid, AccessShareLock);
-	Relation columnarStripePkeyIndex = index_open(ColumnarStripePKeyIndexRelationId(),
-												  AccessShareLock);
 
-	SysScanDesc scanDescriptor = systable_beginscan_ordered(columnarStripes,
-															columnarStripePkeyIndex,
-															&dirtySnapshot, 2, scanKey);
+	Oid indexId = ColumnarStripePKeyIndexRelationId();
+	bool indexOk = OidIsValid(indexId);
+	SysScanDesc scanDescriptor = systable_beginscan(columnarStripes, indexId, indexOk,
+													&dirtySnapshot, 2, scanKey);
 
-	HeapTuple oldTuple = systable_getnext_ordered(scanDescriptor, ForwardScanDirection);
+	static bool loggedSlowMetadataAccessWarning = false;
+	if (!indexOk && !loggedSlowMetadataAccessWarning)
+	{
+		ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING, "stripe_pkey")));
+		loggedSlowMetadataAccessWarning = true;
+	}
+
+	HeapTuple oldTuple = systable_getnext(scanDescriptor);
 	if (!HeapTupleIsValid(oldTuple))
 	{
 		ereport(ERROR, (errmsg("attempted to modify an unexpected stripe, "
@@ -1347,8 +1427,7 @@ UpdateStripeMetadataRow(uint64 storageId, uint64 stripeId, bool *update,
 
 	CommandCounterIncrement();
 
-	systable_endscan_ordered(scanDescriptor);
-	index_close(columnarStripePkeyIndex, AccessShareLock);
+	systable_endscan(scanDescriptor);
 	table_close(columnarStripes, AccessShareLock);
 
 	/* return StripeMetadata object built from modified tuple */
@@ -1359,6 +1438,10 @@ UpdateStripeMetadataRow(uint64 storageId, uint64 stripeId, bool *update,
 /*
  * ReadDataFileStripeList reads the stripe list for a given storageId
  * in the given snapshot.
+ *
+ * Doesn't sort the stripes by their ids before returning if
+ * stripe_first_row_number_idx is not available --normally can only happen
+ * during pg upgrades.
  */
 static List *
 ReadDataFileStripeList(uint64 storageId, Snapshot snapshot)
@@ -1373,22 +1456,27 @@ ReadDataFileStripeList(uint64 storageId, Snapshot snapshot)
 	Oid columnarStripesOid = ColumnarStripeRelationId();
 
 	Relation columnarStripes = table_open(columnarStripesOid, AccessShareLock);
-	Relation index = index_open(ColumnarStripeFirstRowNumberIndexRelationId(),
-								AccessShareLock);
 
-	SysScanDesc scanDescriptor = systable_beginscan_ordered(columnarStripes, index,
-															snapshot, 1,
-															scanKey);
+	Oid indexId = ColumnarStripeFirstRowNumberIndexRelationId();
+	bool indexOk = OidIsValid(indexId);
+	SysScanDesc scanDescriptor = systable_beginscan(columnarStripes, indexId,
+													indexOk, snapshot, 1, scanKey);
 
-	while (HeapTupleIsValid(heapTuple = systable_getnext_ordered(scanDescriptor,
-																 ForwardScanDirection)))
+	static bool loggedSlowMetadataAccessWarning = false;
+	if (!indexOk && !loggedSlowMetadataAccessWarning)
+	{
+		ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING,
+								 "stripe_first_row_number_idx")));
+		loggedSlowMetadataAccessWarning = true;
+	}
+
+	while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor)))
 	{
 		StripeMetadata *stripeMetadata = BuildStripeMetadata(columnarStripes, heapTuple);
 		stripeMetadataList = lappend(stripeMetadataList, stripeMetadata);
 	}
 
-	systable_endscan_ordered(scanDescriptor);
-	index_close(index, AccessShareLock);
+	systable_endscan(scanDescriptor);
 	table_close(columnarStripes, AccessShareLock);
 
 	return stripeMetadataList;
@@ -1499,25 +1587,30 @@ DeleteStorageFromColumnarMetadataTable(Oid metadataTableId,
 		return;
 	}
 
-	Relation index = index_open(storageIdIndexId, AccessShareLock);
+	bool indexOk = OidIsValid(storageIdIndexId);
+	SysScanDesc scanDescriptor = systable_beginscan(metadataTable, storageIdIndexId,
+													indexOk, NULL, 1, scanKey);
 
-	SysScanDesc scanDescriptor = systable_beginscan_ordered(metadataTable, index, NULL,
-															1, scanKey);
+	static bool loggedSlowMetadataAccessWarning = false;
+	if (!indexOk && !loggedSlowMetadataAccessWarning)
+	{
+		ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING,
+								 "on a columnar metadata table")));
+		loggedSlowMetadataAccessWarning = true;
+	}
 
 	ModifyState *modifyState = StartModifyRelation(metadataTable);
 
 	HeapTuple heapTuple;
-	while (HeapTupleIsValid(heapTuple = systable_getnext_ordered(scanDescriptor,
-																 ForwardScanDirection)))
+	while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor)))
 	{
 		DeleteTupleAndEnforceConstraints(modifyState, heapTuple);
 	}
 
-	systable_endscan_ordered(scanDescriptor);
+	systable_endscan(scanDescriptor);
 
 	FinishModifyRelation(modifyState);
 
-	index_close(index, AccessShareLock);
 	table_close(metadataTable, AccessShareLock);
 }
 
@@ -1650,6 +1743,9 @@ create_estate_for_relation(Relation rel)
 
 /*
  * DatumToBytea serializes a datum into a bytea value.
+ *
+ * Since we don't want to limit datum size to RSIZE_MAX unnecessarily,
+ * we use memcpy instead of memcpy_s several places in this function.
  */
 static bytea *
 DatumToBytea(Datum value, Form_pg_attribute attrForm)
@@ -1666,19 +1762,16 @@ DatumToBytea(Datum value, Form_pg_attribute attrForm)
 			Datum tmp;
 			store_att_byval(&tmp, value, attrForm->attlen);
 
-			memcpy_s(VARDATA(result), datumLength + VARHDRSZ,
-					 &tmp, attrForm->attlen);
+			memcpy(VARDATA(result), &tmp, attrForm->attlen); /* IGNORE-BANNED */
 		}
 		else
 		{
-			memcpy_s(VARDATA(result), datumLength + VARHDRSZ,
-					 DatumGetPointer(value), attrForm->attlen);
+			memcpy(VARDATA(result), DatumGetPointer(value), attrForm->attlen); /* IGNORE-BANNED */
 		}
 	}
 	else
 	{
-		memcpy_s(VARDATA(result), datumLength + VARHDRSZ,
-				 DatumGetPointer(value), datumLength);
+		memcpy(VARDATA(result), DatumGetPointer(value), datumLength); /* IGNORE-BANNED */
 	}
 
 	return result;
@@ -1697,8 +1790,12 @@ ByteaToDatum(bytea *bytes, Form_pg_attribute attrForm)
 	 * after the byteaDatum is freed.
 	 */
 	char *binaryDataCopy = palloc0(VARSIZE_ANY_EXHDR(bytes));
-	memcpy_s(binaryDataCopy, VARSIZE_ANY_EXHDR(bytes),
-			 VARDATA_ANY(bytes), VARSIZE_ANY_EXHDR(bytes));
+
+	/*
+	 * We use IGNORE-BANNED here since we don't want to limit datum size to
+	 * RSIZE_MAX unnecessarily.
+	 */
+	memcpy(binaryDataCopy, VARDATA_ANY(bytes), VARSIZE_ANY_EXHDR(bytes)); /* IGNORE-BANNED */
 
 	return fetch_att(binaryDataCopy, attrForm->attbyval, attrForm->attlen);
 }

src/backend/columnar/columnar_tableam.c

@@ -739,7 +739,9 @@ columnar_tuple_insert(Relation relation, TupleTableSlot *slot, CommandId cid,
 	 */
 	ColumnarWriteState *writeState = columnar_init_write_state(relation,
 															   RelationGetDescr(relation),
+															   slot->tts_tableOid,
 															   GetCurrentSubTransactionId());
+
 	MemoryContext oldContext = MemoryContextSwitchTo(ColumnarWritePerTupleContext(
 														 writeState));
 
@@ -781,8 +783,14 @@ columnar_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
 {
 	CheckCitusColumnarVersion(ERROR);
 
+	/*
+	 * The callback to .multi_insert is table_multi_insert() and this is only used for the COPY
+	 * command, so slot[i]->tts_tableoid will always be equal to relation->id. Thus, we can send
+	 * RelationGetRelid(relation) as the tupSlotTableOid
+	 */
 	ColumnarWriteState *writeState = columnar_init_write_state(relation,
 															   RelationGetDescr(relation),
+															   RelationGetRelid(relation),
 															   GetCurrentSubTransactionId());
 
 	ColumnarCheckLogicalReplication(relation);
@@ -2568,8 +2576,13 @@ detoast_values(TupleDesc tupleDesc, Datum *orig_values, bool *isnull)
 			if (values == orig_values)
 			{
 				values = palloc(sizeof(Datum) * natts);
-				memcpy_s(values, sizeof(Datum) * natts,
-						 orig_values, sizeof(Datum) * natts);
+
+				/*
+				 * We use IGNORE-BANNED here since we don't want to limit
+				 * size of the buffer that holds the datum array to RSIZE_MAX
+				 * unnecessarily.
+				 */
+				memcpy(values, orig_values, sizeof(Datum) * natts); /* IGNORE-BANNED */
 			}
 
 			/* will be freed when per-tuple context is reset */

src/backend/columnar/columnar_writer.c

@@ -531,6 +531,9 @@ SerializeBoolArray(bool *boolArray, uint32 boolArrayLength)
 /*
  * SerializeSingleDatum serializes the given datum value and appends it to the
  * provided string info buffer.
+ *
+ * Since we don't want to limit datum buffer size to RSIZE_MAX unnecessarily,
+ * we use memcpy instead of memcpy_s several places in this function.
  */
 static void
 SerializeSingleDatum(StringInfo datumBuffer, Datum datum, bool datumTypeByValue,
@@ -552,15 +555,13 @@ SerializeSingleDatum(StringInfo datumBuffer, Datum datum, bool datumTypeByValue,
 		}
 		else
 		{
-			memcpy_s(currentDatumDataPointer, datumBuffer->maxlen - datumBuffer->len,
-					 DatumGetPointer(datum), datumTypeLength);
+			memcpy(currentDatumDataPointer, DatumGetPointer(datum), datumTypeLength); /* IGNORE-BANNED */
 		}
 	}
 	else
 	{
 		Assert(!datumTypeByValue);
-		memcpy_s(currentDatumDataPointer, datumBuffer->maxlen - datumBuffer->len,
-				 DatumGetPointer(datum), datumLength);
+		memcpy(currentDatumDataPointer, DatumGetPointer(datum), datumLength); /* IGNORE-BANNED */
 	}
 
 	datumBuffer->len += datumLengthAligned;
@@ -714,7 +715,12 @@ DatumCopy(Datum datum, bool datumTypeByValue, int datumTypeLength)
 	{
 		uint32 datumLength = att_addlength_datum(0, datumTypeLength, datum);
 		char *datumData = palloc0(datumLength);
-		memcpy_s(datumData, datumLength, DatumGetPointer(datum), datumLength);
+
+		/*
+		 * We use IGNORE-BANNED here since we don't want to limit datum size to
+		 * RSIZE_MAX unnecessarily.
+		 */
+		memcpy(datumData, DatumGetPointer(datum), datumLength); /* IGNORE-BANNED */
 
 		datumCopy = PointerGetDatum(datumData);
 	}
@@ -737,8 +743,12 @@ CopyStringInfo(StringInfo sourceString)
 		targetString->data = palloc0(sourceString->len);
 		targetString->len = sourceString->len;
 		targetString->maxlen = sourceString->len;
-		memcpy_s(targetString->data, sourceString->len,
-				 sourceString->data, sourceString->len);
+
+		/*
+		 * We use IGNORE-BANNED here since we don't want to limit string
+		 * buffer size to RSIZE_MAX unnecessarily.
+		 */
+		memcpy(targetString->data, sourceString->data, sourceString->len); /* IGNORE-BANNED */
 	}
 
 	return targetString;

src/backend/columnar/sql/columnar--10.2-2--10.2-3.sql

@@ -10,6 +10,17 @@
 --
 -- To do that, drop stripe_first_row_number_idx and create a unique
 -- constraint with the same name to keep the code change at minimum.
+--
+-- If we have a pg_depend entry for this index, we can not drop it as
+-- the extension depends on it. Remove the pg_depend entry if it exists.
+DELETE FROM pg_depend
+WHERE classid = 'pg_am'::regclass::oid
+    AND objid IN (select oid from pg_am where amname = 'columnar')
+    AND objsubid = 0
+    AND refclassid = 'pg_class'::regclass::oid
+    AND refobjid = 'columnar.stripe_first_row_number_idx'::regclass::oid
+    AND refobjsubid = 0
+    AND deptype = 'n';
 DROP INDEX columnar.stripe_first_row_number_idx;
 ALTER TABLE columnar.stripe ADD CONSTRAINT stripe_first_row_number_idx
 UNIQUE (storage_id, first_row_number);

src/backend/columnar/sql/downgrades/columnar--10.2-1--10.1-1.sql

@@ -8,5 +8,16 @@ DROP FUNCTION citus_internal.upgrade_columnar_storage(regclass);
 DROP FUNCTION citus_internal.downgrade_columnar_storage(regclass);
 
 -- drop "first_row_number" column and the index defined on it
+--
+-- If we have a pg_depend entry for this index, we can not drop it as
+-- the extension depends on it. Remove the pg_depend entry if it exists.
+DELETE FROM pg_depend
+WHERE classid = 'pg_am'::regclass::oid
+    AND objid IN (select oid from pg_am where amname = 'columnar')
+    AND objsubid = 0
+    AND refclassid = 'pg_class'::regclass::oid
+    AND refobjid = 'columnar.stripe_first_row_number_idx'::regclass::oid
+    AND refobjsubid = 0
+    AND deptype = 'n';
 DROP INDEX columnar.stripe_first_row_number_idx;
 ALTER TABLE columnar.stripe DROP COLUMN first_row_number;

src/backend/columnar/sql/downgrades/columnar--10.2-3--10.2-2.sql

@@ -1,4 +1,14 @@
 -- columnar--10.2-3--10.2-2.sql
-
+--
+-- If we have a pg_depend entry for this index, we can not drop it as
+-- the extension depends on it. Remove the pg_depend entry if it exists.
+DELETE FROM pg_depend
+WHERE classid = 'pg_am'::regclass::oid
+    AND objid IN (select oid from pg_am where amname = 'columnar')
+    AND objsubid = 0
+    AND refclassid = 'pg_class'::regclass::oid
+    AND refobjid = 'columnar.stripe_first_row_number_idx'::regclass::oid
+    AND refobjsubid = 0
+    AND deptype = 'n';
 ALTER TABLE columnar.stripe DROP CONSTRAINT stripe_first_row_number_idx;
 CREATE INDEX stripe_first_row_number_idx ON columnar.stripe USING BTREE(storage_id, first_row_number);

src/backend/columnar/write_state_management.c

@@ -113,6 +113,7 @@ CleanupWriteStateMap(void *arg)
 
 ColumnarWriteState *
 columnar_init_write_state(Relation relation, TupleDesc tupdesc,
+						  Oid tupSlotRelationId,
 						  SubTransactionId currentSubXid)
 {
 	bool found;
@@ -176,7 +177,16 @@ columnar_init_write_state(Relation relation, TupleDesc tupdesc,
 	MemoryContext oldContext = MemoryContextSwitchTo(WriteStateContext);
 
 	ColumnarOptions columnarOptions = { 0 };
-	ReadColumnarOptions(relation->rd_id, &columnarOptions);
+
+	/*
+	 * In case of a table rewrite, we need to fetch table options based on the
+	 * relation id of the source tuple slot.
+	 *
+	 * For this reason, we always pass tupSlotRelationId here; which should be
+	 * same as the target table if the write operation is not related to a table
+	 * rewrite etc.
+	 */
+	ReadColumnarOptions(tupSlotRelationId, &columnarOptions);
 
 	SubXidWriteState *stackEntry = palloc0(sizeof(SubXidWriteState));
 	stackEntry->writeState = ColumnarBeginWrite(relation->rd_node,

src/backend/distributed/commands/alter_table.c

@@ -1223,8 +1223,15 @@ CreateDistributedTableLike(TableConversionState *con)
 		newShardCount = con->shardCount;
 	}
 
+	/*
+	 * To get the correct column name, we use the original relation id, not the
+	 * new relation id. The reason is that the cached attributes of the original
+	 * and newly created tables are not the same if the original table has
+	 * dropped columns (dropped columns are still present in the attribute cache)
+	 * Detailed example in https://github.com/citusdata/citus/pull/6387
+	 */
 	char *distributionColumnName =
-		ColumnToColumnName(con->newRelationId, (Node *) newDistributionKey);
+		ColumnToColumnName(con->relationId, (Node *) newDistributionKey);
 
 	Oid originalRelationId = con->relationId;
 	if (con->originalDistributionKey != NULL && PartitionTable(originalRelationId))
@@ -1604,6 +1611,8 @@ ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands,
 		}
 		else if (ShouldSyncTableMetadata(sourceId))
 		{
+			char *qualifiedTableName = quote_qualified_identifier(schemaName, sourceName);
+
 			/*
 			 * We are converting a citus local table to a distributed/reference table,
 			 * so we should prevent dropping the sequence on the table. Otherwise, we'd
@@ -1612,8 +1621,8 @@ ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands,
 			StringInfo command = makeStringInfo();
 
 			appendStringInfo(command,
-							 "SELECT pg_catalog.worker_drop_sequence_dependency('%s');",
-							 quote_qualified_identifier(schemaName, sourceName));
+							 "SELECT pg_catalog.worker_drop_sequence_dependency(%s);",
+							 quote_literal_cstr(qualifiedTableName));
 
 			SendCommandToWorkersWithMetadata(command->data);
 		}
@@ -1903,11 +1912,17 @@ CreateWorkerChangeSequenceDependencyCommand(char *sequenceSchemaName, char *sequ
 											char *sourceSchemaName, char *sourceName,
 											char *targetSchemaName, char *targetName)
 {
+	char *qualifiedSchemaName = quote_qualified_identifier(sequenceSchemaName,
+														   sequenceName);
+	char *qualifiedSourceName = quote_qualified_identifier(sourceSchemaName, sourceName);
+	char *qualifiedTargetName = quote_qualified_identifier(targetSchemaName, targetName);
+
 	StringInfo query = makeStringInfo();
-	appendStringInfo(query, "SELECT worker_change_sequence_dependency('%s', '%s', '%s')",
-					 quote_qualified_identifier(sequenceSchemaName, sequenceName),
-					 quote_qualified_identifier(sourceSchemaName, sourceName),
-					 quote_qualified_identifier(targetSchemaName, targetName));
+	appendStringInfo(query, "SELECT worker_change_sequence_dependency(%s, %s, %s)",
+					 quote_literal_cstr(qualifiedSchemaName),
+					 quote_literal_cstr(qualifiedSourceName),
+					 quote_literal_cstr(qualifiedTargetName));
+
 	return query->data;
 }
 

src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c

@@ -623,18 +623,13 @@ ExecuteForeignKeyCreateCommand(const char *commandString, bool skip_validation)
 	 */
 	Assert(IsA(parseTree, AlterTableStmt));
 
-	bool oldSkipConstraintsValidationValue = SkipConstraintValidation;
-
 	if (skip_validation && IsA(parseTree, AlterTableStmt))
 	{
-		EnableSkippingConstraintValidation();
-
+		SkipForeignKeyValidationIfConstraintIsFkey((AlterTableStmt *) parseTree, true);
 		ereport(DEBUG4, (errmsg("skipping validation for foreign key create "
 								"command \"%s\"", commandString)));
 	}
 
 	ProcessUtilityParseTree(parseTree, commandString, PROCESS_UTILITY_QUERY,
 							NULL, None_Receiver, NULL);
-
-	SkipConstraintValidation = oldSkipConstraintsValidationValue;
 }

src/backend/distributed/commands/citus_add_local_table_to_metadata.c

@@ -46,6 +46,7 @@
 #include "utils/lsyscache.h"
 #include "utils/ruleutils.h"
 #include "utils/syscache.h"
+#include "foreign/foreign.h"
 
 
 /*
@@ -61,6 +62,8 @@ static void ErrorIfAddingPartitionTableToMetadata(Oid relationId);
 static void ErrorIfUnsupportedCreateCitusLocalTable(Relation relation);
 static void ErrorIfUnsupportedCitusLocalTableKind(Oid relationId);
 static void ErrorIfUnsupportedCitusLocalColumnDefinition(Relation relation);
+static void EnsureIfPostgresFdwHasTableName(Oid relationId);
+static void ErrorIfOptionListHasNoTableName(List *optionList);
 static void NoticeIfAutoConvertingLocalTables(bool autoConverted, Oid relationId);
 static CascadeOperationType GetCascadeTypeForCitusLocalTables(bool autoConverted);
 static List * GetShellTableDDLEventsForCitusLocalTable(Oid relationId);
@@ -87,8 +90,8 @@ static List * ReversedOidList(List *oidList);
 static void AppendExplicitIndexIdsToList(Form_pg_index indexForm,
 										 List **explicitIndexIdList,
 										 int flags);
-static void DropDefaultExpressionsAndMoveOwnedSequenceOwnerships(Oid sourceRelationId,
-																 Oid targetRelationId);
+static void DropNextValExprsAndMoveOwnedSeqOwnerships(Oid sourceRelationId,
+													  Oid targetRelationId);
 static void DropDefaultColumnDefinition(Oid relationId, char *columnName);
 static void TransferSequenceOwnership(Oid ownedSequenceId, Oid targetRelationId,
 									  char *columnName);
@@ -128,6 +131,9 @@ citus_add_local_table_to_metadata_internal(Oid relationId, bool cascadeViaForeig
 {
 	CheckCitusVersion(ERROR);
 
+	/* enable citus_add_local_table_to_metadata on an empty node */
+	InsertCoordinatorIfClusterEmpty();
+
 	bool autoConverted = false;
 	CreateCitusLocalTable(relationId, cascadeViaForeignKeys, autoConverted);
 }
@@ -363,11 +369,11 @@ CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys, bool autoConve
 
 	/*
 	 * Move sequence ownerships from shard table to shell table and also drop
-	 * DEFAULT expressions from shard relation as we should evaluate such columns
-	 * in shell table when needed.
+	 * DEFAULT expressions based on sequences from shard relation as we should
+	 * evaluate such columns in shell table when needed.
 	 */
-	DropDefaultExpressionsAndMoveOwnedSequenceOwnerships(shardRelationId,
-														 shellRelationId);
+	DropNextValExprsAndMoveOwnedSeqOwnerships(shardRelationId,
+											  shellRelationId);
 
 	InsertMetadataForCitusLocalTable(shellRelationId, shardId, autoConverted);
 
@@ -485,6 +491,16 @@ ErrorIfUnsupportedCreateCitusLocalTable(Relation relation)
 	EnsureTableNotDistributed(relationId);
 	ErrorIfUnsupportedCitusLocalColumnDefinition(relation);
 
+	/*
+	 * Error out with a hint if the foreign table is using postgres_fdw and
+	 * the option table_name is not provided.
+	 * Citus relays all the Citus local foreign table logic to the placement of the
+	 * Citus local table. If table_name is NOT provided, Citus would try to talk to
+	 * the foreign postgres table over the shard's table name, which would not exist
+	 * on the remote server.
+	 */
+	EnsureIfPostgresFdwHasTableName(relationId);
+
 	/*
 	 * When creating other citus table types, we don't need to check that case as
 	 * EnsureTableNotDistributed already errors out if the given relation implies
@@ -500,6 +516,93 @@ ErrorIfUnsupportedCreateCitusLocalTable(Relation relation)
 }
 
 
+/*
+ * ServerUsesPostgresFdw gets a foreign server Oid and returns true if the FDW that
+ * the server depends on is postgres_fdw. Returns false otherwise.
+ */
+bool
+ServerUsesPostgresFdw(Oid serverId)
+{
+	ForeignServer *server = GetForeignServer(serverId);
+	ForeignDataWrapper *fdw = GetForeignDataWrapper(server->fdwid);
+
+	if (strcmp(fdw->fdwname, "postgres_fdw") == 0)
+	{
+		return true;
+	}
+
+	return false;
+}
+
+
+/*
+ * EnsureIfPostgresFdwHasTableName errors out with a hint if the foreign table is using postgres_fdw and
+ * the option table_name is not provided.
+ */
+static void
+EnsureIfPostgresFdwHasTableName(Oid relationId)
+{
+	char relationKind = get_rel_relkind(relationId);
+	if (relationKind == RELKIND_FOREIGN_TABLE)
+	{
+		ForeignTable *foreignTable = GetForeignTable(relationId);
+		if (ServerUsesPostgresFdw(foreignTable->serverid))
+		{
+			ErrorIfOptionListHasNoTableName(foreignTable->options);
+		}
+	}
+}
+
+
+/*
+ * ErrorIfOptionListHasNoTableName gets an option list (DefElem) and errors out
+ * if the list does not contain a table_name element.
+ */
+static void
+ErrorIfOptionListHasNoTableName(List *optionList)
+{
+	char *table_nameString = "table_name";
+	DefElem *option = NULL;
+	foreach_ptr(option, optionList)
+	{
+		char *optionName = option->defname;
+		if (strcmp(optionName, table_nameString) == 0)
+		{
+			return;
+		}
+	}
+
+	ereport(ERROR, (errmsg(
+						"table_name option must be provided when using postgres_fdw with Citus"),
+					errhint("Provide the option \"table_name\" with value target table's"
+							" name")));
+}
+
+
+/*
+ * ForeignTableDropsTableNameOption returns true if given option list contains
+ * (DROP table_name).
+ */
+bool
+ForeignTableDropsTableNameOption(List *optionList)
+{
+	char *table_nameString = "table_name";
+	DefElem *option = NULL;
+	foreach_ptr(option, optionList)
+	{
+		char *optionName = option->defname;
+		DefElemAction optionAction = option->defaction;
+		if (strcmp(optionName, table_nameString) == 0 &&
+			optionAction == DEFELEM_DROP)
+		{
+			return true;
+		}
+	}
+
+	return false;
+}
+
+
 /*
  * ErrorIfUnsupportedCitusLocalTableKind errors out if the relation kind of
  * relation with relationId is not supported for citus local table creation.
@@ -1158,14 +1261,15 @@ GetRenameStatsCommandList(List *statsOidList, uint64 shardId)
 
 
 /*
- * DropDefaultExpressionsAndMoveOwnedSequenceOwnerships drops default column
- * definitions for relation with sourceRelationId. Also, for each column that
- * defaults to an owned sequence, it grants ownership to the same named column
- * of the relation with targetRelationId.
+ * DropNextValExprsAndMoveOwnedSeqOwnerships drops default column definitions
+ * that are based on sequences for relation with sourceRelationId.
+ *
+ * Also, for each such column that owns a sequence, it grants ownership to the
+ * same named column of the relation with targetRelationId.
  */
 static void
-DropDefaultExpressionsAndMoveOwnedSequenceOwnerships(Oid sourceRelationId,
-													 Oid targetRelationId)
+DropNextValExprsAndMoveOwnedSeqOwnerships(Oid sourceRelationId,
+										  Oid targetRelationId)
 {
 	List *columnNameList = NIL;
 	List *ownedSequenceIdList = NIL;
@@ -1176,9 +1280,28 @@ DropDefaultExpressionsAndMoveOwnedSequenceOwnerships(Oid sourceRelationId,
 	Oid ownedSequenceId = InvalidOid;
 	forboth_ptr_oid(columnName, columnNameList, ownedSequenceId, ownedSequenceIdList)
 	{
-		DropDefaultColumnDefinition(sourceRelationId, columnName);
+		/*
+		 * We drop nextval() expressions because Citus currently evaluates
+		 * nextval() on the shell table, not on the shards. Hence, there is
+		 * no reason for keeping nextval(). Also, distributed/reference table
+		 * shards do not have - so be consistent with those.
+		 *
+		 * Note that we keep other kind of DEFAULT expressions on shards
+		 * because we still want to be able to evaluate DEFAULT expressions
+		 * that are not based on sequences on shards, e.g., for foreign key
+		 * - SET DEFAULT actions.
+		 */
+		AttrNumber columnAttrNumber = get_attnum(sourceRelationId, columnName);
+		if (ColumnDefaultsToNextVal(sourceRelationId, columnAttrNumber))
+		{
+			DropDefaultColumnDefinition(sourceRelationId, columnName);
+		}
 
-		/* column might not own a sequence */
+		/*
+		 * Column might own a sequence without having a nextval() expr on it
+		 * --e.g., due to ALTER SEQUENCE OWNED BY .. --, so check if that is
+		 * the case even if the column doesn't have a DEFAULT.
+		 */
 		if (OidIsValid(ownedSequenceId))
 		{
 			TransferSequenceOwnership(ownedSequenceId, targetRelationId, columnName);

src/backend/distributed/commands/create_distributed_table.c

@@ -382,7 +382,6 @@ CreateDistributedTableConcurrently(Oid relationId, char *distributionColumnName,
 							   "citus.shard_replication_factor > 1")));
 	}
 
-	EnsureCoordinatorIsInMetadata();
 	EnsureCitusTableCanBeCreated(relationId);
 
 	EnsureValidDistributionColumn(relationId, distributionColumnName);
@@ -415,6 +414,19 @@ CreateDistributedTableConcurrently(Oid relationId, char *distributionColumnName,
 	if (!IsColocateWithDefault(colocateWithTableName) && !IsColocateWithNone(
 			colocateWithTableName))
 	{
+		if (replicationModel != REPLICATION_MODEL_STREAMING)
+		{
+			ereport(ERROR, (errmsg("cannot create distributed table "
+								   "concurrently because Citus allows "
+								   "concurrent table distribution only when "
+								   "citus.shard_replication_factor = 1"),
+							errhint("table %s is requested to be colocated "
+									"with %s which has "
+									"citus.shard_replication_factor > 1",
+									get_rel_name(relationId),
+									colocateWithTableName)));
+		}
+
 		EnsureColocateWithTableIsValid(relationId, distributionMethod,
 									   distributionColumnName,
 									   colocateWithTableName);
@@ -528,6 +540,14 @@ CreateDistributedTableConcurrently(Oid relationId, char *distributionColumnName,
 		colocatedTableId = ColocatedTableId(colocationId);
 	}
 
+	List *workerNodeList = DistributedTablePlacementNodeList(NoLock);
+	if (workerNodeList == NIL)
+	{
+		ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						errmsg("no worker nodes are available for placing shards"),
+						errhint("Add more worker nodes.")));
+	}
+
 	List *workersForPlacementList;
 	List *shardSplitPointsList;
 
@@ -555,7 +575,6 @@ CreateDistributedTableConcurrently(Oid relationId, char *distributionColumnName,
 		/*
 		 * Place shards in a round-robin fashion across all data nodes.
 		 */
-		List *workerNodeList = DistributedTablePlacementNodeList(NoLock);
 		workersForPlacementList = RoundRobinWorkerNodeList(workerNodeList, shardCount);
 	}
 
@@ -856,6 +875,8 @@ WorkerNodesForShardList(List *shardList)
 static List *
 RoundRobinWorkerNodeList(List *workerNodeList, int listLength)
 {
+	Assert(workerNodeList != NIL);
+
 	List *nodeIdList = NIL;
 
 	for (int idx = 0; idx < listLength; idx++)

src/backend/distributed/commands/foreign_constraint.c

@@ -23,12 +23,14 @@
 #include "catalog/pg_type.h"
 #include "distributed/colocation_utils.h"
 #include "distributed/commands.h"
+#include "distributed/commands/sequence.h"
 #include "distributed/coordinator_protocol.h"
 #include "distributed/listutils.h"
 #include "distributed/coordinator_protocol.h"
 #include "distributed/multi_join_order.h"
 #include "distributed/namespace_utils.h"
 #include "distributed/reference_table_utils.h"
+#include "distributed/utils/array_type.h"
 #include "distributed/version_compat.h"
 #include "miscadmin.h"
 #include "utils/builtins.h"
@@ -57,6 +59,8 @@ typedef bool (*CheckRelationFunc)(Oid);
 /* Local functions forward declarations */
 static void EnsureReferencingTableNotReplicated(Oid referencingTableId);
 static void EnsureSupportedFKeyOnDistKey(Form_pg_constraint constraintForm);
+static bool ForeignKeySetsNextValColumnToDefault(HeapTuple pgConstraintTuple);
+static List * ForeignKeyGetDefaultingAttrs(HeapTuple pgConstraintTuple);
 static void EnsureSupportedFKeyBetweenCitusLocalAndRefTable(Form_pg_constraint
 															constraintForm,
 															char
@@ -256,6 +260,23 @@ ErrorIfUnsupportedForeignConstraintExists(Relation relation, char referencingDis
 			referencedReplicationModel = referencingReplicationModel;
 		}
 
+		/*
+		 * Given that we drop DEFAULT nextval('sequence') expressions from
+		 * shard relation columns, allowing ON DELETE/UPDATE SET DEFAULT
+		 * on such columns causes inserting NULL values to referencing relation
+		 * as a result of a delete/update operation on referenced relation.
+		 *
+		 * For this reason, we disallow ON DELETE/UPDATE SET DEFAULT actions
+		 * on columns that default to sequences.
+		 */
+		if (ForeignKeySetsNextValColumnToDefault(heapTuple))
+		{
+			ereport(ERROR, (errmsg("cannot create foreign key constraint "
+								   "since Citus does not support ON DELETE "
+								   "/ UPDATE SET DEFAULT actions on the "
+								   "columns that default to sequences")));
+		}
+
 		bool referencingIsCitusLocalOrRefTable =
 			(referencingDistMethod == DISTRIBUTE_BY_NONE);
 		bool referencedIsCitusLocalOrRefTable =
@@ -358,6 +379,104 @@ ErrorIfUnsupportedForeignConstraintExists(Relation relation, char referencingDis
 }
 
 
+/*
+ * ForeignKeySetsNextValColumnToDefault returns true if at least one of the
+ * columns specified in ON DELETE / UPDATE SET DEFAULT clauses default to
+ * nextval().
+ */
+static bool
+ForeignKeySetsNextValColumnToDefault(HeapTuple pgConstraintTuple)
+{
+	Form_pg_constraint pgConstraintForm =
+		(Form_pg_constraint) GETSTRUCT(pgConstraintTuple);
+
+	List *setDefaultAttrs = ForeignKeyGetDefaultingAttrs(pgConstraintTuple);
+	AttrNumber setDefaultAttr = InvalidAttrNumber;
+	foreach_int(setDefaultAttr, setDefaultAttrs)
+	{
+		if (ColumnDefaultsToNextVal(pgConstraintForm->conrelid, setDefaultAttr))
+		{
+			return true;
+		}
+	}
+
+	return false;
+}
+
+
+/*
+ * ForeignKeyGetDefaultingAttrs returns a list of AttrNumbers
+ * might be set to default ON DELETE or ON UPDATE.
+ *
+ * For example; if the foreign key has SET DEFAULT clause for
+ * both actions, then returns a superset of the attributes that
+ * might be set to DEFAULT on either of those actions.
+ */
+static List *
+ForeignKeyGetDefaultingAttrs(HeapTuple pgConstraintTuple)
+{
+	bool isNull = false;
+	Datum referencingColumnsDatum = SysCacheGetAttr(CONSTROID, pgConstraintTuple,
+													Anum_pg_constraint_conkey, &isNull);
+	if (isNull)
+	{
+		ereport(ERROR, (errmsg("got NULL conkey from catalog")));
+	}
+
+	List *referencingColumns =
+		IntegerArrayTypeToList(DatumGetArrayTypeP(referencingColumnsDatum));
+
+	Form_pg_constraint pgConstraintForm =
+		(Form_pg_constraint) GETSTRUCT(pgConstraintTuple);
+	if (pgConstraintForm->confupdtype == FKCONSTR_ACTION_SETDEFAULT)
+	{
+		/*
+		 * Postgres doesn't allow specifying SET DEFAULT for a subset of
+		 * the referencing columns for ON UPDATE action, so in that case
+		 * we return all referencing columns regardless of what ON DELETE
+		 * action says.
+		 */
+		return referencingColumns;
+	}
+
+	if (pgConstraintForm->confdeltype != FKCONSTR_ACTION_SETDEFAULT)
+	{
+		return NIL;
+	}
+
+	List *onDeleteSetDefColumnList = NIL;
+#if PG_VERSION_NUM >= PG_VERSION_15
+	Datum onDeleteSetDefColumnsDatum = SysCacheGetAttr(CONSTROID, pgConstraintTuple,
+													   Anum_pg_constraint_confdelsetcols,
+													   &isNull);
+
+	/*
+	 * confdelsetcols being NULL means that "ON DELETE SET DEFAULT" doesn't
+	 * specify which subset of columns should be set to DEFAULT, so fetching
+	 * NULL from the catalog is also possible.
+	 */
+	if (!isNull)
+	{
+		onDeleteSetDefColumnList =
+			IntegerArrayTypeToList(DatumGetArrayTypeP(onDeleteSetDefColumnsDatum));
+	}
+#endif
+
+	if (list_length(onDeleteSetDefColumnList) == 0)
+	{
+		/*
+		 * That means that all referencing columns need to be set to
+		 * DEFAULT.
+		 */
+		return referencingColumns;
+	}
+	else
+	{
+		return onDeleteSetDefColumnList;
+	}
+}
+
+
 /*
  * EnsureSupportedFKeyBetweenCitusLocalAndRefTable is a helper function that
  * takes a foreign key constraint form for a foreign key between two citus
@@ -1192,19 +1311,6 @@ IsTableTypeIncluded(Oid relationId, int flags)
 }
 
 
-/*
- * EnableSkippingConstraintValidation is simply a C interface for setting the following:
- *      SET LOCAL citus.skip_constraint_validation TO on;
- */
-void
-EnableSkippingConstraintValidation()
-{
-	set_config_option("citus.skip_constraint_validation", "true",
-					  PGC_SUSET, PGC_S_SESSION,
-					  GUC_ACTION_LOCAL, true, 0, false);
-}
-
-
 /*
  * RelationInvolvedInAnyNonInheritedForeignKeys returns true if relation involved
  * in a foreign key that is not inherited from its parent relation.

src/backend/distributed/commands/role.c

@@ -354,17 +354,23 @@ ExtractEncryptedPassword(Oid roleOid)
 
 	Datum passwordDatum = heap_getattr(tuple, Anum_pg_authid_rolpassword,
 									   pgAuthIdDescription, &isNull);
-	char *passwordCstring = TextDatumGetCString(passwordDatum);
+
+	/*
+	 * In PG, an empty password is treated the same as NULL.
+	 * So we propagate NULL password to the other nodes, even if
+	 * the user supplied an empty password
+	 */
+
+	char *passwordCstring = NULL;
+	if (!isNull)
+	{
+		passwordCstring = pstrdup(TextDatumGetCString(passwordDatum));
+	}
 
 	table_close(pgAuthId, AccessShareLock);
 	ReleaseSysCache(tuple);
 
-	if (isNull)
-	{
-		return NULL;
-	}
-
-	return pstrdup(passwordCstring);
+	return passwordCstring;
 }
 
 

src/backend/distributed/commands/sequence.c

@@ -27,6 +27,7 @@
 #include "nodes/makefuncs.h"
 #include "distributed/worker_create_or_replace.h"
 #include "nodes/parsenodes.h"
+#include "rewrite/rewriteHandler.h"
 #include "utils/builtins.h"
 #include "utils/lsyscache.h"
 
@@ -213,6 +214,29 @@ ExtractDefaultColumnsAndOwnedSequences(Oid relationId, List **columnNameList,
 }
 
 
+/*
+ * ColumnDefaultsToNextVal returns true if the column with attrNumber
+ * has a default expression that contains nextval().
+ */
+bool
+ColumnDefaultsToNextVal(Oid relationId, AttrNumber attrNumber)
+{
+	AssertArg(AttributeNumberIsValid(attrNumber));
+
+	Relation relation = RelationIdGetRelation(relationId);
+	Node *defExpr = build_column_default(relation, attrNumber);
+	RelationClose(relation);
+
+	if (defExpr == NULL)
+	{
+		/* column doesn't have a DEFAULT expression */
+		return false;
+	}
+
+	return contain_nextval_expression_walker(defExpr, NULL);
+}
+
+
 /*
  * PreprocessDropSequenceStmt gets called during the planning phase of a DROP SEQUENCE statement
  * and returns a list of DDLJob's that will drop any distributed sequences from the

src/backend/distributed/commands/table.c

@@ -40,6 +40,7 @@
 #include "distributed/resource_lock.h"
 #include "distributed/version_compat.h"
 #include "distributed/worker_shard_visibility.h"
+#include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "nodes/parsenodes.h"
 #include "parser/parse_expr.h"
@@ -117,6 +118,8 @@ static char * GetAlterColumnWithNextvalDefaultCmd(Oid sequenceOid, Oid relationI
 												  char *colname);
 static char * GetAddColumnWithNextvalDefaultCmd(Oid sequenceOid, Oid relationId,
 												char *colname, TypeName *typeName);
+static void ErrorIfAlterTableDropTableNameFromPostgresFdw(List *optionList, Oid
+														  relationId);
 
 
 /*
@@ -1875,7 +1878,8 @@ PreprocessAlterTableSchemaStmt(Node *node, const char *queryString,
  * ALTER TABLE ... ADD FOREIGN KEY command to skip the validation step.
  */
 void
-SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStatement)
+SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStatement,
+										   bool processLocalRelation)
 {
 	/* first check whether a distributed relation is affected */
 	if (alterTableStatement->relation == NULL)
@@ -1890,11 +1894,17 @@ SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStatement)
 		return;
 	}
 
-	if (!IsCitusTable(leftRelationId))
+	if (!IsCitusTable(leftRelationId) && !processLocalRelation)
 	{
 		return;
 	}
 
+	/*
+	 * We check if there is a ADD FOREIGN CONSTRAINT command in sub commands
+	 * list. We set skip_validation to true to prevent PostgreSQL to verify
+	 * validity of the foreign constraint. Validity will be checked on the
+	 * shards anyway.
+	 */
 	AlterTableCmd *command = NULL;
 	foreach_ptr(command, alterTableStatement->cmds)
 	{
@@ -1906,9 +1916,8 @@ SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStatement)
 			Constraint *constraint = (Constraint *) command->def;
 			if (constraint->contype == CONSTR_FOREIGN)
 			{
-				/* set the GUC skip_constraint_validation to on */
-				EnableSkippingConstraintValidation();
-				return;
+				/* foreign constraint validations will be done in shards. */
+				constraint->skip_validation = true;
 			}
 		}
 	}
@@ -2643,6 +2652,42 @@ ErrorIfUnsupportedConstraint(Relation relation, char distributionMethod,
 }
 
 
+/*
+ * ErrorIfAlterTableDropTableNameFromPostgresFdw errors if given alter foreign table
+ * option list drops 'table_name' from a postgresfdw foreign table which is
+ * inside metadata.
+ */
+static void
+ErrorIfAlterTableDropTableNameFromPostgresFdw(List *optionList, Oid relationId)
+{
+	char relationKind PG_USED_FOR_ASSERTS_ONLY =
+		get_rel_relkind(relationId);
+	Assert(relationKind == RELKIND_FOREIGN_TABLE);
+
+	ForeignTable *foreignTable = GetForeignTable(relationId);
+	Oid serverId = foreignTable->serverid;
+	if (!ServerUsesPostgresFdw(serverId))
+	{
+		return;
+	}
+
+	if (IsCitusTableType(relationId, CITUS_LOCAL_TABLE) &&
+		ForeignTableDropsTableNameOption(optionList))
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg(
+					 "alter foreign table alter options (drop table_name) command "
+					 "is not allowed for Citus tables"),
+				 errdetail(
+					 "Table_name option can not be dropped from a foreign table "
+					 "which is inside metadata."),
+				 errhint(
+					 "Try to undistribute foreign table before dropping table_name option.")));
+	}
+}
+
+
 /*
  * ErrorIfUnsupportedAlterTableStmt checks if the corresponding alter table
  * statement is supported for distributed tables and errors out if it is not.
@@ -3031,6 +3076,8 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement)
 			{
 				if (IsForeignTable(relationId))
 				{
+					List *optionList = (List *) command->def;
+					ErrorIfAlterTableDropTableNameFromPostgresFdw(optionList, relationId);
 					break;
 				}
 			}

src/backend/distributed/commands/trigger.c

@@ -43,6 +43,7 @@
 
 
 /* local function forward declarations */
+static char * GetAlterTriggerStateCommand(Oid triggerId);
 static bool IsCreateCitusTruncateTriggerStmt(CreateTrigStmt *createTriggerStmt);
 static String * GetAlterTriggerDependsTriggerNameValue(AlterObjectDependsStmt *
 													   alterTriggerDependsStmt);
@@ -99,6 +100,18 @@ GetExplicitTriggerCommandList(Oid relationId)
 		createTriggerCommandList = lappend(
 			createTriggerCommandList,
 			makeTableDDLCommandString(createTriggerCommand));
+
+		/*
+		 * Appends the commands for the trigger settings that are not covered
+		 * by CREATE TRIGGER command, such as ALTER TABLE ENABLE/DISABLE <trigger>.
+		 */
+
+		char *alterTriggerStateCommand =
+			GetAlterTriggerStateCommand(triggerId);
+
+		createTriggerCommandList = lappend(
+			createTriggerCommandList,
+			makeTableDDLCommandString(alterTriggerStateCommand));
 	}
 
 	/* revert back to original search_path */
@@ -108,6 +121,72 @@ GetExplicitTriggerCommandList(Oid relationId)
 }
 
 
+/*
+ * GetAlterTriggerStateCommand returns the DDL command to set enable/disable
+ * state for given trigger. Throws an error if no such trigger exists.
+ */
+static char *
+GetAlterTriggerStateCommand(Oid triggerId)
+{
+	StringInfo alterTriggerStateCommand = makeStringInfo();
+
+	bool missingOk = false;
+	HeapTuple triggerTuple = GetTriggerTupleById(triggerId, missingOk);
+
+	Form_pg_trigger triggerForm = (Form_pg_trigger) GETSTRUCT(triggerTuple);
+
+	char *qualifiedRelName = generate_qualified_relation_name(triggerForm->tgrelid);
+	const char *quotedTrigName = quote_identifier(NameStr(triggerForm->tgname));
+	char enableDisableState = triggerForm->tgenabled;
+
+	const char *alterTriggerStateStr = NULL;
+	switch (enableDisableState)
+	{
+		case TRIGGER_FIRES_ON_ORIGIN:
+		{
+			/* default mode */
+			alterTriggerStateStr = "ENABLE";
+			break;
+		}
+
+		case TRIGGER_FIRES_ALWAYS:
+		{
+			alterTriggerStateStr = "ENABLE ALWAYS";
+			break;
+		}
+
+		case TRIGGER_FIRES_ON_REPLICA:
+		{
+			alterTriggerStateStr = "ENABLE REPLICA";
+			break;
+		}
+
+		case TRIGGER_DISABLED:
+		{
+			alterTriggerStateStr = "DISABLE";
+			break;
+		}
+
+		default:
+		{
+			elog(ERROR, "unexpected trigger state");
+		}
+	}
+
+	appendStringInfo(alterTriggerStateCommand, "ALTER TABLE %s %s TRIGGER %s;",
+					 qualifiedRelName, alterTriggerStateStr, quotedTrigName);
+
+	/*
+	 * Free triggerTuple at the end since quote_identifier() might not return
+	 * a palloc'd string if given identifier doesn't need to be quoted, and in
+	 * that case quotedTrigName would still be bound to triggerTuple.
+	 */
+	heap_freetuple(triggerTuple);
+
+	return alterTriggerStateCommand->data;
+}
+
+
 /*
  * GetTriggerTupleById returns copy of the heap tuple from pg_trigger for
  * the trigger with triggerId. If no such trigger exists, this function returns

src/backend/distributed/commands/utility_hook.c

@@ -117,9 +117,6 @@ static void DecrementUtilityHookCountersIfNecessary(Node *parsetree);
 static bool IsDropSchemaOrDB(Node *parsetree);
 static bool ShouldCheckUndistributeCitusLocalTables(void);
 static bool ShouldAddNewTableToMetadata(Node *parsetree);
-static bool ServerUsesPostgresFDW(char *serverName);
-static void ErrorIfOptionListHasNoTableName(List *optionList);
-
 
 /*
  * ProcessUtilityParseTree is a convenience method to create a PlannedStmt out of
@@ -178,7 +175,9 @@ multi_ProcessUtility(PlannedStmt *pstmt,
 		IsA(parsetree, ExecuteStmt) ||
 		IsA(parsetree, PrepareStmt) ||
 		IsA(parsetree, DiscardStmt) ||
-		IsA(parsetree, DeallocateStmt))
+		IsA(parsetree, DeallocateStmt) ||
+		IsA(parsetree, DeclareCursorStmt) ||
+		IsA(parsetree, FetchStmt))
 	{
 		/*
 		 * Skip additional checks for common commands that do not have any
@@ -378,7 +377,6 @@ ProcessUtilityInternal(PlannedStmt *pstmt,
 	Node *parsetree = pstmt->utilityStmt;
 	List *ddlJobs = NIL;
 	DistOpsValidationState distOpsValidationState = HasNoneValidObject;
-	bool oldSkipConstraintsValidationValue = SkipConstraintValidation;
 
 	if (IsA(parsetree, ExplainStmt) &&
 		IsA(((ExplainStmt *) parsetree)->query, Query))
@@ -597,7 +595,9 @@ ProcessUtilityInternal(PlannedStmt *pstmt,
 		 * Citus intervening. The only exception is partition column drop, in
 		 * which case we error out. Advanced Citus users use this to implement their
 		 * own DDL propagation. We also use it to avoid re-propagating DDL commands
-		 * when changing MX tables on workers.
+		 * when changing MX tables on workers. Below, we also make sure that DDL
+		 * commands don't run queries that might get intercepted by Citus and error
+		 * out during planning, specifically we skip validation in foreign keys.
 		 */
 
 		if (IsA(parsetree, AlterTableStmt))
@@ -616,7 +616,33 @@ ProcessUtilityInternal(PlannedStmt *pstmt,
 				 * Note validation is done on the shard level when DDL propagation
 				 * is enabled. The following eagerly executes some tasks on workers.
 				 */
-				SkipForeignKeyValidationIfConstraintIsFkey(alterTableStmt);
+				SkipForeignKeyValidationIfConstraintIsFkey(alterTableStmt, false);
+			}
+		}
+	}
+
+	/*
+	 * If we've explicitly set the citus.skip_constraint_validation GUC, then
+	 * we skip validation of any added constraints.
+	 */
+	if (IsA(parsetree, AlterTableStmt) && SkipConstraintValidation)
+	{
+		AlterTableStmt *alterTableStmt = (AlterTableStmt *) parsetree;
+		AlterTableCmd *command = NULL;
+		foreach_ptr(command, alterTableStmt->cmds)
+		{
+			AlterTableType alterTableType = command->subtype;
+
+			/*
+			 * XXX: In theory we could probably use this GUC to skip validation
+			 * of VALIDATE CONSTRAINT and ALTER CONSTRAINT too. But currently
+			 * this is not needed, so we make its behaviour only apply to ADD
+			 * CONSTRAINT.
+			 */
+			if (alterTableType == AT_AddConstraint)
+			{
+				Constraint *constraint = (Constraint *) command->def;
+				constraint->skip_validation = true;
 			}
 		}
 	}
@@ -787,18 +813,6 @@ ProcessUtilityInternal(PlannedStmt *pstmt,
 
 		CreateStmt *createTableStmt = (CreateStmt *) (&createForeignTableStmt->base);
 
-		/*
-		 * Error out with a hint if the foreign table is using postgres_fdw and
-		 * the option table_name is not provided.
-		 * Citus relays all the Citus local foreign table logic to the placement of the
-		 * Citus local table. If table_name is NOT provided, Citus would try to talk to
-		 * the foreign postgres table over the shard's table name, which would not exist
-		 * on the remote server.
-		 */
-		if (ServerUsesPostgresFDW(createForeignTableStmt->servername))
-		{
-			ErrorIfOptionListHasNoTableName(createForeignTableStmt->options);
-		}
 
 		PostprocessCreateTableStmt(createTableStmt, queryString);
 	}
@@ -902,8 +916,6 @@ ProcessUtilityInternal(PlannedStmt *pstmt,
 		 */
 		CitusHasBeenLoaded(); /* lgtm[cpp/return-value-ignored] */
 	}
-
-	SkipConstraintValidation = oldSkipConstraintsValidationValue;
 }
 
 
@@ -1088,50 +1100,6 @@ ShouldAddNewTableToMetadata(Node *parsetree)
 }
 
 
-/*
- * ServerUsesPostgresFDW gets a foreign server name and returns true if the FDW that
- * the server depends on is postgres_fdw. Returns false otherwise.
- */
-static bool
-ServerUsesPostgresFDW(char *serverName)
-{
-	ForeignServer *server = GetForeignServerByName(serverName, false);
-	ForeignDataWrapper *fdw = GetForeignDataWrapper(server->fdwid);
-
-	if (strcmp(fdw->fdwname, "postgres_fdw") == 0)
-	{
-		return true;
-	}
-
-	return false;
-}
-
-
-/*
- * ErrorIfOptionListHasNoTableName gets an option list (DefElem) and errors out
- * if the list does not contain a table_name element.
- */
-static void
-ErrorIfOptionListHasNoTableName(List *optionList)
-{
-	char *table_nameString = "table_name";
-	DefElem *option = NULL;
-	foreach_ptr(option, optionList)
-	{
-		char *optionName = option->defname;
-		if (strcmp(optionName, table_nameString) == 0)
-		{
-			return;
-		}
-	}
-
-	ereport(ERROR, (errmsg(
-						"table_name option must be provided when using postgres_fdw with Citus"),
-					errhint("Provide the option \"table_name\" with value target table's"
-							" name")));
-}
-
-
 /*
  * NotifyUtilityHookConstraintDropped sets ConstraintDropped to true to tell us
  * last command dropped a table constraint.

src/backend/distributed/connection/connection_management.c

@@ -16,7 +16,7 @@
 #include "miscadmin.h"
 
 #include "safe_lib.h"
-
+#include "postmaster/postmaster.h"
 #include "access/hash.h"
 #include "commands/dbcommands.h"
 #include "distributed/backend_data.h"
@@ -63,7 +63,6 @@ static void FreeConnParamsHashEntryFields(ConnParamsHashEntry *entry);
 static void AfterXactHostConnectionHandling(ConnectionHashEntry *entry, bool isCommit);
 static bool ShouldShutdownConnection(MultiConnection *connection, const int
 									 cachedConnectionCount);
-static void ResetConnection(MultiConnection *connection);
 static bool RemoteTransactionIdle(MultiConnection *connection);
 static int EventSetSizeForConnectionList(List *connections);
 
@@ -244,6 +243,23 @@ GetNodeUserDatabaseConnection(uint32 flags, const char *hostname, int32 port,
 }
 
 
+/*
+ * GetConnectionForLocalQueriesOutsideTransaction returns a localhost connection for
+ * subtransaction. To avoid creating excessive connections, we reuse an
+ * existing connection.
+ */
+MultiConnection *
+GetConnectionForLocalQueriesOutsideTransaction(char *userName)
+{
+	int connectionFlag = OUTSIDE_TRANSACTION;
+	MultiConnection *connection =
+		GetNodeUserDatabaseConnection(connectionFlag, LocalHostName, PostPortNumber,
+									  userName, get_database_name(MyDatabaseId));
+
+	return connection;
+}
+
+
 /*
  * StartNodeUserDatabaseConnection() initiates a connection to a remote node.
  *
@@ -688,8 +704,8 @@ CloseConnection(MultiConnection *connection)
 		dlist_delete(&connection->connectionNode);
 
 		/* same for transaction state and shard/placement machinery */
-		CloseRemoteTransaction(connection);
 		CloseShardPlacementAssociation(connection);
+		ResetRemoteTransaction(connection);
 
 		/* we leave the per-host entry alive */
 		pfree(connection);
@@ -1433,6 +1449,9 @@ AfterXactHostConnectionHandling(ConnectionHashEntry *entry, bool isCommit)
 		{
 			ShutdownConnection(connection);
 
+			/* remove from transactionlist before free-ing */
+			ResetRemoteTransaction(connection);
+
 			/* unlink from list */
 			dlist_delete(iter.cur);
 
@@ -1443,7 +1462,10 @@ AfterXactHostConnectionHandling(ConnectionHashEntry *entry, bool isCommit)
 			/*
 			 * reset healthy session lifespan connections.
 			 */
-			ResetConnection(connection);
+			ResetRemoteTransaction(connection);
+
+			UnclaimConnection(connection);
+
 
 			cachedConnectionCount++;
 		}
@@ -1482,24 +1504,6 @@ ShouldShutdownConnection(MultiConnection *connection, const int cachedConnection
 }
 
 
-/*
- * ResetConnection preserves the given connection for later usage by
- * resetting its states.
- */
-static void
-ResetConnection(MultiConnection *connection)
-{
-	/* reset per-transaction state */
-	ResetRemoteTransaction(connection);
-	ResetShardPlacementAssociation(connection);
-
-	/* reset copy state */
-	connection->copyBytesWrittenSinceLastFlush = 0;
-
-	UnclaimConnection(connection);
-}
-
-
 /*
  * RemoteTransactionIdle function returns true if we manually
  * set flag on run_commands_on_session_level_connection_to_node to true to

src/backend/distributed/connection/remote_commands.c

@@ -633,14 +633,14 @@ PutRemoteCopyData(MultiConnection *connection, const char *buffer, int nbytes)
 	Assert(PQisnonblocking(pgConn));
 
 	int copyState = PQputCopyData(pgConn, buffer, nbytes);
-	if (copyState == -1)
+	if (copyState <= 0)
 	{
 		return false;
 	}
 
 	/*
 	 * PQputCopyData may have queued up part of the data even if it managed
-	 * to send some of it succesfully. We provide back pressure by waiting
+	 * to send some of it successfully. We provide back pressure by waiting
 	 * until the socket is writable to prevent the internal libpq buffers
 	 * from growing excessively.
 	 *

src/backend/distributed/deparser/ruleutils_13.c

@@ -1405,8 +1405,15 @@ set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte,
 
 	/* Assert we processed the right number of columns */
 #ifdef USE_ASSERT_CHECKING
-	while (i < colinfo->num_cols && colinfo->colnames[i] == NULL)
-		i++;
+	for (int col_index = 0; col_index < colinfo->num_cols; col_index++)
+	{
+		/*
+		 * In the above processing-loops, "i" advances only if
+		 * the column is not new, check if this is a new column.
+		 */
+		if (colinfo->is_new_col[col_index])
+			i++;
+	}
 	Assert(i == colinfo->num_cols);
 	Assert(j == nnewcolumns);
 #endif

src/backend/distributed/deparser/ruleutils_14.c

@@ -482,7 +482,7 @@ get_merged_argument_list(CallStmt *stmt, List **mergedNamedArgList,
 	Oid  functionOid = stmt->funcexpr->funcid;
 	List *namedArgList = NIL;
 	List *finalArgumentList = NIL;
-	Oid  finalArgTypes[FUNC_MAX_ARGS];
+	Oid  *finalArgTypes;
 	Oid  *argTypes = NULL;
 	char *argModes = NULL;
 	char **argNames = NULL;
@@ -519,6 +519,7 @@ get_merged_argument_list(CallStmt *stmt, List **mergedNamedArgList,
 
 	/* Remove the duplicate INOUT counting */
 	numberOfArgs = numberOfArgs - totalInoutArgs;
+	finalArgTypes = palloc0(sizeof(Oid) * numberOfArgs);
 
 	ListCell *inArgCell = list_head(stmt->funcexpr->args);
 	ListCell *outArgCell = list_head(stmt->outargs);
@@ -1527,8 +1528,15 @@ set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte,
 
 	/* Assert we processed the right number of columns */
 #ifdef USE_ASSERT_CHECKING
-	while (i < colinfo->num_cols && colinfo->colnames[i] == NULL)
-		i++;
+	for (int col_index = 0; col_index < colinfo->num_cols; col_index++)
+	{
+		/*
+		 * In the above processing-loops, "i" advances only if
+		 * the column is not new, check if this is a new column.
+		 */
+		if (colinfo->is_new_col[col_index])
+			i++;
+	}
 	Assert(i == colinfo->num_cols);
 	Assert(j == nnewcolumns);
 #endif

src/backend/distributed/deparser/ruleutils_15.c

@@ -491,7 +491,7 @@ get_merged_argument_list(CallStmt *stmt, List **mergedNamedArgList,
 	Oid  functionOid = stmt->funcexpr->funcid;
 	List *namedArgList = NIL;
 	List *finalArgumentList = NIL;
-	Oid  finalArgTypes[FUNC_MAX_ARGS];
+	Oid  *finalArgTypes;
 	Oid  *argTypes = NULL;
 	char *argModes = NULL;
 	char **argNames = NULL;
@@ -528,6 +528,7 @@ get_merged_argument_list(CallStmt *stmt, List **mergedNamedArgList,
 
 	/* Remove the duplicate INOUT counting */
 	numberOfArgs = numberOfArgs - totalInoutArgs;
+	finalArgTypes = palloc0(sizeof(Oid) * numberOfArgs);
 
 	ListCell *inArgCell = list_head(stmt->funcexpr->args);
 	ListCell *outArgCell = list_head(stmt->outargs);
@@ -610,6 +611,7 @@ get_merged_argument_list(CallStmt *stmt, List **mergedNamedArgList,
 
 	return true;
 }
+
 /*
  * pg_get_rule_expr deparses an expression and returns the result as a string.
  */
@@ -1559,8 +1561,15 @@ set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte,
 
 	/* Assert we processed the right number of columns */
 #ifdef USE_ASSERT_CHECKING
-	while (i < colinfo->num_cols && colinfo->colnames[i] == NULL)
-		i++;
+	for (int col_index = 0; col_index < colinfo->num_cols; col_index++)
+	{
+		/*
+		 * In the above processing-loops, "i" advances only if
+		 * the column is not new, check if this is a new column.
+		 */
+		if (colinfo->is_new_col[col_index])
+			i++;
+	}
 	Assert(i == colinfo->num_cols);
 	Assert(j == nnewcolumns);
 #endif

src/backend/distributed/executor/adaptive_executor.c

@@ -511,7 +511,9 @@ typedef enum TaskExecutionState
 /*
  * PlacementExecutionOrder indicates whether a command should be executed
  * on any replica, on all replicas sequentially (in order), or on all
- * replicas in parallel.
+ * replicas in parallel. In other words, EXECUTION_ORDER_ANY is used for
+ * SELECTs, EXECUTION_ORDER_SEQUENTIAL/EXECUTION_ORDER_PARALLEL is used for
+ * DML/DDL.
  */
 typedef enum PlacementExecutionOrder
 {
@@ -4558,6 +4560,7 @@ ReceiveResults(WorkerSession *session, bool storeRows)
 		TupleDesc tupleDescriptor = tupleDest->tupleDescForQuery(tupleDest, queryIndex);
 		if (tupleDescriptor == NULL)
 		{
+			PQclear(result);
 			continue;
 		}
 
@@ -5290,6 +5293,10 @@ TaskExecutionStateMachine(ShardCommandExecution *shardCommandExecution)
 	{
 		currentTaskExecutionState = TASK_EXECUTION_FAILED;
 	}
+	else if (executionOrder != EXECUTION_ORDER_ANY && failedPlacementCount > 0)
+	{
+		currentTaskExecutionState = TASK_EXECUTION_FAILED;
+	}
 	else if (executionOrder == EXECUTION_ORDER_ANY && donePlacementCount > 0)
 	{
 		currentTaskExecutionState = TASK_EXECUTION_FINISHED;

src/backend/distributed/executor/citus_custom_scan.c

@@ -781,7 +781,19 @@ CitusEndScan(CustomScanState *node)
  */
 static void
 CitusReScan(CustomScanState *node)
-{ }
+{
+	if (node->ss.ps.ps_ResultTupleSlot)
+	{
+		ExecClearTuple(node->ss.ps.ps_ResultTupleSlot);
+	}
+	ExecScanReScan(&node->ss);
+
+	CitusScanState *scanState = (CitusScanState *) node;
+	if (scanState->tuplestorestate)
+	{
+		tuplestore_rescan(scanState->tuplestorestate);
+	}
+}
 
 
 /*

src/backend/distributed/executor/multi_executor.c

@@ -861,11 +861,6 @@ AlterTableConstraintCheck(QueryDesc *queryDesc)
 		return false;
 	}
 
-	if (SkipConstraintValidation)
-	{
-		return true;
-	}
-
 	/*
 	 * While an ALTER TABLE is in progress, we might do SELECTs on some
 	 * catalog tables too. For example, when dropping a column, citus_drop_trigger()

src/backend/distributed/metadata/metadata_utility.c

@@ -985,7 +985,7 @@ AppendShardSizeQuery(StringInfo selectQuery, ShardInterval *shardInterval)
 
 	appendStringInfo(selectQuery, "SELECT " UINT64_FORMAT " AS shard_id, ", shardId);
 	appendStringInfo(selectQuery, "%s AS shard_name, ", quotedShardName);
-	appendStringInfo(selectQuery, PG_RELATION_SIZE_FUNCTION, quotedShardName);
+	appendStringInfo(selectQuery, PG_TOTAL_RELATION_SIZE_FUNCTION, quotedShardName);
 }
 
 
@@ -4022,7 +4022,7 @@ CancelTasksForJob(int64 jobid)
 		}
 
 		/* make sure the current user has the rights to cancel this task */
-		Oid taskOwner = DatumGetObjectId(values[Anum_pg_dist_background_task_owner]);
+		Oid taskOwner = DatumGetObjectId(values[Anum_pg_dist_background_task_owner - 1]);
 		if (superuser_arg(taskOwner) && !superuser())
 		{
 			/* must be a superuser to cancel tasks owned by superuser */

src/backend/distributed/operations/shard_cleaner.c

@@ -303,21 +303,40 @@ DropOrphanedShardsForCleanup()
 										   workerNode->workerName,
 										   workerNode->workerPort))
 		{
+			if (record->policy == CLEANUP_DEFERRED_ON_SUCCESS)
+			{
+				ereport(LOG, (errmsg("deferred drop of orphaned shard %s on %s:%d "
+									 "completed",
+									 qualifiedTableName,
+									 workerNode->workerName, workerNode->workerPort)));
+			}
+			else
+			{
+				ereport(LOG, (errmsg("cleaned up orphaned shard %s on %s:%d which "
+									 "was left behind after a failed operation",
+									 qualifiedTableName,
+									 workerNode->workerName, workerNode->workerPort)));
+			}
+
 			/* delete the cleanup record */
 			DeleteCleanupRecordByRecordId(record->recordId);
 			removedShardCountForCleanup++;
 		}
 		else
 		{
+			/*
+			 * We log failures at the end, since they occur repeatedly
+			 * for a large number of objects.
+			 */
 			failedShardCountForCleanup++;
 		}
 	}
 
 	if (failedShardCountForCleanup > 0)
 	{
-		ereport(WARNING, (errmsg("Failed to cleanup %d shards out of %d",
-								 failedShardCountForCleanup, list_length(
-									 cleanupRecordList))));
+		ereport(WARNING, (errmsg("failed to clean up %d orphaned shards out of %d",
+								 failedShardCountForCleanup,
+								 list_length(cleanupRecordList))));
 	}
 
 	return removedShardCountForCleanup;
@@ -396,19 +415,29 @@ DropOrphanedShardsForMove(bool waitForLocks)
 										   shardPlacement->nodeName,
 										   shardPlacement->nodePort))
 		{
+			ereport(LOG, (errmsg("deferred drop of orphaned shard %s on %s:%d "
+								 "after a move completed",
+								 qualifiedTableName,
+								 shardPlacement->nodeName,
+								 shardPlacement->nodePort)));
+
 			/* delete the actual placement */
 			DeleteShardPlacementRow(placement->placementId);
 			removedShardCount++;
 		}
 		else
 		{
+			/*
+			 * We log failures at the end, since they occur repeatedly
+			 * for a large number of objects.
+			 */
 			failedShardDropCount++;
 		}
 	}
 
 	if (failedShardDropCount > 0)
 	{
-		ereport(WARNING, (errmsg("Failed to drop %d orphaned shards out of %d",
+		ereport(WARNING, (errmsg("failed to clean up %d orphaned shards out of %d",
 								 failedShardDropCount, list_length(shardPlacementList))));
 	}
 
@@ -436,7 +465,7 @@ RegisterOperationNeedingCleanup(void)
  * completion with failure. This will trigger cleanup of appropriate resources.
  */
 void
-FinalizeOperationNeedingCleanupOnFailure()
+FinalizeOperationNeedingCleanupOnFailure(const char *operationName)
 {
 	/* We must have a valid OperationId. Any operation requring cleanup
 	 * will call RegisterOperationNeedingCleanup.
@@ -454,8 +483,9 @@ FinalizeOperationNeedingCleanupOnFailure()
 		/* We only supporting cleaning shards right now */
 		if (record->objectType != CLEANUP_OBJECT_SHARD_PLACEMENT)
 		{
-			ereport(WARNING, (errmsg("Invalid object type %d for cleanup record ",
-									 record->objectType)));
+			ereport(WARNING, (errmsg(
+								  "Invalid object type %d on failed operation cleanup",
+								  record->objectType)));
 			continue;
 		}
 
@@ -473,6 +503,12 @@ FinalizeOperationNeedingCleanupOnFailure()
 											   workerNode->workerName,
 											   workerNode->workerPort))
 			{
+				ereport(LOG, (errmsg("cleaned up orphaned shard %s on %s:%d after a "
+									 "%s operation failed",
+									 qualifiedTableName,
+									 workerNode->workerName, workerNode->workerPort,
+									 operationName)));
+
 				/*
 				 * Given the operation is failing and we will abort its transaction, we cannot delete
 				 * records in the current transaction. Delete these records outside of the
@@ -483,23 +519,22 @@ FinalizeOperationNeedingCleanupOnFailure()
 			}
 			else
 			{
+				/*
+				 * We log failures at the end, since they occur repeatedly
+				 * for a large number of objects.
+				 */
 				failedShardCountOnComplete++;
 			}
 		}
 	}
 
-	if (list_length(currentOperationRecordList) > 0)
+	if (failedShardCountOnComplete > 0)
 	{
-		ereport(LOG, (errmsg("Removed %d orphaned shards out of %d",
-							 removedShardCountOnComplete, list_length(
-								 currentOperationRecordList))));
-
-		if (failedShardCountOnComplete > 0)
-		{
-			ereport(WARNING, (errmsg("Failed to cleanup %d shards out of %d",
-									 failedShardCountOnComplete, list_length(
-										 currentOperationRecordList))));
-		}
+		ereport(WARNING, (errmsg("failed to clean up %d orphaned shards out of %d after "
+								 "a %s operation failed",
+								 failedShardCountOnComplete,
+								 list_length(currentOperationRecordList),
+								 operationName)));
 	}
 }
 
@@ -509,7 +544,7 @@ FinalizeOperationNeedingCleanupOnFailure()
  * completion with success. This will trigger cleanup of appropriate resources.
  */
 void
-FinalizeOperationNeedingCleanupOnSuccess()
+FinalizeOperationNeedingCleanupOnSuccess(const char *operationName)
 {
 	/* We must have a valid OperationId. Any operation requring cleanup
 	 * will call RegisterOperationNeedingCleanup.
@@ -527,8 +562,9 @@ FinalizeOperationNeedingCleanupOnSuccess()
 		/* We only supporting cleaning shards right now */
 		if (record->objectType != CLEANUP_OBJECT_SHARD_PLACEMENT)
 		{
-			ereport(WARNING, (errmsg("Invalid object type %d for cleanup record ",
-									 record->objectType)));
+			ereport(WARNING, (errmsg(
+								  "Invalid object type %d on operation cleanup",
+								  record->objectType)));
 			continue;
 		}
 
@@ -546,6 +582,12 @@ FinalizeOperationNeedingCleanupOnSuccess()
 											   workerNode->workerName,
 											   workerNode->workerPort))
 			{
+				ereport(LOG, (errmsg("cleaned up orphaned shard %s on %s:%d after a "
+									 "%s operation completed",
+									 qualifiedTableName,
+									 workerNode->workerName, workerNode->workerPort,
+									 operationName)));
+
 				/*
 				 * Delete cleanup records outside transaction as:
 				 * The resources are marked as 'CLEANUP_ALWAYS' and should be cleaned no matter
@@ -556,6 +598,10 @@ FinalizeOperationNeedingCleanupOnSuccess()
 			}
 			else
 			{
+				/*
+				 * We log failures at the end, since they occur repeatedly
+				 * for a large number of objects.
+				 */
 				failedShardCountOnComplete++;
 			}
 		}
@@ -570,18 +616,14 @@ FinalizeOperationNeedingCleanupOnSuccess()
 		}
 	}
 
-	if (list_length(currentOperationRecordList) > 0)
+	if (failedShardCountOnComplete > 0)
 	{
-		ereport(LOG, (errmsg("Removed %d orphaned shards out of %d",
-							 removedShardCountOnComplete, list_length(
-								 currentOperationRecordList))));
-
-		if (failedShardCountOnComplete > 0)
-		{
-			ereport(WARNING, (errmsg("Failed to cleanup %d shards out of %d",
-									 failedShardCountOnComplete, list_length(
-										 currentOperationRecordList))));
-		}
+		ereport(WARNING, (errmsg(
+							  "failed to clean up %d orphaned shards out of %d after "
+							  "a %s operation completed",
+							  failedShardCountOnComplete,
+							  list_length(currentOperationRecordList),
+							  operationName)));
 	}
 }
 
@@ -670,10 +712,10 @@ InsertCleanupRecordInSubtransaction(CleanupObject objectType,
 					 nodeGroupId,
 					 policy);
 
-	SendCommandListToWorkerOutsideTransaction(LocalHostName,
-											  PostPortNumber,
-											  CitusExtensionOwnerName(),
-											  list_make1(command->data));
+	MultiConnection *connection =
+		GetConnectionForLocalQueriesOutsideTransaction(CitusExtensionOwnerName());
+	SendCommandListToWorkerOutsideTransactionWithConnection(connection,
+															list_make1(command->data));
 }
 
 
@@ -691,10 +733,10 @@ DeleteCleanupRecordByRecordIdOutsideTransaction(uint64 recordId)
 					 PG_DIST_CLEANUP,
 					 recordId);
 
-	SendCommandListToWorkerOutsideTransaction(LocalHostName,
-											  PostPortNumber,
-											  CitusExtensionOwnerName(),
-											  list_make1(command->data));
+	MultiConnection *connection = GetConnectionForLocalQueriesOutsideTransaction(
+		CitusExtensionOwnerName());
+	SendCommandListToWorkerOutsideTransactionWithConnection(connection,
+															list_make1(command->data));
 }
 
 
@@ -727,18 +769,11 @@ TryLockRelationAndPlacementCleanup(Oid relationId, LOCKMODE lockmode)
  * true on success.
  */
 static bool
-TryDropShardOutsideTransaction(OperationId operationId, char *qualifiedTableName,
-							   char *nodeName, int nodePort)
+TryDropShardOutsideTransaction(OperationId operationId,
+							   char *qualifiedTableName,
+							   char *nodeName,
+							   int nodePort)
 {
-	char *operation = (operationId == INVALID_OPERATION_ID) ? "move" : "cleanup";
-
-	ereport(LOG, (errmsg("cleaning up %s on %s:%d which was left "
-						 "after a %s",
-						 qualifiedTableName,
-						 nodeName,
-						 nodePort,
-						 operation)));
-
 	/* prepare sql query to execute to drop the shard */
 	StringInfo dropQuery = makeStringInfo();
 	appendStringInfo(dropQuery, DROP_REGULAR_TABLE_COMMAND, qualifiedTableName);
@@ -756,10 +791,14 @@ TryDropShardOutsideTransaction(OperationId operationId, char *qualifiedTableName
 									   dropQuery->data);
 
 	/* remove the shard from the node */
-	bool success = SendOptionalCommandListToWorkerOutsideTransaction(nodeName,
-																	 nodePort,
-																	 NULL,
-																	 dropCommandList);
+	int connectionFlags = OUTSIDE_TRANSACTION;
+	MultiConnection *workerConnection = GetNodeUserDatabaseConnection(connectionFlags,
+																	  nodeName, nodePort,
+																	  CurrentUserName(),
+																	  NULL);
+	bool success = SendOptionalCommandListToWorkerOutsideTransactionWithConnection(
+		workerConnection,
+		dropCommandList);
 
 	return success;
 }
@@ -800,13 +839,8 @@ GetNextOperationId()
 	appendStringInfo(nextValueCommand, "SELECT nextval(%s);",
 					 quote_literal_cstr(sequenceName->data));
 
-	int connectionFlag = FORCE_NEW_CONNECTION;
-	MultiConnection *connection = GetNodeUserDatabaseConnection(connectionFlag,
-																LocalHostName,
-																PostPortNumber,
-																CitusExtensionOwnerName(),
-																get_database_name(
-																	MyDatabaseId));
+	MultiConnection *connection = GetConnectionForLocalQueriesOutsideTransaction(
+		CitusExtensionOwnerName());
 
 	PGresult *result = NULL;
 	int queryResult = ExecuteOptionalRemoteCommand(connection, nextValueCommand->data,
@@ -821,7 +855,6 @@ GetNextOperationId()
 
 	PQclear(result);
 	ForgetResults(connection);
-	CloseConnection(connection);
 
 	return operationdId;
 }

src/backend/distributed/operations/shard_rebalancer.c

@@ -1818,7 +1818,10 @@ RebalanceTableShardsBackground(RebalanceOptions *options, Oid shardReplicationMo
 
 	if (HasNodesWithMissingReferenceTables(&referenceTableIdList))
 	{
-		VerifyTablesHaveReplicaIdentity(referenceTableIdList);
+		if (shardTransferMode == TRANSFER_MODE_AUTOMATIC)
+		{
+			VerifyTablesHaveReplicaIdentity(referenceTableIdList);
+		}
 
 		/*
 		 * Reference tables need to be copied to (newly-added) nodes, this needs to be the

src/backend/distributed/operations/shard_split.c

@@ -151,7 +151,7 @@ static List * ExecuteSplitShardReplicationSetupUDF(WorkerNode *sourceWorkerNode,
 												   List *destinationWorkerNodesList,
 												   DistributionColumnMap *
 												   distributionColumnOverrides);
-static void ExecuteSplitShardReleaseSharedMemory(WorkerNode *sourceWorkerNode);
+static void ExecuteSplitShardReleaseSharedMemory(MultiConnection *sourceConnection);
 static void AddDummyShardEntryInMap(HTAB *mapOfPlacementToDummyShardList, uint32
 									targetNodeId,
 									ShardInterval *shardInterval);
@@ -169,6 +169,12 @@ static const char *const SplitOperationName[] =
 	[ISOLATE_TENANT_TO_NEW_SHARD] = "isolate",
 	[CREATE_DISTRIBUTED_TABLE] = "create"
 };
+static const char *const SplitOperationAPIName[] =
+{
+	[SHARD_SPLIT_API] = "citus_split_shard_by_split_points",
+	[ISOLATE_TENANT_TO_NEW_SHARD] = "isolate_tenant_to_new_shard",
+	[CREATE_DISTRIBUTED_TABLE] = "create_distributed_table_concurrently"
+};
 static const char *const SplitTargetName[] =
 {
 	[SHARD_SPLIT_API] = "shard",
@@ -469,6 +475,8 @@ SplitShard(SplitMode splitMode,
 		   List *colocatedShardIntervalList,
 		   uint32 targetColocationId)
 {
+	const char *operationName = SplitOperationAPIName[splitOperation];
+
 	ErrorIfModificationAndSplitInTheSameTransaction(splitOperation);
 
 	ShardInterval *shardIntervalToSplit = LoadShardInterval(shardIdToSplit);
@@ -526,6 +534,8 @@ SplitShard(SplitMode splitMode,
 
 	if (splitMode == BLOCKING_SPLIT)
 	{
+		ereport(LOG, (errmsg("performing blocking %s ", operationName)));
+
 		BlockingShardSplit(
 			splitOperation,
 			splitWorkflowId,
@@ -536,6 +546,8 @@ SplitShard(SplitMode splitMode,
 	}
 	else
 	{
+		ereport(LOG, (errmsg("performing non-blocking %s ", operationName)));
+
 		NonBlockingShardSplit(
 			splitOperation,
 			splitWorkflowId,
@@ -548,7 +560,10 @@ SplitShard(SplitMode splitMode,
 		PlacementMovedUsingLogicalReplicationInTX = true;
 	}
 
-	FinalizeOperationNeedingCleanupOnSuccess();
+	/*
+	 * Drop temporary objects that were marked as CLEANUP_ALWAYS.
+	 */
+	FinalizeOperationNeedingCleanupOnSuccess(operationName);
 }
 
 
@@ -569,6 +584,8 @@ BlockingShardSplit(SplitOperation splitOperation,
 				   List *workersForPlacementList,
 				   DistributionColumnMap *distributionColumnOverrides)
 {
+	const char *operationName = SplitOperationAPIName[splitOperation];
+
 	BlockWritesToShardList(sourceColocatedShardIntervalList);
 
 	/* First create shard interval metadata for split children */
@@ -583,10 +600,14 @@ BlockingShardSplit(SplitOperation splitOperation,
 
 	PG_TRY();
 	{
+		ereport(LOG, (errmsg("creating child shards for %s", operationName)));
+
 		/* Physically create split children. */
 		CreateSplitShardsForShardGroup(shardGroupSplitIntervalListList,
 									   workersForPlacementList);
 
+		ereport(LOG, (errmsg("performing copy for %s", operationName)));
+
 		/* For Blocking split, copy isn't snapshotted */
 		char *snapshotName = NULL;
 		DoSplitCopy(sourceShardNode, sourceColocatedShardIntervalList,
@@ -596,6 +617,10 @@ BlockingShardSplit(SplitOperation splitOperation,
 		/* Used for testing */
 		ConflictOnlyWithIsolationTesting();
 
+		ereport(LOG, (errmsg(
+						  "creating auxillary structures (indexes, stats, replicaindentities, triggers) for %s",
+						  operationName)));
+
 		/* Create auxiliary structures (indexes, stats, replicaindentities, triggers) */
 		CreateAuxiliaryStructuresForShardGroup(shardGroupSplitIntervalListList,
 											   workersForPlacementList,
@@ -617,10 +642,16 @@ BlockingShardSplit(SplitOperation splitOperation,
 		 */
 		if (DeferShardDeleteOnSplit)
 		{
+			ereport(LOG, (errmsg("marking deferred cleanup of source shard(s) for %s",
+								 operationName)));
+
 			InsertDeferredDropCleanupRecordsForShards(sourceColocatedShardIntervalList);
 		}
 		else
 		{
+			ereport(LOG, (errmsg("performing cleanup of source shard(s) for %s",
+								 operationName)));
+
 			DropShardList(sourceColocatedShardIntervalList);
 		}
 
@@ -635,6 +666,9 @@ BlockingShardSplit(SplitOperation splitOperation,
 			shardGroupSplitIntervalListList,
 			workersForPlacementList);
 
+		ereport(LOG, (errmsg("creating foreign key constraints (if any) for %s",
+							 operationName)));
+
 		/*
 		 * Create foreign keys if exists after the metadata changes happening in
 		 * DropShardList() and InsertSplitChildrenShardMetadata() because the foreign
@@ -649,7 +683,7 @@ BlockingShardSplit(SplitOperation splitOperation,
 		ShutdownAllConnections();
 
 		/* Do a best effort cleanup of shards created on workers in the above block */
-		FinalizeOperationNeedingCleanupOnFailure();
+		FinalizeOperationNeedingCleanupOnFailure(operationName);
 
 		PG_RE_THROW();
 	}
@@ -670,10 +704,15 @@ CheckIfRelationWithSameNameExists(ShardInterval *shardInterval, WorkerNode *work
 	AppendShardIdToName(&shardName, shardInterval->shardId);
 
 	StringInfo checkShardExistsQuery = makeStringInfo();
+
+	/*
+	 * We pass schemaName and shardName without quote_identifier, since
+	 * they are used as strings here.
+	 */
 	appendStringInfo(checkShardExistsQuery,
-					 "SELECT EXISTS (SELECT FROM pg_catalog.pg_tables WHERE schemaname = '%s' AND tablename = '%s');",
-					 schemaName,
-					 shardName);
+					 "SELECT EXISTS (SELECT FROM pg_catalog.pg_tables WHERE schemaname = %s AND tablename = %s);",
+					 quote_literal_cstr(schemaName),
+					 quote_literal_cstr(shardName));
 
 	int connectionFlags = 0;
 	MultiConnection *connection = GetNodeUserDatabaseConnection(connectionFlags,
@@ -691,11 +730,13 @@ CheckIfRelationWithSameNameExists(ShardInterval *shardInterval, WorkerNode *work
 		ReportResultError(connection, result, ERROR);
 	}
 
-	char *checkExists = PQgetvalue(result, 0, 0);
+	char *existsString = PQgetvalue(result, 0, 0);
+	bool tableExists = strcmp(existsString, "t") == 0;
+
 	PQclear(result);
 	ForgetResults(connection);
 
-	return strcmp(checkExists, "t") == 0;
+	return tableExists;
 }
 
 
@@ -1015,11 +1056,13 @@ static void
 CreateObjectOnPlacement(List *objectCreationCommandList,
 						WorkerNode *workerPlacementNode)
 {
-	char *currentUser = CurrentUserName();
-	SendCommandListToWorkerOutsideTransaction(workerPlacementNode->workerName,
-											  workerPlacementNode->workerPort,
-											  currentUser,
-											  objectCreationCommandList);
+	MultiConnection *connection =
+		GetNodeUserDatabaseConnection(OUTSIDE_TRANSACTION,
+									  workerPlacementNode->workerName,
+									  workerPlacementNode->workerPort,
+									  NULL, NULL);
+	SendCommandListToWorkerOutsideTransactionWithConnection(connection,
+															objectCreationCommandList);
 }
 
 
@@ -1487,6 +1530,8 @@ NonBlockingShardSplit(SplitOperation splitOperation,
 					  DistributionColumnMap *distributionColumnOverrides,
 					  uint32 targetColocationId)
 {
+	const char *operationName = SplitOperationAPIName[splitOperation];
+
 	ErrorIfMultipleNonblockingMoveSplitInTheSameTransaction();
 
 	char *superUser = CitusExtensionOwnerName();
@@ -1529,6 +1574,9 @@ NonBlockingShardSplit(SplitOperation splitOperation,
 	/* Non-Blocking shard split workflow starts here */
 	PG_TRY();
 	{
+		ereport(LOG, (errmsg("creating child shards for %s",
+							 operationName)));
+
 		/* 1) Physically create split children. */
 		CreateSplitShardsForShardGroup(shardGroupSplitIntervalListList,
 									   workersForPlacementList);
@@ -1558,6 +1606,10 @@ NonBlockingShardSplit(SplitOperation splitOperation,
 		 */
 		CreateReplicaIdentitiesForDummyShards(mapOfPlacementToDummyShardList);
 
+		ereport(LOG, (errmsg(
+						  "creating replication artifacts (publications, replication slots, subscriptions for %s",
+						  operationName)));
+
 		/* 4) Create Publications. */
 		CreatePublications(sourceConnection, publicationInfoHash);
 
@@ -1606,11 +1658,35 @@ NonBlockingShardSplit(SplitOperation splitOperation,
 			databaseName,
 			logicalRepTargetList);
 
+		/*
+		 * We have to create the primary key (or any other replica identity)
+		 * before the update/delete operations that are queued will be
+		 * replicated. Because if the replica identity does not exist on the
+		 * target, the replication would fail.
+		 *
+		 * So the latest possible moment we could do this is right after the
+		 * initial data COPY, but before enabling the susbcriptions. It might
+		 * seem like a good idea to it after the initial data COPY, since
+		 * it's generally the rule that it's cheaper to build an index at once
+		 * than to create it incrementally. This general rule, is why we create
+		 * all the regular indexes as late during the move as possible.
+		 *
+		 * But as it turns out in practice it's not as clear cut, and we saw a
+		 * speed degradation in the time it takes to move shards when doing the
+		 * replica identity creation after the initial COPY. So, instead we
+		 * keep it before the COPY.
+		 */
+		CreateReplicaIdentities(logicalRepTargetList);
+
+		ereport(LOG, (errmsg("performing copy for %s", operationName)));
+
 		/* 8) Do snapshotted Copy */
 		DoSplitCopy(sourceShardToCopyNode, sourceColocatedShardIntervalList,
 					shardGroupSplitIntervalListList, workersForPlacementList,
 					snapshot, distributionColumnOverrides);
 
+		ereport(LOG, (errmsg("replicating changes for %s", operationName)));
+
 		/*
 		 * 9) Logically replicate all the changes and do most of the table DDL,
 		 * like index and foreign key creation.
@@ -1631,10 +1707,16 @@ NonBlockingShardSplit(SplitOperation splitOperation,
 		 */
 		if (DeferShardDeleteOnSplit)
 		{
+			ereport(LOG, (errmsg("marking deferred cleanup of source shard(s) for %s",
+								 operationName)));
+
 			InsertDeferredDropCleanupRecordsForShards(sourceColocatedShardIntervalList);
 		}
 		else
 		{
+			ereport(LOG, (errmsg("performing cleanup of source shard(s) for %s",
+								 operationName)));
+
 			DropShardList(sourceColocatedShardIntervalList);
 		}
 
@@ -1683,6 +1765,9 @@ NonBlockingShardSplit(SplitOperation splitOperation,
 		 */
 		CreatePartitioningHierarchy(logicalRepTargetList);
 
+		ereport(LOG, (errmsg("creating foreign key constraints (if any) for %s",
+							 operationName)));
+
 		/*
 		 * 14) Create foreign keys if exists after the metadata changes happening in
 		 * DropShardList() and InsertSplitChildrenShardMetadata() because the foreign
@@ -1694,7 +1779,7 @@ NonBlockingShardSplit(SplitOperation splitOperation,
 		 * 15) Release shared memory allocated by worker_split_shard_replication_setup udf
 		 * at source node.
 		 */
-		ExecuteSplitShardReleaseSharedMemory(sourceShardToCopyNode);
+		ExecuteSplitShardReleaseSharedMemory(sourceConnection);
 
 		/* 16) Close source connection */
 		CloseConnection(sourceConnection);
@@ -1716,7 +1801,11 @@ NonBlockingShardSplit(SplitOperation splitOperation,
 		 */
 		DropAllLogicalReplicationLeftovers(SHARD_SPLIT);
 
-		FinalizeOperationNeedingCleanupOnFailure();
+		/*
+		 * Drop temporary objects that were marked as CLEANUP_ON_FAILURE
+		 * or CLEANUP_ALWAYS.
+		 */
+		FinalizeOperationNeedingCleanupOnFailure(operationName);
 
 		PG_RE_THROW();
 	}
@@ -1900,7 +1989,7 @@ CreateWorkerForPlacementSet(List *workersForPlacementList)
 	/* we don't have value field as it's a set */
 	info.entrysize = info.keysize;
 
-	uint32 hashFlags = (HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT);
+	uint32 hashFlags = (HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT | HASH_COMPARE);
 
 	HTAB *workerForPlacementSet = hash_create("worker placement set", 32, &info,
 											  hashFlags);
@@ -1987,19 +2076,8 @@ ExecuteSplitShardReplicationSetupUDF(WorkerNode *sourceWorkerNode,
  * shared memory to store split information. This has to be released after split completes(or fails).
  */
 static void
-ExecuteSplitShardReleaseSharedMemory(WorkerNode *sourceWorkerNode)
+ExecuteSplitShardReleaseSharedMemory(MultiConnection *sourceConnection)
 {
-	char *superUser = CitusExtensionOwnerName();
-	char *databaseName = get_database_name(MyDatabaseId);
-
-	int connectionFlag = FORCE_NEW_CONNECTION;
-	MultiConnection *sourceConnection = GetNodeUserDatabaseConnection(
-		connectionFlag,
-		sourceWorkerNode->workerName,
-		sourceWorkerNode->workerPort,
-		superUser,
-		databaseName);
-
 	StringInfo splitShardReleaseMemoryUDF = makeStringInfo();
 	appendStringInfo(splitShardReleaseMemoryUDF,
 					 "SELECT pg_catalog.worker_split_shard_release_dsm();");
@@ -2214,14 +2292,8 @@ GetNextShardIdForSplitChild()
 	appendStringInfo(nextValueCommand, "SELECT nextval(%s);", quote_literal_cstr(
 						 "pg_catalog.pg_dist_shardid_seq"));
 
-	int connectionFlag = FORCE_NEW_CONNECTION;
-	MultiConnection *connection = GetNodeUserDatabaseConnection(connectionFlag,
-																LocalHostName,
-																PostPortNumber,
-																CitusExtensionOwnerName(),
-																get_database_name(
-																	MyDatabaseId));
-
+	MultiConnection *connection = GetConnectionForLocalQueriesOutsideTransaction(
+		CitusExtensionOwnerName());
 	PGresult *result = NULL;
 	int queryResult = ExecuteOptionalRemoteCommand(connection, nextValueCommand->data,
 												   &result);
@@ -2238,7 +2310,8 @@ GetNextShardIdForSplitChild()
 	}
 
 	shardId = SafeStringToUint64(PQgetvalue(result, 0, 0 /* nodeId column*/));
-	CloseConnection(connection);
+	PQclear(result);
+	ForgetResults(connection);
 
 	return shardId;
 }

src/backend/distributed/operations/shard_transfer.c

@@ -1485,6 +1485,15 @@ EnsureShardCanBeCopied(int64 shardId, const char *sourceNodeName, int32 sourceNo
 								shardId)));
 		}
 	}
+
+	/*
+	 * Make sure the relation exists. In some cases the relation is actually dropped but
+	 * the metadata remains, such as dropping table while citus.enable_ddl_propagation
+	 * is set to off.
+	 */
+	ShardInterval *shardInterval = LoadShardInterval(shardId);
+	Oid distributedTableId = shardInterval->relationId;
+	EnsureRelationExists(distributedTableId);
 }
 
 

src/backend/distributed/operations/worker_copy_table_to_node_udf.c

@@ -53,8 +53,14 @@ worker_copy_table_to_node(PG_FUNCTION_ARGS)
 		targetNodeId);
 
 	StringInfo selectShardQueryForCopy = makeStringInfo();
+
+	/*
+	 * Even though we do COPY(SELECT ...) all the columns, we can't just do SELECT * because we need to not COPY generated colums.
+	 */
+	const char *columnList = CopyableColumnNamesFromRelationName(relationSchemaName,
+																 relationName);
 	appendStringInfo(selectShardQueryForCopy,
-					 "SELECT * FROM %s;", relationQualifiedName);
+					 "SELECT %s FROM %s;", columnList, relationQualifiedName);
 
 	ParamListInfo params = NULL;
 	ExecuteQueryStringIntoDestReceiver(selectShardQueryForCopy->data, params,

src/backend/distributed/operations/worker_node_manager.c

@@ -267,8 +267,11 @@ ErrorIfCoordinatorNotAddedAsWorkerNode()
 		return;
 	}
 
-	ereport(ERROR, (errmsg("could not find the coordinator node in "
-						   "metadata as it is not added as a worker")));
+	ereport(ERROR, (errmsg("operation is not allowed when coordinator "
+						   "is not added into metadata"),
+					errhint("Use \"SELECT citus_set_coordinator_host('"
+							"<hostname>', '<port>')\" to configure the "
+							"coordinator hostname and port")));
 }
 
 

src/backend/distributed/operations/worker_shard_copy.c

@@ -73,7 +73,7 @@ static void ShardCopyDestReceiverDestroy(DestReceiver *destReceiver);
 static bool CanUseLocalCopy(uint32_t destinationNodeId);
 static StringInfo ConstructShardCopyStatement(List *destinationShardFullyQualifiedName,
 											  bool
-											  useBinaryFormat);
+											  useBinaryFormat, TupleDesc tupleDesc);
 static void WriteLocalTuple(TupleTableSlot *slot, ShardCopyDestReceiver *copyDest);
 static int ReadFromLocalBufferCallback(void *outBuf, int minRead, int maxRead);
 static void LocalCopyToShard(ShardCopyDestReceiver *copyDest, CopyOutState
@@ -105,7 +105,8 @@ ConnectToRemoteAndStartCopy(ShardCopyDestReceiver *copyDest)
 
 	StringInfo copyStatement = ConstructShardCopyStatement(
 		copyDest->destinationShardFullyQualifiedName,
-		copyDest->copyOutState->binary);
+		copyDest->copyOutState->binary,
+		copyDest->tupleDescriptor);
 
 	if (!SendRemoteCommand(copyDest->connection, copyStatement->data))
 	{
@@ -344,21 +345,80 @@ ShardCopyDestReceiverDestroy(DestReceiver *dest)
 }
 
 
+/*
+ *  CopyableColumnNamesFromTupleDesc function creates and returns a comma seperated column names string  to be used in COPY
+ *  and SELECT statements when copying a table. The COPY and SELECT statements should filter out the GENERATED columns since COPY
+ *  statement fails to handle them. Iterating over the attributes of the table we also need to skip the dropped columns.
+ */
+const char *
+CopyableColumnNamesFromTupleDesc(TupleDesc tupDesc)
+{
+	StringInfo columnList = makeStringInfo();
+	bool firstInList = true;
+
+	for (int i = 0; i < tupDesc->natts; i++)
+	{
+		Form_pg_attribute att = TupleDescAttr(tupDesc, i);
+		if (att->attgenerated || att->attisdropped)
+		{
+			continue;
+		}
+		if (!firstInList)
+		{
+			appendStringInfo(columnList, ",");
+		}
+
+		firstInList = false;
+
+		appendStringInfo(columnList, "%s", quote_identifier(NameStr(att->attname)));
+	}
+
+	return columnList->data;
+}
+
+
+/*
+ *  CopyableColumnNamesFromRelationName function is a wrapper for CopyableColumnNamesFromTupleDesc.
+ */
+const char *
+CopyableColumnNamesFromRelationName(const char *schemaName, const char *relationName)
+{
+	Oid namespaceOid = get_namespace_oid(schemaName, true);
+
+	Oid relationId = get_relname_relid(relationName, namespaceOid);
+
+	Relation relation = relation_open(relationId, AccessShareLock);
+
+	TupleDesc tupleDesc = RelationGetDescr(relation);
+
+	const char *columnList = CopyableColumnNamesFromTupleDesc(tupleDesc);
+
+	relation_close(relation, NoLock);
+
+	return columnList;
+}
+
+
 /*
  * ConstructShardCopyStatement constructs the text of a COPY statement
  * for copying into a result table
  */
 static StringInfo
 ConstructShardCopyStatement(List *destinationShardFullyQualifiedName, bool
-							useBinaryFormat)
+							useBinaryFormat,
+							TupleDesc tupleDesc)
 {
 	char *destinationShardSchemaName = linitial(destinationShardFullyQualifiedName);
 	char *destinationShardRelationName = lsecond(destinationShardFullyQualifiedName);
 
+
 	StringInfo command = makeStringInfo();
-	appendStringInfo(command, "COPY %s.%s FROM STDIN",
+
+	const char *columnList = CopyableColumnNamesFromTupleDesc(tupleDesc);
+
+	appendStringInfo(command, "COPY %s.%s (%s) FROM STDIN",
 					 quote_identifier(destinationShardSchemaName), quote_identifier(
-						 destinationShardRelationName));
+						 destinationShardRelationName), columnList);
 
 	if (useBinaryFormat)
 	{

src/backend/distributed/operations/worker_split_copy_udf.c

@@ -43,6 +43,10 @@ static DestReceiver *  CreatePartitionedSplitCopyDestReceiver(EState *executor,
 															  List *splitCopyInfoList);
 static void BuildMinMaxRangeArrays(List *splitCopyInfoList, ArrayType **minValueArray,
 								   ArrayType **maxValueArray);
+static char * TraceWorkerSplitCopyUdf(char *sourceShardToCopySchemaName,
+									  char *sourceShardToCopyPrefix,
+									  char *sourceShardToCopyQualifiedName,
+									  List *splitCopyInfoList);
 
 /*
  * worker_split_copy(source_shard_id bigint, splitCopyInfo pg_catalog.split_copy_info[])
@@ -93,15 +97,26 @@ worker_split_copy(PG_FUNCTION_ARGS)
 	Oid sourceShardToCopySchemaOId = get_rel_namespace(
 		shardIntervalToSplitCopy->relationId);
 	char *sourceShardToCopySchemaName = get_namespace_name(sourceShardToCopySchemaOId);
-	char *sourceShardToCopyName = get_rel_name(shardIntervalToSplitCopy->relationId);
+	char *sourceShardPrefix = get_rel_name(shardIntervalToSplitCopy->relationId);
+	char *sourceShardToCopyName = pstrdup(sourceShardPrefix);
 	AppendShardIdToName(&sourceShardToCopyName, shardIdToSplitCopy);
 	char *sourceShardToCopyQualifiedName = quote_qualified_identifier(
 		sourceShardToCopySchemaName,
 		sourceShardToCopyName);
 
+	ereport(LOG, (errmsg("%s", TraceWorkerSplitCopyUdf(sourceShardToCopySchemaName,
+													   sourceShardPrefix,
+													   sourceShardToCopyQualifiedName,
+													   splitCopyInfoList))));
+
 	StringInfo selectShardQueryForCopy = makeStringInfo();
+	const char *columnList = CopyableColumnNamesFromRelationName(
+		sourceShardToCopySchemaName,
+		sourceShardToCopyName);
+
 	appendStringInfo(selectShardQueryForCopy,
-					 "SELECT * FROM %s;", sourceShardToCopyQualifiedName);
+					 "SELECT %s FROM %s;", columnList,
+					 sourceShardToCopyQualifiedName);
 
 	ParamListInfo params = NULL;
 	ExecuteQueryStringIntoDestReceiver(selectShardQueryForCopy->data, params,
@@ -113,6 +128,48 @@ worker_split_copy(PG_FUNCTION_ARGS)
 }
 
 
+/* Trace split copy udf */
+static char *
+TraceWorkerSplitCopyUdf(char *sourceShardToCopySchemaName,
+						char *sourceShardToCopyPrefix,
+						char *sourceShardToCopyQualifiedName,
+						List *splitCopyInfoList)
+{
+	StringInfo splitCopyTrace = makeStringInfo();
+	appendStringInfo(splitCopyTrace, "performing copy from shard %s to [",
+					 sourceShardToCopyQualifiedName);
+
+	/* split copy always has atleast two destinations */
+	int index = 1;
+	int splitWayCount = list_length(splitCopyInfoList);
+	SplitCopyInfo *splitCopyInfo = NULL;
+	foreach_ptr(splitCopyInfo, splitCopyInfoList)
+	{
+		char *shardNameCopy = pstrdup(sourceShardToCopyPrefix);
+		AppendShardIdToName(&shardNameCopy, splitCopyInfo->destinationShardId);
+
+		char *shardNameCopyQualifiedName = quote_qualified_identifier(
+			sourceShardToCopySchemaName,
+			shardNameCopy);
+
+		appendStringInfo(splitCopyTrace, "%s (nodeId: %u)", shardNameCopyQualifiedName,
+						 splitCopyInfo->destinationShardNodeId);
+		pfree(shardNameCopy);
+
+		if (index < splitWayCount)
+		{
+			appendStringInfo(splitCopyTrace, ", ");
+		}
+
+		index++;
+	}
+
+	appendStringInfo(splitCopyTrace, "]");
+
+	return splitCopyTrace->data;
+}
+
+
 /* Parse a single SplitCopyInfo Tuple */
 static void
 ParseSplitCopyInfoDatum(Datum splitCopyInfoDatum, SplitCopyInfo **splitCopyInfo)

src/backend/distributed/operations/worker_split_shard_replication_setup_udf.c

@@ -281,7 +281,8 @@ PopulateShardSplitInfoInSM(ShardSplitInfoSMHeader *shardSplitInfoSMHeader)
 	{
 		uint32_t nodeId = entry->key.nodeId;
 		uint32_t tableOwnerId = entry->key.tableOwnerId;
-		char *derivedSlotName = ReplicationSlotName(SHARD_SPLIT, nodeId, tableOwnerId);
+		char *derivedSlotName = ReplicationSlotNameForNodeAndOwner(SHARD_SPLIT, nodeId,
+																   tableOwnerId);
 
 		List *shardSplitInfoList = entry->shardSplitInfoList;
 		ShardSplitInfo *splitShardInfo = NULL;
@@ -389,8 +390,9 @@ ReturnReplicationSlotInfo(Tuplestorestate *tupleStore, TupleDesc
 		char *tableOwnerName = GetUserNameFromId(entry->key.tableOwnerId, false);
 		values[1] = CStringGetTextDatum(tableOwnerName);
 
-		char *slotName = ReplicationSlotName(SHARD_SPLIT, entry->key.nodeId,
-											 entry->key.tableOwnerId);
+		char *slotName = ReplicationSlotNameForNodeAndOwner(SHARD_SPLIT,
+															entry->key.nodeId,
+															entry->key.tableOwnerId);
 		values[2] = CStringGetTextDatum(slotName);
 
 		tuplestore_putvalues(tupleStore, tupleDescriptor, values, nulls);

src/backend/distributed/planner/distributed_planner.c

@@ -1897,14 +1897,14 @@ multi_relation_restriction_hook(PlannerInfo *root, RelOptInfo *relOptInfo,
 	MemoryContext restrictionsMemoryContext = plannerRestrictionContext->memoryContext;
 	MemoryContext oldMemoryContext = MemoryContextSwitchTo(restrictionsMemoryContext);
 
-	bool distributedTable = IsCitusTable(rte->relid);
+	bool isCitusTable = IsCitusTable(rte->relid);
 
 	RelationRestriction *relationRestriction = palloc0(sizeof(RelationRestriction));
 	relationRestriction->index = restrictionIndex;
 	relationRestriction->relationId = rte->relid;
 	relationRestriction->rte = rte;
 	relationRestriction->relOptInfo = relOptInfo;
-	relationRestriction->distributedRelation = distributedTable;
+	relationRestriction->citusTable = isCitusTable;
 	relationRestriction->plannerInfo = root;
 
 	/* see comments on GetVarFromAssignedParam() */
@@ -1919,10 +1919,42 @@ multi_relation_restriction_hook(PlannerInfo *root, RelOptInfo *relOptInfo,
 	 * We're also keeping track of whether all participant
 	 * tables are reference tables.
 	 */
-	if (distributedTable)
+	if (isCitusTable)
 	{
 		cacheEntry = GetCitusTableCacheEntry(rte->relid);
 
+		/*
+		 * The statistics objects of the distributed table are not relevant
+		 * for the distributed planning, so we can override it.
+		 *
+		 * Normally, we should not need this. However, the combination of
+		 * Postgres commit 269b532aef55a579ae02a3e8e8df14101570dfd9 and
+		 * Citus function AdjustPartitioningForDistributedPlanning()
+		 * forces us to do this. The commit expects statistics objects
+		 * of partitions to have "inh" flag set properly. Whereas, the
+		 * function overrides "inh" flag. To avoid Postgres to throw error,
+		 * we override statlist such that Postgres does not try to process
+		 * any statistics objects during the standard_planner() on the
+		 * coordinator. In the end, we do not need the standard_planner()
+		 * on the coordinator to generate an optimized plan. We call
+		 * into standard_planner() for other purposes, such as generating the
+		 * relationRestrictionContext here.
+		 *
+		 * AdjustPartitioningForDistributedPlanning() is a hack that we use
+		 * to prevent Postgres' standard_planner() to expand all the partitions
+		 * for the distributed planning when a distributed partitioned table
+		 * is queried. It is required for both correctness and performance
+		 * reasons. Although we can eliminate the use of the function for
+		 * the correctness (e.g., make sure that rest of the planner can handle
+		 * partitions), it's performance implication is hard to avoid. Certain
+		 * planning logic of Citus (such as router or query pushdown) relies
+		 * heavily on the relationRestrictionList. If
+		 * AdjustPartitioningForDistributedPlanning() is removed, all the
+		 * partitions show up in the, causing high planning times for
+		 * such queries.
+		 */
+		relOptInfo->statlist = NIL;
+
 		relationRestrictionContext->allReferenceTables &=
 			IsCitusTableTypeCacheEntry(cacheEntry, REFERENCE_TABLE);
 	}

src/backend/distributed/planner/multi_router_planner.c

@@ -3692,7 +3692,7 @@ CopyRelationRestrictionContext(RelationRestrictionContext *oldContext)
 
 		newRestriction->index = oldRestriction->index;
 		newRestriction->relationId = oldRestriction->relationId;
-		newRestriction->distributedRelation = oldRestriction->distributedRelation;
+		newRestriction->citusTable = oldRestriction->citusTable;
 		newRestriction->rte = copyObject(oldRestriction->rte);
 
 		/* can't be copied, we copy (flatly) a RelOptInfo, and then decouple baserestrictinfo */

src/backend/distributed/planner/relation_restriction_equivalence.c

@@ -224,7 +224,7 @@ ContextContainsLocalRelation(RelationRestrictionContext *restrictionContext)
 	{
 		RelationRestriction *relationRestriction = lfirst(relationRestrictionCell);
 
-		if (!relationRestriction->distributedRelation)
+		if (!relationRestriction->citusTable)
 		{
 			return true;
 		}

src/backend/distributed/replication/multi_logical_replication.c

@@ -66,6 +66,7 @@
 #include "utils/syscache.h"
 
 #define STR_ERRCODE_UNDEFINED_OBJECT "42704"
+#define STR_ERRCODE_OBJECT_IN_USE "55006"
 
 
 #define REPLICATION_SLOT_CATALOG_TABLE_NAME "pg_replication_slots"
@@ -156,6 +157,10 @@ static void WaitForGroupedLogicalRepTargetsToBecomeReady(
 static void WaitForGroupedLogicalRepTargetsToCatchUp(XLogRecPtr sourcePosition,
 													 GroupedLogicalRepTargets *
 													 groupedLogicalRepTargets);
+static void RecreateGroupedLogicalRepTargetsConnections(
+	HTAB *groupedLogicalRepTargetsHash,
+	char *user,
+	char *databaseName);
 
 /*
  * LogicallyReplicateShards replicates a list of shards from one node to another
@@ -233,6 +238,26 @@ LogicallyReplicateShards(List *shardList, char *sourceNodeName, int sourceNodePo
 		/* only useful for isolation testing, see the function comment for the details */
 		ConflictOnlyWithIsolationTesting();
 
+		/*
+		 * We have to create the primary key (or any other replica identity)
+		 * before the update/delete operations that are queued will be
+		 * replicated. Because if the replica identity does not exist on the
+		 * target, the replication would fail.
+		 *
+		 * So the latest possible moment we could do this is right after the
+		 * initial data COPY, but before enabling the susbcriptions. It might
+		 * seem like a good idea to it after the initial data COPY, since
+		 * it's generally the rule that it's cheaper to build an index at once
+		 * than to create it incrementally. This general rule, is why we create
+		 * all the regular indexes as late during the move as possible.
+		 *
+		 * But as it turns out in practice it's not as clear cut, and we saw a
+		 * speed degradation in the time it takes to move shards when doing the
+		 * replica identity creation after the initial COPY. So, instead we
+		 * keep it before the COPY.
+		 */
+		CreateReplicaIdentities(logicalRepTargetList);
+
 		CopyShardsToNode(sourceNode, targetNode, shardList, snapshot);
 
 		/*
@@ -346,20 +371,6 @@ CompleteNonBlockingShardTransfer(List *shardList,
 								 HTAB *groupedLogicalRepTargetsHash,
 								 LogicalRepType type)
 {
-	/*
-	 * We have to create the primary key (or any other replica identity)
-	 * before the update/delete operations that are queued will be
-	 * replicated. Because if the replica identity does not exist on the
-	 * target, the replication would fail.
-	 *
-	 * So we it right after the initial data COPY, but before enabling the
-	 * susbcriptions. We do it at this latest possible moment, because its
-	 * much cheaper to build an index at once than to create it
-	 * incrementally. So this way we create the primary key index in one go
-	 * for all data from the initial COPY.
-	 */
-	CreateReplicaIdentities(logicalRepTargetList);
-
 	/* Start applying the changes from the replication slots to catch up. */
 	EnableSubscriptions(logicalRepTargetList);
 
@@ -490,9 +501,9 @@ CreateShardMoveLogicalRepTargetList(HTAB *publicationInfoHash, List *shardList)
 		target->newShards = NIL;
 		target->subscriptionOwnerName = SubscriptionRoleName(SHARD_MOVE, ownerId);
 		target->replicationSlot = palloc0(sizeof(ReplicationSlotInfo));
-		target->replicationSlot->name = ReplicationSlotName(SHARD_MOVE,
-															nodeId,
-															ownerId);
+		target->replicationSlot->name = ReplicationSlotNameForNodeAndOwner(SHARD_MOVE,
+																		   nodeId,
+																		   ownerId);
 		target->replicationSlot->targetNodeId = nodeId;
 		target->replicationSlot->tableOwnerId = ownerId;
 		logicalRepTargetList = lappend(logicalRepTargetList, target);
@@ -559,10 +570,10 @@ DropAllLogicalReplicationLeftovers(LogicalRepType type)
 	char *databaseName = get_database_name(MyDatabaseId);
 
 	/*
-	 * We open new connections to all nodes. The reason for this is that
-	 * operations on subscriptions, publications and replication slotscannot be
-	 * run in a transaction. By forcing a new connection we make sure no
-	 * transaction is active on the connection.
+	 * We need connections that are not currently inside a transaction. The
+	 * reason for this is that operations on subscriptions, publications and
+	 * replication slots cannot be run in a transaction. By forcing a new
+	 * connection we make sure no transaction is active on the connection.
 	 */
 	int connectionFlags = FORCE_NEW_CONNECTION;
 
@@ -600,7 +611,9 @@ DropAllLogicalReplicationLeftovers(LogicalRepType type)
 		/*
 		 * We close all connections that we opened for the dropping here. That
 		 * way we don't keep these connections open unnecessarily during the
-		 * 'LogicalRepType' operation (which can take a long time).
+		 * 'LogicalRepType' operation (which can take a long time). We might
+		 * need to reopen a few later on, but that seems better than keeping
+		 * many open for no reason for a long time.
 		 */
 		CloseConnection(cleanupConnection);
 	}
@@ -1150,11 +1163,14 @@ CreatePartitioningHierarchy(List *logicalRepTargetList)
 				 * parallel, so create them sequentially. Also attaching partition
 				 * is a quick operation, so it is fine to execute sequentially.
 				 */
-				SendCommandListToWorkerOutsideTransaction(
-					target->superuserConnection->hostname,
-					target->superuserConnection->port,
-					tableOwner,
-					list_make1(attachPartitionCommand));
+
+				MultiConnection *connection =
+					GetNodeUserDatabaseConnection(OUTSIDE_TRANSACTION,
+												  target->superuserConnection->hostname,
+												  target->superuserConnection->port,
+												  tableOwner, NULL);
+				ExecuteCriticalRemoteCommand(connection, attachPartitionCommand);
+
 				MemoryContextReset(localContext);
 			}
 		}
@@ -1203,10 +1219,8 @@ CreateUncheckedForeignKeyConstraints(List *logicalRepTargetList)
 				list_make1("SET LOCAL citus.skip_constraint_validation TO ON;"),
 				commandList);
 
-			SendCommandListToWorkerOutsideTransaction(
-				target->superuserConnection->hostname,
-				target->superuserConnection->port,
-				target->superuserConnection->user,
+			SendCommandListToWorkerOutsideTransactionWithConnection(
+				target->superuserConnection,
 				commandList);
 
 			MemoryContextReset(localContext);
@@ -1281,18 +1295,64 @@ DropPublications(MultiConnection *sourceConnection, HTAB *publicationInfoHash)
 
 /*
  * DropReplicationSlot drops the replication slot with the given name
- * if it exists.
+ * if it exists. It retries if the command fails with an OBJECT_IN_USE error.
  */
 static void
 DropReplicationSlot(MultiConnection *connection, char *replicationSlotName)
 {
-	ExecuteCriticalRemoteCommand(
-		connection,
-		psprintf(
-			"select pg_drop_replication_slot(slot_name) from "
-			REPLICATION_SLOT_CATALOG_TABLE_NAME
-			" where slot_name = %s",
-			quote_literal_cstr(replicationSlotName)));
+	int maxSecondsToTryDropping = 20;
+	bool raiseInterrupts = true;
+	PGresult *result = NULL;
+
+	/* we'll retry in case of an OBJECT_IN_USE error */
+	while (maxSecondsToTryDropping >= 0)
+	{
+		int querySent = SendRemoteCommand(
+			connection,
+			psprintf(
+				"select pg_drop_replication_slot(slot_name) from "
+				REPLICATION_SLOT_CATALOG_TABLE_NAME
+				" where slot_name = %s",
+				quote_literal_cstr(replicationSlotName))
+			);
+
+		if (querySent == 0)
+		{
+			ReportConnectionError(connection, ERROR);
+		}
+
+		result = GetRemoteCommandResult(connection, raiseInterrupts);
+
+		if (IsResponseOK(result))
+		{
+			/* no error, we are good to go */
+			break;
+		}
+
+		char *errorcode = PQresultErrorField(result, PG_DIAG_SQLSTATE);
+		if (errorcode != NULL && strcmp(errorcode, STR_ERRCODE_OBJECT_IN_USE) == 0 &&
+			maxSecondsToTryDropping > 0)
+		{
+			/* retry dropping the replication slot after sleeping for one sec */
+			maxSecondsToTryDropping--;
+			pg_usleep(1000);
+		}
+		else
+		{
+			/*
+			 * Report error if:
+			 * - Error code is not 55006 (Object In Use)
+			 * - Or, we have made enough number of retries (currently 20), but didn't work
+			 */
+			ReportResultError(connection, result, ERROR);
+		}
+
+		PQclear(result);
+		ForgetResults(connection);
+	}
+
+	PQclear(result);
+	ForgetResults(connection);
 }
 
 
@@ -1321,11 +1381,14 @@ PublicationName(LogicalRepType type, uint32_t nodeId, Oid ownerId)
 
 
 /*
- * ReplicationSlotName returns the name of the replication slot for the given
- * node and table owner.
+ * ReplicationSlotNameForNodeAndOwner returns the name of the replication slot for the
+ * given node and table owner.
+ *
+ * Note that PG15 introduced a new ReplicationSlotName function that caused name conflicts
+ * and we renamed this function.
  */
 char *
-ReplicationSlotName(LogicalRepType type, uint32_t nodeId, Oid ownerId)
+ReplicationSlotNameForNodeAndOwner(LogicalRepType type, uint32_t nodeId, Oid ownerId)
 {
 	StringInfo slotName = makeStringInfo();
 	appendStringInfo(slotName, "%s%u_%u", replicationSlotPrefix[type], nodeId,
@@ -1348,7 +1411,7 @@ ReplicationSlotName(LogicalRepType type, uint32_t nodeId, Oid ownerId)
 char *
 SubscriptionName(LogicalRepType type, Oid ownerId)
 {
-	return psprintf("%s%i", subscriptionPrefix[type], ownerId);
+	return psprintf("%s%u", subscriptionPrefix[type], ownerId);
 }
 
 
@@ -1359,7 +1422,7 @@ SubscriptionName(LogicalRepType type, Oid ownerId)
 char *
 SubscriptionRoleName(LogicalRepType type, Oid ownerId)
 {
-	return psprintf("%s%i", subscriptionRolePrefix[type], ownerId);
+	return psprintf("%s%u", subscriptionRolePrefix[type], ownerId);
 }
 
 
@@ -1585,11 +1648,11 @@ DropUser(MultiConnection *connection, char *username)
 	 * The DROP USER command should not propagate, so we temporarily disable
 	 * DDL propagation.
 	 */
-	SendCommandListToWorkerOutsideTransaction(
-		connection->hostname, connection->port, connection->user,
+	SendCommandListToWorkerOutsideTransactionWithConnection(
+		connection,
 		list_make2(
 			"SET LOCAL citus.enable_ddl_propagation TO OFF;",
-			psprintf("DROP USER IF EXISTS %s",
+			psprintf("DROP USER IF EXISTS %s;",
 					 quote_identifier(username))));
 }
 
@@ -1611,7 +1674,7 @@ CreatePublications(MultiConnection *connection,
 		bool prefixWithComma = false;
 
 		appendStringInfo(createPublicationCommand, "CREATE PUBLICATION %s FOR TABLE ",
-						 entry->name);
+						 quote_identifier(entry->name));
 
 		ShardInterval *shard = NULL;
 		foreach_ptr(shard, entry->shardIntervals)
@@ -1771,16 +1834,14 @@ CreateSubscriptions(MultiConnection *sourceConnection,
 		 * create a user with SUPERUSER permissions and then alter it to NOSUPERUSER.
 		 * This prevents permission escalations.
 		 */
-		SendCommandListToWorkerOutsideTransaction(
-			target->superuserConnection->hostname,
-			target->superuserConnection->port,
-			target->superuserConnection->user,
+		SendCommandListToWorkerOutsideTransactionWithConnection(
+			target->superuserConnection,
 			list_make2(
 				"SET LOCAL citus.enable_ddl_propagation TO OFF;",
 				psprintf(
-					"CREATE USER %s SUPERUSER IN ROLE %s",
-					target->subscriptionOwnerName,
-					GetUserNameFromId(ownerId, false)
+					"CREATE USER %s SUPERUSER IN ROLE %s;",
+					quote_identifier(target->subscriptionOwnerName),
+					quote_identifier(GetUserNameFromId(ownerId, false))
 					)));
 
 		StringInfo conninfo = makeStringInfo();
@@ -1824,23 +1885,21 @@ CreateSubscriptions(MultiConnection *sourceConnection,
 		pfree(createSubscriptionCommand);
 		ExecuteCriticalRemoteCommand(target->superuserConnection, psprintf(
 										 "ALTER SUBSCRIPTION %s OWNER TO %s",
-										 target->subscriptionName,
-										 target->subscriptionOwnerName
+										 quote_identifier(target->subscriptionName),
+										 quote_identifier(target->subscriptionOwnerName)
 										 ));
 
 		/*
 		 * The ALTER ROLE command should not propagate, so we temporarily
 		 * disable DDL propagation.
 		 */
-		SendCommandListToWorkerOutsideTransaction(
-			target->superuserConnection->hostname,
-			target->superuserConnection->port,
-			target->superuserConnection->user,
+		SendCommandListToWorkerOutsideTransactionWithConnection(
+			target->superuserConnection,
 			list_make2(
 				"SET LOCAL citus.enable_ddl_propagation TO OFF;",
 				psprintf(
-					"ALTER ROLE %s NOSUPERUSER",
-					target->subscriptionOwnerName
+					"ALTER ROLE %s NOSUPERUSER;",
+					quote_identifier(target->subscriptionOwnerName)
 					)));
 	}
 }
@@ -2001,8 +2060,12 @@ CreateGroupedLogicalRepTargetsConnections(HTAB *groupedLogicalRepTargetsHash,
  * RecreateGroupedLogicalRepTargetsConnections recreates connections for all of the
  * nodes in the groupedLogicalRepTargetsHash where the old connection is broken or
  * currently running a query.
+ *
+ * IMPORTANT: When it recreates the connection, it doesn't close the existing
+ * connection. This means that this function should only be called when we know
+ * we'll throw an error afterwards, otherwise we would leak these connections.
  */
-void
+static void
 RecreateGroupedLogicalRepTargetsConnections(HTAB *groupedLogicalRepTargetsHash,
 											char *user,
 											char *databaseName)
@@ -2012,10 +2075,11 @@ RecreateGroupedLogicalRepTargetsConnections(HTAB *groupedLogicalRepTargetsHash,
 	GroupedLogicalRepTargets *groupedLogicalRepTargets = NULL;
 	foreach_htab(groupedLogicalRepTargets, &status, groupedLogicalRepTargetsHash)
 	{
-		if (groupedLogicalRepTargets->superuserConnection &&
-			PQstatus(groupedLogicalRepTargets->superuserConnection->pgConn) ==
-			CONNECTION_OK &&
-			!PQisBusy(groupedLogicalRepTargets->superuserConnection->pgConn)
+		MultiConnection *superuserConnection =
+			groupedLogicalRepTargets->superuserConnection;
+		if (superuserConnection &&
+			PQstatus(superuserConnection->pgConn) == CONNECTION_OK &&
+			!PQisBusy(superuserConnection->pgConn)
 			)
 		{
 			continue;
@@ -2023,12 +2087,12 @@ RecreateGroupedLogicalRepTargetsConnections(HTAB *groupedLogicalRepTargetsHash,
 		WorkerNode *targetWorkerNode = FindNodeWithNodeId(
 			groupedLogicalRepTargets->nodeId,
 			false);
-		MultiConnection *superuserConnection =
-			GetNodeUserDatabaseConnection(connectionFlags,
-										  targetWorkerNode->workerName,
-										  targetWorkerNode->workerPort,
-										  user,
-										  databaseName);
+		superuserConnection = GetNodeUserDatabaseConnection(
+			connectionFlags,
+			targetWorkerNode->workerName,
+			targetWorkerNode->workerPort,
+			user,
+			databaseName);
 
 		/*
 		 * Operations on subscriptions cannot run in a transaction block. We

src/backend/distributed/shardsplit/shardsplit_decoder.c

@@ -34,6 +34,10 @@ static Oid FindTargetRelationOid(Relation sourceShardRelation,
 								 HeapTuple tuple,
 								 char *currentSlotName);
 
+static HeapTuple GetTupleForTargetSchema(HeapTuple sourceRelationTuple,
+										 TupleDesc sourceTupleDesc,
+										 TupleDesc targetTupleDesc);
+
 /*
  * Postgres uses 'pgoutput' as default plugin for logical replication.
  * We want to reuse Postgres pgoutput's functionality as much as possible.
@@ -129,6 +133,71 @@ split_change_cb(LogicalDecodingContext *ctx, ReorderBufferTXN *txn,
 	}
 
 	Relation targetRelation = RelationIdGetRelation(targetRelationOid);
+
+	/*
+	 * If any columns from source relation have been dropped, then the tuple needs to
+	 * be formatted according to the target relation.
+	 */
+	TupleDesc sourceRelationDesc = RelationGetDescr(relation);
+	TupleDesc targetRelationDesc = RelationGetDescr(targetRelation);
+	if (sourceRelationDesc->natts > targetRelationDesc->natts)
+	{
+		switch (change->action)
+		{
+			case REORDER_BUFFER_CHANGE_INSERT:
+			{
+				HeapTuple sourceRelationNewTuple = &(change->data.tp.newtuple->tuple);
+				HeapTuple targetRelationNewTuple = GetTupleForTargetSchema(
+					sourceRelationNewTuple, sourceRelationDesc, targetRelationDesc);
+
+				change->data.tp.newtuple->tuple = *targetRelationNewTuple;
+				break;
+			}
+
+			case REORDER_BUFFER_CHANGE_UPDATE:
+			{
+				HeapTuple sourceRelationNewTuple = &(change->data.tp.newtuple->tuple);
+				HeapTuple targetRelationNewTuple = GetTupleForTargetSchema(
+					sourceRelationNewTuple, sourceRelationDesc, targetRelationDesc);
+
+				change->data.tp.newtuple->tuple = *targetRelationNewTuple;
+
+				/*
+				 * Format oldtuple according to the target relation. If the column values of replica
+				 * identiy change, then the old tuple is non-null and needs to be formatted according
+				 * to the target relation schema.
+				 */
+				if (change->data.tp.oldtuple != NULL)
+				{
+					HeapTuple sourceRelationOldTuple = &(change->data.tp.oldtuple->tuple);
+					HeapTuple targetRelationOldTuple = GetTupleForTargetSchema(
+						sourceRelationOldTuple,
+						sourceRelationDesc,
+						targetRelationDesc);
+
+					change->data.tp.oldtuple->tuple = *targetRelationOldTuple;
+				}
+				break;
+			}
+
+			case REORDER_BUFFER_CHANGE_DELETE:
+			{
+				HeapTuple sourceRelationOldTuple = &(change->data.tp.oldtuple->tuple);
+				HeapTuple targetRelationOldTuple = GetTupleForTargetSchema(
+					sourceRelationOldTuple, sourceRelationDesc, targetRelationDesc);
+
+				change->data.tp.oldtuple->tuple = *targetRelationOldTuple;
+				break;
+			}
+
+			/* Only INSERT/DELETE/UPDATE actions are visible in the replication path of split shard */
+			default:
+				ereport(ERROR, errmsg(
+							"Unexpected Action :%d. Expected action is INSERT/DELETE/UPDATE",
+							change->action));
+		}
+	}
+
 	pgoutputChangeCB(ctx, txn, targetRelation, change);
 	RelationClose(targetRelation);
 }
@@ -223,3 +292,51 @@ GetHashValueForIncomingTuple(Relation sourceShardRelation,
 
 	return DatumGetInt32(hashedValueDatum);
 }
+
+
+/*
+ * GetTupleForTargetSchema returns a tuple with the schema of the target relation.
+ * If some columns within the source relations are dropped, we would have to reformat
+ * the tuple to match the schema of the target relation.
+ *
+ * Consider the below scenario:
+ * Session1 : Drop column followed by create_distributed_table_concurrently
+ * Session2 : Concurrent insert workload
+ *
+ * The child shards created by create_distributed_table_concurrently will have less columns
+ * than the source shard because some column were dropped.
+ * The incoming tuple from session2 will have more columns as the writes
+ * happened on source shard. But now the tuple needs to be applied on child shard. So we need to format
+ * it according to child schema.
+ */
+static HeapTuple
+GetTupleForTargetSchema(HeapTuple sourceRelationTuple,
+						TupleDesc sourceRelDesc,
+						TupleDesc targetRelDesc)
+{
+	/* Deform the tuple */
+	Datum *oldValues = (Datum *) palloc0(sourceRelDesc->natts * sizeof(Datum));
+	bool *oldNulls = (bool *) palloc0(sourceRelDesc->natts * sizeof(bool));
+	heap_deform_tuple(sourceRelationTuple, sourceRelDesc, oldValues,
+					  oldNulls);
+
+
+	/* Create new tuple by skipping dropped columns */
+	int nextAttributeIndex = 0;
+	Datum *newValues = (Datum *) palloc0(targetRelDesc->natts * sizeof(Datum));
+	bool *newNulls = (bool *) palloc0(targetRelDesc->natts * sizeof(bool));
+	for (int i = 0; i < sourceRelDesc->natts; i++)
+	{
+		if (TupleDescAttr(sourceRelDesc, i)->attisdropped)
+		{
+			continue;
+		}
+
+		newValues[nextAttributeIndex] = oldValues[i];
+		newNulls[nextAttributeIndex] = oldNulls[i];
+		nextAttributeIndex++;
+	}
+
+	HeapTuple targetRelationTuple = heap_form_tuple(targetRelDesc, newValues, newNulls);
+	return targetRelationTuple;
+}

src/backend/distributed/sql/citus--11.0-4--11.1-1.sql

@@ -5,14 +5,23 @@
 #include "udfs/citus_internal_delete_partition_metadata/11.1-1.sql"
 #include "udfs/citus_copy_shard_placement/11.1-1.sql"
 
+-- We should not introduce breaking sql changes to upgrade files after they are released.
+-- We did that for worker_fetch_foreign_file in v9.0.0 and worker_repartition_cleanup in v9.2.0.
+-- When we try to drop those udfs in that file, they were missing for some clients unexpectedly
+-- due to buggy changes in old upgrade scripts. For that case, the fix is to change DROP statements
+-- with DROP IF EXISTS for those 2 udfs in 11.0-4--11.1-1.
+-- Fixes an upgrade problem for worker_fetch_foreign_file when upgrade starts from 8.3 up to 11.1
+-- Fixes an upgrade problem for worker_repartition_cleanup when upgrade starts from 9.1 up to 11.1
+-- Refer the related PR https://github.com/citusdata/citus/pull/6441 for more information
+DROP FUNCTION IF EXISTS pg_catalog.worker_fetch_foreign_file(text, text, bigint, text[], integer[]);
+DROP FUNCTION IF EXISTS pg_catalog.worker_repartition_cleanup(bigint);
+
 DROP FUNCTION pg_catalog.worker_create_schema(bigint,text);
 DROP FUNCTION pg_catalog.worker_cleanup_job_schema_cache();
-DROP FUNCTION pg_catalog.worker_fetch_foreign_file(text, text, bigint, text[], integer[]);
 DROP FUNCTION pg_catalog.worker_fetch_partition_file(bigint, integer, integer, integer, text, integer);
 DROP FUNCTION pg_catalog.worker_hash_partition_table(bigint, integer, text, text, oid, anyarray);
 DROP FUNCTION pg_catalog.worker_merge_files_into_table(bigint, integer, text[], text[]);
 DROP FUNCTION pg_catalog.worker_range_partition_table(bigint, integer, text, text, oid, anyarray);
-DROP FUNCTION pg_catalog.worker_repartition_cleanup(bigint);
 
 DO $check_columnar$
 BEGIN

src/backend/distributed/test/pg_send_cancellation.c

@@ -1,70 +0,0 @@
-/*-------------------------------------------------------------------------
- *
- * pg_send_cancellation.c
- *
- * This file contains functions to test setting pg_send_cancellation.
- *
- * Copyright (c) Citus Data, Inc.
- *
- *-------------------------------------------------------------------------
- */
-
-#include "postgres.h"
-#include "miscadmin.h"
-#include "fmgr.h"
-#include "port.h"
-
-#include "postmaster/postmaster.h"
-
-
-#define PG_SEND_CANCELLATION_VERSION \
-	"pg_send_cancellation (PostgreSQL) " PG_VERSION "\n"
-
-
-/* exports for SQL callable functions */
-PG_FUNCTION_INFO_V1(get_cancellation_key);
-PG_FUNCTION_INFO_V1(run_pg_send_cancellation);
-
-
-/*
- * get_cancellation_key returns the cancellation key of the current process
- * as an integer.
- */
-Datum
-get_cancellation_key(PG_FUNCTION_ARGS)
-{
-	PG_RETURN_INT32(MyCancelKey);
-}
-
-
-/*
- * run_pg_send_cancellation runs the pg_send_cancellation program with
- * the specified arguments
- */
-Datum
-run_pg_send_cancellation(PG_FUNCTION_ARGS)
-{
-	int pid = PG_GETARG_INT32(0);
-	int cancelKey = PG_GETARG_INT32(1);
-
-	char sendCancellationPath[MAXPGPATH];
-	char command[1024];
-
-	/* Locate executable backend before we change working directory */
-	if (find_other_exec(my_exec_path, "pg_send_cancellation",
-						PG_SEND_CANCELLATION_VERSION,
-						sendCancellationPath) < 0)
-	{
-		ereport(ERROR, (errmsg("could not locate pg_send_cancellation")));
-	}
-
-	pg_snprintf(command, sizeof(command), "%s %d %d %s %d",
-				sendCancellationPath, pid, cancelKey, "localhost", PostPortNumber);
-
-	if (system(command) != 0)
-	{
-		ereport(ERROR, (errmsg("failed to run command: %s", command)));
-	}
-
-	PG_RETURN_VOID();
-}

src/backend/distributed/transaction/remote_transaction.c

@@ -66,6 +66,7 @@ StartRemoteTransactionBegin(struct MultiConnection *connection)
 
 	/* remember transaction as being in-progress */
 	dlist_push_tail(&InProgressTransactions, &connection->transactionNode);
+	connection->transactionInProgress = true;
 
 	transaction->transactionState = REMOTE_TRANS_STARTING;
 
@@ -750,26 +751,6 @@ MarkRemoteTransactionCritical(struct MultiConnection *connection)
 }
 
 
-/*
- * CloseRemoteTransaction handles closing a connection that, potentially, is
- * part of a coordinated transaction.  This should only ever be called from
- * connection_management.c, while closing a connection during a transaction.
- */
-void
-CloseRemoteTransaction(struct MultiConnection *connection)
-{
-	RemoteTransaction *transaction = &connection->remoteTransaction;
-
-	/* unlink from list of open transactions, if necessary */
-	if (transaction->transactionState != REMOTE_TRANS_NOT_STARTED)
-	{
-		/* XXX: Should we error out for a critical transaction? */
-
-		dlist_delete(&connection->transactionNode);
-	}
-}
-
-
 /*
  * ResetRemoteTransaction resets the state of the transaction after the end of
  * the main transaction, if the connection is being reused.
@@ -779,8 +760,23 @@ ResetRemoteTransaction(struct MultiConnection *connection)
 {
 	RemoteTransaction *transaction = &connection->remoteTransaction;
 
+	/* unlink from list of open transactions, if necessary */
+	if (connection->transactionInProgress)
+	{
+		/* XXX: Should we error out for a critical transaction? */
+
+		dlist_delete(&connection->transactionNode);
+		connection->transactionInProgress = false;
+		memset(&connection->transactionNode, 0, sizeof(connection->transactionNode));
+	}
+
 	/* just reset the entire state, relying on 0 being invalid/false */
 	memset(transaction, 0, sizeof(*transaction));
+
+	ResetShardPlacementAssociation(connection);
+
+	/* reset copy state */
+	connection->copyBytesWrittenSinceLastFlush = 0;
 }
 
 

src/backend/distributed/transaction/worker_transaction.c

@@ -340,6 +340,25 @@ SendCommandListToWorkerOutsideTransaction(const char *nodeName, int32 nodePort,
 																	  nodeName, nodePort,
 																	  nodeUser, NULL);
 
+	SendCommandListToWorkerOutsideTransactionWithConnection(workerConnection,
+															commandList);
+	CloseConnection(workerConnection);
+}
+
+
+/*
+ * SendCommandListToWorkerOutsideTransactionWithConnection sends the command list
+ * over the specified connection. This opens a new transaction on the
+ * connection, thus it's important that no transaction is currently open.
+ * This function is mainly useful to avoid opening an closing
+ * connections excessively by allowing reusing a single connection to send
+ * multiple separately committing transactions. The function raises an error if
+ * any of the queries fail.
+ */
+void
+SendCommandListToWorkerOutsideTransactionWithConnection(MultiConnection *workerConnection,
+														List *commandList)
+{
 	MarkRemoteTransactionCritical(workerConnection);
 	RemoteTransactionBegin(workerConnection);
 
@@ -351,7 +370,7 @@ SendCommandListToWorkerOutsideTransaction(const char *nodeName, int32 nodePort,
 	}
 
 	RemoteTransactionCommit(workerConnection);
-	CloseConnection(workerConnection);
+	ResetRemoteTransaction(workerConnection);
 }
 
 
@@ -430,21 +449,18 @@ SendMetadataCommandListToWorkerListInCoordinatedTransaction(List *workerNodeList
 
 
 /*
- * SendOptionalCommandListToWorkerOutsideTransaction sends the given command
- * list to the given worker in a single transaction that is outside of the
- * coordinated tranaction. If any of the commands fail, it rollbacks the
- * transaction, and otherwise commits.
+ * SendOptionalCommandListToWorkerOutsideTransactionWithConnection sends the
+ * given command list over a specified connection in a single transaction that
+ * is outside of the coordinated tranaction.
+ *
+ * If any of the commands fail, it rollbacks the transaction, and otherwise commits.
+ * A successful commit is indicated by returning true, and a failed commit by returning
+ * false.
  */
 bool
-SendOptionalCommandListToWorkerOutsideTransaction(const char *nodeName, int32 nodePort,
-												  const char *nodeUser, List *commandList)
+SendOptionalCommandListToWorkerOutsideTransactionWithConnection(
+	MultiConnection *workerConnection, List *commandList)
 {
-	int connectionFlags = FORCE_NEW_CONNECTION;
-	bool failed = false;
-
-	MultiConnection *workerConnection = GetNodeUserDatabaseConnection(connectionFlags,
-																	  nodeName, nodePort,
-																	  nodeUser, NULL);
 	if (PQstatus(workerConnection->pgConn) != CONNECTION_OK)
 	{
 		return false;
@@ -452,6 +468,7 @@ SendOptionalCommandListToWorkerOutsideTransaction(const char *nodeName, int32 no
 	RemoteTransactionBegin(workerConnection);
 
 	/* iterate over the commands and execute them in the same connection */
+	bool failed = false;
 	const char *commandString = NULL;
 	foreach_ptr(commandString, commandList)
 	{
@@ -471,6 +488,30 @@ SendOptionalCommandListToWorkerOutsideTransaction(const char *nodeName, int32 no
 		RemoteTransactionCommit(workerConnection);
 	}
 
+	ResetRemoteTransaction(workerConnection);
+
+	return !failed;
+}
+
+
+/*
+ * SendOptionalCommandListToWorkerOutsideTransaction sends the given command
+ * list to the given worker in a single transaction that is outside of the
+ * coordinated tranaction. If any of the commands fail, it rollbacks the
+ * transaction, and otherwise commits.
+ */
+bool
+SendOptionalCommandListToWorkerOutsideTransaction(const char *nodeName, int32 nodePort,
+												  const char *nodeUser, List *commandList)
+{
+	int connectionFlags = FORCE_NEW_CONNECTION;
+
+	MultiConnection *workerConnection = GetNodeUserDatabaseConnection(connectionFlags,
+																	  nodeName, nodePort,
+																	  nodeUser, NULL);
+	bool failed = SendOptionalCommandListToWorkerOutsideTransactionWithConnection(
+		workerConnection,
+		commandList);
 	CloseConnection(workerConnection);
 
 	return !failed;

src/backend/distributed/utils/citus_depended_object.c

@@ -9,6 +9,7 @@
 #include "postgres.h"
 #include "miscadmin.h"
 
+#include "access/xact.h"
 #include "catalog/namespace.h"
 #include "catalog/pg_aggregate.h"
 #include "catalog/pg_am.h"
@@ -442,21 +443,25 @@ HasDropCommandViolatesOwnership(Node *node)
 static bool
 AnyObjectViolatesOwnership(DropStmt *dropStmt)
 {
+	bool hasOwnershipViolation = false;
 	volatile ObjectAddress objectAddress = { 0 };
 	Relation relation = NULL;
-	bool objectViolatesOwnership = false;
 	ObjectType objectType = dropStmt->removeType;
 	bool missingOk = dropStmt->missing_ok;
 
-	Node *object = NULL;
-	foreach_ptr(object, dropStmt->objects)
+	MemoryContext savedContext = CurrentMemoryContext;
+	ResourceOwner savedOwner = CurrentResourceOwner;
+	BeginInternalSubTransaction(NULL);
+	MemoryContextSwitchTo(savedContext);
+
+	PG_TRY();
 	{
-		PG_TRY();
+		Node *object = NULL;
+		foreach_ptr(object, dropStmt->objects)
 		{
 			objectAddress = get_object_address(objectType, object,
 											   &relation, AccessShareLock, missingOk);
 
-
 			if (OidIsValid(objectAddress.objectId))
 			{
 				/*
@@ -467,29 +472,39 @@ AnyObjectViolatesOwnership(DropStmt *dropStmt)
 									   objectAddress,
 									   object, relation);
 			}
-		}
-		PG_CATCH();
-		{
-			if (OidIsValid(objectAddress.objectId))
+
+			if (relation != NULL)
 			{
-				/* ownership violation */
-				objectViolatesOwnership = true;
+				relation_close(relation, NoLock);
+				relation = NULL;
 			}
 		}
-		PG_END_TRY();
 
+		ReleaseCurrentSubTransaction();
+		MemoryContextSwitchTo(savedContext);
+		CurrentResourceOwner = savedOwner;
+	}
+	PG_CATCH();
+	{
+		MemoryContextSwitchTo(savedContext);
+		ErrorData *edata = CopyErrorData();
+		FlushErrorState();
+
+		hasOwnershipViolation = true;
 		if (relation != NULL)
 		{
-			relation_close(relation, AccessShareLock);
+			relation_close(relation, NoLock);
 			relation = NULL;
 		}
+		RollbackAndReleaseCurrentSubTransaction();
+		MemoryContextSwitchTo(savedContext);
+		CurrentResourceOwner = savedOwner;
 
-		/* we found ownership violation, so can return here */
-		if (objectViolatesOwnership)
-		{
-			return true;
-		}
+		/* Rethrow error with LOG_SERVER_ONLY to prevent log to be sent to client */
+		edata->elevel = LOG_SERVER_ONLY;
+		ThrowErrorData(edata);
 	}
+	PG_END_TRY();
 
-	return false;
+	return hasOwnershipViolation;
 }

src/backend/distributed/utils/citus_nodefuncs.c

@@ -141,7 +141,17 @@ SetRangeTblExtraData(RangeTblEntry *rte, CitusRTEKind rteKind, char *fragmentSch
 	fauxFunction->funcexpr = (Node *) fauxFuncExpr;
 
 	/* set the column count to pass ruleutils checks, not used elsewhere */
-	fauxFunction->funccolcount = list_length(rte->eref->colnames);
+	if (rte->relid != 0)
+	{
+		Relation rel = RelationIdGetRelation(rte->relid);
+		fauxFunction->funccolcount = RelationGetNumberOfAttributes(rel);
+		RelationClose(rel);
+	}
+	else
+	{
+		fauxFunction->funccolcount = list_length(rte->eref->colnames);
+	}
+
 	fauxFunction->funccolnames = funcColumnNames;
 	fauxFunction->funccoltypes = funcColumnTypes;
 	fauxFunction->funccoltypmods = funcColumnTypeMods;

src/backend/distributed/utils/colocation_utils.c

@@ -170,12 +170,11 @@ BreakColocation(Oid sourceRelationId)
 	 */
 	Relation pgDistColocation = table_open(DistColocationRelationId(), ExclusiveLock);
 
-	uint32 newColocationId = GetNextColocationId();
-	bool localOnly = false;
-	UpdateRelationColocationGroup(sourceRelationId, newColocationId, localOnly);
+	uint32 oldColocationId = TableColocationId(sourceRelationId);
+	CreateColocationGroupForRelation(sourceRelationId);
 
-	/* if there is not any remaining table in the colocation group, delete it */
-	DeleteColocationGroupIfNoTablesBelong(sourceRelationId);
+	/* if there is not any remaining table in the old colocation group, delete it */
+	DeleteColocationGroupIfNoTablesBelong(oldColocationId);
 
 	table_close(pgDistColocation, NoLock);
 }

src/backend/distributed/utils/enable_ssl.c

@@ -9,20 +9,22 @@
  */
 
 
+#include "postgres.h"
+
 /*
  * Make sure that functions marked as deprecated in OpenSSL 3.0 don't trigger
  * deprecation warnings by indicating that we're using the OpenSSL 1.0.1
  * compatibile API. Postgres does this by already in PG14, so we should not do
- * it otherwise we get warnings about redefining this value.
+ * it otherwise we get warnings about redefining this value. This needs to be
+ * done before including libpq.h.
  */
+#include "distributed/pg_version_constants.h"
 #if PG_VERSION_NUM < PG_VERSION_14
 #ifndef OPENSSL_API_COMPAT
 #define OPENSSL_API_COMPAT 0x1000100L
 #endif
 #endif
 
-#include "postgres.h"
-
 #include "distributed/connection_management.h"
 #include "distributed/memutils.h"
 #include "distributed/worker_protocol.h"

src/backend/distributed/utils/foreign_key_relationship.c

@@ -28,6 +28,7 @@
 #include "distributed/version_compat.h"
 #include "nodes/pg_list.h"
 #include "storage/lockdefs.h"
+#include "utils/catcache.h"
 #include "utils/fmgroids.h"
 #include "utils/hsearch.h"
 #include "common/hashfn.h"
@@ -96,6 +97,8 @@ static List * GetConnectedListHelper(ForeignConstraintRelationshipNode *node,
 									 bool isReferencing);
 static List * GetForeignConstraintRelationshipHelper(Oid relationId, bool isReferencing);
 
+MemoryContext ForeignConstraintRelationshipMemoryContext = NULL;
+
 
 /*
  * GetForeignKeyConnectedRelationIdList returns a list of relation id's for
@@ -321,17 +324,36 @@ CreateForeignConstraintRelationshipGraph()
 		return;
 	}
 
-	ClearForeignConstraintRelationshipGraphContext();
+	/*
+	 * Lazily create our memory context once and reset on every reuse.
+	 * Since we have cleared and invalidated the fConstraintRelationshipGraph, right
+	 * before we can simply reset the context if it was already existing.
+	 */
+	if (ForeignConstraintRelationshipMemoryContext == NULL)
+	{
+		/* make sure we've initialized CacheMemoryContext */
+		if (CacheMemoryContext == NULL)
+		{
+			CreateCacheMemoryContext();
+		}
 
-	MemoryContext fConstraintRelationshipMemoryContext = AllocSetContextCreateInternal(
-		CacheMemoryContext,
-		"Forign Constraint Relationship Graph Context",
-		ALLOCSET_DEFAULT_MINSIZE,
-		ALLOCSET_DEFAULT_INITSIZE,
-		ALLOCSET_DEFAULT_MAXSIZE);
+		ForeignConstraintRelationshipMemoryContext = AllocSetContextCreate(
+			CacheMemoryContext,
+			"Foreign Constraint Relationship Graph Context",
+			ALLOCSET_DEFAULT_MINSIZE,
+			ALLOCSET_DEFAULT_INITSIZE,
+			ALLOCSET_DEFAULT_MAXSIZE);
+	}
+	else
+	{
+		fConstraintRelationshipGraph = NULL;
+		MemoryContextReset(ForeignConstraintRelationshipMemoryContext);
+	}
+
+	Assert(fConstraintRelationshipGraph == NULL);
 
 	MemoryContext oldContext = MemoryContextSwitchTo(
-		fConstraintRelationshipMemoryContext);
+		ForeignConstraintRelationshipMemoryContext);
 
 	fConstraintRelationshipGraph = (ForeignConstraintRelationshipGraph *) palloc(
 		sizeof(ForeignConstraintRelationshipGraph));
@@ -631,22 +653,3 @@ CreateOrFindNode(HTAB *adjacencyLists, Oid relid)
 
 	return node;
 }
-
-
-/*
- * ClearForeignConstraintRelationshipGraphContext clear all the allocated memory obtained
- * for foreign constraint relationship graph. Since all the variables of relationship
- * graph was obtained within the same context, destroying hash map is enough as
- * it deletes the context.
- */
-void
-ClearForeignConstraintRelationshipGraphContext()
-{
-	if (fConstraintRelationshipGraph == NULL)
-	{
-		return;
-	}
-
-	hash_destroy(fConstraintRelationshipGraph->nodeMap);
-	fConstraintRelationshipGraph = NULL;
-}

src/backend/distributed/utils/reference_table_utils.c

@@ -723,6 +723,12 @@ ReplicateAllReferenceTablesToNode(WorkerNode *workerNode)
 		ShardInterval *shardInterval = NULL;
 		foreach_ptr(shardInterval, referenceShardIntervalList)
 		{
+			/*
+			 * Make sure the relation exists. In some cases the relation is
+			 * actually dropped but the metadata remains, such as dropping table
+			 * while citus.enable_ddl_propagation is set to off.
+			 */
+			EnsureRelationExists(shardInterval->relationId);
 			uint64 shardId = shardInterval->shardId;
 
 			LockShardDistributionMetadata(shardId, ExclusiveLock);

src/backend/distributed/utils/resource_lock.c

@@ -781,14 +781,7 @@ SerializeNonCommutativeWrites(List *shardIntervalList, LOCKMODE lockMode)
 static bool
 AnyTableReplicated(List *shardIntervalList, List **replicatedShardIntervalList)
 {
-	if (replicatedShardIntervalList == NULL)
-	{
-		/* the caller is not interested in the replicatedShardIntervalList */
-		List *localList = NIL;
-		replicatedShardIntervalList = &localList;
-	}
-
-	*replicatedShardIntervalList = NIL;
+	List *localList = NIL;
 
 	ShardInterval *shardInterval = NULL;
 	foreach_ptr(shardInterval, shardIntervalList)
@@ -798,17 +791,22 @@ AnyTableReplicated(List *shardIntervalList, List **replicatedShardIntervalList)
 		Oid relationId = RelationIdForShard(shardId);
 		if (ReferenceTableShardId(shardId))
 		{
-			*replicatedShardIntervalList =
-				lappend(*replicatedShardIntervalList, LoadShardInterval(shardId));
+			localList =
+				lappend(localList, LoadShardInterval(shardId));
 		}
 		else if (!SingleReplicatedTable(relationId))
 		{
-			*replicatedShardIntervalList =
-				lappend(*replicatedShardIntervalList, LoadShardInterval(shardId));
+			localList =
+				lappend(localList, LoadShardInterval(shardId));
 		}
 	}
 
-	return list_length(*replicatedShardIntervalList) > 0;
+	if (replicatedShardIntervalList != NULL)
+	{
+		*replicatedShardIntervalList = localList;
+	}
+
+	return list_length(localList) > 0;
 }
 
 

src/bin/pg_send_cancellation/.gitignore

@@ -1 +0,0 @@
-pg_send_cancellation

src/bin/pg_send_cancellation/Makefile

@@ -1,20 +0,0 @@
-citus_top_builddir = ../../..
-
-PROGRAM = pg_send_cancellation
-PGFILEDESC = "pg_send_cancellation sends a custom cancellation message"
-OBJS = $(citus_abs_srcdir)/src/bin/pg_send_cancellation/pg_send_cancellation.o
-PG_CPPFLAGS = -I$(libpq_srcdir)
-PG_LIBS_INTERNAL = $(libpq_pgport)
-PG_LDFLAGS += $(LDFLAGS)
-
-include $(citus_top_builddir)/Makefile.global
-
-# We reuse all the Citus flags (incl. security flags), but we are building a program not a shared library
-override CFLAGS := $(filter-out -shared,$(CFLAGS))
-
-# Filter out unneeded dependencies
-override LIBS := $(filter-out -lz -lreadline -ledit -ltermcap -lncurses -lcurses -lpam, $(LIBS))
-
-clean: clean-pg_send_cancellation
-clean-pg_send_cancellation:
-	rm -f $(PROGRAM) $(OBJS)

src/bin/pg_send_cancellation/README.md

@@ -1,47 +0,0 @@
-# pg_send_cancellation
-
-pg_send_cancellation is a program for manually sending a cancellation
-to a Postgres endpoint. It is effectively a command-line version of
-PQcancel in libpq, but it can use any PID or cancellation key.
-
-We use pg_send_cancellation primarily to propagate cancellations between pgbouncers
-behind a load balancer. Since the cancellation protocol involves
-opening a new connection, the new connection may go to a different
-node that does not recognize the cancellation key. To handle that
-scenario, we modified pgbouncer to pass unrecognized cancellation
-keys to a shell command.
-
-Users can configure the cancellation_command, which will be run with:
-```
-<cancellation_command> <client ip> <client port> <pid> <cancel key>
-```
-
-Note that pgbouncer does not use actual PIDs. Instead, it generates PID and cancellation key together a random 8-byte number. This makes the chance of collisions exceedingly small.
-
-By providing pg_send_cancellation as part of Citus, we can use a shell script that pgbouncer invokes to propagate the cancellation to all *other* worker nodes in the same cluster, for example:
-
-```bash
-#!/bin/sh
-remote_ip=$1
-remote_port=$2
-pid=$3
-cancel_key=$4
-
-postgres_path=/usr/pgsql-14/bin
-pgbouncer_port=6432
-
-nodes_query="select nodename from pg_dist_node where groupid > 0 and groupid not in (select groupid from pg_dist_local_group) and nodecluster = current_setting('citus.cluster_name')"
-
-# Get hostnames of other worker nodes in the cluster, and send cancellation to their pgbouncers
-$postgres_path/psql -c "$nodes_query" -tAX | xargs -n 1 sh -c "$postgres_path/pg_send_cancellation $pid $cancel_key \$0 $pgbouncer_port"
-```
-
-One thing we need to be careful about is that the cancellations do not get forwarded
-back-and-forth. This is handled in pgbouncer by setting the last bit of all generated
-cancellation keys (sent to clients) to 1, and setting the last bit of all forwarded bits to 0.
-That way, when a pgbouncer receives a cancellation key with the last bit set to 0,
-it knows it is from another pgbouncer and should not forward further, and should set
-the last bit to 1 when comparing to stored cancellation keys.
-
-Another thing we need to be careful about is that the integers should be encoded
-as big endian on the wire.

src/bin/pg_send_cancellation/pg_send_cancellation.c

@@ -1,261 +0,0 @@
-/*
- * pg_send_cancellation is a program for manually sending a cancellation
- * to a Postgres endpoint. It is effectively a command-line version of
- * PQcancel in libpq, but it can use any PID or cancellation key.
- *
- * Portions Copyright (c) Citus Data, Inc.
- *
- * For the internal_cancel function:
- *
- * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
- * Portions Copyright (c) 1994, Regents of the University of California
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation for any purpose, without fee, and without a written agreement
- * is hereby granted, provided that the above copyright notice and this
- * paragraph and the following two paragraphs appear in all copies.
- *
- * IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR
- * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
- * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
- * DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- *
- * THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
- * AND FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
- * ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO
- * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
- *
- */
-#include "postgres_fe.h"
-
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <ctype.h>
-#include <time.h>
-#include <unistd.h>
-
-#include "common/ip.h"
-#include "common/link-canary.h"
-#include "common/scram-common.h"
-#include "common/string.h"
-#include "libpq-fe.h"
-#include "libpq-int.h"
-#include "mb/pg_wchar.h"
-#include "port/pg_bswap.h"
-
-
-#define ERROR_BUFFER_SIZE 256
-
-
-static int internal_cancel(SockAddr *raddr, int be_pid, int be_key,
-						   char *errbuf, int errbufsize);
-
-
-/*
- * main entry point into the pg_send_cancellation program.
- */
-int
-main(int argc, char *argv[])
-{
-	if (argc == 2 && strcmp(argv[1], "-V") == 0)
-	{
-		pg_fprintf(stdout, "pg_send_cancellation (PostgreSQL) " PG_VERSION "\n");
-		return 0;
-	}
-
-	if (argc < 4 || argc > 5)
-	{
-		char *program = argv[0];
-		pg_fprintf(stderr, "%s requires 4 arguments\n\n", program);
-		pg_fprintf(stderr, "Usage: %s <pid> <cancel key> <hostname> [port]\n", program);
-		return 1;
-	}
-
-	char *pidString = argv[1];
-	char *cancelKeyString = argv[2];
-	char *host = argv[3];
-	char *portString = "5432";
-
-	if (argc >= 5)
-	{
-		portString = argv[4];
-	}
-
-	/* parse the PID and cancellation key */
-	int pid = strtol(pidString, NULL, 10);
-	int cancelAuthCode = strtol(cancelKeyString, NULL, 10);
-
-	char errorBuffer[ERROR_BUFFER_SIZE] = { 0 };
-
-	struct addrinfo *ipAddressList;
-	struct addrinfo hint;
-	int ipAddressListFamily = AF_UNSPEC;
-	SockAddr socketAddress;
-
-	memset(&hint, 0, sizeof(hint));
-	hint.ai_socktype = SOCK_STREAM;
-	hint.ai_family = ipAddressListFamily;
-
-	/* resolve the hostname to an IP */
-	int ret = pg_getaddrinfo_all(host, portString, &hint, &ipAddressList);
-	if (ret || !ipAddressList)
-	{
-		pg_fprintf(stderr, "could not translate host name \"%s\" to address: %s\n",
-				   host, gai_strerror(ret));
-		return 1;
-	}
-
-	if (ipAddressList->ai_addrlen > sizeof(socketAddress.addr))
-	{
-		pg_fprintf(stderr, "invalid address length");
-		return 1;
-	}
-
-	/*
-	 * Explanation of IGNORE-BANNED:
-	 * This is a common pattern when using getaddrinfo. The system guarantees
-	 * that ai_addrlen < sizeof(socketAddress.addr). Out of an abundance of
-	 * caution. We also check it above.
-	 */
-	memcpy(&socketAddress.addr, ipAddressList->ai_addr, ipAddressList->ai_addrlen); /* IGNORE-BANNED */
-	socketAddress.salen = ipAddressList->ai_addrlen;
-
-	/* send the cancellation */
-	bool cancelSucceeded = internal_cancel(&socketAddress, pid, cancelAuthCode,
-										   errorBuffer, sizeof(errorBuffer));
-	if (!cancelSucceeded)
-	{
-		pg_fprintf(stderr, "sending cancellation to %s:%s failed: %s",
-				   host, portString, errorBuffer);
-		return 1;
-	}
-
-	pg_freeaddrinfo_all(ipAddressListFamily, ipAddressList);
-
-	return 0;
-}
-
-
-/* *INDENT-OFF* */
-
-/*
- * internal_cancel is copied from fe-connect.c
- *
- * The return value is true if the cancel request was successfully
- * dispatched, false if not (in which case an error message is available).
- * Note: successful dispatch is no guarantee that there will be any effect at
- * the backend.  The application must read the operation result as usual.
- *
- * CAUTION: we want this routine to be safely callable from a signal handler
- * (for example, an application might want to call it in a SIGINT handler).
- * This means we cannot use any C library routine that might be non-reentrant.
- * malloc/free are often non-reentrant, and anything that might call them is
- * just as dangerous.  We avoid sprintf here for that reason.  Building up
- * error messages with strcpy/strcat is tedious but should be quite safe.
- * We also save/restore errno in case the signal handler support doesn't.
- *
- * internal_cancel() is an internal helper function to make code-sharing
- * between the two versions of the cancel function possible.
- */
-static int
-internal_cancel(SockAddr *raddr, int be_pid, int be_key,
-				char *errbuf, int errbufsize)
-{
-	int			save_errno = SOCK_ERRNO;
-	pgsocket	tmpsock = PGINVALID_SOCKET;
-	char		sebuf[PG_STRERROR_R_BUFLEN];
-	int			maxlen;
-	struct
-	{
-		uint32		packetlen;
-		CancelRequestPacket cp;
-	}			crp;
-
-	/*
-	 * We need to open a temporary connection to the postmaster. Do this with
-	 * only kernel calls.
-	 */
-	if ((tmpsock = socket(raddr->addr.ss_family, SOCK_STREAM, 0)) == PGINVALID_SOCKET)
-	{
-		strlcpy(errbuf, "PQcancel() -- socket() failed: ", errbufsize);
-		goto cancel_errReturn;
-	}
-retry3:
-	if (connect(tmpsock, (struct sockaddr *) &raddr->addr, raddr->salen) < 0)
-	{
-		if (SOCK_ERRNO == EINTR)
-			/* Interrupted system call - we'll just try again */
-			goto retry3;
-		strlcpy(errbuf, "PQcancel() -- connect() failed: ", errbufsize);
-		goto cancel_errReturn;
-	}
-
-	/*
-	 * We needn't set nonblocking I/O or NODELAY options here.
-	 */
-
-	/* Create and send the cancel request packet. */
-
-	crp.packetlen = pg_hton32((uint32) sizeof(crp));
-	crp.cp.cancelRequestCode = (MsgType) pg_hton32(CANCEL_REQUEST_CODE);
-	crp.cp.backendPID = pg_hton32(be_pid);
-	crp.cp.cancelAuthCode = pg_hton32(be_key);
-
-retry4:
-	if (send(tmpsock, (char *) &crp, sizeof(crp), 0) != (int) sizeof(crp))
-	{
-		if (SOCK_ERRNO == EINTR)
-			/* Interrupted system call - we'll just try again */
-			goto retry4;
-		strlcpy(errbuf, "PQcancel() -- send() failed: ", errbufsize);
-		goto cancel_errReturn;
-	}
-
-	/*
-	 * Wait for the postmaster to close the connection, which indicates that
-	 * it's processed the request.  Without this delay, we might issue another
-	 * command only to find that our cancel zaps that command instead of the
-	 * one we thought we were canceling.  Note we don't actually expect this
-	 * read to obtain any data, we are just waiting for EOF to be signaled.
-	 */
-retry5:
-	if (recv(tmpsock, (char *) &crp, 1, 0) < 0)
-	{
-		if (SOCK_ERRNO == EINTR)
-			/* Interrupted system call - we'll just try again */
-			goto retry5;
-		/* we ignore other error conditions */
-	}
-
-	/* All done */
-	closesocket(tmpsock);
-	SOCK_ERRNO_SET(save_errno);
-	return true;
-
-cancel_errReturn:
-
-	/*
-	 * Make sure we don't overflow the error buffer. Leave space for the \n at
-	 * the end, and for the terminating zero.
-	 */
-	maxlen = errbufsize - strlen(errbuf) - 2;
-	if (maxlen >= 0)
-	{
-		/*
-		 * Explanation of IGNORE-BANNED:
-		 * This is well-tested libpq code that we would like to preserve in its
-		 * original form. The appropriate length calculation is done above.
-		 */
-		strncat(errbuf, SOCK_STRERROR(SOCK_ERRNO, sebuf, sizeof(sebuf)), /* IGNORE-BANNED */
-				maxlen);
-		strcat(errbuf, "\n"); /* IGNORE-BANNED */
-	}
-	if (tmpsock != PGINVALID_SOCKET)
-		closesocket(tmpsock);
-	SOCK_ERRNO_SET(save_errno);
-	return false;
-}
-
-/* *INDENT-ON* */

src/include/columnar/columnar.h

@@ -314,6 +314,7 @@ extern Datum columnar_relation_storageid(PG_FUNCTION_ARGS);
 /* write_state_management.c */
 extern ColumnarWriteState * columnar_init_write_state(Relation relation, TupleDesc
 													  tupdesc,
+													  Oid tupSlotRelationId,
 													  SubTransactionId currentSubXid);
 extern void FlushWriteStateForRelfilenode(Oid relfilenode, SubTransactionId
 										  currentSubXid);

src/include/distributed/commands.h

@@ -285,7 +285,6 @@ extern bool TableHasExternalForeignKeys(Oid relationId);
 extern List * GetForeignKeyOids(Oid relationId, int flags);
 extern Oid GetReferencedTableId(Oid foreignKeyId);
 extern Oid GetReferencingTableId(Oid foreignKeyId);
-extern void EnableSkippingConstraintValidation(void);
 extern bool RelationInvolvedInAnyNonInheritedForeignKeys(Oid relationId);
 
 
@@ -541,7 +540,8 @@ extern List * PreprocessAlterTableMoveAllStmt(Node *node, const char *queryStrin
 											  ProcessUtilityContext processUtilityContext);
 extern List * PreprocessAlterTableSchemaStmt(Node *node, const char *queryString,
 											 ProcessUtilityContext processUtilityContext);
-extern void SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStmt);
+extern void SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStmt,
+													   bool processLocalRelation);
 extern bool IsAlterTableRenameStmt(RenameStmt *renameStmt);
 extern void ErrorIfAlterDropsPartitionColumn(AlterTableStmt *alterTableStatement);
 extern void PostprocessAlterTableStmt(AlterTableStmt *pStmt);

src/include/distributed/commands/sequence.h

@@ -10,9 +10,11 @@
 #ifndef CITUS_SEQUENCE_H
 #define CITUS_SEQUENCE_H
 
+#include "access/attnum.h"
 #include "nodes/pg_list.h"
 
 
+extern bool ColumnDefaultsToNextVal(Oid relationId, AttrNumber attrNumber);
 extern void ExtractDefaultColumnsAndOwnedSequences(Oid relationId,
 												   List **columnNameList,
 												   List **ownedSequenceIdList);

src/include/distributed/connection_management.h

@@ -189,8 +189,12 @@ typedef struct MultiConnection
 	/* information about the associated remote transaction */
 	RemoteTransaction remoteTransaction;
 
-	/* membership in list of in-progress transactions */
+	/*
+	 * membership in list of in-progress transactions and a flag to indicate
+	 * that the connection was added to this list
+	 */
 	dlist_node transactionNode;
+	bool transactionInProgress;
 
 	/* list of all placements referenced by this connection */
 	dlist_head referencedPlacements;
@@ -289,6 +293,7 @@ extern MultiConnection * StartNodeConnection(uint32 flags, const char *hostname,
 extern MultiConnection * GetNodeUserDatabaseConnection(uint32 flags, const char *hostname,
 													   int32 port, const char *user,
 													   const char *database);
+extern MultiConnection * GetConnectionForLocalQueriesOutsideTransaction(char *userName);
 extern MultiConnection * StartNodeUserDatabaseConnection(uint32 flags,
 														 const char *hostname,
 														 int32 port,

src/include/distributed/distributed_planner.h

@@ -56,7 +56,7 @@ typedef struct RelationRestriction
 {
 	Index index;
 	Oid relationId;
-	bool distributedRelation;
+	bool citusTable;
 	RangeTblEntry *rte;
 	RelOptInfo *relOptInfo;
 	PlannerInfo *plannerInfo;

src/include/distributed/foreign_key_relationship.h

@@ -20,7 +20,6 @@ extern bool ShouldUndistributeCitusLocalTable(Oid relationId);
 extern List * ReferencedRelationIdList(Oid relationId);
 extern List * ReferencingRelationIdList(Oid relationId);
 extern void SetForeignConstraintRelationshipGraphInvalid(void);
-extern void ClearForeignConstraintRelationshipGraphContext(void);
 extern bool OidVisited(HTAB *oidVisitedMap, Oid oid);
 extern void VisitOid(HTAB *oidVisitedMap, Oid oid);
 

src/include/distributed/metadata_utility.h

@@ -358,6 +358,8 @@ extern bool RegularTable(Oid relationId);
 extern bool TableEmpty(Oid tableId);
 extern bool IsForeignTable(Oid relationId);
 extern bool RelationUsesIdentityColumns(TupleDesc relationDesc);
+extern bool ForeignTableDropsTableNameOption(List *optionList);
+extern bool ServerUsesPostgresFdw(Oid serverId);
 extern char * ConstructQualifiedShardName(ShardInterval *shardInterval);
 extern uint64 GetFirstShardId(Oid relationId);
 extern Datum StringToDatum(char *inputString, Oid dataType);

src/include/distributed/multi_logical_replication.h

@@ -157,7 +157,8 @@ extern void DropPublications(MultiConnection *sourceConnection,
 extern void DropAllLogicalReplicationLeftovers(LogicalRepType type);
 
 extern char * PublicationName(LogicalRepType type, uint32_t nodeId, Oid ownerId);
-extern char * ReplicationSlotName(LogicalRepType type, uint32_t nodeId, Oid ownerId);
+extern char * ReplicationSlotNameForNodeAndOwner(LogicalRepType type, uint32_t nodeId, Oid
+												 ownerId);
 extern char * SubscriptionName(LogicalRepType type, Oid ownerId);
 extern char * SubscriptionRoleName(LogicalRepType type, Oid ownerId);
 
@@ -172,10 +173,6 @@ extern HTAB * CreateGroupedLogicalRepTargetsHash(List *subscriptionInfoList);
 extern void CreateGroupedLogicalRepTargetsConnections(HTAB *groupedLogicalRepTargetsHash,
 													  char *user,
 													  char *databaseName);
-extern void RecreateGroupedLogicalRepTargetsConnections(
-	HTAB *groupedLogicalRepTargetsHash,
-	char *user,
-	char *databaseName);
 extern void CloseGroupedLogicalRepTargetsConnections(HTAB *groupedLogicalRepTargetsHash);
 extern void CompleteNonBlockingShardTransfer(List *shardList,
 											 MultiConnection *sourceConnection,

src/include/distributed/remote_transaction.h

@@ -130,7 +130,6 @@ extern void MarkRemoteTransactionCritical(struct MultiConnection *connection);
  * transaction managment code.
  */
 
-extern void CloseRemoteTransaction(struct MultiConnection *connection);
 extern void ResetRemoteTransaction(struct MultiConnection *connection);
 
 /* perform handling for all in-progress transactions */

src/include/distributed/shard_cleaner.h

@@ -103,13 +103,13 @@ extern void InsertCleanupRecordInSubtransaction(CleanupObject objectType,
  * completion on failure. This will trigger cleanup of appropriate resources
  * and cleanup records.
  */
-extern void FinalizeOperationNeedingCleanupOnFailure(void);
+extern void FinalizeOperationNeedingCleanupOnFailure(const char *operationName);
 
 /*
  * FinalizeOperationNeedingCleanupOnSuccess is be called by an operation to signal
  * completion on success. This will trigger cleanup of appropriate resources
  * and cleanup records.
  */
-extern void FinalizeOperationNeedingCleanupOnSuccess(void);
+extern void FinalizeOperationNeedingCleanupOnSuccess(const char *operationName);
 
 #endif /*CITUS_SHARD_CLEANER_H */

src/include/distributed/worker_shard_copy.h

@@ -19,4 +19,9 @@ extern DestReceiver * CreateShardCopyDestReceiver(EState *executorState,
 												  List *destinationShardFullyQualifiedName,
 												  uint32_t destinationNodeId);
 
+extern const char * CopyableColumnNamesFromRelationName(const char *schemaName, const
+														char *relationName);
+
+extern const char * CopyableColumnNamesFromTupleDesc(TupleDesc tupdesc);
+
 #endif /* WORKER_SHARD_COPY_H_ */

src/include/distributed/worker_transaction.h

@@ -12,6 +12,7 @@
 #ifndef WORKER_TRANSACTION_H
 #define WORKER_TRANSACTION_H
 
+#include "distributed/connection_management.h"
 #include "distributed/worker_manager.h"
 #include "storage/lockdefs.h"
 
@@ -59,6 +60,10 @@ extern bool SendOptionalCommandListToWorkerOutsideTransaction(const char *nodeNa
 															  int32 nodePort,
 															  const char *nodeUser,
 															  List *commandList);
+extern bool SendOptionalCommandListToWorkerOutsideTransactionWithConnection(
+	MultiConnection *workerConnection,
+	List *
+	commandList);
 extern bool SendOptionalMetadataCommandListToWorkerInCoordinatedTransaction(const
 																			char *nodeName,
 																			int32 nodePort,
@@ -74,6 +79,9 @@ extern void SendCommandListToWorkerOutsideTransaction(const char *nodeName,
 													  int32 nodePort,
 													  const char *nodeUser,
 													  List *commandList);
+extern void SendCommandListToWorkerOutsideTransactionWithConnection(
+	MultiConnection *workerConnection,
+	List *commandList);
 extern void SendMetadataCommandListToWorkerListInCoordinatedTransaction(
 	List *workerNodeList,
 	const char *

src/include/pg_version_compat.h

@@ -55,6 +55,14 @@ pg_strtoint64(char *s)
 }
 
 
+/*
+ * RelationGetSmgr got backported in 13.10 and 14.7 so redefining it for any
+ * version higher causes compilation errors due to redefining of the function.
+ * We want to use it in all versions. So we backport it ourselves in earlier
+ * versions, and rely on the Postgres provided version in the later versions.
+ */
+#if PG_VERSION_NUM >= PG_VERSION_13 && PG_VERSION_NUM < 130010 \
+	|| PG_VERSION_NUM >= PG_VERSION_14 && PG_VERSION_NUM < 140007
 static inline SMgrRelation
 RelationGetSmgr(Relation rel)
 {
@@ -66,6 +74,9 @@ RelationGetSmgr(Relation rel)
 }
 
 
+#endif
+
+
 #define CREATE_SEQUENCE_COMMAND \
 	"CREATE SEQUENCE IF NOT EXISTS %s AS %s INCREMENT BY " INT64_FORMAT \
 	" MINVALUE " INT64_FORMAT " MAXVALUE " INT64_FORMAT \

src/test/regress/enterprise_isolation_logicalrep_1_schedule

@@ -6,7 +6,9 @@ test: isolation_setup
 test: isolation_cluster_management
 
 test: isolation_logical_replication_single_shard_commands
+test: isolation_logical_replication_nonsu_nonbypassrls
 test: isolation_logical_replication_multi_shard_commands
 test: isolation_non_blocking_shard_split
+test: isolation_create_distributed_concurrently_after_drop_column
 test: isolation_non_blocking_shard_split_with_index_as_replicaIdentity
 test: isolation_non_blocking_shard_split_fkey

src/test/regress/enterprise_isolation_logicalrep_2_schedule

@@ -7,3 +7,4 @@ test: isolation_cluster_management
 
 test: isolation_logical_replication_single_shard_commands_on_mx
 test: isolation_logical_replication_multi_shard_commands_on_mx
+test: isolation_logical_replication_skip_fk_validation

src/test/regress/enterprise_isolation_schedule

@@ -10,7 +10,6 @@ test: isolation_move_placement_vs_modification
 test: isolation_move_placement_vs_modification_fk
 test: isolation_tenant_isolation_with_fkey_to_reference
 test: isolation_ref2ref_foreign_keys_enterprise
-test: isolation_pg_send_cancellation
 test: isolation_shard_move_vs_start_metadata_sync
 test: isolation_tenant_isolation
 test: isolation_tenant_isolation_nonblocking

src/test/regress/expected/alter_distributed_table.out

@@ -100,6 +100,161 @@ SELECT STRING_AGG(table_name::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM
  dist_table
 (3 rows)
 
+-- right now dist_table has columns a, b, dist_column is b, it has 6 shards
+-- column cache is: a pos 1, b pos 2
+-- let's add another column
+ALTER TABLE dist_table ADD COLUMN c int DEFAULT 1;
+-- right now column cache is: a pos 1, b pos 2, c pos 3
+-- test using alter_distributed_table to change shard count after dropping one column
+ALTER TABLE dist_table DROP COLUMN a;
+-- right now column cache is: a pos 1 attisdropped=true, b pos 2, c pos 3
+-- let's try changing the shard count
+SELECT alter_distributed_table('dist_table', shard_count := 7, cascade_to_colocated := false);
+NOTICE:  creating a new table for alter_distributed_table.dist_table
+NOTICE:  moving the data of alter_distributed_table.dist_table
+NOTICE:  dropping the old alter_distributed_table.dist_table
+NOTICE:  renaming the new table to alter_distributed_table.dist_table
+ alter_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- right now column cache is: b pos 1, c pos 2 because a new table has been created
+-- check that b is still distribution column
+SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables
+    WHERE table_name = 'dist_table'::regclass;
+ table_name | citus_table_type | distribution_column | shard_count
+---------------------------------------------------------------------
+ dist_table | distributed      | b                   |           7
+(1 row)
+
+-- let's add another column
+ALTER TABLE dist_table ADD COLUMN d int DEFAULT 2;
+-- right now column cache is: b pos 1, c pos 2, d pos 3, dist_column is b
+-- test using alter_distributed_table to change dist. column after dropping one column
+ALTER TABLE dist_table DROP COLUMN c;
+-- right now column cache is: b pos 1, c pos 2 attisdropped=true, d pos 3
+-- let's try changing the distribution column
+SELECT alter_distributed_table('dist_table', distribution_column := 'd');
+NOTICE:  creating a new table for alter_distributed_table.dist_table
+NOTICE:  moving the data of alter_distributed_table.dist_table
+NOTICE:  dropping the old alter_distributed_table.dist_table
+NOTICE:  renaming the new table to alter_distributed_table.dist_table
+ alter_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- right now column cache is: b pos 1, d pos 2 because a new table has been created
+-- check that d is the distribution column
+SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables
+    WHERE table_name = 'dist_table'::regclass;
+ table_name | citus_table_type | distribution_column | shard_count
+---------------------------------------------------------------------
+ dist_table | distributed      | d                   |           7
+(1 row)
+
+-- add another column and undistribute
+ALTER TABLE dist_table ADD COLUMN e int DEFAULT 3;
+SELECT undistribute_table('dist_table');
+NOTICE:  creating a new table for alter_distributed_table.dist_table
+NOTICE:  moving the data of alter_distributed_table.dist_table
+NOTICE:  dropping the old alter_distributed_table.dist_table
+NOTICE:  renaming the new table to alter_distributed_table.dist_table
+ undistribute_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- right now column cache is: b pos 1, d pos 2, e pos 3, table is not Citus table
+-- try dropping column and then distributing
+ALTER TABLE dist_table DROP COLUMN b;
+-- right now column cache is: b pos 1 attisdropped=true, d pos 2, e pos 3
+-- distribute with d
+SELECT create_distributed_table ('dist_table', 'd', colocate_with := 'none');
+NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is no longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$alter_distributed_table.dist_table$$)
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- check that d is the distribution column
+SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables
+    WHERE table_name = 'dist_table'::regclass;
+ table_name | citus_table_type | distribution_column | shard_count
+---------------------------------------------------------------------
+ dist_table | distributed      | d                   |           4
+(1 row)
+
+-- alter distribution column to e
+SELECT alter_distributed_table('dist_table', distribution_column := 'e');
+NOTICE:  creating a new table for alter_distributed_table.dist_table
+NOTICE:  moving the data of alter_distributed_table.dist_table
+NOTICE:  dropping the old alter_distributed_table.dist_table
+NOTICE:  renaming the new table to alter_distributed_table.dist_table
+ alter_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- right now column cache is: d pos 1, e pos 2
+-- check that e is the distribution column
+SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables
+    WHERE table_name = 'dist_table'::regclass;
+ table_name | citus_table_type | distribution_column | shard_count
+---------------------------------------------------------------------
+ dist_table | distributed      | e                   |           4
+(1 row)
+
+ALTER TABLE dist_table ADD COLUMN a int DEFAULT 4;
+ALTER TABLE dist_table ADD COLUMN b int DEFAULT 5;
+-- right now column cache is: d pos 1, e pos 2, a pos 3, b pos 4
+-- alter distribution column to a
+SELECT alter_distributed_table('dist_table', distribution_column := 'a');
+NOTICE:  creating a new table for alter_distributed_table.dist_table
+NOTICE:  moving the data of alter_distributed_table.dist_table
+NOTICE:  dropping the old alter_distributed_table.dist_table
+NOTICE:  renaming the new table to alter_distributed_table.dist_table
+ alter_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- right now column cache hasn't changed
+-- check that a is the distribution column
+SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables
+    WHERE table_name = 'dist_table'::regclass;
+ table_name | citus_table_type | distribution_column | shard_count
+---------------------------------------------------------------------
+ dist_table | distributed      | a                   |           4
+(1 row)
+
+ALTER TABLE dist_table DROP COLUMN d;
+ALTER TABLE dist_table DROP COLUMN e;
+-- right now column cache is: d pos 1 attisdropped=true, e pos 2 attisdropped=true, a pos 3, b pos 4
+-- alter distribution column to b
+SELECT alter_distributed_table('dist_table', distribution_column := 'b');
+NOTICE:  creating a new table for alter_distributed_table.dist_table
+NOTICE:  moving the data of alter_distributed_table.dist_table
+NOTICE:  dropping the old alter_distributed_table.dist_table
+NOTICE:  renaming the new table to alter_distributed_table.dist_table
+ alter_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- column cache is: a pos 1, b pos 2 -> configuration with which we started these drop column tests
+-- check that b is the distribution column
+SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables
+    WHERE table_name = 'dist_table'::regclass;
+ table_name | citus_table_type | distribution_column | shard_count
+---------------------------------------------------------------------
+ dist_table | distributed      | b                   |           4
+(1 row)
+
 -- test altering colocation, note that shard count will also change
 SELECT alter_distributed_table('dist_table', colocate_with := 'alter_distributed_table.colocation_table');
 NOTICE:  creating a new table for alter_distributed_table.dist_table

src/test/regress/expected/alter_role_propagation.out

@@ -319,5 +319,42 @@ SELECT COUNT(*) FROM public.test_search_path;
 (1 row)
 
 ALTER USER current_user RESET search_path;
+-- test empty/null password: it is treated the same as no password
+SET password_encryption TO md5;
+CREATE ROLE new_role;
+SELECT workers.result AS worker_password, pg_authid.rolpassword AS coord_password FROM run_command_on_workers($$SELECT rolpassword FROM pg_authid WHERE rolname = 'new_role'$$) workers, pg_authid WHERE pg_authid.rolname = 'new_role';
+ worker_password | coord_password
+---------------------------------------------------------------------
+                 |
+                 |
+(2 rows)
+
+ALTER ROLE new_role PASSWORD '';
+NOTICE:  empty string is not a valid password, clearing password
+SELECT workers.result AS worker_password, pg_authid.rolpassword AS coord_password FROM run_command_on_workers($$SELECT rolpassword FROM pg_authid WHERE rolname = 'new_role'$$) workers, pg_authid WHERE pg_authid.rolname = 'new_role';
+ worker_password | coord_password
+---------------------------------------------------------------------
+                 |
+                 |
+(2 rows)
+
+ALTER ROLE new_role PASSWORD 'new_password';
+SELECT workers.result AS worker_password, pg_authid.rolpassword AS coord_password, workers.result = pg_authid.rolpassword AS password_is_same FROM run_command_on_workers($$SELECT rolpassword FROM pg_authid WHERE rolname = 'new_role'$$) workers, pg_authid WHERE pg_authid.rolname = 'new_role';
+           worker_password           |           coord_password            | password_is_same
+---------------------------------------------------------------------
+ md51a28da0f1a2416525eec435bdce8cbbe | md51a28da0f1a2416525eec435bdce8cbbe | t
+ md51a28da0f1a2416525eec435bdce8cbbe | md51a28da0f1a2416525eec435bdce8cbbe | t
+(2 rows)
+
+ALTER ROLE new_role PASSWORD NULL;
+SELECT workers.result AS worker_password, pg_authid.rolpassword AS coord_password FROM run_command_on_workers($$SELECT rolpassword FROM pg_authid WHERE rolname = 'new_role'$$) workers, pg_authid WHERE pg_authid.rolname = 'new_role';
+ worker_password | coord_password
+---------------------------------------------------------------------
+                 |
+                 |
+(2 rows)
+
+RESET password_encryption;
+DROP ROLE new_role;
 DROP TABLE test_search_path;
 DROP SCHEMA alter_role, ",CitUs,.TeeN!?", test_sp CASCADE;

src/test/regress/expected/background_rebalance.out

@@ -176,5 +176,128 @@ SELECT citus_rebalance_wait();
 
 (1 row)
 
+DROP TABLE t1;
+-- make sure a non-super user can stop rebalancing
+CREATE USER non_super_user_rebalance WITH LOGIN;
+GRANT ALL ON SCHEMA background_rebalance TO non_super_user_rebalance;
+SET ROLE non_super_user_rebalance;
+CREATE TABLE non_super_user_t1 (a int PRIMARY KEY);
+SELECT create_distributed_table('non_super_user_t1', 'a', shard_count => 4, colocate_with => 'none');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT citus_move_shard_placement(85674008, 'localhost', :worker_1_port, 'localhost', :worker_2_port, shard_transfer_mode => 'block_writes');
+ citus_move_shard_placement
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT 1 FROM citus_rebalance_start();
+NOTICE:  Scheduled 1 moves as job xxx
+DETAIL:  Rebalance scheduled as background job
+HINT:  To monitor progress, run: SELECT * FROM pg_dist_background_task WHERE job_id = xxx ORDER BY task_id ASC; or SELECT * FROM get_rebalance_progress();
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+SELECT citus_rebalance_stop();
+ citus_rebalance_stop
+---------------------------------------------------------------------
+
+(1 row)
+
+RESET ROLE;
+SET citus.replicate_reference_tables_on_activate = false;
+CREATE TABLE ref_no_pk(a int);
+SELECT create_reference_table('ref_no_pk');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE ref_with_pk(a int primary key);
+SELECT create_reference_table('ref_with_pk');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- Add coordinator so there's a node which doesn't have the reference tables
+SELECT 1 FROM citus_add_node('localhost', :master_port, groupId=>0);
+NOTICE:  localhost:xxxxx is the coordinator and already contains metadata, skipping syncing the metadata
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+-- fails
+BEGIN;
+SELECT 1 FROM citus_rebalance_start();
+ERROR:  cannot use logical replication to transfer shards of the relation ref_no_pk since it doesn't have a REPLICA IDENTITY or PRIMARY KEY
+DETAIL:  UPDATE and DELETE commands on the shard will error out during logical replication unless there is a REPLICA IDENTITY or PRIMARY KEY.
+HINT:  If you wish to continue without a replica identity set the shard_transfer_mode to 'force_logical' or 'block_writes'.
+ROLLBACK;
+-- success
+BEGIN;
+SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'force_logical');
+NOTICE:  Scheduled 1 moves as job xxx
+DETAIL:  Rebalance scheduled as background job
+HINT:  To monitor progress, run: SELECT * FROM pg_dist_background_task WHERE job_id = xxx ORDER BY task_id ASC; or SELECT * FROM get_rebalance_progress();
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+ROLLBACK;
+-- success
+BEGIN;
+SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'block_writes');
+NOTICE:  Scheduled 1 moves as job xxx
+DETAIL:  Rebalance scheduled as background job
+HINT:  To monitor progress, run: SELECT * FROM pg_dist_background_task WHERE job_id = xxx ORDER BY task_id ASC; or SELECT * FROM get_rebalance_progress();
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+ROLLBACK;
+-- fails
+SELECT 1 FROM citus_rebalance_start();
+ERROR:  cannot use logical replication to transfer shards of the relation ref_no_pk since it doesn't have a REPLICA IDENTITY or PRIMARY KEY
+DETAIL:  UPDATE and DELETE commands on the shard will error out during logical replication unless there is a REPLICA IDENTITY or PRIMARY KEY.
+HINT:  If you wish to continue without a replica identity set the shard_transfer_mode to 'force_logical' or 'block_writes'.
+-- succeeds
+SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'force_logical');
+NOTICE:  Scheduled 1 moves as job xxx
+DETAIL:  Rebalance scheduled as background job
+HINT:  To monitor progress, run: SELECT * FROM pg_dist_background_task WHERE job_id = xxx ORDER BY task_id ASC; or SELECT * FROM get_rebalance_progress();
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+-- wait for success
+SELECT citus_rebalance_wait();
+ citus_rebalance_wait
+---------------------------------------------------------------------
+
+(1 row)
+
+-- Remove coordinator again to allow rerunning of this test
+SELECT 1 FROM citus_remove_node('localhost', :master_port);
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+SELECT public.wait_until_metadata_sync(30000);
+ wait_until_metadata_sync
+---------------------------------------------------------------------
+
+(1 row)
+
 SET client_min_messages TO WARNING;
 DROP SCHEMA background_rebalance CASCADE;

src/test/regress/expected/citus_local_table_triggers.out

@@ -232,6 +232,30 @@ SELECT * FROM citus_local_table_triggers
  truncate_trigger_xxxxxxx  | "interesting!schema"."citus_local!_table"         | O
 (3 rows)
 
+-- ALTER TABLE ENABLE REPLICA trigger
+ALTER TABLE "interesting!schema"."citus_local!_table" ENABLE REPLICA TRIGGER "trigger\'name22";
+NOTICE:  executing the command locally: SELECT worker_apply_shard_ddl_command (1507008, 'interesting!schema', E'ALTER TABLE "interesting!schema"."citus_local!_table" ENABLE REPLICA TRIGGER "trigger\\''name22";')
+SELECT * FROM citus_local_table_triggers
+    WHERE tgname NOT LIKE 'RI_ConstraintTrigger%';
+         tgname          |                      tgrelid                      | tgenabled
+---------------------------------------------------------------------
+ trigger\'name22         | "interesting!schema"."citus_local!_table"         | R
+ trigger\'name22_1507008 | "interesting!schema"."citus_local!_table_1507008" | R
+ truncate_trigger_xxxxxxx  | "interesting!schema"."citus_local!_table"         | O
+(3 rows)
+
+-- ALTER TABLE ENABLE ALWAYS trigger
+ALTER TABLE "interesting!schema"."citus_local!_table" ENABLE ALWAYS TRIGGER "trigger\'name22";
+NOTICE:  executing the command locally: SELECT worker_apply_shard_ddl_command (1507008, 'interesting!schema', E'ALTER TABLE "interesting!schema"."citus_local!_table" ENABLE ALWAYS TRIGGER "trigger\\''name22";')
+SELECT * FROM citus_local_table_triggers
+    WHERE tgname NOT LIKE 'RI_ConstraintTrigger%';
+         tgname          |                      tgrelid                      | tgenabled
+---------------------------------------------------------------------
+ trigger\'name22         | "interesting!schema"."citus_local!_table"         | A
+ trigger\'name22_1507008 | "interesting!schema"."citus_local!_table_1507008" | A
+ truncate_trigger_xxxxxxx  | "interesting!schema"."citus_local!_table"         | O
+(3 rows)
+
 -- ALTER TABLE DISABLE trigger
 ALTER TABLE "interesting!schema"."citus_local!_table" DISABLE TRIGGER "trigger\'name22";
 NOTICE:  executing the command locally: SELECT worker_apply_shard_ddl_command (1507008, 'interesting!schema', E'ALTER TABLE "interesting!schema"."citus_local!_table" DISABLE TRIGGER "trigger\\''name22";')

src/test/regress/expected/citus_local_tables.out

@@ -40,7 +40,13 @@ SELECT 1 FROM master_remove_node('localhost', :master_port);
 CREATE TABLE citus_local_table_1 (a int primary key);
 -- this should fail as coordinator is removed from pg_dist_node
 SELECT citus_add_local_table_to_metadata('citus_local_table_1');
-ERROR:  could not find the coordinator node in metadata as it is not added as a worker
+ERROR:  operation is not allowed when coordinator is not added into metadata
+-- This should also fail as coordinator is removed from pg_dist_node.
+--
+-- This is not a great place to test this but is one of those places that we
+-- have workers in metadata but not the coordinator.
+SELECT create_distributed_table_concurrently('citus_local_table_1', 'a');
+ERROR:  operation is not allowed when coordinator is not added into metadata
 -- let coordinator have citus local tables again for next tests
 set client_min_messages to ERROR;
 SELECT 1 FROM master_add_node('localhost', :master_port, groupId => 0);
@@ -223,7 +229,7 @@ ROLLBACK;
 CREATE FOREIGN TABLE foreign_table (
   id bigint not null,
   full_name text not null default ''
-) SERVER fake_fdw_server OPTIONS (encoding 'utf-8', compression 'true');
+) SERVER fake_fdw_server OPTIONS (encoding 'utf-8', compression 'true', table_name 'foreign_table');
 -- observe that we do not create fdw server for shell table, both shard relation
 -- & shell relation points to the same same server object
 -- Disable metadata sync since citus doesn't support distributing

src/test/regress/expected/citus_non_blocking_split_shards.out

@@ -60,7 +60,7 @@ SELECT create_reference_table('reference_table');
 
 (1 row)
 
-CREATE TABLE colocated_dist_table (measureid integer PRIMARY KEY);
+CREATE TABLE colocated_dist_table (measureid integer PRIMARY KEY, genid integer GENERATED ALWAYS AS ( measureid + 3 ) stored, value varchar(44), col_todrop integer);
 CLUSTER colocated_dist_table USING colocated_dist_table_pkey;
 SELECT create_distributed_table('colocated_dist_table', 'measureid', colocate_with:='sensors');
  create_distributed_table
@@ -84,8 +84,9 @@ ALTER TABLE sensors ADD CONSTRAINT fkey_table_to_dist FOREIGN KEY (measureid) RE
 -- END : Create Foreign key constraints.
 -- BEGIN : Load data into tables.
 INSERT INTO reference_table SELECT i FROM generate_series(0,1000)i;
-INSERT INTO colocated_dist_table SELECT i FROM generate_series(0,1000)i;
+INSERT INTO colocated_dist_table(measureid, value, col_todrop) SELECT i,'Value',i FROM generate_series(0,1000)i;
 INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(0,1000)i;
+ALTER TABLE colocated_dist_table DROP COLUMN col_todrop;
 SELECT COUNT(*) FROM sensors;
  count
 ---------------------------------------------------------------------

src/test/regress/expected/citus_split_shard_by_split_points.out

@@ -56,7 +56,7 @@ SELECT create_reference_table('reference_table');
 
 (1 row)
 
-CREATE TABLE colocated_dist_table (measureid integer PRIMARY KEY);
+CREATE TABLE colocated_dist_table (measureid integer PRIMARY KEY, genid integer GENERATED ALWAYS AS ( measureid + 3 ) stored, value varchar(44), col_todrop integer);
 CLUSTER colocated_dist_table USING colocated_dist_table_pkey;
 SELECT create_distributed_table('colocated_dist_table', 'measureid', colocate_with:='sensors');
  create_distributed_table
@@ -80,8 +80,9 @@ ALTER TABLE sensors ADD CONSTRAINT fkey_table_to_dist FOREIGN KEY (measureid) RE
 -- END : Create Foreign key constraints.
 -- BEGIN : Load data into tables.
 INSERT INTO reference_table SELECT i FROM generate_series(0,1000)i;
-INSERT INTO colocated_dist_table SELECT i FROM generate_series(0,1000)i;
+INSERT INTO colocated_dist_table(measureid, value, col_todrop) SELECT i,'Value',i FROM generate_series(0,1000)i;
 INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(0,1000)i;
+ALTER TABLE colocated_dist_table DROP COLUMN col_todrop;
 SELECT COUNT(*) FROM sensors;
  count
 ---------------------------------------------------------------------

src/test/regress/expected/citus_table_triggers.out

@@ -146,9 +146,12 @@ SELECT master_get_table_ddl_events('test_table');
  CREATE TABLE table_triggers_schema.test_table (id integer, text_number text, text_col text)
  ALTER TABLE table_triggers_schema.test_table OWNER TO postgres
  CREATE TRIGGER test_table_delete AFTER DELETE ON table_triggers_schema.test_table FOR EACH STATEMENT EXECUTE FUNCTION table_triggers_schema.test_table_trigger_function()
+ ALTER TABLE table_triggers_schema.test_table ENABLE TRIGGER test_table_delete;
  CREATE CONSTRAINT TRIGGER test_table_insert AFTER INSERT ON table_triggers_schema.test_table DEFERRABLE INITIALLY IMMEDIATE FOR EACH ROW WHEN (((new.id > 5) OR ((new.text_col IS NOT NULL) AND ((new.id)::numeric < to_number(new.text_number, '9999'::text))))) EXECUTE FUNCTION table_triggers_schema.test_table_trigger_function()
+ ALTER TABLE table_triggers_schema.test_table ENABLE TRIGGER test_table_insert;
  CREATE CONSTRAINT TRIGGER test_table_update AFTER UPDATE OF id ON table_triggers_schema.test_table NOT DEFERRABLE INITIALLY IMMEDIATE FOR EACH ROW WHEN (((NOT (old.* IS DISTINCT FROM new.*)) AND (old.text_number IS NOT NULL))) EXECUTE FUNCTION table_triggers_schema.test_table_trigger_function()
-(5 rows)
+ ALTER TABLE table_triggers_schema.test_table ENABLE TRIGGER test_table_update;
+(8 rows)
 
 -- cleanup at exit
 DROP SCHEMA table_triggers_schema CASCADE;

src/test/regress/expected/citus_update_table_statistics.out

@@ -64,11 +64,11 @@ SET citus.multi_shard_modify_mode TO sequential;
 SELECT citus_update_table_statistics('test_table_statistics_hash');
 NOTICE:  issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing SELECT 981000 AS shard_id, 'public.test_table_statistics_hash_981000' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981000') UNION ALL SELECT 981001 AS shard_id, 'public.test_table_statistics_hash_981001' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981001') UNION ALL SELECT 981002 AS shard_id, 'public.test_table_statistics_hash_981002' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981002') UNION ALL SELECT 981003 AS shard_id, 'public.test_table_statistics_hash_981003' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981003') UNION ALL SELECT 981004 AS shard_id, 'public.test_table_statistics_hash_981004' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981004') UNION ALL SELECT 981005 AS shard_id, 'public.test_table_statistics_hash_981005' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981005') UNION ALL SELECT 981006 AS shard_id, 'public.test_table_statistics_hash_981006' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981006') UNION ALL SELECT 981007 AS shard_id, 'public.test_table_statistics_hash_981007' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981007') UNION ALL SELECT 0::bigint, NULL::text, 0::bigint;
+NOTICE:  issuing SELECT 981000 AS shard_id, 'public.test_table_statistics_hash_981000' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981000') UNION ALL SELECT 981001 AS shard_id, 'public.test_table_statistics_hash_981001' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981001') UNION ALL SELECT 981002 AS shard_id, 'public.test_table_statistics_hash_981002' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981002') UNION ALL SELECT 981003 AS shard_id, 'public.test_table_statistics_hash_981003' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981003') UNION ALL SELECT 981004 AS shard_id, 'public.test_table_statistics_hash_981004' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981004') UNION ALL SELECT 981005 AS shard_id, 'public.test_table_statistics_hash_981005' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981005') UNION ALL SELECT 981006 AS shard_id, 'public.test_table_statistics_hash_981006' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981006') UNION ALL SELECT 981007 AS shard_id, 'public.test_table_statistics_hash_981007' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981007') UNION ALL SELECT 0::bigint, NULL::text, 0::bigint;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing SELECT 981000 AS shard_id, 'public.test_table_statistics_hash_981000' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981000') UNION ALL SELECT 981001 AS shard_id, 'public.test_table_statistics_hash_981001' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981001') UNION ALL SELECT 981002 AS shard_id, 'public.test_table_statistics_hash_981002' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981002') UNION ALL SELECT 981003 AS shard_id, 'public.test_table_statistics_hash_981003' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981003') UNION ALL SELECT 981004 AS shard_id, 'public.test_table_statistics_hash_981004' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981004') UNION ALL SELECT 981005 AS shard_id, 'public.test_table_statistics_hash_981005' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981005') UNION ALL SELECT 981006 AS shard_id, 'public.test_table_statistics_hash_981006' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981006') UNION ALL SELECT 981007 AS shard_id, 'public.test_table_statistics_hash_981007' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981007') UNION ALL SELECT 0::bigint, NULL::text, 0::bigint;
+NOTICE:  issuing SELECT 981000 AS shard_id, 'public.test_table_statistics_hash_981000' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981000') UNION ALL SELECT 981001 AS shard_id, 'public.test_table_statistics_hash_981001' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981001') UNION ALL SELECT 981002 AS shard_id, 'public.test_table_statistics_hash_981002' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981002') UNION ALL SELECT 981003 AS shard_id, 'public.test_table_statistics_hash_981003' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981003') UNION ALL SELECT 981004 AS shard_id, 'public.test_table_statistics_hash_981004' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981004') UNION ALL SELECT 981005 AS shard_id, 'public.test_table_statistics_hash_981005' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981005') UNION ALL SELECT 981006 AS shard_id, 'public.test_table_statistics_hash_981006' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981006') UNION ALL SELECT 981007 AS shard_id, 'public.test_table_statistics_hash_981007' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981007') UNION ALL SELECT 0::bigint, NULL::text, 0::bigint;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing COMMIT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
@@ -152,11 +152,11 @@ SET citus.multi_shard_modify_mode TO sequential;
 SELECT citus_update_table_statistics('test_table_statistics_append');
 NOTICE:  issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing SELECT 981008 AS shard_id, 'public.test_table_statistics_append_981008' AS shard_name, pg_relation_size('public.test_table_statistics_append_981008') UNION ALL SELECT 981009 AS shard_id, 'public.test_table_statistics_append_981009' AS shard_name, pg_relation_size('public.test_table_statistics_append_981009') UNION ALL SELECT 0::bigint, NULL::text, 0::bigint;
+NOTICE:  issuing SELECT 981008 AS shard_id, 'public.test_table_statistics_append_981008' AS shard_name, pg_total_relation_size('public.test_table_statistics_append_981008') UNION ALL SELECT 981009 AS shard_id, 'public.test_table_statistics_append_981009' AS shard_name, pg_total_relation_size('public.test_table_statistics_append_981009') UNION ALL SELECT 0::bigint, NULL::text, 0::bigint;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing SELECT 981008 AS shard_id, 'public.test_table_statistics_append_981008' AS shard_name, pg_relation_size('public.test_table_statistics_append_981008') UNION ALL SELECT 981009 AS shard_id, 'public.test_table_statistics_append_981009' AS shard_name, pg_relation_size('public.test_table_statistics_append_981009') UNION ALL SELECT 0::bigint, NULL::text, 0::bigint;
+NOTICE:  issuing SELECT 981008 AS shard_id, 'public.test_table_statistics_append_981008' AS shard_name, pg_total_relation_size('public.test_table_statistics_append_981008') UNION ALL SELECT 981009 AS shard_id, 'public.test_table_statistics_append_981009' AS shard_name, pg_total_relation_size('public.test_table_statistics_append_981009') UNION ALL SELECT 0::bigint, NULL::text, 0::bigint;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing COMMIT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx

src/test/regress/expected/columnar_alter_set_type.out

@@ -50,3 +50,28 @@ SELECT * FROM test_alter_table ORDER BY a;
 (4 rows)
 
 DROP TABLE test_alter_table;
+--  Make sure that the correct table options are used when rewriting the table.
+-- This is reflected by the VACUUM VERBOSE output right after a rewrite showing
+-- that all chunks are compressed with the configured compression algorithm
+-- https://github.com/citusdata/citus/issues/5927
+CREATE TABLE test(i int) USING columnar;
+ALTER TABLE test SET (columnar.compression = lz4);
+INSERT INTO test VALUES(1);
+VACUUM VERBOSE test;
+INFO:  statistics for "test":
+storage id: xxxxx
+total file size: 24576, total data size: 6
+compression rate: 0.83x
+total row count: 1, stripe count: 1, average rows per stripe: 1
+chunk count: 1, containing data for dropped columns: 0, lz4 compressed: 1
+
+ALTER TABLE test ALTER COLUMN i TYPE int8;
+VACUUM VERBOSE test;
+INFO:  statistics for "test":
+storage id: xxxxx
+total file size: 24576, total data size: 10
+compression rate: 0.90x
+total row count: 1, stripe count: 1, average rows per stripe: 1
+chunk count: 1, containing data for dropped columns: 0, lz4 compressed: 1
+
+DROP TABLE test;

src/test/regress/expected/columnar_create.out

@@ -55,6 +55,89 @@ SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_table_1_
  t
 (1 row)
 
+BEGIN;
+  INSERT INTO columnar_table_1 VALUES (2);
+ROLLBACK;
+INSERT INTO columnar_table_1 VALUES (3),(4);
+INSERT INTO columnar_table_1 VALUES (5),(6);
+INSERT INTO columnar_table_1 VALUES (7),(8);
+-- Test whether columnar metadata accessors are still fine even
+-- when the metadata indexes are not available to them.
+BEGIN;
+  ALTER INDEX columnar_internal.stripe_first_row_number_idx RENAME TO new_index_name;
+  ALTER INDEX columnar_internal.chunk_pkey RENAME TO new_index_name_1;
+  ALTER INDEX columnar_internal.stripe_pkey RENAME TO new_index_name_2;
+  ALTER INDEX columnar_internal.chunk_group_pkey RENAME TO new_index_name_3;
+  CREATE INDEX columnar_table_1_idx ON columnar_table_1(a);
+WARNING:  Metadata index stripe_first_row_number_idx is not available, this might mean slower read/writes on columnar tables. This is expected during Postgres upgrades and not expected otherwise.
+WARNING:  Metadata index stripe_first_row_number_idx is not available, this might mean slower read/writes on columnar tables. This is expected during Postgres upgrades and not expected otherwise.
+WARNING:  Metadata index chunk_pkey is not available, this might mean slower read/writes on columnar tables. This is expected during Postgres upgrades and not expected otherwise.
+WARNING:  Metadata index chunk_group_pkey is not available, this might mean slower read/writes on columnar tables. This is expected during Postgres upgrades and not expected otherwise.
+  -- make sure that we test index scan
+  SET LOCAL columnar.enable_custom_scan TO 'off';
+  SET LOCAL enable_seqscan TO off;
+  SET LOCAL seq_page_cost TO 10000000;
+  SELECT * FROM columnar_table_1 WHERE a = 6;
+WARNING:  Metadata index stripe_first_row_number_idx is not available, this might mean slower read/writes on columnar tables. This is expected during Postgres upgrades and not expected otherwise.
+ a
+---------------------------------------------------------------------
+ 6
+(1 row)
+
+  SELECT * FROM columnar_table_1 WHERE a = 5;
+ a
+---------------------------------------------------------------------
+ 5
+(1 row)
+
+  SELECT * FROM columnar_table_1 WHERE a = 7;
+ a
+---------------------------------------------------------------------
+ 7
+(1 row)
+
+  SELECT * FROM columnar_table_1 WHERE a = 3;
+ a
+---------------------------------------------------------------------
+ 3
+(1 row)
+
+  DROP INDEX columnar_table_1_idx;
+  -- Re-shuffle some metadata records to test whether we can
+  -- rely on sequential metadata scan when the metadata records
+  -- are not ordered by their "first_row_number"s.
+  WITH cte AS (
+      DELETE FROM columnar_internal.stripe
+      WHERE storage_id = columnar.get_storage_id('columnar_table_1')
+      RETURNING *
+  )
+  INSERT INTO columnar_internal.stripe SELECT * FROM cte ORDER BY first_row_number DESC;
+  SELECT SUM(a) FROM columnar_table_1;
+ sum
+---------------------------------------------------------------------
+  34
+(1 row)
+
+  SELECT * FROM columnar_table_1 WHERE a = 6;
+ a
+---------------------------------------------------------------------
+ 6
+(1 row)
+
+  -- Run a SELECT query after the INSERT command to force flushing the
+  -- data within the xact block.
+  INSERT INTO columnar_table_1 VALUES (20);
+  SELECT COUNT(*) FROM columnar_table_1;
+WARNING:  Metadata index stripe_pkey is not available, this might mean slower read/writes on columnar tables. This is expected during Postgres upgrades and not expected otherwise.
+ count
+---------------------------------------------------------------------
+     8
+(1 row)
+
+  DROP TABLE columnar_table_1 CASCADE;
+NOTICE:  drop cascades to materialized view columnar_table_1_mv
+WARNING:  Metadata index on a columnar metadata table is not available, this might mean slower read/writes on columnar tables. This is expected during Postgres upgrades and not expected otherwise.
+ROLLBACK;
 -- test dropping columnar table
 DROP TABLE columnar_table_1 CASCADE;
 NOTICE:  drop cascades to materialized view columnar_table_1_mv