From c940bda953ea7e658d76869913b825e0ce0b8b41 Mon Sep 17 00:00:00 2001 From: Mehmet Yilmaz Date: Thu, 3 Apr 2025 10:34:53 +0000 Subject: [PATCH 1/3] Add GitHub Actions workflow for building and testing PG master --- .../workflows/build-and-test-pg-master.yml | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 .github/workflows/build-and-test-pg-master.yml diff --git a/.github/workflows/build-and-test-pg-master.yml b/.github/workflows/build-and-test-pg-master.yml new file mode 100644 index 000000000..79d0c5c00 --- /dev/null +++ b/.github/workflows/build-and-test-pg-master.yml @@ -0,0 +1,103 @@ +name: Build & Test PG Master + +on: + workflow_dispatch: + push: + branches: + - "main" + pull_request: + +jobs: + params: + runs-on: ubuntu-latest + name: Initialize parameters + outputs: + build_image_name: "ghcr.io/citusdata/extbuilder" + test_image_name: "ghcr.io/citusdata/exttester" + pg_master_version: '{ "major": "master", "full": "master" }' + image_suffix: "-veab367a" + steps: + - name: Set up parameters + run: echo 'noop' + + build_pg_master: + needs: params + name: Build for PG${{ fromJson(matrix.pg_version).major }} + strategy: + fail-fast: false + matrix: + image_name: + - ${{ needs.params.outputs.build_image_name }} + pg_version: + - ${{ needs.params.outputs.pg_master_version }} + image_suffix: + - ${{ needs.params.outputs.image_suffix }} + runs-on: ubuntu-20.04 + container: + image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}" + options: --user root + steps: + - uses: actions/checkout@v4 + - name: Expose PG_MAJOR to GitHub Environment + run: echo "PG_MAJOR=${{ fromJson(matrix.pg_version).major }}" >> $GITHUB_ENV + shell: bash + - name: Build Citus with PG master branch + run: "./ci/build-citus.sh" + shell: bash + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: build-${{ env.PG_MAJOR }} + path: |- + ./build-${{ env.PG_MAJOR }}/* + ./install-${{ env.PG_MAJOR }}.tar + + test_citus_pg_master: + needs: build_pg_master + name: PG${{ fromJson(matrix.pg_version).major }} - Test Citus + strategy: + fail-fast: false + matrix: + suite: + - regress + image_name: + - ${{ needs.params.outputs.test_image_name }} + pg_version: + - ${{ needs.params.outputs.pg_master_version }} + make: + - check-split + - check-multi + - check-multi-1 + - check-multi-mx + - check-vanilla + - check-isolation + - check-operations + - check-follower-cluster + - check-columnar + - check-columnar-isolation + - check-enterprise + - check-enterprise-isolation + - check-enterprise-isolation-logicalrep-1 + - check-enterprise-isolation-logicalrep-2 + - check-enterprise-isolation-logicalrep-3 + runs-on: ubuntu-20.04 + container: + image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}" + options: --user root --dns=8.8.8.8 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup_extension + - name: Run tests + run: gosu circleci make -C src/test/${{ matrix.suite }} ${{ matrix.make }} + timeout-minutes: 20 + - name: Save logs and results + if: always() + uses: ./.github/actions/save_logs_and_results + with: + folder: ${{ fromJson(matrix.pg_version).major }}_${{ matrix.make }} + - name: Upload coverage results + if: always() + uses: ./.github/actions/upload_coverage + with: + flags: ${{ env.PG_MAJOR }}_${{ matrix.suite }}_${{ matrix.make }} + codecov_token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file From e2f882cc8932c8ad411139e4f79ea7007918719b Mon Sep 17 00:00:00 2001 From: Mehmet Yilmaz Date: Thu, 3 Apr 2025 10:46:36 +0000 Subject: [PATCH 2/3] Update GitHub Actions workflow to build and test Citus with PostgreSQL master --- .../workflows/build-and-test-pg-master.yml | 137 ++++++------------ 1 file changed, 47 insertions(+), 90 deletions(-) diff --git a/.github/workflows/build-and-test-pg-master.yml b/.github/workflows/build-and-test-pg-master.yml index 79d0c5c00..ab7e0cbd0 100644 --- a/.github/workflows/build-and-test-pg-master.yml +++ b/.github/workflows/build-and-test-pg-master.yml @@ -1,4 +1,4 @@ -name: Build & Test PG Master +name: Build & Test Citus with PG Master on: workflow_dispatch: @@ -8,96 +8,53 @@ on: pull_request: jobs: - params: - runs-on: ubuntu-latest - name: Initialize parameters - outputs: - build_image_name: "ghcr.io/citusdata/extbuilder" - test_image_name: "ghcr.io/citusdata/exttester" - pg_master_version: '{ "major": "master", "full": "master" }' - image_suffix: "-veab367a" - steps: - - name: Set up parameters - run: echo 'noop' - - build_pg_master: - needs: params - name: Build for PG${{ fromJson(matrix.pg_version).major }} - strategy: - fail-fast: false - matrix: - image_name: - - ${{ needs.params.outputs.build_image_name }} - pg_version: - - ${{ needs.params.outputs.pg_master_version }} - image_suffix: - - ${{ needs.params.outputs.image_suffix }} + build-and-test-pg-master: runs-on: ubuntu-20.04 - container: - image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}" - options: --user root steps: - uses: actions/checkout@v4 - - name: Expose PG_MAJOR to GitHub Environment - run: echo "PG_MAJOR=${{ fromJson(matrix.pg_version).major }}" >> $GITHUB_ENV - shell: bash - - name: Build Citus with PG master branch - run: "./ci/build-citus.sh" - shell: bash - - name: Upload build artifacts - uses: actions/upload-artifact@v4 - with: - name: build-${{ env.PG_MAJOR }} - path: |- - ./build-${{ env.PG_MAJOR }}/* - ./install-${{ env.PG_MAJOR }}.tar - test_citus_pg_master: - needs: build_pg_master - name: PG${{ fromJson(matrix.pg_version).major }} - Test Citus - strategy: - fail-fast: false - matrix: - suite: - - regress - image_name: - - ${{ needs.params.outputs.test_image_name }} - pg_version: - - ${{ needs.params.outputs.pg_master_version }} - make: - - check-split - - check-multi - - check-multi-1 - - check-multi-mx - - check-vanilla - - check-isolation - - check-operations - - check-follower-cluster - - check-columnar - - check-columnar-isolation - - check-enterprise - - check-enterprise-isolation - - check-enterprise-isolation-logicalrep-1 - - check-enterprise-isolation-logicalrep-2 - - check-enterprise-isolation-logicalrep-3 - runs-on: ubuntu-20.04 - container: - image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}" - options: --user root --dns=8.8.8.8 - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup_extension - - name: Run tests - run: gosu circleci make -C src/test/${{ matrix.suite }} ${{ matrix.make }} - timeout-minutes: 20 - - name: Save logs and results - if: always() - uses: ./.github/actions/save_logs_and_results - with: - folder: ${{ fromJson(matrix.pg_version).major }}_${{ matrix.make }} - - name: Upload coverage results - if: always() - uses: ./.github/actions/upload_coverage - with: - flags: ${{ env.PG_MAJOR }}_${{ matrix.suite }}_${{ matrix.make }} - codecov_token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file + - name: Install Build Dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential \ + libreadline-dev \ + zlib1g-dev \ + flex \ + bison \ + libxml2-dev \ + libxslt-dev \ + libssl-dev \ + git + + - name: Clone PostgreSQL Repository + run: | + # Clone PostgreSQL master branch + git clone https://github.com/postgres/postgres.git + cd postgres + git checkout master + + - name: Build PostgreSQL from Source + run: | + cd postgres + ./configure --prefix=/usr/local/pgmaster + make -j$(nproc) + sudo make install + + - name: Set PG Binaries Path + run: echo "/usr/local/pgmaster/bin" >> $GITHUB_PATH + + - name: Verify PostgreSQL Build + run: | + pg_config --version + + - name: Build Citus + run: | + # Ensure the newly built PostgreSQL is used + which pg_config + ./ci/build-citus.sh + + - name: Run Citus Tests + run: | + # Example test – adjust the make target and test directory as needed + make -C src/test/regress check-vanilla From ab53e235711db4485073068deaef7f0b6e8d48c4 Mon Sep 17 00:00:00 2001 From: Mehmet Yilmaz Date: Thu, 3 Apr 2025 10:53:41 +0000 Subject: [PATCH 3/3] Refactor GitHub Actions workflow for PostgreSQL build and Citus setup --- .../workflows/build-and-test-pg-master.yml | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-test-pg-master.yml b/.github/workflows/build-and-test-pg-master.yml index ab7e0cbd0..37a48d58f 100644 --- a/.github/workflows/build-and-test-pg-master.yml +++ b/.github/workflows/build-and-test-pg-master.yml @@ -29,7 +29,6 @@ jobs: - name: Clone PostgreSQL Repository run: | - # Clone PostgreSQL master branch git clone https://github.com/postgres/postgres.git cd postgres git checkout master @@ -41,20 +40,32 @@ jobs: make -j$(nproc) sudo make install - - name: Set PG Binaries Path + - name: Add PostgreSQL to PATH run: echo "/usr/local/pgmaster/bin" >> $GITHUB_PATH - name: Verify PostgreSQL Build run: | pg_config --version - - name: Build Citus + - name: Set up PG config for Citus build run: | - # Ensure the newly built PostgreSQL is used + # Set PG_MAJOR to "master" + export PG_MAJOR=master + echo "PG_MAJOR is set to $PG_MAJOR" + # Create the directory structure expected by build-citus.sh and symlink the pg_config + sudo mkdir -p /usr/lib/postgresql/$PG_MAJOR/bin + sudo ln -sf /usr/local/pgmaster/bin/pg_config /usr/lib/postgresql/$PG_MAJOR/bin/pg_config + + - name: Build Citus + env: + PG_MAJOR: master + run: | + # Confirm that the expected pg_config is being used which pg_config + ls -l /usr/lib/postgresql/$PG_MAJOR/bin/pg_config ./ci/build-citus.sh - name: Run Citus Tests run: | - # Example test – adjust the make target and test directory as needed + # Adjust the test target as needed. make -C src/test/regress check-vanilla