Reorganize env variables and inputs.

.github/actions/configure/action.yml

@@ -5,5 +5,6 @@ runs:
   - name: Configure
     run: |-
       chown -R circleci .
+      git config --global --add safe.directory ${GITHUB_WORKSPACE}
       gosu circleci ./configure --without-pg-version-check
-    shell: bash
+    shell: bash

.github/actions/install_extension/action.yml

@@ -6,5 +6,5 @@ runs:
   using: composite
   steps:
   - name: Install Extension
-    run: tar xfv "${{ github.workspace }}/install-${{ inputs.pg_major }}.tar" --directory /
+    run: tar xfv "artifact/install-${{ inputs.pg_major }}.tar" --directory /
-    shell: bash
+    shell: bash

.github/actions/parallelization/action.yml

@@ -0,0 +1,23 @@
+name: 'Parallelization matrix'
+inputs:
+  count:
+    required: false
+    default: 32
+outputs:
+  json:
+    value: ${{ steps.generate_matrix.outputs.json }}
+runs:
+  using: "composite"
+  steps:
+    - name: Generate parallelization matrix
+      id: generate_matrix
+      shell: bash
+      run: |-
+        json_array="{\"include\": ["
+        for ((i = 1; i <= ${{ inputs.count }}; i++)); do
+            json_array+="{\"id\":\"$i\"},"
+        done
+        json_array=${json_array%,}
+        json_array+=" ]}"
+        echo "json=$json_array" >> "$GITHUB_OUTPUT"
+        echo "json=$json_array"

.github/actions/save_logs_and_results/action.yml

@@ -3,17 +3,52 @@ runs:
   using: composite
   steps:
   - uses: actions/upload-artifact@v3.1.1
+    if: ${{ matrix.id == '' }}
     with:
+      name: proxy.output
       path: src/test/regress/proxy.output
   - uses: actions/upload-artifact@v3.1.1
+    if: ${{ matrix.id == '' }}
     with:
+      name: regress
       path: src/test/regress/results/
   - uses: actions/upload-artifact@v3.1.1
+    if: ${{ matrix.id == '' }}
     with:
+      name: master.log
       path: src/test/regress/tmp_check/master/log
   - uses: actions/upload-artifact@v3.1.1
+    if: ${{ matrix.id == '' }}
     with:
+      name: worker.57637.log
       path: src/test/regress/tmp_check/worker.57637/log
   - uses: actions/upload-artifact@v3.1.1
+    if: ${{ matrix.id == '' }}
     with:
-      path: src/test/regress/tmp_check/worker.57638/log
+      name: worker.57638.log
+      path: src/test/regress/tmp_check/worker.57638/log
+  - uses: actions/upload-artifact@v3.1.1
+    if: ${{ matrix.id != '' }}
+    with:
+      name: proxy_${{ matrix.id }}.output
+      path: src/test/regress/proxy.output
+  - uses: actions/upload-artifact@v3.1.1
+    if: ${{ matrix.id != '' }}
+    with:
+      name: regress_${{ matrix.id }}
+      path: src/test/regress/results/
+  - uses: actions/upload-artifact@v3.1.1
+    if: ${{ matrix.id != '' }}
+    with:
+      name: master_${{ matrix.id }}.log
+      path: src/test/regress/tmp_check/master/log
+  - uses: actions/upload-artifact@v3.1.1
+    if: ${{ matrix.id != '' }}
+    with:
+      name: worker_${{ matrix.id }}.57637.log
+      path: src/test/regress/tmp_check/worker.57637/log
+  - uses: actions/upload-artifact@v3.1.1
+    if: ${{ matrix.id != '' }}
+    with:
+      name: worker_${{ matrix.id }}.57638.log
+      path: src/test/regress/tmp_check/worker.57638/log

.github/actions/save_regressions/action.yml

@@ -11,5 +11,12 @@ runs:
     if: failure()
     shell: bash
   - uses: actions/upload-artifact@v3.1.1
+    if: ${{ matrix.id == '' }}
     with:
-      path: src/test/regress/regression.diffs
+      name: regression.diffs
+      path: src/test/regress/regression.diffs
+  - uses: actions/upload-artifact@v3.1.1
+    if: ${{ matrix.id != '' }}
+    with:
+      path: src/test/regress/regression.diffs
+      name: regression_${{ matrix.id }}.diffs

.github/workflows/flaky_test_debugging.yml

@@ -1,38 +1,49 @@
-name: citusdata/citus/flaky_test_debugging
+name: Flaky test detector
+run-name: Flaky test detector - ${{ inputs.flaky_test }} (${{ inputs.flaky_test_runs_per_job }}x${{ inputs.flaky_test_parallel_jobs }})
 on:
   workflow_dispatch:
     inputs:
       flaky_test:
         required: true
+        type: string
+        description: Test to run
       flaky_test_runs_per_job:
         required: false
-        default: 50
+        default: 8
         type: number
-      image_suffix:
+        description: Number of times to run the test
+      flaky_test_parallel_jobs:
         required: false
-        default: "-vbab548a"
+        default: 32
-      pg14_version:
+        type: number
-        required: false
+        description: Number of parallel jobs to run
-        default: '14.8'
+env:
-      pg15_version:
+  pg_major: 15
-        required: false
+  image_suffix: -v1b94240
-        default: '15.3'
+  pg15_version: 15.4
-      upgrade_pg_versions:
+  build_image_name: citus/extbuilder
-        required: false
+  test_image_name: citus/failtester
-        default: 14.8-15.3
-      style_checker_tools_version:
-        required: false
-        default: 0.8.18
 jobs:
-  build:
+  set_image_names:
-    if: ${{ inputs.flaky_test }}
+    name: Set CI image names
     runs-on: ubuntu-latest
+    outputs:
+      build_image: ${{ steps.set_build_image_name.outputs.image }}
+      test_image: ${{ steps.set_test_image_name.outputs.image }}
+    steps:
+    - id: set_build_image_name
+      run: |
+        echo "image=${{ env.build_image_name }}:${{ env.pg15_version  }}${{ env.image_suffix }}" >> "$GITHUB_OUTPUT"
+    - id: set_test_image_name
+      run: |
+        echo "image=${{ env.test_image_name }}:${{ env.pg15_version  }}${{ env.image_suffix }}" >> "$GITHUB_OUTPUT"
+  build:
+    name: Build Citus
+    runs-on: ubuntu-latest
+    needs: set_image_names
     container:
-      image: "${{ vars.image }}"
+      image: ${{ needs.set_image_names.outputs.build_image }}
-    env:
+      options: --user root
-      pg_major: 15
-      image: citus/extbuilder
-      image_tag: "${{ inputs.pg15_version }}"
     steps:
     - uses: actions/checkout@v3.5.0
     - name: Configure, Build, and Install
@@ -42,20 +53,33 @@ jobs:
         path: |-
           ./build-${{ env.pg_major }}/*
           ./install-${{ env.pg_major }}.tar
-  test-flakyness:
+  prepare_parallelization_matrix:
+    name: Prepare parallelization matrix
+    runs-on: ubuntu-latest
+    outputs:
+      json: ${{ steps.parallelization.outputs.json }}
+    steps:
+      - uses: actions/checkout@v3.5.0
+      - uses: "./.github/actions/parallelization"
+        id: parallelization
+        with:
+          count: ${{ inputs.flaky_test_parallel_jobs }}
+  test_flakyness:
+    name: Test flakyness
     if: ${{ inputs.flaky_test }}
     runs-on: ubuntu-latest
     container:
-      image: '${{ vars.image }}:${{ vars.image_tag }}${{ vars.image_suffix }}'
+      image: ${{ needs.set_image_names.outputs.test_image }}
+      options: --user root
     needs:
-    - build
+      [set_image_names,  build, prepare_parallelization_matrix]
     env:
-      pg_major: 15
-      image: citus/failtester
-      image_tag: "${{ inputs.pg15_version }}"
       test: "${{ inputs.flaky_test }}"
       runs: "${{ inputs.flaky_test_runs_per_job }}"
       skip: false
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.prepare_parallelization_matrix.outputs.json) }}
     steps:
     - uses: actions/checkout@v3.5.0
     - uses: actions/download-artifact@v3.0.1
@@ -73,7 +97,8 @@ jobs:
         fi
         if [ -z "$tests" ]; then
             echo "No test found."
-            circleci-agent step halt
+            gh run cancel ${{ github.run_id }}
+            gh run watch ${{ github.run_id }}
         else
           echo "Detected tests " $tests
         fi
@@ -91,6 +116,10 @@ jobs:
             test_name=$(echo "$test" | sed -r "s/.+\/(.+)\..+/\1/")
             gosu circleci src/test/regress/citus_tests/run_test.py $test_name --repeat ${{ env.runs }} --use-base-schedule --use-whole-schedule-line
         done
+      shell: bash
     - uses: "./.github/actions/save_logs_and_results"
+      if: always()
     - uses: "./.github/actions/save_regressions"
+      if: always()
     - uses: "./.github/actions/stack_trace"
+      if: always()

ci/build-citus.sh

@@ -15,9 +15,6 @@ PG_MAJOR=${PG_MAJOR:?please provide the postgres major version}
 codename=${VERSION#*(}
 codename=${codename%)*}
 
-# get project from argument
-project="${CIRCLE_PROJECT_REPONAME}"
-
 # we'll do everything with absolute paths
 basedir="$(pwd)"
 
@@ -28,7 +25,7 @@ build_ext() {
   pg_major="$1"
 
   builddir="${basedir}/build-${pg_major}"
-  echo "Beginning build of ${project} for PostgreSQL ${pg_major}..." >&2
+  echo "Beginning build for PostgreSQL ${pg_major}..." >&2
 
   # do everything in a subdirectory to avoid clutter in current directory
   mkdir -p "${builddir}" && cd "${builddir}"