PG-292: Automate the Q/A and implement tap testcases.

This commit brings following changes:
1) Implementation of tap based testing using perl language for different
scenarios that could not be covered under traditional SQL based diff testing,
or require server start/shutdown. At this point of time, tap testing is only
enabled for Postgres 14 & 13, for rest of back branches it will be done at
laster time as there is substantial change in number of columns and their
names.
2) Changes to github action workflows for Postgres 14 & 13 to accomodate the
requirements for tap testing.
3) Similarly, minor changes to Makefile are also done.
4) Testing of supported GUCs using tap tests for different possible
configuration.
5) pg_stat_monitor_reset_errors testing using the tap testcases.
6) Insufficient shared space and buffer overflow testing via tap testcases.
7) Some sql scripts under 'scripts' folder to generate some work load requried
for tap test cases.
8) Everything under 't' folder is specific to perl based test cases. It houses
perl files and folders for some expected files and result folder.
9) 90%+ code coverage for LOC and functions.
10) PG-339 Fix by diego, change in pgsm_errors.c.

.github/workflows/coverage_test.yml

@@ -10,13 +10,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           repository: 'postgres/postgres'
-          ref: 'REL_13_STABLE'
-
-      - name: Clone pg_stat_monitor repository
-        uses: actions/checkout@v2
-        with:
-          path: 'src/pg_stat_monitor'
-          ref: 'master'
+          ref: 'REL_14_STABLE'
 
       - name: Install dependencies
         run: |
@@ -33,6 +27,10 @@ jobs:
           sudo rm -rf /usr/share/postgresql
           sudo rm -rf /etc/postgresql
           sudo rm -f /usr/bin/pg_config
+          sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
+          sudo /usr/bin/perl -MCPAN -e 'install String::Util'
+          sudo /usr/bin/perl -MCPAN -e 'install Data::Str2Num'
+
       - name: Create pgsql dir
         run: mkdir -p /opt/pgsql
 
@@ -46,12 +44,12 @@ jobs:
             '--libexecdir=${prefix}/lib/x86_64-linux-gnu' '--disable-maintainer-mode' \
             '--disable-dependency-tracking' '--with-icu' '--with-tcl' '--with-perl' \
             '--with-python' '--with-pam' '--with-openssl' '--with-libxml' '--with-libxslt' \
-            'PYTHON=/usr/bin/python3' '--mandir=/usr/share/postgresql/13/man' \
-            '--docdir=/usr/share/doc/postgresql-doc-13' \
+            'PYTHON=/usr/bin/python3' '--mandir=/usr/share/postgresql/14/man' \
+            '--docdir=/usr/share/doc/postgresql-doc-14' \
             '--sysconfdir=/etc/postgresql-common' '--datarootdir=/usr/share/' \
-            '--datadir=/usr/share/postgresql/13' '--bindir=/usr/lib/postgresql/13/bin' \
+            '--datadir=/usr/share/postgresql/14' '--bindir=/usr/lib/postgresql/14/bin' \
             '--libdir=/usr/lib/x86_64-linux-gnu/' '--libexecdir=/usr/lib/postgresql/' \
-            '--includedir=/usr/include/postgresql/' '--with-extra-version= (Ubuntu 2:13-x.focal)' \
+            '--includedir=/usr/include/postgresql/' '--with-extra-version= (Ubuntu 2:14-x.focal)' \
             '--enable-nls' '--enable-thread-safety' '--enable-tap-tests' '--enable-debug' \
             '--enable-dtrace' '--disable-rpath' '--with-uuid=e2fs' '--with-gnu-ld' \
             '--with-pgport=5432' '--with-system-tzdata=/usr/share/zoneinfo' '--with-llvm' \
@@ -61,13 +59,13 @@ jobs:
             'LDFLAGS=-Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-z,now' '--with-gssapi' '--with-ldap' \
             'build_alias=x86_64-linux-gnu' 'CPPFLAGS=-Wdate-time -D_FORTIFY_SOURCE=2' \
             'CXXFLAGS=-g -O2 -fstack-protector-strong -Wformat -Werror=format-security'
-           make
-           sudo make install
+           make world
+           sudo make install-world
 
       - name: Start postgresql cluster
         run: |
-           /usr/lib/postgresql/13/bin/initdb -D /opt/pgsql/data
-           /usr/lib/postgresql/13/bin/pg_ctl -D /opt/pgsql/data -l logfile start
+           /usr/lib/postgresql/14/bin/initdb -D /opt/pgsql/data
+           /usr/lib/postgresql/14/bin/pg_ctl -D /opt/pgsql/data -l logfile start
 
       - name: Clone pg_stat_monitor repository
         uses: actions/checkout@v2
@@ -76,18 +74,23 @@ jobs:
 
       - name: Build pg_stat_monitor
         run: |
-          export PATH="/usr/lib/postgresql/13/bin:$PATH"
-          sudo cp /usr/lib/postgresql/13/bin/pg_config /usr/bin
+          export PATH="/usr/lib/postgresql/14/bin:$PATH"
+          sudo cp /usr/lib/postgresql/14/bin/pg_config /usr/bin
           make USE_PGXS=1
           sudo make USE_PGXS=1 install
         working-directory: src/pg_stat_monitor/
 
-      - name: Start pg_stat_monitor_tests & Run code coverage 
+      - name: Load pg_stat_monitor library and Restart Server
         run: |
-          /usr/lib/postgresql/13/bin/pg_ctl -D /opt/pgsql/data -l logfile stop
+          /usr/lib/postgresql/14/bin/pg_ctl -D /opt/pgsql/data -l logfile stop
           echo "shared_preload_libraries = 'pg_stat_monitor'" >> /opt/pgsql/data/postgresql.conf
-          /usr/lib/postgresql/13/bin/pg_ctl -D /opt/pgsql/data -l logfile start
+          /usr/lib/postgresql/14/bin/pg_ctl -D /opt/pgsql/data -l logfile start
+        working-directory: src/pg_stat_monitor/
+
+      - name: Start pg_stat_monitor_tests & Run code coverage
+        run: |
           make installcheck
+          /usr/lib/postgresql/14/bin/psql -d postgres -p 5432 -c "\list"
           make coverage-html
           lcov --capture --directory . --output-file coverage/lcov.info
           pip install cpp-coveralls
@@ -95,3 +98,30 @@ jobs:
           coveralls --verbose
         working-directory: src/pg_stat_monitor/
 
+      - name: Change dir permissions on fail
+        if: ${{ failure() }}
+        run: |
+          sudo chmod -R ugo+rwx t
+          sudo chmod -R ugo+rwx tmp_check
+          exit 2 # regenerate error so that we can upload files in next step
+        working-directory: src/pg_stat_monitor/
+
+      - name: Upload logs on fail
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: Regressions diff and postgresql log
+          path: |
+            src/pg_stat_monitor/regression.diffs
+            src/pg_stat_monitor/regression.out
+            src/pg_stat_monitor/logfile
+            src/pg_stat_monitor/t/results/
+            src/pg_stat_monitor/tmp_check/log/
+            !src/pg_stat_monitor/tmp_check/**/archives/*
+            !src/pg_stat_monitor/tmp_check/**/backup/*
+            !src/pg_stat_monitor/tmp_check/**/pgdata/*
+            !src/pg_stat_monitor/tmp_check/**/archives/
+            !src/pg_stat_monitor/tmp_check/**/backup/
+            !src/pg_stat_monitor/tmp_check/**/pgdata/
+          if-no-files-found: warn
+          retention-days: 1

.github/workflows/postgresql-13-build.yml

@@ -27,6 +27,10 @@ jobs:
           sudo rm -rf /usr/share/postgresql
           sudo rm -rf /etc/postgresql
           sudo rm -f /usr/bin/pg_config
+          sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
+          sudo /usr/bin/perl -MCPAN -e 'install String::Util'
+          sudo /usr/bin/perl -MCPAN -e 'install Data::Str2Num'
+
       - name: Create pgsql dir
         run: mkdir -p /opt/pgsql
 
@@ -41,7 +45,7 @@ jobs:
             '--disable-dependency-tracking' '--with-icu' '--with-tcl' '--with-perl' \
             '--with-python' '--with-pam' '--with-openssl' '--with-libxml' '--with-libxslt' \
             'PYTHON=/usr/bin/python3' '--mandir=/usr/share/postgresql/13/man' \
-            '--docdir=/usr/share/doc/postgresql-doc-13' \
+            '--docdir=/usr/share/doc/postgresql-doc-13' 'AWK=mawk' \
             '--sysconfdir=/etc/postgresql-common' '--datarootdir=/usr/share/' \
             '--datadir=/usr/share/postgresql/13' '--bindir=/usr/lib/postgresql/13/bin' \
             '--libdir=/usr/lib/x86_64-linux-gnu/' '--libexecdir=/usr/lib/postgresql/' \
@@ -49,7 +53,7 @@ jobs:
             '--enable-nls' '--enable-thread-safety' '--enable-tap-tests' '--enable-debug' \
             '--enable-dtrace' '--disable-rpath' '--with-uuid=e2fs' '--with-gnu-ld' \
             '--with-pgport=5432' '--with-system-tzdata=/usr/share/zoneinfo' '--with-llvm' \
-            'LLVM_CONFIG=/usr/bin/llvm-config-11' 'CLANG=/usr/bin/clang-11' \
+            'LLVM_CONFIG=/usr/bin/llvm-config-9' 'CLANG=/usr/bin/clang-9' \
             '--with-systemd' '--with-selinux' 'MKDIR_P=/bin/mkdir -p' 'PROVE=/usr/bin/prove' \
             'TAR=/bin/tar' 'XSLTPROC=xsltproc --nonet' 'CFLAGS=-g -O2 -fstack-protector-strong -Wformat -Werror=format-security -fno-omit-frame-pointer' \
             'LDFLAGS=-Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-z,now' '--with-gssapi' '--with-ldap' \
@@ -72,6 +76,7 @@ jobs:
         run: |
           export PATH="/usr/lib/postgresql/13/bin:$PATH"
           sudo cp /usr/lib/postgresql/13/bin/pg_config /usr/bin
+          pg_config
           make USE_PGXS=1
           sudo make USE_PGXS=1 install
         working-directory: src/pg_stat_monitor/
@@ -83,7 +88,7 @@ jobs:
           /usr/lib/postgresql/13/bin/pg_ctl -D /opt/pgsql/data -l logfile start
         working-directory: src/pg_stat_monitor/
 
-      - name: Start Server installcheck-world tests (without TAP)
+      - name: Start Server installcheck-world tests
         run: |
           make installcheck-world
       - name: Report on installcheck-world test suites fail
@@ -102,12 +107,30 @@ jobs:
           make installcheck
         working-directory: src/pg_stat_monitor/
 
-      - name: Report on pg_stat_monitor test fail
-        uses: actions/upload-artifact@v2
+      - name: Change dir permissions on fail
         if: ${{ failure() }}
+        run: |
+          sudo chmod -R ugo+rwx t
+          sudo chmod -R ugo+rwx tmp_check
+          exit 2 # regenerate error so that we can upload files in next step
+        working-directory: src/pg_stat_monitor/
+
+      - name: Upload logs on fail
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
         with:
           name: Regressions diff and postgresql log
           path: |
             src/pg_stat_monitor/regression.diffs
+            src/pg_stat_monitor/regression.out
             src/pg_stat_monitor/logfile
+            src/pg_stat_monitor/t/results/
+            src/pg_stat_monitor/tmp_check/log/
+            !src/pg_stat_monitor/tmp_check/**/archives/*
+            !src/pg_stat_monitor/tmp_check/**/backup/*
+            !src/pg_stat_monitor/tmp_check/**/pgdata/*
+            !src/pg_stat_monitor/tmp_check/**/archives/
+            !src/pg_stat_monitor/tmp_check/**/backup/
+            !src/pg_stat_monitor/tmp_check/**/pgdata/
+          if-no-files-found: warn
           retention-days: 1

.github/workflows/postgresql-13-package.yml

@@ -24,6 +24,9 @@ jobs:
           sudo rm -rf /usr/share/postgresql
           sudo rm -rf /etc/postgresql
           sudo rm -f /usr/bin/pg_config
+          sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
+          sudo /usr/bin/perl -MCPAN -e 'install String::Util'
+          sudo /usr/bin/perl -MCPAN -e 'install Data::Str2Num'
 
       - name: Install postgresql 13
         run: |
@@ -39,22 +42,40 @@ jobs:
         working-directory: src/pg_stat_monitor/
 
       - name: Change sources owner to postgres
-        run: sudo chown -R postgres:postgres src/pg_stat_monitor/
-
+        run: |
+          sudo chown -R postgres:postgres src/pg_stat_monitor/
       - name: Start pg_stat_monitor_tests
         run: |
           sudo service postgresql stop
           echo "shared_preload_libraries = 'pg_stat_monitor'" | sudo tee -a /etc/postgresql/13/main/postgresql.conf
           sudo service postgresql start
-          sudo -u postgres bash -c 'make installcheck USE_PGXS=1'
+          sudo -u postgres bash -c 'make installcheck USE_PGXS=1' 
         working-directory: src/pg_stat_monitor/
 
-      - name: Report on test fail
-        uses: actions/upload-artifact@v2
+      - name: Change dir permissions on fail
         if: ${{ failure() }}
+        run: |
+          sudo chmod -R ugo+rwx t
+          sudo chmod -R ugo+rwx tmp_check
+          exit 2 # regenerate error so that we can upload files in next step
+        working-directory: src/pg_stat_monitor/
+
+      - name: Upload logs on fail
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
         with:
           name: Regressions diff and postgresql log
           path: |
             src/pg_stat_monitor/regression.diffs
+            src/pg_stat_monitor/regression.out
             src/pg_stat_monitor/logfile
+            src/pg_stat_monitor/t/results/
+            src/pg_stat_monitor/tmp_check/log/
+            !src/pg_stat_monitor/tmp_check/**/archives/*
+            !src/pg_stat_monitor/tmp_check/**/backup/*
+            !src/pg_stat_monitor/tmp_check/**/pgdata/*
+            !src/pg_stat_monitor/tmp_check/**/archives/
+            !src/pg_stat_monitor/tmp_check/**/backup/
+            !src/pg_stat_monitor/tmp_check/**/pgdata/
+          if-no-files-found: warn
           retention-days: 1

.github/workflows/postgresql-13-pgdg-package.yml

@@ -23,6 +23,10 @@ jobs:
           sudo rm -rf /usr/share/postgresql
           sudo rm -rf /etc/postgresql
           sudo rm -f /usr/bin/pg_config
+          sudo apt-get install libreadline6-dev systemtap-sdt-dev zlib1g-dev libssl-dev libpam0g-dev python-dev bison flex libipc-run-perl wget -y
+          sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
+          sudo /usr/bin/perl -MCPAN -e 'install String::Util'
+          sudo /usr/bin/perl -MCPAN -e 'install Data::Str2Num'
 
       - name: Install PG Distribution Postgresql 13
         run: |
@@ -31,6 +35,8 @@ jobs:
           sudo apt-get update
           sudo apt-get -y install postgresql-13 postgresql-client-13 postgresql-contrib postgresql-server-dev-13
           sudo chown -R postgres:postgres src/
+          sudo cpan install IPC::RUN
+          sudo cpan install String::Util
 
       - name: Build pg_stat_monitor
         run: |
@@ -48,12 +54,30 @@ jobs:
           sudo -u postgres bash -c 'make installcheck USE_PGXS=1'
         working-directory: src/pg_stat_monitor/
 
-      - name: Report on test fail
-        uses: actions/upload-artifact@v2
+      - name: Change dir permissions on fail
         if: ${{ failure() }}
+        run: |
+          sudo chmod -R ugo+rwx t
+          sudo chmod -R ugo+rwx tmp_check
+          exit 2 # regenerate error so that we can upload files in next step
+        working-directory: src/pg_stat_monitor/
+
+      - name: Upload logs on fail
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
         with:
           name: Regressions diff and postgresql log
           path: |
             src/pg_stat_monitor/regression.diffs
+            src/pg_stat_monitor/regression.out
             src/pg_stat_monitor/logfile
+            src/pg_stat_monitor/t/results/
+            src/pg_stat_monitor/tmp_check/log/
+            !src/pg_stat_monitor/tmp_check/**/archives/*
+            !src/pg_stat_monitor/tmp_check/**/backup/*
+            !src/pg_stat_monitor/tmp_check/**/pgdata/*
+            !src/pg_stat_monitor/tmp_check/**/archives/
+            !src/pg_stat_monitor/tmp_check/**/backup/
+            !src/pg_stat_monitor/tmp_check/**/pgdata/
+          if-no-files-found: warn
           retention-days: 1

.github/workflows/postgresql-14-build.yml

@@ -27,6 +27,10 @@ jobs:
           sudo rm -rf /usr/share/postgresql
           sudo rm -rf /etc/postgresql
           sudo rm -f /usr/bin/pg_config
+          sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
+          sudo /usr/bin/perl -MCPAN -e 'install String::Util'
+          sudo /usr/bin/perl -MCPAN -e 'install Data::Str2Num'
+
       - name: Create pgsql dir
         run: mkdir -p /opt/pgsql
 
@@ -88,12 +92,30 @@ jobs:
           make installcheck
         working-directory: src/pg_stat_monitor/
 
-      - name: Report on pg_stat_monitor test fail
-        uses: actions/upload-artifact@v2
+      - name: Change dir permissions on fail
         if: ${{ failure() }}
+        run: |
+          sudo chmod -R ugo+rwx t
+          sudo chmod -R ugo+rwx tmp_check
+          exit 2 # regenerate error so that we can upload files in next step
+        working-directory: src/pg_stat_monitor/
+
+      - name: Upload logs on fail
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
         with:
           name: Regressions diff and postgresql log
           path: |
             src/pg_stat_monitor/regression.diffs
+            src/pg_stat_monitor/regression.out
             src/pg_stat_monitor/logfile
+            src/pg_stat_monitor/t/results/
+            src/pg_stat_monitor/tmp_check/log/
+            !src/pg_stat_monitor/tmp_check/**/archives/*
+            !src/pg_stat_monitor/tmp_check/**/backup/*
+            !src/pg_stat_monitor/tmp_check/**/pgdata/*
+            !src/pg_stat_monitor/tmp_check/**/archives/
+            !src/pg_stat_monitor/tmp_check/**/backup/
+            !src/pg_stat_monitor/tmp_check/**/pgdata/
+          if-no-files-found: warn
           retention-days: 1