Normalize test result/expected files before doing diff.

src/test/regress/Makefile

@@ -10,6 +10,11 @@ ifndef MAJORVERSION
 MAJORVERSION := $(basename $(VERSION))
 endif
 
+# Add ./bin to $PATH so we use our custom diff instead of the default diff tool.
+# We do this to be able to mask shard Ids, placement Ids, node ports, etc.
+MAKEFILE_DIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
+export PATH := $(MAKEFILE_DIR)/bin:$(PATH)
+
 ##
 ## Citus regression support
 ##

src/test/regress/bin/diff

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# Our custom diff tool which normalizes result and expected files before
+# doing the actual diff. Rules for normalization are in normalize.sed. See
+# the comments there for more information.
+#
+# We only normalize tests listed in normalized_tests.lst. We use the bare
+# diff for other tests.
+#
+# Note that src/test/regress/Makefile adds this directory to $PATH so
+# pg_regress uses this diff tool instead of the system diff tool.
+
+file1="${@:(-2):1}"
+file2="${@:(-1):1}"
+test=$(basename $file1 .out)
+args=${@:1:$#-2}
+BASEDIR=$(dirname "$0")
+
+# whereis searches for standard unix places before $PATH. So select the first
+# entry as the original diff tool.
+DIFF=$(whereis diff | sed "s/diff://g" | awk '{print $1}')
+if [ -z "$DIFF" ]
+then
+    DIFF=/usr/bin/diff
+fi
+
+grep -q "^$test$" $BASEDIR/normalized_tests.lst
+grep_result=$?
+
+if [ "$grep_result" -eq "0" ]
+then
+    sed -Ef $BASEDIR/normalize.sed < $file1 > $file1.modified
+    sed -Ef $BASEDIR/normalize.sed < $file2 > $file2.modified
+
+    $DIFF $args $file1.modified $file2.modified
+    exitcode=$?
+
+    rm -rf $file1.modified $file2.modified
+else
+    # if test is not in normalized_tests.lst, just diff without normalizing.
+    $DIFF $args $file1 $file2
+    exitcode=$?
+fi
+
+exit $exitcode

src/test/regress/bin/normalize.sed

@@ -0,0 +1,17 @@
+# Rules to normalize test outputs. Our custom diff tool passes test output
+# of tests in normalized_tests.lst through the substitution rules in this file
+# before doing the actual comparison.
+#
+# An example of when this is useful is when an error happens on a different
+# port number, or a different worker shard, or a different placement, etc.
+# because we are running the tests in a different configuration.
+
+# In all tests, normalize worker ports, placement ids, and shard ids
+s/localhost:[0-9]+/localhost:xxxxx/g
+s/placement [0-9]+/placement xxxxx/g
+s/shard [0-9]+/shard xxxxx/g
+
+# In foreign_key_to_reference_table, normalize shard table names, etc in
+# the generated plan
+s/"(fkey_ref_|referenced_table_|referencing_table_)[0-9]+"/"\1xxxxxxx"/g
+s/\(id\)=\([0-9]+\)/(id)=(X)/g

src/test/regress/bin/normalized_tests.lst

@@ -0,0 +1,4 @@
+# List of tests whose output we want to normalize, one per line
+multi_alter_table_add_constraints
+multi_alter_table_statements
+foreign_key_to_reference_table