From a9f7c1e8cbd47084eb2ea4fd464fa8a4c2082890 Mon Sep 17 00:00:00 2001
From: Hadi Moshayedi <hadi.moshayedi@microsoft.com>
Date: Mon, 29 Apr 2019 15:03:08 -0700
Subject: [PATCH] Normalize test result/expected files before doing diff.

---
 src/test/regress/Makefile                 |  5 +++
 src/test/regress/bin/diff                 | 45 +++++++++++++++++++++++
 src/test/regress/bin/normalize.sed        | 17 +++++++++
 src/test/regress/bin/normalized_tests.lst |  4 ++
 4 files changed, 71 insertions(+)
 create mode 100755 src/test/regress/bin/diff
 create mode 100644 src/test/regress/bin/normalize.sed
 create mode 100644 src/test/regress/bin/normalized_tests.lst

diff --git a/src/test/regress/Makefile b/src/test/regress/Makefile
index f64774ea8..b046e496d 100644
--- a/src/test/regress/Makefile
+++ b/src/test/regress/Makefile
@@ -10,6 +10,11 @@ ifndef MAJORVERSION
 MAJORVERSION := $(basename $(VERSION))
 endif
 
+# Add ./bin to $PATH so we use our custom diff instead of the default diff tool.
+# We do this to be able to mask shard Ids, placement Ids, node ports, etc.
+MAKEFILE_DIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
+export PATH := $(MAKEFILE_DIR)/bin:$(PATH)
+
 ##
 ## Citus regression support
 ##
diff --git a/src/test/regress/bin/diff b/src/test/regress/bin/diff
new file mode 100755
index 000000000..c1628144a
--- /dev/null
+++ b/src/test/regress/bin/diff
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# Our custom diff tool which normalizes result and expected files before
+# doing the actual diff. Rules for normalization are in normalize.sed. See
+# the comments there for more information.
+#
+# We only normalize tests listed in normalized_tests.lst. We use the bare
+# diff for other tests.
+#
+# Note that src/test/regress/Makefile adds this directory to $PATH so
+# pg_regress uses this diff tool instead of the system diff tool.
+
+file1="${@:(-2):1}"
+file2="${@:(-1):1}"
+test=$(basename $file1 .out)
+args=${@:1:$#-2}
+BASEDIR=$(dirname "$0")
+
+# whereis searches for standard unix places before $PATH. So select the first
+# entry as the original diff tool.
+DIFF=$(whereis diff | sed "s/diff://g" | awk '{print $1}')
+if [ -z "$DIFF" ]
+then
+    DIFF=/usr/bin/diff
+fi
+
+grep -q "^$test$" $BASEDIR/normalized_tests.lst
+grep_result=$?
+
+if [ "$grep_result" -eq "0" ]
+then
+    sed -Ef $BASEDIR/normalize.sed < $file1 > $file1.modified
+    sed -Ef $BASEDIR/normalize.sed < $file2 > $file2.modified
+
+    $DIFF $args $file1.modified $file2.modified
+    exitcode=$?
+
+    rm -rf $file1.modified $file2.modified
+else
+    # if test is not in normalized_tests.lst, just diff without normalizing.
+    $DIFF $args $file1 $file2
+    exitcode=$?
+fi
+
+exit $exitcode
diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed
new file mode 100644
index 000000000..d4a1133a7
--- /dev/null
+++ b/src/test/regress/bin/normalize.sed
@@ -0,0 +1,17 @@
+# Rules to normalize test outputs. Our custom diff tool passes test output
+# of tests in normalized_tests.lst through the substitution rules in this file
+# before doing the actual comparison.
+#
+# An example of when this is useful is when an error happens on a different
+# port number, or a different worker shard, or a different placement, etc.
+# because we are running the tests in a different configuration.
+
+# In all tests, normalize worker ports, placement ids, and shard ids
+s/localhost:[0-9]+/localhost:xxxxx/g
+s/placement [0-9]+/placement xxxxx/g
+s/shard [0-9]+/shard xxxxx/g
+
+# In foreign_key_to_reference_table, normalize shard table names, etc in
+# the generated plan
+s/"(fkey_ref_|referenced_table_|referencing_table_)[0-9]+"/"\1xxxxxxx"/g
+s/\(id\)=\([0-9]+\)/(id)=(X)/g
diff --git a/src/test/regress/bin/normalized_tests.lst b/src/test/regress/bin/normalized_tests.lst
new file mode 100644
index 000000000..528775d8c
--- /dev/null
+++ b/src/test/regress/bin/normalized_tests.lst
@@ -0,0 +1,4 @@
+# List of tests whose output we want to normalize, one per line
+multi_alter_table_add_constraints
+multi_alter_table_statements
+foreign_key_to_reference_table