Merge pull request #4811 from citusdata/fix-gitignore

.circleci/config.yml

@@ -64,6 +64,12 @@ jobs:
       - run:
           name: 'Check if changed'
           command: git diff --exit-code
+      - run:
+          name: 'Check for gitignore entries .for source files'
+          command: ci/fix_gitignore.sh
+      - run:
+          name: 'Check if changed'
+          command: git diff --exit-code
       - run:
           name: 'Check for lengths of changelog entries'
           command: ci/disallow_long_changelog_entries.sh

ci/README.md

@@ -346,3 +346,15 @@ foo = 2
 #endif
 ```
 This was deemed to be error prone and not worth the effort.
+
+## `fix_gitignore.sh`
+
+This script checks and fixes issues with `.gitignore` rules:
+
+1. Makes sure git ignores the `.sql` files and expected output files that are generated
+   from `.source` template files. If you created or deleted a `.source` file in a commit,
+   git ignore rules should be updated to reflect this change.
+
+2. Makes sure we do not commit any generated files that should be ignored. If there is an
+   ignored file in the git tree, the user is expected to review the files that are removed
+   from the git tree and commit them.

ci/fix_gitignore.sh

@@ -0,0 +1,34 @@
+#! /bin/bash
+# shellcheck disable=SC2012
+
+set -euo pipefail
+# shellcheck disable=SC1091
+source ci/ci_helpers.sh
+
+# We list all the .source files in alphabetical order, and do a substitution
+# before writing the resulting file names that are created by those templates in
+# relevant .gitignore files
+#
+# 1. Capture the file name without the .source extension
+# 2. Add the desired extension at the end
+# 3. Add a / character at the beginning of each line to conform to .gitignore file format
+#
+# e.g. multi_copy.source -> /multi_copy.sql
+ls -1 src/test/regress/input | sed -E "s#(.*)\.source#/\1.sql#" > src/test/regress/sql/.gitignore
+
+# e.g. multi_copy.source -> /multi_copy.out
+ls -1 src/test/regress/output | sed -E "s#(.*)\.source#/\1.out#" > src/test/regress/expected/.gitignore
+
+# Remove all the ignored files from git tree, and error out
+# find all ignored files in git tree, and use quotation marks to prevent word splitting on filenames with spaces in them
+ignored_lines_in_git_tree=$(git ls-files --ignored --exclude-standard | sed 's/.*/"&"/')
+
+if [[ -n $ignored_lines_in_git_tree ]]
+then
+    echo "Ignored files should not be in git tree!"
+    echo "${ignored_lines_in_git_tree}"
+
+    echo "Removing these files from git tree, please review and commit"
+    echo "$ignored_lines_in_git_tree" | xargs git rm -r --cached
+    exit 1
+fi

ci/fix_style.sh

@@ -14,3 +14,4 @@ ci/remove_useless_declarations.sh
 ci/disallow_c_comments_in_migrations.sh
 ci/disallow_long_changelog_entries.sh
 ci/normalize_expected.sh
+ci/fix_gitignore.sh

src/test/regress/expected/.gitignore

@@ -1,36 +1,24 @@
-/multi_agg_distinct.out
-/multi_agg_type_conversion.out
-/multi_alter_table_statements.out
-/multi_alter_table_statements_0.out
-/multi_append_table_to_shard.out
-/multi_behavioral_analytics_create_table.out
-/multi_behavioral_analytics_create_table_superuser.out
-/multi_copy.out
-/multi_create_schema.out
-/multi_large_shardid.out
-/multi_master_delete_protocol.out
-/multi_outer_join.out
-/multi_outer_join_reference.out
-/multi_load_data.out
-/multi_load_data_superuser.out
-/multi_load_large_records.out
-/multi_load_more_data.out
-/worker_copy.out
-/multi_complex_count_distinct.out
-/multi_mx_copy_data.out
-/multi_behavioral_analytics_create_table.out
-/multi_insert_select_behavioral_analytics_create_table.out
-/hyperscale_tutorial.out
-/am_chunk_filtering.out
-/am_copyto.out
-/am_data_types.out
-/am_load.out
-/fdw_block_filtering.out
-/fdw_copyto.out
-/fdw_create.out
-/fdw_data_types.out
-/fdw_load.out
 /columnar_chunk_filtering.out
 /columnar_copyto.out
 /columnar_data_types.out
 /columnar_load.out
+/hyperscale_tutorial.out
+/multi_agg_distinct.out
+/multi_agg_type_conversion.out
+/multi_alter_table_statements.out
+/multi_append_table_to_shard.out
+/multi_behavioral_analytics_create_table.out
+/multi_behavioral_analytics_create_table_superuser.out
+/multi_complex_count_distinct.out
+/multi_copy.out
+/multi_create_schema.out
+/multi_large_shardid.out
+/multi_load_data.out
+/multi_load_data_superuser.out
+/multi_load_large_records.out
+/multi_load_more_data.out
+/multi_master_delete_protocol.out
+/multi_mx_copy_data.out
+/multi_outer_join.out
+/multi_outer_join_reference.out
+/worker_copy.out

src/test/regress/sql/.gitignore

@@ -1,35 +1,24 @@
+/columnar_chunk_filtering.sql
+/columnar_copyto.sql
+/columnar_data_types.sql
+/columnar_load.sql
+/hyperscale_tutorial.sql
 /multi_agg_distinct.sql
 /multi_agg_type_conversion.sql
 /multi_alter_table_statements.sql
 /multi_append_table_to_shard.sql
 /multi_behavioral_analytics_create_table.sql
 /multi_behavioral_analytics_create_table_superuser.sql
+/multi_complex_count_distinct.sql
 /multi_copy.sql
 /multi_create_schema.sql
 /multi_large_shardid.sql
-/multi_master_delete_protocol.sql
-/multi_outer_join.sql
-/multi_outer_join_reference.sql
 /multi_load_data.sql
 /multi_load_data_superuser.sql
 /multi_load_large_records.sql
 /multi_load_more_data.sql
-/worker_copy.sql
-/multi_complex_count_distinct.sql
+/multi_master_delete_protocol.sql
 /multi_mx_copy_data.sql
-/multi_behavioral_analytics_create_table.sql
-/multi_insert_select_behavioral_analytics_create_table.sql
-/hyperscale_tutorial.sql
-/am_chunk_filtering.sql
-/am_copyto.sql
-/am_data_types.sql
-/am_load.sql
-/fdw_block_filtering.sql
-/fdw_copyto.sql
-/fdw_create.sql
-/fdw_data_types.sql
-/fdw_load.sql
-/columnar_chunk_filtering.sql
-/columnar_copyto.sql
-/columnar_data_types.sql
-/columnar_load.sql
+/multi_outer_join.sql
+/multi_outer_join_reference.sql
+/worker_copy.sql