From c96b439a481dbc63eaf2caac790b5893668cc0b2 Mon Sep 17 00:00:00 2001 From: Hanefi Onaldi Date: Thu, 11 Mar 2021 13:27:05 +0300 Subject: [PATCH 1/3] Introduce scripts to sync gitignore rules for .source files --- .circleci/config.yml | 6 ++++++ ci/README.md | 6 ++++++ ci/fix_gitignore.sh | 20 ++++++++++++++++++++ ci/fix_style.sh | 1 + 4 files changed, 33 insertions(+) create mode 100755 ci/fix_gitignore.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index c6f2ee445..a286b7ba7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -64,6 +64,12 @@ jobs: - run: name: 'Check if changed' command: git diff --exit-code + - run: + name: 'Check for gitignore entries .for source files' + command: ci/fix_gitignore.sh + - run: + name: 'Check if changed' + command: git diff --exit-code - run: name: 'Check for lengths of changelog entries' command: ci/disallow_long_changelog_entries.sh diff --git a/ci/README.md b/ci/README.md index 0d1be65f2..73962f80a 100644 --- a/ci/README.md +++ b/ci/README.md @@ -346,3 +346,9 @@ foo = 2 #endif ``` This was deemed to be error prone and not worth the effort. + +## `fix_gitignore.sh` + +This script makes sure that git ignores the sql files and expected output files +that are generated from `.source` template files. If you created or deleted a +`.source` file in a commit, git ignore rules should be updated. diff --git a/ci/fix_gitignore.sh b/ci/fix_gitignore.sh new file mode 100755 index 000000000..c8d6b29ac --- /dev/null +++ b/ci/fix_gitignore.sh @@ -0,0 +1,20 @@ +#! /bin/bash +# shellcheck disable=SC2012 + +set -euo pipefail +# shellcheck disable=SC1091 +source ci/ci_helpers.sh + +# We list all the .source files in alphabetical order, and do a substitution +# before writing the resulting file names that are created by those templates in +# relevant .gitignore files +# +# 1. Capture the file name without the .source extension +# 2. Add the desired extension at the end +# 3. Add a / character at the beginning of each line to conform to .gitignore file format +# +# e.g. multi_copy.source -> /multi_copy.sql +ls -1 src/test/regress/input | sed -E "s#(.*)\.source#/\1.sql#" > src/test/regress/sql/.gitignore + +# e.g. multi_copy.source -> /multi_copy.out +ls -1 src/test/regress/output | sed -E "s#(.*)\.source#/\1.out#" > src/test/regress/expected/.gitignore diff --git a/ci/fix_style.sh b/ci/fix_style.sh index 8846eda38..c15eb3def 100755 --- a/ci/fix_style.sh +++ b/ci/fix_style.sh @@ -14,3 +14,4 @@ ci/remove_useless_declarations.sh ci/disallow_c_comments_in_migrations.sh ci/disallow_long_changelog_entries.sh ci/normalize_expected.sh +ci/fix_gitignore.sh From 13808b60cf5ac19185f2c4639f583cac993c37fc Mon Sep 17 00:00:00 2001 From: Hanefi Onaldi Date: Thu, 11 Mar 2021 13:28:50 +0300 Subject: [PATCH 2/3] Update gitignore files --- src/test/regress/expected/.gitignore | 52 +++++++++++----------------- src/test/regress/sql/.gitignore | 31 ++++++----------- 2 files changed, 30 insertions(+), 53 deletions(-) diff --git a/src/test/regress/expected/.gitignore b/src/test/regress/expected/.gitignore index c63944e0d..f500ab38e 100644 --- a/src/test/regress/expected/.gitignore +++ b/src/test/regress/expected/.gitignore @@ -1,36 +1,24 @@ -/multi_agg_distinct.out -/multi_agg_type_conversion.out -/multi_alter_table_statements.out -/multi_alter_table_statements_0.out -/multi_append_table_to_shard.out -/multi_behavioral_analytics_create_table.out -/multi_behavioral_analytics_create_table_superuser.out -/multi_copy.out -/multi_create_schema.out -/multi_large_shardid.out -/multi_master_delete_protocol.out -/multi_outer_join.out -/multi_outer_join_reference.out -/multi_load_data.out -/multi_load_data_superuser.out -/multi_load_large_records.out -/multi_load_more_data.out -/worker_copy.out -/multi_complex_count_distinct.out -/multi_mx_copy_data.out -/multi_behavioral_analytics_create_table.out -/multi_insert_select_behavioral_analytics_create_table.out -/hyperscale_tutorial.out -/am_chunk_filtering.out -/am_copyto.out -/am_data_types.out -/am_load.out -/fdw_block_filtering.out -/fdw_copyto.out -/fdw_create.out -/fdw_data_types.out -/fdw_load.out /columnar_chunk_filtering.out /columnar_copyto.out /columnar_data_types.out /columnar_load.out +/hyperscale_tutorial.out +/multi_agg_distinct.out +/multi_agg_type_conversion.out +/multi_alter_table_statements.out +/multi_append_table_to_shard.out +/multi_behavioral_analytics_create_table.out +/multi_behavioral_analytics_create_table_superuser.out +/multi_complex_count_distinct.out +/multi_copy.out +/multi_create_schema.out +/multi_large_shardid.out +/multi_load_data.out +/multi_load_data_superuser.out +/multi_load_large_records.out +/multi_load_more_data.out +/multi_master_delete_protocol.out +/multi_mx_copy_data.out +/multi_outer_join.out +/multi_outer_join_reference.out +/worker_copy.out diff --git a/src/test/regress/sql/.gitignore b/src/test/regress/sql/.gitignore index d0085e0d7..093d9b90b 100644 --- a/src/test/regress/sql/.gitignore +++ b/src/test/regress/sql/.gitignore @@ -1,35 +1,24 @@ +/columnar_chunk_filtering.sql +/columnar_copyto.sql +/columnar_data_types.sql +/columnar_load.sql +/hyperscale_tutorial.sql /multi_agg_distinct.sql /multi_agg_type_conversion.sql /multi_alter_table_statements.sql /multi_append_table_to_shard.sql /multi_behavioral_analytics_create_table.sql /multi_behavioral_analytics_create_table_superuser.sql +/multi_complex_count_distinct.sql /multi_copy.sql /multi_create_schema.sql /multi_large_shardid.sql -/multi_master_delete_protocol.sql -/multi_outer_join.sql -/multi_outer_join_reference.sql /multi_load_data.sql /multi_load_data_superuser.sql /multi_load_large_records.sql /multi_load_more_data.sql -/worker_copy.sql -/multi_complex_count_distinct.sql +/multi_master_delete_protocol.sql /multi_mx_copy_data.sql -/multi_behavioral_analytics_create_table.sql -/multi_insert_select_behavioral_analytics_create_table.sql -/hyperscale_tutorial.sql -/am_chunk_filtering.sql -/am_copyto.sql -/am_data_types.sql -/am_load.sql -/fdw_block_filtering.sql -/fdw_copyto.sql -/fdw_create.sql -/fdw_data_types.sql -/fdw_load.sql -/columnar_chunk_filtering.sql -/columnar_copyto.sql -/columnar_data_types.sql -/columnar_load.sql +/multi_outer_join.sql +/multi_outer_join_reference.sql +/worker_copy.sql From 6b2c9d3567a63a7ed41147691f97643f3c412317 Mon Sep 17 00:00:00 2001 From: Hanefi Onaldi Date: Tue, 11 May 2021 11:17:55 +0300 Subject: [PATCH 3/3] Remove ignored files from git tree --- ci/README.md | 12 +++++++++--- ci/fix_gitignore.sh | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/ci/README.md b/ci/README.md index 73962f80a..52a8925cc 100644 --- a/ci/README.md +++ b/ci/README.md @@ -349,6 +349,12 @@ This was deemed to be error prone and not worth the effort. ## `fix_gitignore.sh` -This script makes sure that git ignores the sql files and expected output files -that are generated from `.source` template files. If you created or deleted a -`.source` file in a commit, git ignore rules should be updated. +This script checks and fixes issues with `.gitignore` rules: + +1. Makes sure git ignores the `.sql` files and expected output files that are generated + from `.source` template files. If you created or deleted a `.source` file in a commit, + git ignore rules should be updated to reflect this change. + +2. Makes sure we do not commit any generated files that should be ignored. If there is an + ignored file in the git tree, the user is expected to review the files that are removed + from the git tree and commit them. diff --git a/ci/fix_gitignore.sh b/ci/fix_gitignore.sh index c8d6b29ac..a2258c472 100755 --- a/ci/fix_gitignore.sh +++ b/ci/fix_gitignore.sh @@ -18,3 +18,17 @@ ls -1 src/test/regress/input | sed -E "s#(.*)\.source#/\1.sql#" > src/test/regre # e.g. multi_copy.source -> /multi_copy.out ls -1 src/test/regress/output | sed -E "s#(.*)\.source#/\1.out#" > src/test/regress/expected/.gitignore + +# Remove all the ignored files from git tree, and error out +# find all ignored files in git tree, and use quotation marks to prevent word splitting on filenames with spaces in them +ignored_lines_in_git_tree=$(git ls-files --ignored --exclude-standard | sed 's/.*/"&"/') + +if [[ -n $ignored_lines_in_git_tree ]] +then + echo "Ignored files should not be in git tree!" + echo "${ignored_lines_in_git_tree}" + + echo "Removing these files from git tree, please review and commit" + echo "$ignored_lines_in_git_tree" | xargs git rm -r --cached + exit 1 +fi