From d99deab7d9a584e8ab9572c754f4f6a9b53451d5 Mon Sep 17 00:00:00 2001 From: SaitTalhaNisanci Date: Tue, 10 Sep 2019 17:56:04 +0300 Subject: [PATCH] Add upgrade postgres version test (#2940) * Add creating a citus cluster script Creating a citus cluster is automated. Before running this script: - Citus should be installed and its control file should be added to postgres. (make install) - Postgres should be installed. * Initialize upgrade test table and fill * Finalize the layout of upgrade tests Postgres upgrade function is added. The newly added UDFs(citus_prepare_pg_upgrade, citus_finish_pg_upgrade) are used to perform upgrade. * Refactor upgrade test and add config file * Add schedules for upgrade testing * Use pg_regress for upgrade tests pg_regress is used for creating a simple distributed table in upgrade tests. After upgrading another schedule is used to verify that the distributed table exists. Router and realtime queries are used for verifying. * Run upgrade tests as a postgres user in a temp dir postgres user is used for psql to be consistent at running tests. A temp dir is created and the temp dir's permissions are changed so that postgres user can access it. All psql commands are now run with postgres user. "Select * from t" query is changed as "Select * from t order by a" so that the result is always in the same order. * Add docopt and arguments for the upgrade script Docopt dependency is added to parse flags in script. Some refactoring in variable names is done. * Add readme for upgrade tests * Refactor upgrade tests Use relative data path instead of absolute assuming that this script will always be run from 'src/test/regress' Remove 'citus-path' flag Use specific version for docopt instead of * Use named args in string formatting * Resolve a security problem Instead of using string formatting in subprocess.call, arguments list is used. Otherwise users could do shell injection. Shell = True is removed from subprocess call as it is not recommended to use this. * Add how the test works to readme * Refactor some variables to be consistent * Update upgrade script based on the reviews It was possible that postgres server would stay running even when the script crashes, atexit library is used to ensure that we always do a teardown where we stop the databases. Some formatting is done in the code for better readability. Config class is used instead of a dictonary. A target for upgrade test is added to makefile. Unused flags/functions/variables are removed. * Format commands and remove unnecessary flag from readme --- src/test/regress/.gitignore | 4 + src/test/regress/Makefile | 5 + src/test/regress/Pipfile | 1 + src/test/regress/Pipfile.lock | 328 ++++++++---------- src/test/regress/after_upgrade_schedule | 1 + src/test/regress/before_upgrade_schedule | 1 + .../upgrade_distributed_table_after.out | 19 + .../upgrade_distributed_table_before.out | 10 + .../sql/upgrade_distributed_table_after.sql | 6 + .../sql/upgrade_distributed_table_before.sql | 6 + src/test/regress/upgrade/README.md | 44 +++ src/test/regress/upgrade/config.py | 32 ++ src/test/regress/upgrade/upgrade_test.py | 179 ++++++++++ src/test/regress/upgrade/utils.py | 29 ++ 14 files changed, 486 insertions(+), 179 deletions(-) create mode 100644 src/test/regress/after_upgrade_schedule create mode 100644 src/test/regress/before_upgrade_schedule create mode 100644 src/test/regress/expected/upgrade_distributed_table_after.out create mode 100644 src/test/regress/expected/upgrade_distributed_table_before.out create mode 100644 src/test/regress/sql/upgrade_distributed_table_after.sql create mode 100644 src/test/regress/sql/upgrade_distributed_table_before.sql create mode 100644 src/test/regress/upgrade/README.md create mode 100644 src/test/regress/upgrade/config.py create mode 100755 src/test/regress/upgrade/upgrade_test.py create mode 100644 src/test/regress/upgrade/utils.py diff --git a/src/test/regress/.gitignore b/src/test/regress/.gitignore index 1aab2767f..38f055cfc 100644 --- a/src/test/regress/.gitignore +++ b/src/test/regress/.gitignore @@ -3,6 +3,7 @@ # Generated subdirectories /tmp_check/ +/tmp_upgrade/ /results/ /log/ @@ -15,3 +16,6 @@ # Failure test side effets /proxy.output + +# python +**.pyc diff --git a/src/test/regress/Makefile b/src/test/regress/Makefile index d91f5faf5..dc8d2e33d 100644 --- a/src/test/regress/Makefile +++ b/src/test/regress/Makefile @@ -22,6 +22,8 @@ MULTI_INSTALLDIR=$(CURDIR)/tmp_check/install pg_regress_multi_check = $(PERL) $(citus_abs_srcdir)/pg_regress_multi.pl --pgxsdir="$(pgxsdir)" --bindir="$(bindir)" --libdir="$(libdir)" --majorversion="$(MAJORVERSION)" --postgres-builddir="$(postgres_abs_builddir)" --postgres-srcdir="$(postgres_abs_srcdir)" MULTI_REGRESS_OPTS = --inputdir=$(citus_abs_srcdir) $(pg_regress_locale_flags) --launcher="$(citus_abs_srcdir)/log_test_times" +pg_upgrade_check = $(citus_abs_srcdir)/upgrade/upgrade_test.py + # XXX: Can't actually do useful testruns against install - $libdir # etc will point to the directory configured during postgres' # build. We could copy the installed tree around, but that's quite @@ -125,6 +127,9 @@ check-failure-base: all $(pg_regress_multi_check) --load-extension=citus --mitmproxy \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/failure_base_schedule $(EXTRA_TESTS) +check-upgrade: + $(pg_upgrade_check) --old-bindir=$(old-bindir) --new-bindir=$(new-bindir) --pgxsdir=$(pgxsdir) + clean distclean maintainer-clean: rm -f $(output_files) $(input_files) rm -rf tmp_check/ diff --git a/src/test/regress/Pipfile b/src/test/regress/Pipfile index bc45fe737..7cc0276c1 100644 --- a/src/test/regress/Pipfile +++ b/src/test/regress/Pipfile @@ -6,6 +6,7 @@ verify_ssl = true [packages] mitmproxy = "==4.0.4" construct = "==2.9.45" +docopt = "==0.6.2" [dev-packages] diff --git a/src/test/regress/Pipfile.lock b/src/test/regress/Pipfile.lock index 305b226c8..143c4e25a 100644 --- a/src/test/regress/Pipfile.lock +++ b/src/test/regress/Pipfile.lock @@ -1,20 +1,7 @@ { "_meta": { "hash": { - "sha256": "1d99f17bd98cc90bbebdf6da999b3d82e1530ba5dbbeeceaf50cf090b78bbd12" - }, - "host-environment-markers": { - "implementation_name": "cpython", - "implementation_version": "3.6.6", - "os_name": "posix", - "platform_machine": "x86_64", - "platform_python_implementation": "CPython", - "platform_release": "4.4.0-130-generic", - "platform_system": "Linux", - "platform_version": "#156-Ubuntu SMP Thu Jun 14 08:53:28 UTC 2018", - "python_full_version": "3.6.6", - "python_version": "3.6", - "sys_platform": "linux" + "sha256": "705f2b4bd5a49fcd8d7a7d0b4f1bc90f92f8fa3640764b36689296365d41aefe" }, "pipfile-spec": 6, "requires": { @@ -44,81 +31,79 @@ }, "brotlipy": { "hashes": [ - "sha256:af65d2699cb9f13b26ec3ba09e75e80d31ff422c03675fcb36ee4dabe588fdc2", + "sha256:07194f4768eb62a4f4ea76b6d0df6ade185e24ebd85877c351daa0a069f1111a", + "sha256:091b299bf36dd6ef7a06570dbc98c0f80a504a56c5b797f31934d2ad01ae7d17", + "sha256:09ec3e125d16749b31c74f021aba809541b3564e5359f8c265cbae442810b41a", + "sha256:0be698678a114addcf87a4b9496c552c68a2c99bf93cf8e08f5738b392e82057", + "sha256:0fa6088a9a87645d43d7e21e32b4a6bf8f7c3939015a50158c10972aa7f425b7", + "sha256:1379347337dc3d20b2d61456d44ccce13e0625db2611c368023b4194d5e2477f", + "sha256:1ea4e578241504b58f2456a6c69952c88866c794648bdc74baee74839da61d44", + "sha256:2699945a0a992c04fc7dc7fa2f1d0575a2c8b4b769f2874a08e8eae46bef36ae", + "sha256:2a80319ae13ea8dd60ecdc4f5ccf6da3ae64787765923256b62c598c5bba4121", + "sha256:2e5c64522364a9ebcdf47c5744a5ddeb3f934742d31e61ebfbbc095460b47162", + "sha256:36def0b859beaf21910157b4c33eb3b06d8ce459c942102f16988cca6ea164df", + "sha256:3a3e56ced8b15fbbd363380344f70f3b438e0fd1fcf27b7526b6172ea950e867", + "sha256:3c1d5e2cf945a46975bdb11a19257fa057b67591eb232f393d260e7246d9e571", + "sha256:4e4638b49835d567d447a2cfacec109f9a777f219f071312268b351b6839436d", "sha256:50ca336374131cfad20612f26cc43c637ac0bfd2be3361495e99270883b52962", - "sha256:fd1d1c64214af5d90014d82cee5d8141b13d44c92ada7a0c0ec0679c6f15a471", - "sha256:b4c98b0d2c9c7020a524ca5bbff42027db1004c6571f8bc7b747f2b843128e7a", - "sha256:8b39abc3256c978f575df5cd7893153277216474f303e26f0e43ba3d3969ef96", "sha256:5de6f7d010b7558f72f4b061a07395c5c3fd57f0285c5af7f126a677b976a868", "sha256:637847560d671657f993313ecc6c6c6666a936b7a925779fd044065c7bc035b9", - "sha256:96bc59ff9b5b5552843dc67999486a220e07a0522dddd3935da05dc194fa485c", - "sha256:091b299bf36dd6ef7a06570dbc98c0f80a504a56c5b797f31934d2ad01ae7d17", - "sha256:0be698678a114addcf87a4b9496c552c68a2c99bf93cf8e08f5738b392e82057", - "sha256:d2c1c724c4ac375feb2110f1af98ecdc0e5a8ea79d068efb5891f621a5b235cb", - "sha256:3a3e56ced8b15fbbd363380344f70f3b438e0fd1fcf27b7526b6172ea950e867", "sha256:653faef61241bf8bf99d73ca7ec4baa63401ba7b2a2aa88958394869379d67c7", - "sha256:0fa6088a9a87645d43d7e21e32b4a6bf8f7c3939015a50158c10972aa7f425b7", - "sha256:79aaf217072840f3e9a3b641cccc51f7fc23037496bd71e26211856b93f4b4cb", - "sha256:a07647886e24e2fb2d68ca8bf3ada398eb56fd8eac46c733d4d95c64d17f743b", - "sha256:c6cc0036b1304dd0073eec416cb2f6b9e37ac8296afd9e481cac3b1f07f9db25", - "sha256:07194f4768eb62a4f4ea76b6d0df6ade185e24ebd85877c351daa0a069f1111a", - "sha256:7e31f7adcc5851ca06134705fcf3478210da45d35ad75ec181e1ce9ce345bb38", - "sha256:9448227b0df082e574c45c983fa5cd4bda7bfb11ea6b59def0940c1647be0c3c", - "sha256:dc6c5ee0df9732a44d08edab32f8a616b769cc5a4155a12d2d010d248eb3fb07", - "sha256:3c1d5e2cf945a46975bdb11a19257fa057b67591eb232f393d260e7246d9e571", - "sha256:2a80319ae13ea8dd60ecdc4f5ccf6da3ae64787765923256b62c598c5bba4121", - "sha256:2699945a0a992c04fc7dc7fa2f1d0575a2c8b4b769f2874a08e8eae46bef36ae", - "sha256:1ea4e578241504b58f2456a6c69952c88866c794648bdc74baee74839da61d44", - "sha256:2e5c64522364a9ebcdf47c5744a5ddeb3f934742d31e61ebfbbc095460b47162", - "sha256:09ec3e125d16749b31c74f021aba809541b3564e5359f8c265cbae442810b41a", "sha256:786afc8c9bd67de8d31f46e408a3386331e126829114e4db034f91eacb05396d", - "sha256:36def0b859beaf21910157b4c33eb3b06d8ce459c942102f16988cca6ea164df" + "sha256:79aaf217072840f3e9a3b641cccc51f7fc23037496bd71e26211856b93f4b4cb", + "sha256:7e31f7adcc5851ca06134705fcf3478210da45d35ad75ec181e1ce9ce345bb38", + "sha256:8b39abc3256c978f575df5cd7893153277216474f303e26f0e43ba3d3969ef96", + "sha256:9448227b0df082e574c45c983fa5cd4bda7bfb11ea6b59def0940c1647be0c3c", + "sha256:96bc59ff9b5b5552843dc67999486a220e07a0522dddd3935da05dc194fa485c", + "sha256:a07647886e24e2fb2d68ca8bf3ada398eb56fd8eac46c733d4d95c64d17f743b", + "sha256:af65d2699cb9f13b26ec3ba09e75e80d31ff422c03675fcb36ee4dabe588fdc2", + "sha256:b4c98b0d2c9c7020a524ca5bbff42027db1004c6571f8bc7b747f2b843128e7a", + "sha256:c6cc0036b1304dd0073eec416cb2f6b9e37ac8296afd9e481cac3b1f07f9db25", + "sha256:d2c1c724c4ac375feb2110f1af98ecdc0e5a8ea79d068efb5891f621a5b235cb", + "sha256:dc6c5ee0df9732a44d08edab32f8a616b769cc5a4155a12d2d010d248eb3fb07", + "sha256:fd1d1c64214af5d90014d82cee5d8141b13d44c92ada7a0c0ec0679c6f15a471" ], "version": "==0.7.0" }, "certifi": { "hashes": [ - "sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0", - "sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7" + "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", + "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695" ], - "version": "==2018.4.16" + "version": "==2019.6.16" }, "cffi": { "hashes": [ - "sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50", - "sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596", - "sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef", - "sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743", - "sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f", - "sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31", - "sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04", - "sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6", - "sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3", - "sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6", - "sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b", - "sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca", - "sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e", - "sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb", - "sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd", - "sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1", - "sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917", - "sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359", - "sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f", - "sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95", - "sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801", - "sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257", - "sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184", - "sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc", - "sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085", - "sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93", - "sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2", - "sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30", - "sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5", - "sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e", - "sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b", - "sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4" + "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774", + "sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d", + "sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90", + "sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b", + "sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63", + "sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45", + "sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25", + "sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3", + "sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b", + "sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647", + "sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016", + "sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4", + "sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb", + "sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753", + "sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7", + "sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9", + "sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f", + "sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8", + "sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f", + "sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc", + "sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42", + "sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3", + "sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909", + "sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45", + "sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d", + "sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512", + "sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff", + "sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201" ], - "version": "==1.11.5" + "version": "==1.12.3" }, "click": { "hashes": [ @@ -131,45 +116,53 @@ "hashes": [ "sha256:2271a0efd0798679dea825ff47e22a4c550456a5db0ba8baa82f7eae0af0118c" ], + "index": "pypi", "version": "==2.9.45" }, "cryptography": { "hashes": [ - "sha256:87d092a7c2a44e5f7414ab02fb4145723ebba411425e1a99773531dd4c0e9b8d", - "sha256:67f7f57eae8dede577f3f7775957f5bec93edd6bdb6ce597bb5b28e1bdf3d4fb", - "sha256:27bb401a20a838d6d0ea380f08c6ead3ccd8c9d8a0232dc9adcc0e4994576a66", - "sha256:c3d945b7b577f07a477700f618f46cbc287af3a9222cd73035c6ef527ef2c363", - "sha256:bc2301170986ad82d9349a91eb8884e0e191209c45f5541b16aa7c0cfb135978", - "sha256:7ba834564daef87557e7fcd35c3c3183a4147b0b3a57314e53317360b9b201b3", - "sha256:9449f5d4d7c516a6118fa9210c4a00f34384cb1d2028672100ee0c6cce49d7f6", - "sha256:d01dfc5c2b3495184f683574e03c70022674ca9a7be88589c5aba130d835ea90", - "sha256:2a35b7570d8f247889784010aac8b384fd2e4a47b33e15c4a60b45a7c1944120", - "sha256:5eb86f03f9c4f0ac2336ac5431271072ddf7ecc76b338e26366732cfac58aa19", - "sha256:29720c4253263cff9aea64585adbbe85013ba647f6e98367efff9db2d7193ded", - "sha256:82409f5150e529d699e5c33fa8fd85e965104db03bc564f5f4b6a9199e591f7c", - "sha256:7d7f084cbe1fdb82be5a0545062b59b1ad3637bc5a48612ac2eb428ff31b31ea", - "sha256:6ec84edcbc966ae460560a51a90046503ff0b5b66157a9efc61515c68059f6c8", - "sha256:8c56ef989342e42b9fcaba7c74b446f0cc9bed546dd00034fa7ad66fc00307ef", - "sha256:42c531a6a354407f42ee07fda5c2c0dc822cf6d52744949c182f2b295fbd4183", - "sha256:21af753934f2f6d1a10fe8f4c0a64315af209ef6adeaee63ca349797d747d687", - "sha256:cee18beb4c807b5c0b178f4fa2fae03cef9d51821a358c6890f8b23465b7e5d2", - "sha256:c132bab45d4bd0fff1d3fe294d92b0a6eb8404e93337b3127bdec9f21de117e6" + "sha256:02602e1672b62e803e08617ec286041cc453e8d43f093a5f4162095506bc0beb", + "sha256:10b48e848e1edb93c1d3b797c83c72b4c387ab0eb4330aaa26da8049a6cbede0", + "sha256:17db09db9d7c5de130023657be42689d1a5f60502a14f6f745f6f65a6b8195c0", + "sha256:227da3a896df1106b1a69b1e319dce218fa04395e8cc78be7e31ca94c21254bc", + "sha256:2cbaa03ac677db6c821dac3f4cdfd1461a32d0615847eedbb0df54bb7802e1f7", + "sha256:31db8febfc768e4b4bd826750a70c79c99ea423f4697d1dab764eb9f9f849519", + "sha256:4a510d268e55e2e067715d728e4ca6cd26a8e9f1f3d174faf88e6f2cb6b6c395", + "sha256:6a88d9004310a198c474d8a822ee96a6dd6c01efe66facdf17cb692512ae5bc0", + "sha256:76936ec70a9b72eb8c58314c38c55a0336a2b36de0c7ee8fb874a4547cadbd39", + "sha256:7e3b4aecc4040928efa8a7cdaf074e868af32c58ffc9bb77e7bf2c1a16783286", + "sha256:8168bcb08403ef144ff1fb880d416f49e2728101d02aaadfe9645883222c0aa5", + "sha256:8229ceb79a1792823d87779959184a1bf95768e9248c93ae9f97c7a2f60376a1", + "sha256:8a19e9f2fe69f6a44a5c156968d9fc8df56d09798d0c6a34ccc373bb186cee86", + "sha256:8d10113ca826a4c29d5b85b2c4e045ffa8bad74fb525ee0eceb1d38d4c70dfd6", + "sha256:be495b8ec5a939a7605274b6e59fbc35e76f5ad814ae010eb679529671c9e119", + "sha256:dc2d3f3b1548f4d11786616cf0f4415e25b0fbecb8a1d2cd8c07568f13fdde38", + "sha256:e4aecdd9d5a3d06c337894c9a6e2961898d3f64fe54ca920a72234a3de0f9cb3", + "sha256:e79ab4485b99eacb2166f3212218dd858258f374855e1568f728462b0e6ee0d9", + "sha256:f995d3667301e1754c57b04e0bae6f0fa9d710697a9f8d6712e8cca02550910f" ], - "version": "==2.3" + "version": "==2.3.1" + }, + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "index": "pypi", + "version": "==0.6.2" }, "h11": { "hashes": [ - "sha256:af77d5d82fa027c032650fb8afdef3cd0a3735ba01480bee908cddad9be1bdce", - "sha256:1c0fbb1cba6f809fe3e6b27f8f6d517ca171f848922708871403636143d530d9" + "sha256:1c0fbb1cba6f809fe3e6b27f8f6d517ca171f848922708871403636143d530d9", + "sha256:af77d5d82fa027c032650fb8afdef3cd0a3735ba01480bee908cddad9be1bdce" ], "version": "==0.7.0" }, "h2": { "hashes": [ - "sha256:4be613e35caad5680dc48f98f3bf4e7338c7c429e6375a5137be7fbe45219981", - "sha256:b2962f883fa392a23cbfcc4ad03c335bcc661be0cf9627657b589f0df2206e64" + "sha256:ac377fcf586314ef3177bfd90c12c7826ab0840edeb03f0f24f511858326049e", + "sha256:b8a32bd282594424c0ac55845377eea13fa54fe4a8db012f3a198ed923dc3ab4" ], - "version": "==3.0.1" + "version": "==3.1.1" }, "hpack": { "hashes": [ @@ -180,17 +173,17 @@ }, "hyperframe": { "hashes": [ - "sha256:87567c9eb1540de1e7f48805adf00e87856409342fdebd0cd20cf5d381c38b69", - "sha256:a25944539db36d6a2e47689e7915dcee562b3f8d10c6cdfa0d53c91ed692fb04" + "sha256:5187962cb16dcc078f23cb5a4b110098d546c3f41ff2d4038a9896893bbd0b40", + "sha256:a9f5c17f2cc3c719b917c4f33ed1c61bd1f8dfac4b1bd23b7c80b3400971b41f" ], - "version": "==5.1.0" + "version": "==5.2.0" }, "idna": { "hashes": [ - "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", - "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" ], - "version": "==2.7" + "version": "==2.8" }, "kaitaistruct": { "hashes": [ @@ -200,51 +193,37 @@ }, "ldap3": { "hashes": [ - "sha256:31192a209ddb4d979fa43e37a130eb9b5945a588d8c992f3412c0ac50a4a3e8a", - "sha256:a6c4b127b35845e214b203c05c77a7faf086324d4603725b8d72f910523bdd7b", - "sha256:5a25d825de9c2e9f9bb1eeb8a96822fd1d4739ee00e797beb400f45e497a502e", - "sha256:4e3d5b4861e27eaf65280fbab2b6aac8a116f0937fb0fb98224f644c375364bf", - "sha256:55078bbc981f715a8867b4c040402627fdfccf5664e0277a621416559748e384", - "sha256:5ec40b8619c46e0f0dc8d2e965fa566777dfbca9dee48210f6b4a6c02489c24e" + "sha256:3f67c83185b1f0df8fdf6b52fa42c55bc9e9b7120c8b7fec60f0d6003c536d18", + "sha256:dd9be8ea27773c4ffc18ede0b95c3ca1eb12513a184590b9f8ae423db3f71eb9" ], - "version": "==2.5" + "version": "==2.5.2" }, "mitmproxy": { "hashes": [ "sha256:e74869c7bf4e5b988fbe3a3d0039f430d1e1eeb5927abf2097183a711bf5b312" ], + "index": "pypi", "version": "==4.0.4" }, "passlib": { "hashes": [ - "sha256:43526aea08fa32c6b6dbbbe9963c4c767285b78147b7437597f992812f69d280", - "sha256:3d948f64138c25633613f303bcc471126eae67c04d5e3f6b7b8ce6242f8653e0" + "sha256:3d948f64138c25633613f303bcc471126eae67c04d5e3f6b7b8ce6242f8653e0", + "sha256:43526aea08fa32c6b6dbbbe9963c4c767285b78147b7437597f992812f69d280" ], "version": "==1.7.1" }, "pyasn1": { "hashes": [ - "sha256:dea873d6c907c1cf1341fd88742a61efce33227d7743cb37564ab7d7e77dd9fd", - "sha256:13794d835643ee970b2c059dbfe4eb5d751e16c693c8baee61c526abd209e5c7", - "sha256:edad117649643230493aeb4955456ce19ab4b12e94489dde6f7094cdb5a3c87e", - "sha256:e8b69ea2200d42201cbedd486eedb8980f320d4534f83ce2fb468e96aa5545d0", - "sha256:b9d3abc5031e61927c82d4d96c1cec1e55676c1a991623cfed28faea73cdd7ca", - "sha256:d3bbd726c1a760d4ca596a4d450c380b81737612fe0182f5bb3caebc17461fd9", - "sha256:74ac8521a0480f228549be20bea555ae35678f0e754c2fbc6f1576b0959bec43", - "sha256:ded5eea5cb88bc1ce9aa074b5a3092f95ce4741887e317e9b49c7ece75d7ea0e", - "sha256:0ad0fe0593dde1e599cac0bf65bb1a4ec663032f0bc68ee44850db4251e8c501", - "sha256:49a8ed515f26913049113820b462f698e6ed26df62c389dafb6fa3685ddca8de", - "sha256:89399ca8ecd4524f974e926d4ef9e7a787903e01f0a9cdff3131ad1361792fe5", - "sha256:8f291e0338d519a1a0d07f0b9d03c9265f6be26eb32fdd21af6d3259d14ea49c", - "sha256:f58f2a3d12fd754aa123e9fa74fb7345333000a035f3921dbdaa08597aa53137" + "sha256:62cdade8b5530f0b185e09855dd422bc05c0bbff6b72ff61381c09dac7befd8c", + "sha256:a9495356ca1d66ed197a0f72b41eb1823cf7ea8b5bd07191673e8147aecf8604" ], - "version": "==0.4.4" + "version": "==0.4.7" }, "pycparser": { "hashes": [ - "sha256:99a8ca03e29851d96616ad0404b4aad7d9ee16f25c9f9708a11faf2810f7b226" + "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" ], - "version": "==2.18" + "version": "==2.19" }, "pyopenssl": { "hashes": [ @@ -255,74 +234,65 @@ }, "pyparsing": { "hashes": [ - "sha256:fee43f17a9c4087e7ed1605bd6df994c6173c1e977d7ade7b651292fab2bd010", - "sha256:0832bcf47acd283788593e7a0f542407bd9550a55a8a8435214a1960e04bcb04", - "sha256:9e8143a3e15c13713506886badd96ca4b579a87fbdf49e550dbfc057d6cb218e", - "sha256:281683241b25fe9b80ec9d66017485f6deff1af5cde372469134b56ca8447a07", - "sha256:b8b3117ed9bdf45e14dcc89345ce638ec7e0e29b2b579fa1ecf32ce45ebac8a5", - "sha256:8f1e18d3fd36c6795bb7e02a39fd05c611ffc2596c1e0d995d34d67630426c18", - "sha256:e4d45427c6e20a59bf4f88c639dcc03ce30d193112047f94012102f235853a58" + "sha256:bc6c7146b91af3f567cf6daeaec360bc07d45ffec4cf5353f4d7a208ce7ca30a", + "sha256:d29593d8ebe7b57d6967b62494f8c72b03ac0262b1eed63826c6f788b3606401" ], - "version": "==2.2.0" + "version": "==2.2.2" }, "pyperclip": { "hashes": [ - "sha256:f70e83d27c445795b6bf98c2bc826bbf2d0d63d4c7f83091c8064439042ba0dc" + "sha256:406bc020d4b8e60d8673876271b815befc4c02fd8d919e4aacc667d69fab99ea" ], - "version": "==1.6.4" + "version": "==1.6.5" }, "ruamel.yaml": { "hashes": [ - "sha256:7b8bcfda46d807d5cc85bf25aa5bebf04abad21e3ff5b98445e6df50675d2dad", - "sha256:566273fcdc63795eb1341dc018f129200353d655217f674334f1b2382654fe28", - "sha256:4abe0c556ad8b1427b1e5ef22499f145645672a428e8ae9ebcd0adfe930cfa65", - "sha256:b3f92041e59b207d1fe84067b22eed1f54b6c2547533a26cea10f3f36139085d", - "sha256:b6d608aa7634ad9811688cdb00bb42283c8792551ced0d54d9ee469f215a1bb4", - "sha256:0223a923c368a3d60f9c113e9d53973aaa9e67031b68b9345194d0f672a9fad3", - "sha256:b365b9761a3365b3287bd335d385ec1a76559438dde941c81960ed955d8ab9d9", - "sha256:df2db416963f06e7a1654b6c0159d60a468f2d308860ec67e0916257c46b2b7a", - "sha256:ac017f15fb7a8d6a9afb16df79a1e85c9c706a36a2effefbfa147cccdf6afaf9", - "sha256:f39f747b8d65487e253682a94697d8e352d9b6a02bda02b0f73cd352d8c627f9", - "sha256:aad048a0b80c538aa82ea885c3ca48329037abebb33e10f3d93bfed79b25675c", - "sha256:376fc396dafa2541d656c410fec23e03f9818dbcedfe8f855c09199660474cf3", - "sha256:8ba5cb735dd11eb9ba5651520d8a4ec83758b38df3f790efb71a2d12d2beff0c", - "sha256:18449746049b8eab51c8730f8a7ebc314af2ee471fdf542ec4affb67bd8d5c42", - "sha256:d93ba3163c87ddbd043940ff7a738a27655c3818fdadda14e5f8149d8662d616", - "sha256:b3e445cd15c545c47f9c8abb755cc27e084fe06081cffd37b6cebb800afb26ed", - "sha256:2da38bdfe7021c30dbef2c710cad67f341507225613ea1a48a1d9dec33fb943e", - "sha256:041335981d89f999244f58ca538c55d2c5c3ca882121f3d54c25fa353cf441ad", - "sha256:d08c39d18c8a8153707f438172e208fc9d2e71c815236134b39f67f3005e2a96", - "sha256:4db8af1cefad214920297c178923eb161d405555f05e0bbbc2ef94461a6d6d6c", - "sha256:642f7728605e0adb528987d55ebefe53a9602cd485325e227cdfadecb4f96d3d", - "sha256:4b9de78a8b54ce0fcc04cdddeba16cd506f879189efc15ec865061ea7c93843a" + "sha256:08aaaa74ff66565024ecabf9ba2db212712382a21c0458f9a91c623a1fa83b34", + "sha256:23f2efb872d2ebe3d5428b4f1a8f30cbf59f56e780c4981c155411ee65572673", + "sha256:38718e69270141c403b5fc539f774ed394568f8a5195b507991f5b690356facb", + "sha256:44da2be1153e173f90ad8775d4ac4237a3c06cfbb9660c1c1980271621833faa", + "sha256:4b1674a936cdae9735578d4fd64bcbc6cfbb77a1a8f7037a50c6e3874ba4c9d8", + "sha256:51d49c870aca850e652e2cd1c9bea9b52b77d13ad52b0556de496c1d264ea65f", + "sha256:63dc8c6147a4cf77efadf2ae0f34e89e03de79289298bb941b7ae333d5d4020b", + "sha256:6672798c6b52a976a7b24e20665055852388c83198d88029d3c76e2197ac221a", + "sha256:6b6025f9b6a557e15e9fdfda4d9af0b57cd8d59ff98e23a0097ab2d7c0540f07", + "sha256:7b750252e3d1ec5b53d03be508796c04a907060900c7d207280b7456650ebbfc", + "sha256:847177699994f9c31adf78d1ef1ff8f069ef0241e744a3ee8b30fbdaa914cc1e", + "sha256:8e42f3067a59e819935a2926e247170ed93c8f0b2ab64526f888e026854db2e4", + "sha256:922d9e483c05d9000256640026f277fcc0c2e1e9271d05acada8e6cfb4c8b721", + "sha256:92a8ca79f9173cca29ca9663b49d9c936aefc4c8a76f39318b0218c8f3626438", + "sha256:ab8eeca4de4decf0d0a42cb6949d354da9fc70a2d9201f0dd55186c599b2e3a5", + "sha256:bd4b60b649f4a81086f70cd56eff4722018ef36a28094c396f1a53bf450bd579", + "sha256:fc6471ef15b69e454cca82433ac5f84929d9f3e2d72b9e54b06850b6b7133cc0", + "sha256:ffc89770339191acbe5a15041950b5ad9daec7d659619b0ed9dad8c9c80c26f3" ], - "version": "==0.15.47" + "version": "==0.15.100" }, "six": { "hashes": [ - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb", - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9" + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], - "version": "==1.11.0" + "version": "==1.12.0" }, "sortedcontainers": { "hashes": [ - "sha256:ef38b128302ee8f65d81e31c9d8fbf10d81df4d6d06c9c0b66f01d33747525bb", - "sha256:607294c6e291a270948420f7ffa1fb3ed47384a4c08db6d1e9c92d08a6981982" + "sha256:220bb2e3e1886297fd7cdd6d164cb5cf237be1cfae1a3a3e526d149c52816682", + "sha256:b74f2756fb5e23512572cc76f0fe0832fd86310f77dfee54335a35fb33f6b950" ], - "version": "==2.0.4" + "version": "==2.0.5" }, "tornado": { "hashes": [ - "sha256:d8984742ce86c0855cccecd5c6f54a9f7532c983947cff06f3a0e2115b47f85c", - "sha256:6a7e8657618268bb007646b9eae7661d0b57f13efc94faa33cd2588eae5912c9", - "sha256:5327ba1a6c694e0149e7d9126426b3704b1d9d520852a3e4aa9fc8fe989e4046", - "sha256:a9b14804783a1d77c0bd6c66f7a9b1196cbddfbdf8bceb64683c5ae60bd1ec6f", - "sha256:c58757e37c4a3172949c99099d4d5106e4d7b63aa0617f9bb24bfbff712c7866", - "sha256:1c0816fc32b7d31b98781bd8ebc7a9726d7dce67407dc353a2e66e697e138448", - "sha256:4f66a2172cb947387193ca4c2c3e19131f1c70fa8be470ddbbd9317fd0801582" + "sha256:0662d28b1ca9f67108c7e3b77afabfb9c7e87bde174fbda78186ecedc2499a9d", + "sha256:4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409", + "sha256:732e836008c708de2e89a31cb2fa6c0e5a70cb60492bee6f1ea1047500feaf7f", + "sha256:8154ec22c450df4e06b35f131adc4f2f3a12ec85981a203301d310abf580500f", + "sha256:8e9d728c4579682e837c92fdd98036bd5cdefa1da2aaf6acf26947e6dd0c01c5", + "sha256:d4b3e5329f572f055b587efc57d29bd051589fb5a43ec8898c77a47ec2fa2bbb", + "sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444" ], - "version": "==5.1" + "version": "==5.1.1" }, "urwid": { "hashes": [ @@ -332,8 +302,8 @@ }, "wsproto": { "hashes": [ - "sha256:d2a7f718ab3144ec956a3267d57b5c172f0668827f5803e7d670837b0125b9fa", - "sha256:02f214f6bb43cda62a511e2e8f1d5fa4703ed83d376d18d042bd2bbf2e995824" + "sha256:02f214f6bb43cda62a511e2e8f1d5fa4703ed83d376d18d042bd2bbf2e995824", + "sha256:d2a7f718ab3144ec956a3267d57b5c172f0668827f5803e7d670837b0125b9fa" ], "version": "==0.11.0" } diff --git a/src/test/regress/after_upgrade_schedule b/src/test/regress/after_upgrade_schedule new file mode 100644 index 000000000..1fc486810 --- /dev/null +++ b/src/test/regress/after_upgrade_schedule @@ -0,0 +1 @@ +test: upgrade_distributed_table_after diff --git a/src/test/regress/before_upgrade_schedule b/src/test/regress/before_upgrade_schedule new file mode 100644 index 000000000..bf8f4c13f --- /dev/null +++ b/src/test/regress/before_upgrade_schedule @@ -0,0 +1 @@ +test: upgrade_distributed_table_before diff --git a/src/test/regress/expected/upgrade_distributed_table_after.out b/src/test/regress/expected/upgrade_distributed_table_after.out new file mode 100644 index 000000000..c80db1eb9 --- /dev/null +++ b/src/test/regress/expected/upgrade_distributed_table_after.out @@ -0,0 +1,19 @@ +SET search_path TO upgrade_distributed_table_before, public; +SELECT * FROM t ORDER BY a; + a +--- + 1 + 2 + 3 + 4 + 5 +(5 rows) + +SELECT * FROM t WHERE a = 1; + a +--- + 1 +(1 row) + +DROP SCHEMA upgrade_distributed_table_before CASCADE; +NOTICE: drop cascades to table t diff --git a/src/test/regress/expected/upgrade_distributed_table_before.out b/src/test/regress/expected/upgrade_distributed_table_before.out new file mode 100644 index 000000000..fb276b15a --- /dev/null +++ b/src/test/regress/expected/upgrade_distributed_table_before.out @@ -0,0 +1,10 @@ +CREATE SCHEMA upgrade_distributed_table_before; +SET search_path TO upgrade_distributed_table_before, public; +CREATE TABLE t(a int); +SELECT create_distributed_table('t', 'a'); + create_distributed_table +-------------------------- + +(1 row) + +INSERT INTO t SELECT * FROM generate_series(1, 5); diff --git a/src/test/regress/sql/upgrade_distributed_table_after.sql b/src/test/regress/sql/upgrade_distributed_table_after.sql new file mode 100644 index 000000000..50f6e4e48 --- /dev/null +++ b/src/test/regress/sql/upgrade_distributed_table_after.sql @@ -0,0 +1,6 @@ +SET search_path TO upgrade_distributed_table_before, public; + +SELECT * FROM t ORDER BY a; +SELECT * FROM t WHERE a = 1; + +DROP SCHEMA upgrade_distributed_table_before CASCADE; diff --git a/src/test/regress/sql/upgrade_distributed_table_before.sql b/src/test/regress/sql/upgrade_distributed_table_before.sql new file mode 100644 index 000000000..72da0c0ff --- /dev/null +++ b/src/test/regress/sql/upgrade_distributed_table_before.sql @@ -0,0 +1,6 @@ +CREATE SCHEMA upgrade_distributed_table_before; +SET search_path TO upgrade_distributed_table_before, public; + +CREATE TABLE t(a int); +SELECT create_distributed_table('t', 'a'); +INSERT INTO t SELECT * FROM generate_series(1, 5); diff --git a/src/test/regress/upgrade/README.md b/src/test/regress/upgrade/README.md new file mode 100644 index 000000000..1504a21f1 --- /dev/null +++ b/src/test/regress/upgrade/README.md @@ -0,0 +1,44 @@ +Upgrade test is used for testing postgres version upgrade with citus installed. +Before running the script, make sure that: +- You have downloaded citus. +- You have two different postgres versions. +- Citus is installed to both of the postgres versions. For each postgres version: + - In citus source directory run: + ``` + make clean + ./configure PG_CONFIG= + PG_CONFIG= make + sudo PG_CONFIG= make install + ``` + Make sure you do this for both postgres versions, pg_config should be different for each postgres version. +- Install `pipenv` and run in `citus/src/test/regress`: +``` + pipenv install + pipenv shell +``` + +- Finally run upgrade test in `citus/src/test/regress`: +``` + pipenv run make check-upgrade old-bindir= new-bindir= +``` + +To see full command list: + +``` + pipenv run upgrade/upgrade_test.py -help +``` + + +How the upgrade test works: +- Temporary folder `tmp_upgrade` is created in `src/test/regress/`, if one exists it is removed first. +- Database is initialized and citus cluster is created(1 coordinator + 2 workers) with old postgres. +- `before_upgrade_schedule` is run with `pg_regress`. This schedule does not drop any tables or data so that we can verify upgrade. +- `citus_prepare_pg_upgrade` is run in coordinators and workers. +- Old database is stopped. +- A new database is initialized with new postgres under `tmp_upgrade`. +- Postgres upgrade is performed. +- New database is started in both coordinators and workers. +- `citus_finish_pg_upgrade` is run in coordinators and workers to finalize the upgrade step. +- `after_upgrade_schedule` is run with `pg_regress` to verify that the previously created tables, and data still exist. Router and realtime queries are used to verify this. + + diff --git a/src/test/regress/upgrade/config.py b/src/test/regress/upgrade/config.py new file mode 100644 index 000000000..8eb19a8da --- /dev/null +++ b/src/test/regress/upgrade/config.py @@ -0,0 +1,32 @@ +BEFORE_UPGRADE_SCHEDULE = './before_upgrade_schedule' +AFTER_UPGRADE_SCHEDULE = './after_upgrade_schedule' + + +class Config(): + def __init__(self, arguments): + self.old_bindir = arguments['--old-bindir'] + self.new_bindir = arguments['--new-bindir'] + self.pg_srcdir = arguments['--pgxsdir'] + self.temp_dir = './tmp_upgrade' + self.old_datadir = self.temp_dir + '/oldData' + self.new_datadir = self.temp_dir + '/newData' + self.settings = { + 'shared_preload_libraries': 'citus', + 'citus.node_conninfo': 'sslmode=prefer' + } + + +USER = 'postgres' +DBNAME = 'postgres' + +COORDINATOR_NAME = 'coordinator' +WORKER1 = 'worker1' +WORKER2 = 'worker2' +NODE_NAMES = [COORDINATOR_NAME, WORKER1, WORKER2] + +WORKER_PORTS = [9701, 9702] +NODE_PORTS = { + COORDINATOR_NAME: 9700, + WORKER1: 9701, + WORKER2: 9702, +} diff --git a/src/test/regress/upgrade/upgrade_test.py b/src/test/regress/upgrade/upgrade_test.py new file mode 100755 index 000000000..eb9f1a565 --- /dev/null +++ b/src/test/regress/upgrade/upgrade_test.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 + +"""upgrade_test +Usage: + upgrade_test --old-bindir= --new-bindir= --pgxsdir= + +Options: + --old-bindir= The old PostgreSQL executable directory;(ex: '~/.pgenv/pgsql/bin') + --new-bindir= New postgres binary absolute path(ex: '~/.pgenv/pgsql-11.3/bin') + --pgxsdir= Path to the PGXS directory(ex: ~/.pgenv/src/postgresql-11.3) +""" + +import utils +import atexit +import subprocess +import sys +import shutil +import os + +from docopt import docopt + +from config import ( + Config, USER, NODE_PORTS, + NODE_NAMES, DBNAME, COORDINATOR_NAME, + WORKER_PORTS, AFTER_UPGRADE_SCHEDULE, BEFORE_UPGRADE_SCHEDULE +) + + +def initialize_temp_dir(temp_dir): + if os.path.exists(temp_dir): + shutil.rmtree(temp_dir) + os.mkdir(temp_dir) + # Give full access to TEMP_DIR so that postgres user can use it. + os.chmod(temp_dir, 0o777) + + +def initialize_db_for_cluster(pg_path, rel_data_path, settings): + subprocess.call(['mkdir', rel_data_path]) + for node_name in NODE_NAMES: + abs_data_path = os.path.abspath(os.path.join(rel_data_path, node_name)) + command = [ + os.path.join(pg_path, 'initdb'), + '--pgdata', abs_data_path, + '--username', USER + ] + subprocess.call(command) + add_settings(abs_data_path, settings) + + +def add_settings(abs_data_path, settings): + conf_path = os.path.join(abs_data_path, 'postgresql.conf') + with open(conf_path, 'a') as conf_file: + for setting_key, setting_val in settings.items(): + setting = "{setting_key} = \'{setting_val}\'\n".format( + setting_key=setting_key, + setting_val=setting_val) + conf_file.write(setting) + + +def start_databases(pg_path, rel_data_path): + for node_name in NODE_NAMES: + abs_data_path = os.path.abspath(os.path.join(rel_data_path, node_name)) + command = [ + os.path.join(pg_path, 'pg_ctl'), 'start', + '--pgdata', abs_data_path, + '-U', USER, + '-o', '-p {}'.format(NODE_PORTS[node_name]), + '--log', os.path.join(abs_data_path, 'logfile_' + node_name) + ] + subprocess.call(command) + + +def create_citus_extension(pg_path): + for port in NODE_PORTS.values(): + utils.psql(pg_path, port, "CREATE EXTENSION citus;") + + +def add_workers(pg_path): + for port in WORKER_PORTS: + command = "SELECT * from master_add_node('localhost', {port});".format( + port=port) + utils.psql(pg_path, NODE_PORTS[COORDINATOR_NAME], command) + + +def run_pg_regress(pg_path, PG_SRCDIR, port, schedule): + command = [ + os.path.join(PG_SRCDIR, 'src/test/regress/pg_regress'), + '--port', str(port), + '--schedule', schedule, + '--bindir', pg_path, + '--user', USER, + '--dbname', DBNAME, + '--use-existing' + ] + exit_code = subprocess.call(command) + if exit_code != 0: + sys.exit(exit_code) + + +def citus_prepare_pg_upgrade(pg_path): + for port in NODE_PORTS.values(): + utils.psql(pg_path, port, "SELECT citus_prepare_pg_upgrade();") + + +def stop_databases(pg_path, rel_data_path): + for node_name in NODE_NAMES: + abs_data_path = os.path.abspath(os.path.join(rel_data_path, node_name)) + command = [ + os.path.join(pg_path, 'pg_ctl'), 'stop', + '--pgdata', abs_data_path, + '-U', USER, + '-o', '-p {}'.format(NODE_PORTS[node_name]), + '--log', os.path.join(abs_data_path, 'logfile_' + node_name) + ] + subprocess.call(command) + + +def perform_postgres_upgrade(old_bindir, new_bindir, old_datadir, new_datadir): + for node_name in NODE_NAMES: + base_new_data_path = os.path.abspath(new_datadir) + base_old_data_path = os.path.abspath(old_datadir) + with utils.cd(base_new_data_path): + abs_new_data_path = os.path.join(base_new_data_path, node_name) + abs_old_data_path = os.path.join(base_old_data_path, node_name) + command = [ + os.path.join(new_bindir, 'pg_upgrade'), + '--username', USER, + '--old-bindir', old_bindir, + '--new-bindir', new_bindir, + '--old-datadir', abs_old_data_path, + '--new-datadir', abs_new_data_path + ] + subprocess.call(command) + + +def citus_finish_pg_upgrade(pg_path): + for port in NODE_PORTS.values(): + utils.psql(pg_path, port, "SELECT citus_finish_pg_upgrade();") + + +def initialize_citus_cluster(old_bindir, old_datadir, settings): + initialize_db_for_cluster(old_bindir, old_datadir, settings) + start_databases(old_bindir, old_datadir) + create_citus_extension(old_bindir) + add_workers(old_bindir) + + +def stop_all_databases(old_bindir, new_bindir, old_datadir, new_datadir): + stop_databases(old_bindir, old_datadir) + stop_databases(new_bindir, new_datadir) + + +def main(config): + initialize_temp_dir(config.temp_dir) + initialize_citus_cluster( + config.old_bindir, config.old_datadir, config.settings) + + run_pg_regress(config.old_bindir, config.pg_srcdir, + NODE_PORTS[COORDINATOR_NAME], BEFORE_UPGRADE_SCHEDULE) + + citus_prepare_pg_upgrade(config.old_bindir) + stop_databases(config.old_bindir, config.old_datadir) + + initialize_db_for_cluster( + config.new_bindir, config.new_datadir, config.settings) + perform_postgres_upgrade( + config.old_bindir, config.new_bindir, config.old_datadir, config.new_datadir) + start_databases(config.new_bindir, config.new_datadir) + citus_finish_pg_upgrade(config.new_bindir) + + run_pg_regress(config.new_bindir, config.pg_srcdir, + NODE_PORTS[COORDINATOR_NAME], AFTER_UPGRADE_SCHEDULE) + + +if __name__ == '__main__': + config = Config(docopt(__doc__, version='upgrade_test')) + atexit.register(stop_all_databases, config.old_bindir, + config.new_bindir, config.old_datadir, config.new_datadir) + main(config) diff --git a/src/test/regress/upgrade/utils.py b/src/test/regress/upgrade/utils.py new file mode 100644 index 000000000..733825c8d --- /dev/null +++ b/src/test/regress/upgrade/utils.py @@ -0,0 +1,29 @@ +import subprocess +import os +from config import USER + + +def psql(pg_path, port, command): + + return subprocess.call([ + os.path.join(pg_path, 'psql'), + '-U', USER, + '-p', str(port), + '-c', command] + ) + +# Taken from https://stackoverflow.com/questions/431684/how-do-i-change-directory-cd-in-python/13197763#13197763 + + +class cd(object): + """Context manager for changing the current working directory""" + + def __init__(self, newPath): + self.newPath = os.path.expanduser(newPath) + + def __enter__(self): + self.savedPath = os.getcwd() + os.chdir(self.newPath) + + def __exit__(self, etype, value, traceback): + os.chdir(self.savedPath)