From 08d82ce8bbe4d66d30698f36bcecfcc8ed9c6d41 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 9 Mar 2023 11:18:46 +0300 Subject: [PATCH 01/14] upgrade list --- src/test/regress/expected/upgrade_list_citus_objects.out | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/test/regress/expected/upgrade_list_citus_objects.out b/src/test/regress/expected/upgrade_list_citus_objects.out index 7cd2f63c8..6733a28fc 100644 --- a/src/test/regress/expected/upgrade_list_citus_objects.out +++ b/src/test/regress/expected/upgrade_list_citus_objects.out @@ -121,6 +121,8 @@ ORDER BY 1; function citus_stat_activity() function citus_stat_statements() function citus_stat_statements_reset() + function citus_stats_tenants(boolean) + function citus_stats_tenants_storage() function citus_table_is_visible(oid) function citus_table_size(regclass) function citus_task_wait(bigint,citus_task_status) @@ -316,7 +318,9 @@ ORDER BY 1; view citus_shards_on_worker view citus_stat_activity view citus_stat_statements + view citus_stats_tenants + view citus_stats_tenants_storage view pg_dist_shard_placement view time_partitions -(310 rows) +(314 rows) From ea9cb7756969d7965e71cf665859a5dd57bec71c Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 9 Mar 2023 11:39:03 +0300 Subject: [PATCH 02/14] failure 1 --- .../regress/expected/failure_multi_dml.out | 4 ++-- .../expected/failure_replicated_partitions.out | 2 +- .../regress/expected/failure_single_select.out | 18 +++++++++--------- src/test/regress/sql/failure_multi_dml.sql | 4 ++-- .../sql/failure_replicated_partitions.sql | 2 +- src/test/regress/sql/failure_single_select.sql | 18 +++++++++--------- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/test/regress/expected/failure_multi_dml.out b/src/test/regress/expected/failure_multi_dml.out index 7ca8a8f91..f4449587a 100644 --- a/src/test/regress/expected/failure_multi_dml.out +++ b/src/test/regress/expected/failure_multi_dml.out @@ -25,7 +25,7 @@ SELECT citus.clear_network_traffic(); ---- test multiple statements spanning multiple shards, ---- at each significant point. These transactions are 2pc -- fail at DELETE -SELECT citus.mitmproxy('conn.onQuery(query="^DELETE").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="DELETE").kill()'); mitmproxy --------------------------------------------------------------------- @@ -54,7 +54,7 @@ SELECT * FROM dml_test ORDER BY id ASC; (4 rows) -- cancel at DELETE -SELECT citus.mitmproxy('conn.onQuery(query="^DELETE").cancel(' || pg_backend_pid() || ')'); +SELECT citus.mitmproxy('conn.onQuery(query="DELETE").cancel(' || pg_backend_pid() || ')'); mitmproxy --------------------------------------------------------------------- diff --git a/src/test/regress/expected/failure_replicated_partitions.out b/src/test/regress/expected/failure_replicated_partitions.out index 4ae2d604c..7294df98b 100644 --- a/src/test/regress/expected/failure_replicated_partitions.out +++ b/src/test/regress/expected/failure_replicated_partitions.out @@ -21,7 +21,7 @@ CREATE TABLE partitioned_table_0 PARTITION OF partitioned_table (dist_key, partition_id) FOR VALUES IN ( 0 ); INSERT INTO partitioned_table VALUES (0, 0); -SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="INSERT").kill()'); mitmproxy --------------------------------------------------------------------- diff --git a/src/test/regress/expected/failure_single_select.out b/src/test/regress/expected/failure_single_select.out index 5d17cc4ad..1b60f3125 100644 --- a/src/test/regress/expected/failure_single_select.out +++ b/src/test/regress/expected/failure_single_select.out @@ -23,7 +23,7 @@ SELECT create_distributed_table('select_test', 'key'); -- put data in shard for which mitm node is first placement INSERT INTO select_test VALUES (3, 'test data'); -SELECT citus.mitmproxy('conn.onQuery(query="^SELECT.*select_test").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").kill()'); mitmproxy --------------------------------------------------------------------- @@ -45,7 +45,7 @@ WARNING: connection to the remote node localhost:xxxxx failed with the followin -- kill after first SELECT; txn should fail as INSERT triggers -- 2PC (and placementis not marked bad) -SELECT citus.mitmproxy('conn.onQuery(query="^SELECT.*select_test").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").kill()'); mitmproxy --------------------------------------------------------------------- @@ -66,7 +66,7 @@ TRUNCATE select_test; -- now the same tests with query cancellation -- put data in shard for which mitm node is first placement INSERT INTO select_test VALUES (3, 'test data'); -SELECT citus.mitmproxy('conn.onQuery(query="^SELECT.*select_test").cancel(' || pg_backend_pid() || ')'); +SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").cancel(' || pg_backend_pid() || ')'); mitmproxy --------------------------------------------------------------------- @@ -77,7 +77,7 @@ ERROR: canceling statement due to user request SELECT * FROM select_test WHERE key = 3; ERROR: canceling statement due to user request -- cancel after first SELECT; txn should fail and nothing should be marked as invalid -SELECT citus.mitmproxy('conn.onQuery(query="^SELECT.*select_test").cancel(' || pg_backend_pid() || ')'); +SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").cancel(' || pg_backend_pid() || ')'); mitmproxy --------------------------------------------------------------------- @@ -107,7 +107,7 @@ SELECT citus.mitmproxy('conn.allow()'); TRUNCATE select_test; -- cancel the second query -- error after second SELECT; txn should fail -SELECT citus.mitmproxy('conn.onQuery(query="^SELECT.*select_test").after(1).cancel(' || pg_backend_pid() || ')'); +SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").after(1).cancel(' || pg_backend_pid() || ')'); mitmproxy --------------------------------------------------------------------- @@ -126,7 +126,7 @@ SELECT * FROM select_test WHERE key = 3; ERROR: canceling statement due to user request COMMIT; -- error after second SELECT; txn should fails the transaction -SELECT citus.mitmproxy('conn.onQuery(query="^SELECT.*select_test").after(1).reset()'); +SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").after(1).reset()'); mitmproxy --------------------------------------------------------------------- @@ -144,7 +144,7 @@ INSERT INTO select_test VALUES (3, 'even more data'); SELECT * FROM select_test WHERE key = 3; ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open COMMIT; -SELECT citus.mitmproxy('conn.onQuery(query="^SELECT.*pg_prepared_xacts").after(2).kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*pg_prepared_xacts").after(2).kill()'); mitmproxy --------------------------------------------------------------------- @@ -173,7 +173,7 @@ SELECT create_distributed_table('select_test', 'key'); SET citus.max_cached_conns_per_worker TO 1; -- allow connection to be cached INSERT INTO select_test VALUES (1, 'test data'); -SELECT citus.mitmproxy('conn.onQuery(query="^SELECT.*select_test").after(1).kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").after(1).kill()'); mitmproxy --------------------------------------------------------------------- @@ -188,7 +188,7 @@ SELECT * FROM select_test WHERE key = 1; SELECT * FROM select_test WHERE key = 1; ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open -- now the same test with query cancellation -SELECT citus.mitmproxy('conn.onQuery(query="^SELECT.*select_test").after(1).cancel(' || pg_backend_pid() || ')'); +SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").after(1).cancel(' || pg_backend_pid() || ')'); mitmproxy --------------------------------------------------------------------- diff --git a/src/test/regress/sql/failure_multi_dml.sql b/src/test/regress/sql/failure_multi_dml.sql index 390c01461..9befd74c7 100644 --- a/src/test/regress/sql/failure_multi_dml.sql +++ b/src/test/regress/sql/failure_multi_dml.sql @@ -21,7 +21,7 @@ SELECT citus.clear_network_traffic(); ---- at each significant point. These transactions are 2pc -- fail at DELETE -SELECT citus.mitmproxy('conn.onQuery(query="^DELETE").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="DELETE").kill()'); BEGIN; DELETE FROM dml_test WHERE id = 1; @@ -35,7 +35,7 @@ COMMIT; SELECT * FROM dml_test ORDER BY id ASC; -- cancel at DELETE -SELECT citus.mitmproxy('conn.onQuery(query="^DELETE").cancel(' || pg_backend_pid() || ')'); +SELECT citus.mitmproxy('conn.onQuery(query="DELETE").cancel(' || pg_backend_pid() || ')'); BEGIN; DELETE FROM dml_test WHERE id = 1; diff --git a/src/test/regress/sql/failure_replicated_partitions.sql b/src/test/regress/sql/failure_replicated_partitions.sql index 1ea79fc83..fbe6ec7a0 100644 --- a/src/test/regress/sql/failure_replicated_partitions.sql +++ b/src/test/regress/sql/failure_replicated_partitions.sql @@ -19,7 +19,7 @@ CREATE TABLE partitioned_table_0 INSERT INTO partitioned_table VALUES (0, 0); -SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="INSERT").kill()'); INSERT INTO partitioned_table VALUES (0, 0); diff --git a/src/test/regress/sql/failure_single_select.sql b/src/test/regress/sql/failure_single_select.sql index 8dfb33d3e..c8218c950 100644 --- a/src/test/regress/sql/failure_single_select.sql +++ b/src/test/regress/sql/failure_single_select.sql @@ -13,13 +13,13 @@ SELECT create_distributed_table('select_test', 'key'); -- put data in shard for which mitm node is first placement INSERT INTO select_test VALUES (3, 'test data'); -SELECT citus.mitmproxy('conn.onQuery(query="^SELECT.*select_test").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").kill()'); SELECT * FROM select_test WHERE key = 3; SELECT * FROM select_test WHERE key = 3; -- kill after first SELECT; txn should fail as INSERT triggers -- 2PC (and placementis not marked bad) -SELECT citus.mitmproxy('conn.onQuery(query="^SELECT.*select_test").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").kill()'); BEGIN; INSERT INTO select_test VALUES (3, 'more data'); @@ -35,12 +35,12 @@ TRUNCATE select_test; -- put data in shard for which mitm node is first placement INSERT INTO select_test VALUES (3, 'test data'); -SELECT citus.mitmproxy('conn.onQuery(query="^SELECT.*select_test").cancel(' || pg_backend_pid() || ')'); +SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").cancel(' || pg_backend_pid() || ')'); SELECT * FROM select_test WHERE key = 3; SELECT * FROM select_test WHERE key = 3; -- cancel after first SELECT; txn should fail and nothing should be marked as invalid -SELECT citus.mitmproxy('conn.onQuery(query="^SELECT.*select_test").cancel(' || pg_backend_pid() || ')'); +SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").cancel(' || pg_backend_pid() || ')'); BEGIN; INSERT INTO select_test VALUES (3, 'more data'); @@ -58,7 +58,7 @@ TRUNCATE select_test; -- cancel the second query -- error after second SELECT; txn should fail -SELECT citus.mitmproxy('conn.onQuery(query="^SELECT.*select_test").after(1).cancel(' || pg_backend_pid() || ')'); +SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").after(1).cancel(' || pg_backend_pid() || ')'); BEGIN; INSERT INTO select_test VALUES (3, 'more data'); @@ -68,7 +68,7 @@ SELECT * FROM select_test WHERE key = 3; COMMIT; -- error after second SELECT; txn should fails the transaction -SELECT citus.mitmproxy('conn.onQuery(query="^SELECT.*select_test").after(1).reset()'); +SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").after(1).reset()'); BEGIN; INSERT INTO select_test VALUES (3, 'more data'); @@ -77,7 +77,7 @@ INSERT INTO select_test VALUES (3, 'even more data'); SELECT * FROM select_test WHERE key = 3; COMMIT; -SELECT citus.mitmproxy('conn.onQuery(query="^SELECT.*pg_prepared_xacts").after(2).kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*pg_prepared_xacts").after(2).kill()'); SELECT recover_prepared_transactions(); SELECT recover_prepared_transactions(); @@ -93,12 +93,12 @@ SELECT create_distributed_table('select_test', 'key'); SET citus.max_cached_conns_per_worker TO 1; -- allow connection to be cached INSERT INTO select_test VALUES (1, 'test data'); -SELECT citus.mitmproxy('conn.onQuery(query="^SELECT.*select_test").after(1).kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").after(1).kill()'); SELECT * FROM select_test WHERE key = 1; SELECT * FROM select_test WHERE key = 1; -- now the same test with query cancellation -SELECT citus.mitmproxy('conn.onQuery(query="^SELECT.*select_test").after(1).cancel(' || pg_backend_pid() || ')'); +SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").after(1).cancel(' || pg_backend_pid() || ')'); SELECT * FROM select_test WHERE key = 1; SELECT * FROM select_test WHERE key = 1; From 963712897b2a70e618da2e97199b2568a2e2bbeb Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 9 Mar 2023 11:54:57 +0300 Subject: [PATCH 03/14] failure 2 --- src/test/regress/expected/failure_multi_dml.out | 8 ++++---- src/test/regress/expected/failure_multi_row_insert.out | 2 +- src/test/regress/expected/failure_ref_tables.out | 6 +++--- src/test/regress/expected/failure_single_mod.out | 6 +++--- src/test/regress/sql/failure_multi_dml.sql | 8 ++++---- src/test/regress/sql/failure_multi_row_insert.sql | 2 +- src/test/regress/sql/failure_ref_tables.sql | 6 +++--- src/test/regress/sql/failure_single_mod.sql | 6 +++--- 8 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/test/regress/expected/failure_multi_dml.out b/src/test/regress/expected/failure_multi_dml.out index f4449587a..bbea2c999 100644 --- a/src/test/regress/expected/failure_multi_dml.out +++ b/src/test/regress/expected/failure_multi_dml.out @@ -83,7 +83,7 @@ SELECT * FROM dml_test ORDER BY id ASC; (4 rows) -- fail at INSERT -SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="INSERT").kill()'); mitmproxy --------------------------------------------------------------------- @@ -110,7 +110,7 @@ SELECT * FROM dml_test ORDER BY id ASC; (4 rows) -- cancel at INSERT -SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || pg_backend_pid() || ')'); +SELECT citus.mitmproxy('conn.onQuery(query="INSERT").cancel(' || pg_backend_pid() || ')'); mitmproxy --------------------------------------------------------------------- @@ -137,7 +137,7 @@ SELECT * FROM dml_test ORDER BY id ASC; (4 rows) -- fail at UPDATE -SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="UPDATE").kill()'); mitmproxy --------------------------------------------------------------------- @@ -163,7 +163,7 @@ SELECT * FROM dml_test ORDER BY id ASC; (4 rows) -- cancel at UPDATE -SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").cancel(' || pg_backend_pid() || ')'); +SELECT citus.mitmproxy('conn.onQuery(query="UPDATE").cancel(' || pg_backend_pid() || ')'); mitmproxy --------------------------------------------------------------------- diff --git a/src/test/regress/expected/failure_multi_row_insert.out b/src/test/regress/expected/failure_multi_row_insert.out index 8948be94e..f3cd4919a 100644 --- a/src/test/regress/expected/failure_multi_row_insert.out +++ b/src/test/regress/expected/failure_multi_row_insert.out @@ -36,7 +36,7 @@ SELECT create_reference_table('reference_table'); -- (d) multi-row INSERT that hits multiple shards in multiple workers -- (e) multi-row INSERT to a reference table -- Failure and cancellation on multi-row INSERT that hits the same shard with the same value -SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="INSERT").kill()'); mitmproxy --------------------------------------------------------------------- diff --git a/src/test/regress/expected/failure_ref_tables.out b/src/test/regress/expected/failure_ref_tables.out index 6485691af..4984cc1bf 100644 --- a/src/test/regress/expected/failure_ref_tables.out +++ b/src/test/regress/expected/failure_ref_tables.out @@ -26,7 +26,7 @@ SELECT COUNT(*) FROM ref_table; (1 row) -- verify behavior of single INSERT; should fail to execute -SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="INSERT").kill()'); mitmproxy --------------------------------------------------------------------- @@ -41,7 +41,7 @@ SELECT COUNT(*) FROM ref_table WHERE key=5; (1 row) -- verify behavior of UPDATE ... RETURNING; should not execute -SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="UPDATE").kill()'); mitmproxy --------------------------------------------------------------------- @@ -56,7 +56,7 @@ SELECT COUNT(*) FROM ref_table WHERE key=7; (1 row) -- verify fix to #2214; should raise error and fail to execute -SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="UPDATE").kill()'); mitmproxy --------------------------------------------------------------------- diff --git a/src/test/regress/expected/failure_single_mod.out b/src/test/regress/expected/failure_single_mod.out index 54db33ff6..2a6ed2d77 100644 --- a/src/test/regress/expected/failure_single_mod.out +++ b/src/test/regress/expected/failure_single_mod.out @@ -20,7 +20,7 @@ SELECT create_distributed_table('mod_test', 'key'); (1 row) -- verify behavior of single INSERT; should mark shard as failed -SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="INSERT").kill()'); mitmproxy --------------------------------------------------------------------- @@ -52,7 +52,7 @@ SELECT citus.mitmproxy('conn.allow()'); (1 row) INSERT INTO mod_test VALUES (2, 6); -SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="UPDATE").kill()'); mitmproxy --------------------------------------------------------------------- @@ -78,7 +78,7 @@ WHERE shardid IN ( TRUNCATE mod_test; -- verify behavior of multi-statement modifications to a single shard -- should fail the transaction and never mark placements inactive -SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="UPDATE").kill()'); mitmproxy --------------------------------------------------------------------- diff --git a/src/test/regress/sql/failure_multi_dml.sql b/src/test/regress/sql/failure_multi_dml.sql index 9befd74c7..f62ede4d5 100644 --- a/src/test/regress/sql/failure_multi_dml.sql +++ b/src/test/regress/sql/failure_multi_dml.sql @@ -49,7 +49,7 @@ COMMIT; SELECT * FROM dml_test ORDER BY id ASC; -- fail at INSERT -SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="INSERT").kill()'); BEGIN; DELETE FROM dml_test WHERE id = 1; @@ -63,7 +63,7 @@ COMMIT; SELECT * FROM dml_test ORDER BY id ASC; -- cancel at INSERT -SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || pg_backend_pid() || ')'); +SELECT citus.mitmproxy('conn.onQuery(query="INSERT").cancel(' || pg_backend_pid() || ')'); BEGIN; DELETE FROM dml_test WHERE id = 1; @@ -77,7 +77,7 @@ COMMIT; SELECT * FROM dml_test ORDER BY id ASC; -- fail at UPDATE -SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="UPDATE").kill()'); BEGIN; DELETE FROM dml_test WHERE id = 1; @@ -91,7 +91,7 @@ COMMIT; SELECT * FROM dml_test ORDER BY id ASC; -- cancel at UPDATE -SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").cancel(' || pg_backend_pid() || ')'); +SELECT citus.mitmproxy('conn.onQuery(query="UPDATE").cancel(' || pg_backend_pid() || ')'); BEGIN; DELETE FROM dml_test WHERE id = 1; diff --git a/src/test/regress/sql/failure_multi_row_insert.sql b/src/test/regress/sql/failure_multi_row_insert.sql index 53ab8a84d..cfc98f719 100644 --- a/src/test/regress/sql/failure_multi_row_insert.sql +++ b/src/test/regress/sql/failure_multi_row_insert.sql @@ -30,7 +30,7 @@ SELECT create_reference_table('reference_table'); -- Failure and cancellation on multi-row INSERT that hits the same shard with the same value -SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="INSERT").kill()'); INSERT INTO distributed_table VALUES (1,1), (1,2), (1,3); -- this test is broken, see https://github.com/citusdata/citus/issues/2460 diff --git a/src/test/regress/sql/failure_ref_tables.sql b/src/test/regress/sql/failure_ref_tables.sql index 0088a375e..29b90dc22 100644 --- a/src/test/regress/sql/failure_ref_tables.sql +++ b/src/test/regress/sql/failure_ref_tables.sql @@ -17,19 +17,19 @@ SELECT citus.clear_network_traffic(); SELECT COUNT(*) FROM ref_table; -- verify behavior of single INSERT; should fail to execute -SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="INSERT").kill()'); INSERT INTO ref_table VALUES (5, 6); SELECT COUNT(*) FROM ref_table WHERE key=5; -- verify behavior of UPDATE ... RETURNING; should not execute -SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="UPDATE").kill()'); UPDATE ref_table SET key=7 RETURNING value; SELECT COUNT(*) FROM ref_table WHERE key=7; -- verify fix to #2214; should raise error and fail to execute -SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="UPDATE").kill()'); BEGIN; DELETE FROM ref_table WHERE key=5; diff --git a/src/test/regress/sql/failure_single_mod.sql b/src/test/regress/sql/failure_single_mod.sql index e4dfc8f9f..48fdddcc6 100644 --- a/src/test/regress/sql/failure_single_mod.sql +++ b/src/test/regress/sql/failure_single_mod.sql @@ -8,7 +8,7 @@ CREATE TABLE mod_test (key int, value text); SELECT create_distributed_table('mod_test', 'key'); -- verify behavior of single INSERT; should mark shard as failed -SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="INSERT").kill()'); INSERT INTO mod_test VALUES (2, 6); SELECT COUNT(*) FROM mod_test WHERE key=2; @@ -24,7 +24,7 @@ TRUNCATE mod_test; SELECT citus.mitmproxy('conn.allow()'); INSERT INTO mod_test VALUES (2, 6); -SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="UPDATE").kill()'); UPDATE mod_test SET value='ok' WHERE key=2 RETURNING key; SELECT COUNT(*) FROM mod_test WHERE value='ok'; @@ -38,7 +38,7 @@ TRUNCATE mod_test; -- verify behavior of multi-statement modifications to a single shard -- should fail the transaction and never mark placements inactive -SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()'); +SELECT citus.mitmproxy('conn.onQuery(query="UPDATE").kill()'); BEGIN; INSERT INTO mod_test VALUES (2, 6); From e8b31058e7588f7d54a3fb01ac51cd5b60bad756 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 9 Mar 2023 12:12:28 +0300 Subject: [PATCH 04/14] open locks --- src/backend/distributed/utils/attribute.c | 45 +++++++++++++---------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/src/backend/distributed/utils/attribute.c b/src/backend/distributed/utils/attribute.c index 8eadcb3c8..ed276845f 100644 --- a/src/backend/distributed/utils/attribute.c +++ b/src/backend/distributed/utils/attribute.c @@ -93,10 +93,11 @@ citus_stats_tenants(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } - //!!!!!!!LWLockAcquire(&monitor->lock, LW_EXCLUSIVE); + LWLockAcquire(&monitor->lock, LW_EXCLUSIVE); monitor->periodStart = monitor->periodStart + - ((monitoringTime - monitor->periodStart) / CitusStatsTenantsPeriod) * + ((monitoringTime - monitor->periodStart) / + CitusStatsTenantsPeriod) * CitusStatsTenantsPeriod; int numberOfRowsToReturn = 0; @@ -132,7 +133,7 @@ citus_stats_tenants(PG_FUNCTION_ARGS) tuplestore_putvalues(tupleStore, tupleDescriptor, values, isNulls); } - //!!!!!!!LWLockRelease(&monitor->lock); + LWLockRelease(&monitor->lock); PG_RETURN_VOID(); } @@ -144,7 +145,7 @@ citus_stats_tenants(PG_FUNCTION_ARGS) void AttributeQueryIfAnnotated(const char *query_string, CmdType commandType) { -// attributeToTenant = NULL; +/* attributeToTenant = NULL; */ attributeCommandType = commandType; @@ -183,9 +184,10 @@ AttributeQueryIfAnnotated(const char *query_string, CmdType commandType) while (t) { colocationGroupId *= 10; - colocationGroupId += t%10; - t/=10; + colocationGroupId += t % 10; + t /= 10; } + /* hack to get a clean copy of the tenant id string */ char tenantEndTmp = *tenantEnd; *tenantEnd = '\0'; @@ -198,12 +200,12 @@ AttributeQueryIfAnnotated(const char *query_string, CmdType commandType) quote_literal_cstr(tenantId)))); } - // attributeToTenant = (char *) malloc(strlen(tenantId)); + /* attributeToTenant = (char *) malloc(strlen(tenantId)); */ strcpy(attributeToTenant, tenantId); } else { - //Assert(attributeToTenant == NULL); + /*Assert(attributeToTenant == NULL); */ } /*DetachSegment(); */ @@ -275,10 +277,11 @@ AttributeMetricsIfApplicable() MultiTenantMonitor *monitor = GetMultiTenantMonitor(); - //!!!!!!!LWLockAcquire(&monitor->lock, LW_SHARED); + LWLockAcquire(&monitor->lock, LW_SHARED); monitor->periodStart = monitor->periodStart + - ((queryTime - monitor->periodStart) / CitusStatsTenantsPeriod) * + ((queryTime - monitor->periodStart) / + CitusStatsTenantsPeriod) * CitusStatsTenantsPeriod; int tenantIndex = FindTenantStats(monitor); @@ -289,7 +292,7 @@ AttributeMetricsIfApplicable() } TenantStats *tenantStats = &monitor->tenants[tenantIndex]; - //!!!!!!!LWLockAcquire(&tenantStats->lock, LW_EXCLUSIVE); + LWLockAcquire(&tenantStats->lock, LW_EXCLUSIVE); UpdatePeriodsIfNecessary(monitor, tenantStats); tenantStats->lastQueryTime = queryTime; @@ -308,15 +311,16 @@ AttributeMetricsIfApplicable() while (tenantIndex != 0 && monitor->tenants[tenantIndex - 1].score < tenantStats->score) { - //!!!!!!!LWLockAcquire(&monitor->tenants[tenantIndex - 1].lock, LW_EXCLUSIVE); + LWLockAcquire(&monitor->tenants[tenantIndex - 1].lock, LW_EXCLUSIVE); - ReduceScoreIfNecessary(monitor, &monitor->tenants[tenantIndex - 1], queryTime); + ReduceScoreIfNecessary(monitor, &monitor->tenants[tenantIndex - 1], + queryTime); TenantStats tempTenant = monitor->tenants[tenantIndex]; monitor->tenants[tenantIndex] = monitor->tenants[tenantIndex - 1]; monitor->tenants[tenantIndex - 1] = tempTenant; - //!!!!!!!LWLockRelease(&monitor->tenants[tenantIndex - 1].lock); + LWLockRelease(&monitor->tenants[tenantIndex - 1].lock); tenantIndex--; } @@ -334,8 +338,8 @@ AttributeMetricsIfApplicable() tenantStats->totalInsertTime += cpu_time_used; } - //!!!!!!!LWLockRelease(&tenantStats->lock); - //!!!!!!!LWLockRelease(&monitor->lock); + LWLockRelease(&tenantStats->lock); + LWLockRelease(&monitor->lock); /* * We keep up to CitusStatsTenantsLimit * 3 tenants instead of CitusStatsTenantsLimit, @@ -345,19 +349,20 @@ AttributeMetricsIfApplicable() */ if (monitor->tenantCount >= CitusStatsTenantsLimit * 3) { - //!!!!!!!LWLockAcquire(&monitor->lock, LW_EXCLUSIVE); + LWLockAcquire(&monitor->lock, LW_EXCLUSIVE); monitor->tenantCount = CitusStatsTenantsLimit * 2; - //!!!!!!!LWLockRelease(&monitor->lock); + LWLockRelease(&monitor->lock); } if (MultiTenantMonitoringLogLevel != CITUS_LOG_LEVEL_OFF) { - ereport(NOTICE, (errmsg("total select count = %d, total CPU time = %f to tenant: %s", + ereport(NOTICE, (errmsg("total select count = %d, total CPU time = %f " + "to tenant: %s", tenantStats->selectCount, tenantStats->totalSelectTime, tenantStats->tenantAttribute))); } } - //attributeToTenant = NULL; + /*attributeToTenant = NULL; */ } From e9017b085bb16aebcc2c274cfd4018e4f0bea811 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 9 Mar 2023 12:16:00 +0300 Subject: [PATCH 05/14] indent --- src/backend/distributed/utils/attribute.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/utils/attribute.c b/src/backend/distributed/utils/attribute.c index ed276845f..bc7a3f7cd 100644 --- a/src/backend/distributed/utils/attribute.c +++ b/src/backend/distributed/utils/attribute.c @@ -358,10 +358,12 @@ AttributeMetricsIfApplicable() { ereport(NOTICE, (errmsg("total select count = %d, total CPU time = %f " "to tenant: %s", - tenantStats->selectCount, tenantStats->totalSelectTime, - tenantStats->tenantAttribute))); + tenantStats->selectCount, + tenantStats->totalSelectTime, + tenantStats->tenantAttribute))); } } + /*attributeToTenant = NULL; */ } From 6ea9d0097195a981d507ac414d5855c85adcc44d Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 9 Mar 2023 12:19:29 +0300 Subject: [PATCH 06/14] new line --- src/test/regress/bin/normalize.sed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index ea70ed41c..a6072945a 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -304,4 +304,4 @@ s/(NOTICE: issuing SET LOCAL application_name TO 'citus_rebalancer gpid=)[0-9]+ # shard_rebalancer output, flaky improvement number s/improvement of 0.1[0-9]* is lower/improvement of 0.1xxxxx is lower/g -s/\/\* attributeTo.*\*\///g \ No newline at end of file +s/\/\* attributeTo.*\*\///g From 2b55ecc167ef0a7dd5a2968a989b57b1cc5ca2f8 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 9 Mar 2023 12:22:04 +0300 Subject: [PATCH 07/14] declaration fix --- src/backend/distributed/utils/attribute.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/backend/distributed/utils/attribute.c b/src/backend/distributed/utils/attribute.c index bc7a3f7cd..0563ea6ed 100644 --- a/src/backend/distributed/utils/attribute.c +++ b/src/backend/distributed/utils/attribute.c @@ -263,11 +263,10 @@ AttributeMetricsIfApplicable() if (strcmp(attributeToTenant, "") != 0) { clock_t end = { 0 }; - double cpu_time_used = 0; end = clock(); time_t queryTime = time(0); - cpu_time_used = ((double) (end - attributeToTenantStart)) / CLOCKS_PER_SEC; + double cpu_time_used = ((double) (end - attributeToTenantStart)) / CLOCKS_PER_SEC; if (MultiTenantMonitoringLogLevel != CITUS_LOG_LEVEL_OFF) { From c1a67c9acad41bf29768a99de55c8c9ec6dee746 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Fri, 10 Mar 2023 11:00:09 +0300 Subject: [PATCH 08/14] Fix lock --- src/backend/distributed/utils/attribute.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/backend/distributed/utils/attribute.c b/src/backend/distributed/utils/attribute.c index 0563ea6ed..72d4d5058 100644 --- a/src/backend/distributed/utils/attribute.c +++ b/src/backend/distributed/utils/attribute.c @@ -42,6 +42,9 @@ clock_t attributeToTenantStart = { 0 }; const char *SharedMemoryNameForMultiTenantMonitor = "Shared memory for multi tenant monitor"; +char *tenantTrancheName = "Tenant Tranche"; +char *monitorTrancheName = "Multi Tenant Monitor Tranche"; + static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static void UpdatePeriodsIfNecessary(MultiTenantMonitor *monitor, @@ -319,9 +322,11 @@ AttributeMetricsIfApplicable() monitor->tenants[tenantIndex] = monitor->tenants[tenantIndex - 1]; monitor->tenants[tenantIndex - 1] = tempTenant; - LWLockRelease(&monitor->tenants[tenantIndex - 1].lock); + LWLockRelease(&monitor->tenants[tenantIndex].lock); tenantIndex--; + + tenantStats = &monitor->tenants[tenantIndex]; } if (attributeCommandType == CMD_SELECT) @@ -337,8 +342,8 @@ AttributeMetricsIfApplicable() tenantStats->totalInsertTime += cpu_time_used; } - LWLockRelease(&tenantStats->lock); LWLockRelease(&monitor->lock); + LWLockRelease(&tenantStats->lock); /* * We keep up to CitusStatsTenantsLimit * 3 tenants instead of CitusStatsTenantsLimit, @@ -474,11 +479,10 @@ CreateSharedMemoryForMultiTenantMonitor() return monitor; } - char *trancheName = "Multi Tenant Monitor Tranche"; - monitor->namedLockTranche.trancheId = LWLockNewTrancheId(); + monitor->namedLockTranche.trancheName = monitorTrancheName; - LWLockRegisterTranche(monitor->namedLockTranche.trancheId, trancheName); + LWLockRegisterTranche(monitor->namedLockTranche.trancheId, monitor->namedLockTranche.trancheName); LWLockInitialize(&monitor->lock, monitor->namedLockTranche.trancheId); return monitor; @@ -547,12 +551,11 @@ CreateTenantStats(MultiTenantMonitor *monitor) strcpy(monitor->tenants[tenantIndex].tenantAttribute, attributeToTenant); monitor->tenants[tenantIndex].colocationGroupId = colocationGroupId; - char *trancheName = "Tenant Tranche"; - monitor->tenants[tenantIndex].namedLockTranche.trancheId = LWLockNewTrancheId(); + monitor->tenants[tenantIndex].namedLockTranche.trancheName = tenantTrancheName; LWLockRegisterTranche(monitor->tenants[tenantIndex].namedLockTranche.trancheId, - trancheName); + monitor->tenants[tenantIndex].namedLockTranche.trancheName); LWLockInitialize(&monitor->tenants[tenantIndex].lock, monitor->tenants[tenantIndex].namedLockTranche.trancheId); From ba32f827a9b9b71126578dd6c32c01a93202e3f8 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Fri, 10 Mar 2023 11:14:05 +0300 Subject: [PATCH 09/14] fix --- src/backend/distributed/utils/attribute.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/backend/distributed/utils/attribute.c b/src/backend/distributed/utils/attribute.c index 72d4d5058..40d76fade 100644 --- a/src/backend/distributed/utils/attribute.c +++ b/src/backend/distributed/utils/attribute.c @@ -311,7 +311,7 @@ AttributeMetricsIfApplicable() * After updating the score we might need to change the rank of the tenant in the monitor */ while (tenantIndex != 0 && - monitor->tenants[tenantIndex - 1].score < tenantStats->score) + monitor->tenants[tenantIndex - 1].score < monitor->tenants[tenantIndex]->score) { LWLockAcquire(&monitor->tenants[tenantIndex - 1].lock, LW_EXCLUSIVE); @@ -325,9 +325,8 @@ AttributeMetricsIfApplicable() LWLockRelease(&monitor->tenants[tenantIndex].lock); tenantIndex--; - - tenantStats = &monitor->tenants[tenantIndex]; } + tenantStats = &monitor->tenants[tenantIndex]; if (attributeCommandType == CMD_SELECT) { From 124960697282ff9330580e82763bcae76ae72edd Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Fri, 10 Mar 2023 11:15:27 +0300 Subject: [PATCH 10/14] fix indent --- src/backend/distributed/utils/attribute.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/utils/attribute.c b/src/backend/distributed/utils/attribute.c index 40d76fade..4ec8decaa 100644 --- a/src/backend/distributed/utils/attribute.c +++ b/src/backend/distributed/utils/attribute.c @@ -311,7 +311,8 @@ AttributeMetricsIfApplicable() * After updating the score we might need to change the rank of the tenant in the monitor */ while (tenantIndex != 0 && - monitor->tenants[tenantIndex - 1].score < monitor->tenants[tenantIndex]->score) + monitor->tenants[tenantIndex - 1].score < + monitor->tenants[tenantIndex]->score) { LWLockAcquire(&monitor->tenants[tenantIndex - 1].lock, LW_EXCLUSIVE); @@ -481,7 +482,8 @@ CreateSharedMemoryForMultiTenantMonitor() monitor->namedLockTranche.trancheId = LWLockNewTrancheId(); monitor->namedLockTranche.trancheName = monitorTrancheName; - LWLockRegisterTranche(monitor->namedLockTranche.trancheId, monitor->namedLockTranche.trancheName); + LWLockRegisterTranche(monitor->namedLockTranche.trancheId, + monitor->namedLockTranche.trancheName); LWLockInitialize(&monitor->lock, monitor->namedLockTranche.trancheId); return monitor; From e32a24e94a0c81550f44029320f1cb1b442fb87e Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Fri, 10 Mar 2023 11:20:44 +0300 Subject: [PATCH 11/14] fix --- src/backend/distributed/utils/attribute.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/utils/attribute.c b/src/backend/distributed/utils/attribute.c index 4ec8decaa..34791a81a 100644 --- a/src/backend/distributed/utils/attribute.c +++ b/src/backend/distributed/utils/attribute.c @@ -312,7 +312,7 @@ AttributeMetricsIfApplicable() */ while (tenantIndex != 0 && monitor->tenants[tenantIndex - 1].score < - monitor->tenants[tenantIndex]->score) + monitor->tenants[tenantIndex].score) { LWLockAcquire(&monitor->tenants[tenantIndex - 1].lock, LW_EXCLUSIVE); From 4289d4a981aa992f108860a7977e454dac498a25 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Fri, 10 Mar 2023 12:20:15 +0300 Subject: [PATCH 12/14] safe functions --- src/backend/distributed/utils/attribute.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/utils/attribute.c b/src/backend/distributed/utils/attribute.c index 34791a81a..54b2d5b22 100644 --- a/src/backend/distributed/utils/attribute.c +++ b/src/backend/distributed/utils/attribute.c @@ -11,6 +11,7 @@ #include "postgres.h" #include "unistd.h" +#include "distributed/citus_safe_lib.h" #include "distributed/log_utils.h" #include "distributed/listutils.h" #include "distributed/tuplestore.h" @@ -204,7 +205,7 @@ AttributeQueryIfAnnotated(const char *query_string, CmdType commandType) } /* attributeToTenant = (char *) malloc(strlen(tenantId)); */ - strcpy(attributeToTenant, tenantId); + strcpy_s(attributeToTenant, tenantId); } else { @@ -549,7 +550,7 @@ CreateTenantStats(MultiTenantMonitor *monitor) { int tenantIndex = monitor->tenantCount; - strcpy(monitor->tenants[tenantIndex].tenantAttribute, attributeToTenant); + strcpy_s(monitor->tenants[tenantIndex].tenantAttribute, attributeToTenant); monitor->tenants[tenantIndex].colocationGroupId = colocationGroupId; monitor->tenants[tenantIndex].namedLockTranche.trancheId = LWLockNewTrancheId(); From e8d516f0f0fe989cc623de7c339762c6140e55bf Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Fri, 10 Mar 2023 12:32:50 +0300 Subject: [PATCH 13/14] fix strcpy_s --- src/backend/distributed/utils/attribute.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/utils/attribute.c b/src/backend/distributed/utils/attribute.c index 54b2d5b22..274a4f709 100644 --- a/src/backend/distributed/utils/attribute.c +++ b/src/backend/distributed/utils/attribute.c @@ -205,7 +205,7 @@ AttributeQueryIfAnnotated(const char *query_string, CmdType commandType) } /* attributeToTenant = (char *) malloc(strlen(tenantId)); */ - strcpy_s(attributeToTenant, tenantId); + strcpy_s(attributeToTenant, sizeof(attributeToTenant), tenantId); } else { @@ -550,7 +550,7 @@ CreateTenantStats(MultiTenantMonitor *monitor) { int tenantIndex = monitor->tenantCount; - strcpy_s(monitor->tenants[tenantIndex].tenantAttribute, attributeToTenant); + strcpy_s(monitor->tenants[tenantIndex].tenantAttribute, sizeof(monitor->tenants[tenantIndex].tenantAttribute), attributeToTenant); monitor->tenants[tenantIndex].colocationGroupId = colocationGroupId; monitor->tenants[tenantIndex].namedLockTranche.trancheId = LWLockNewTrancheId(); From fa181a8bfdb8fcbe869e8600d4a9b7c00fb2b615 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Fri, 10 Mar 2023 12:38:56 +0300 Subject: [PATCH 14/14] fix indent --- src/backend/distributed/utils/attribute.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/utils/attribute.c b/src/backend/distributed/utils/attribute.c index 274a4f709..074c2385e 100644 --- a/src/backend/distributed/utils/attribute.c +++ b/src/backend/distributed/utils/attribute.c @@ -550,7 +550,8 @@ CreateTenantStats(MultiTenantMonitor *monitor) { int tenantIndex = monitor->tenantCount; - strcpy_s(monitor->tenants[tenantIndex].tenantAttribute, sizeof(monitor->tenants[tenantIndex].tenantAttribute), attributeToTenant); + strcpy_s(monitor->tenants[tenantIndex].tenantAttribute, + sizeof(monitor->tenants[tenantIndex].tenantAttribute), attributeToTenant); monitor->tenants[tenantIndex].colocationGroupId = colocationGroupId; monitor->tenants[tenantIndex].namedLockTranche.trancheId = LWLockNewTrancheId();