update rounding errors on latest master

feature/batch-add-tdigest
Nils Dijk 2021-06-01 16:27:18 +02:00
parent 5dda26f332
commit 572899b359
No known key found for this signature in database
GPG Key ID: CA1177EF9434F241
1 changed files with 12 additions and 12 deletions

View File

@ -369,31 +369,31 @@ GROUP BY b;
SELECT tdigest(latency, 100) FROM latencies;
tdigest
---------------------------------------------------------------------
flags 0 count 10000 compression 100 centroids 46 (0.287235, 1) (1.025106, 1) (2.058216, 1) (5.335597, 1) (12.707263, 2) (25.302479, 3) (43.435063, 4) (77.987860, 5) (269.478664, 10) (509.417419, 13) (1227.158879, 22) (3408.256171, 35) (7772.721988, 55) (13840.275516, 65) (32937.127607, 108) (64476.403332, 148) (118260.230644, 199) (239584.293240, 292) (562119.836766, 463) (944722.686313, 547) (1751089.620493, 749) (3751264.745959, 1128) (5877270.108576, 1300) (6224557.402567, 1104) (5804999.258033, 874) (5632316.697114, 755) (4648651.050740, 573) (3460055.227950, 402) (2820271.404686, 314) (2676501.012955, 288) (1649845.166017, 173) (1269335.942008, 131) (813964.853243, 83) (484144.878702, 49) (337179.763016, 34) (198775.241901, 20) (149353.499704, 15) (109688.319223, 11) (79855.926155, 8) (49937.731689, 5) (29971.046175, 3) (19982.538737, 2) (9991.467422, 1) (9992.337047, 1) (9995.578357, 1) (9999.700339, 1)
flags 1 count 10000 compression 100 centroids 46 (0.287235, 1) (1.025106, 1) (2.058216, 1) (5.335597, 1) (6.353632, 2) (8.434160, 3) (10.858766, 4) (15.597572, 5) (26.947867, 10) (39.185955, 13) (55.779949, 22) (97.378748, 35) (141.322218, 55) (212.927315, 65) (304.973404, 108) (435.651374, 148) (594.272516, 199) (820.494155, 292) (1214.081721, 463) (1727.098147, 547) (2337.903365, 749) (3325.589314, 1128) (4520.977007, 1300) (5638.186053, 1104) (6641.875582, 874) (7460.022116, 755) (8112.829059, 573) (8607.102557, 402) (8981.756066, 314) (9293.406295, 288) (9536.677260, 173) (9689.587343, 131) (9806.805461, 83) (9880.507729, 49) (9917.051853, 34) (9938.762095, 20) (9956.899980, 15) (9971.665384, 11) (9981.990769, 8) (9987.546338, 5) (9990.348725, 3) (9991.269368, 2) (9991.467422, 1) (9992.337047, 1) (9995.578357, 1) (9999.700339, 1)
(1 row)
SELECT tdigest_percentile(latency, 100, 0.99) FROM latencies;
tdigest_percentile
---------------------------------------------------------------------
9903.84313360471
9903.84313364562
(1 row)
SELECT tdigest_percentile(latency, 100, ARRAY[0.99, 0.95]) FROM latencies;
tdigest_percentile
---------------------------------------------------------------------
{9903.84313360471,9485.49009399385}
{9903.84313364562,9485.4900941881}
(1 row)
SELECT tdigest_percentile_of(latency, 100, 9000) FROM latencies;
tdigest_percentile_of
---------------------------------------------------------------------
0.903462047211138
0.90346204719982
(1 row)
SELECT tdigest_percentile_of(latency, 100, ARRAY[9000, 9500]) FROM latencies;
tdigest_percentile_of
---------------------------------------------------------------------
{0.903462047211138,0.95137481812975}
{0.90346204719982,0.951374818110352}
(1 row)
CREATE TABLE latencies_rollup (a int, tdigest tdigest);
@ -620,31 +620,31 @@ GROUP BY a;
SELECT tdigest(tdigest) FROM latencies_rollup;
tdigest
---------------------------------------------------------------------
flags 0 count 10000 compression 100 centroids 47 (0.287235, 1) (1.025106, 1) (2.058216, 1) (5.335597, 1) (12.707263, 2) (25.302479, 3) (43.435063, 4) (77.987860, 5) (241.681030, 9) (402.696604, 11) (999.675875, 20) (2310.848640, 27) (4374.387978, 37) (9722.896547, 56) (21713.805492, 87) (39735.065966, 112) (87335.860853, 177) (182744.906162, 262) (336766.886786, 338) (661263.339724, 464) (1228663.222377, 623) (2146097.038498, 805) (2854487.701653, 827) (5292830.156590, 1195) (6168185.834602, 1104) (6399734.303813, 966) (5778088.854724, 773) (5213381.984997, 637) (3763042.148296, 431) (3036786.646485, 333) (1948238.134602, 207) (1456568.605821, 152) (999888.715345, 103) (715935.892988, 73) (543464.906535, 55) (327339.982973, 33) (198853.838033, 20) (159362.743852, 16) (79807.827301, 8) (69877.414438, 7) (49937.731689, 5) (29971.046175, 3) (19982.538737, 2) (9991.467422, 1) (9992.337047, 1) (9995.578357, 1) (9999.700339, 1)
flags 1 count 10000 compression 100 centroids 47 (0.287235, 1) (1.025106, 1) (2.058216, 1) (5.335597, 1) (6.353632, 2) (8.434160, 3) (10.858766, 4) (15.597572, 5) (26.853448, 9) (36.608782, 11) (49.983794, 20) (85.586987, 27) (118.226702, 37) (173.623153, 56) (249.583971, 87) (354.777375, 112) (493.422943, 177) (697.499642, 262) (996.351736, 338) (1425.136508, 464) (1972.172106, 623) (2665.959054, 805) (3451.617535, 827) (4429.146574, 1195) (5587.124850, 1104) (6624.983751, 966) (7474.888557, 773) (8184.273132, 637) (8730.956261, 431) (9119.479419, 333) (9411.778428, 207) (9582.688196, 152) (9707.657430, 103) (9807.341000, 73) (9881.180119, 55) (9919.393423, 33) (9942.691902, 20) (9960.171491, 16) (9975.978413, 8) (9982.487777, 7) (9987.546338, 5) (9990.348725, 3) (9991.269368, 2) (9991.467422, 1) (9992.337047, 1) (9995.578357, 1) (9999.700339, 1)
(1 row)
SELECT tdigest_percentile(tdigest, 0.99) FROM latencies_rollup;
tdigest_percentile
---------------------------------------------------------------------
9903.32646580579
9903.32646566529
(1 row)
SELECT tdigest_percentile(tdigest, ARRAY[0.99, 0.95]) FROM latencies_rollup;
tdigest_percentile
---------------------------------------------------------------------
{9903.32646580579,9492.23455844499}
{9903.32646566529,9492.23455830869}
(1 row)
SELECT tdigest_percentile_of(tdigest, 9000) FROM latencies_rollup;
tdigest_percentile_of
---------------------------------------------------------------------
0.902852659582396
0.902852659584438
(1 row)
SELECT tdigest_percentile_of(tdigest, ARRAY[9000, 9500]) FROM latencies_rollup;
tdigest_percentile_of
---------------------------------------------------------------------
{0.902852659582396,0.950865574659141}
{0.902852659584438,0.950865574671753}
(1 row)
SET client_min_messages TO WARNING; -- suppress cascade messages