From 193ebcd603b321ec952f3ef43130282ab3c699bf Mon Sep 17 00:00:00 2001 From: James Coleman Date: Fri, 27 Mar 2020 10:39:04 -0400 Subject: [PATCH v42 08/12] explain output munging and bug fixes --- src/backend/commands/explain.c | 22 +- .../regress/expected/incremental_sort.out | 320 ++++++++++-------- src/test/regress/sql/incremental_sort.sql | 118 ++++++- 3 files changed, 311 insertions(+), 149 deletions(-) diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 56f8e1fd21..39d51848b6 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -2784,26 +2784,38 @@ show_incremental_sort_group_info(IncrementalSortGroupInfo *groupInfo, { long avgSpace = groupInfo->totalMemorySpaceUsed / groupInfo->groupCount; const char *spaceTypeName; + StringInfoData memoryName; + + spaceTypeName = tuplesort_space_type_name(SORT_SPACE_TYPE_MEMORY); + initStringInfo(&memoryName); + appendStringInfo(&memoryName, "Sort Space %s", spaceTypeName); + ExplainOpenGroup("Sort Space", memoryName.data, true, es); ExplainPropertyInteger("Average Sort Space Used", "kB", avgSpace, es); ExplainPropertyInteger("Maximum Sort Space Used", "kB", groupInfo->maxMemorySpaceUsed, es); - spaceTypeName = tuplesort_space_type_name(SORT_SPACE_TYPE_MEMORY); - ExplainPropertyText("Sort Space Type", spaceTypeName, es); + + ExplainCloseGroup("Sort Spaces", memoryName.data, true, es); } if (groupInfo->maxDiskSpaceUsed > 0) { long avgSpace = groupInfo->totalDiskSpaceUsed / groupInfo->groupCount; const char *spaceTypeName; + StringInfoData diskName; + + spaceTypeName = tuplesort_space_type_name(SORT_SPACE_TYPE_DISK); + initStringInfo(&diskName); + appendStringInfo(&diskName, "Sort Space %s", spaceTypeName); + ExplainOpenGroup("Sort Space", diskName.data, true, es); ExplainPropertyInteger("Average Sort Space Used", "kB", avgSpace, es); ExplainPropertyInteger("Maximum Sort Space Used", "kB", groupInfo->maxDiskSpaceUsed, es); - spaceTypeName = tuplesort_space_type_name(SORT_SPACE_TYPE_MEMORY); - ExplainPropertyText("Sort Space Type", spaceTypeName, es); + + ExplainCloseGroup("Sort Spaces", diskName.data, true, es); } - ExplainCloseGroup("Incremental Sort Groups", "XXX Groups", true, es); + ExplainCloseGroup("Incremental Sort Groups", groupName.data, true, es); } } diff --git a/src/test/regress/expected/incremental_sort.out b/src/test/regress/expected/incremental_sort.out index 65604b3429..ebb8412237 100644 --- a/src/test/regress/expected/incremental_sort.out +++ b/src/test/regress/expected/incremental_sort.out @@ -44,6 +44,102 @@ select * from (select * from tenk1 order by four) t order by four, ten; reset work_mem; create table t(a integer, b integer); +create or replace function explain_analyze_without_memory(query text) +returns table (out_line text) language plpgsql +as +$$ +declare + line text; +begin + for line in + execute 'explain (analyze, costs off, summary off, timing off) ' || query + loop + out_line := regexp_replace(line, '\d+kB', 'NNkB', 'g'); + return next; + end loop; +end; +$$; +create or replace function explain_analyze_inc_sort_nodes(query text) +returns jsonb language plpgsql +as +$$ +declare + elements jsonb; + element jsonb; + matching_nodes jsonb := '[]'::jsonb; +begin + execute 'explain (analyze, costs off, summary off, timing off, format ''json'') ' || query into strict elements; + while jsonb_array_length(elements) > 0 loop + element := elements->0; + elements := elements - 0; + case jsonb_typeof(element) + when 'array' then + if jsonb_array_length(element) > 0 then + elements := elements || element; + end if; + when 'object' then + if element ? 'Plan' then + elements := elements || jsonb_build_array(element->'Plan'); + element := element - 'Plan'; + else + if element ? 'Plans' then + elements := elements || jsonb_build_array(element->'Plans'); + element := element - 'Plans'; + end if; + if (element->>'Node Type')::text = 'Incremental Sort' then + matching_nodes := matching_nodes || element; + end if; + end if; + end case; + end loop; + return matching_nodes; +end; +$$; +create or replace function explain_analyze_inc_sort_nodes_without_memory(query text) +returns jsonb language plpgsql +as +$$ +declare + nodes jsonb := '[]'::jsonb; + node jsonb; + group_key text; + space_key text; +begin + for node in select * from jsonb_array_elements(explain_analyze_inc_sort_nodes(query)) t loop + for group_key in select unnest(array['Full-sort Groups', 'Presorted Groups']::text[]) t loop + for space_key in select unnest(array['Sort Space Memory', 'Sort Space Disk']::text[]) t loop + node := jsonb_set(node, array[group_key, space_key, 'Average Sort Space Used'], '"NN"', false); + node := jsonb_set(node, array[group_key, space_key, 'Maximum Sort Space Used'], '"NN"', false); + end loop; + end loop; + nodes := nodes || node; + end loop; + return nodes; +end; +$$; +create or replace function explain_analyze_inc_sort_nodes_verify_invariants(query text) +returns bool language plpgsql +as +$$ +declare + node jsonb; + group_stats jsonb; + group_key text; + space_key text; +begin + for node in select * from jsonb_array_elements(explain_analyze_inc_sort_nodes(query)) t loop + for group_key in select unnest(array['Full-sort Groups', 'Presorted Groups']::text[]) t loop + group_stats := node->group_key; + for space_key in select unnest(array['Sort Space Memory', 'Sort Space Disk']::text[]) t loop + if (group_stats->space_key->'Maximum Sort Space Used')::bigint < (group_stats->space_key->'Maximum Sort Space Used')::bigint then + raise exception '% has invalid max space < average space', group_key; + end if; + end loop; + end loop; + end loop; + return true; +end; +$$; -- A single large group tested around each mode transition point. insert into t(a, b) select 1, i from generate_series(1, 100) n(i); explain (costs off) select * from (select * from t order by a) s order by a, b limit 31; @@ -433,82 +529,59 @@ select * from (select * from t order by a) s order by a, b limit 55; 2 | 55 (55 rows) --- Test EXPLAIN ANALYZE (text output) with only a fullsort group. -explain (analyze, costs off, summary off, timing off) -select * from (select * from t order by a) s order by a, b limit 55; - QUERY PLAN +-- Test EXPLAIN ANALYZE with only a fullsort group. +select explain_analyze_without_memory('select * from (select * from t order by a) s order by a, b limit 55'); + explain_analyze_without_memory ------------------------------------------------------------------------------------------------- Limit (actual rows=55 loops=1) -> Incremental Sort (actual rows=55 loops=1) Sort Key: t.a, t.b Presorted Key: t.a - Full-sort Groups: 2 (Methods: quicksort, top-N heapsort) Memory: 27kB (avg), 27kB (max) + Full-sort Groups: 2 (Methods: quicksort, top-N heapsort) Memory: NNkB (avg), NNkB (max) -> Sort (actual rows=100 loops=1) Sort Key: t.a - Sort Method: quicksort Memory: 30kB + Sort Method: quicksort Memory: NNkB -> Seq Scan on t (actual rows=100 loops=1) (9 rows) -explain (analyze, costs off, summary off, timing off, format json) -select * from (select * from t order by a) s order by a, b limit 55; - QUERY PLAN -------------------------------------------------------------------- - [ + - { + - "Plan": { + - "Node Type": "Limit", + - "Parallel Aware": false, + - "Actual Rows": 55, + - "Actual Loops": 1, + - "Plans": [ + - { + - "Node Type": "Incremental Sort", + - "Parent Relationship": "Outer", + - "Parallel Aware": false, + - "Actual Rows": 55, + - "Actual Loops": 1, + - "Sort Key": ["t.a", "t.b"], + - "Presorted Key": ["t.a"], + - "Full-sort Groups": { + - "Group Count": 2, + - "Sort Methods Used": ["quicksort", "top-N heapsort"],+ - "Average Sort Space Used": 27, + - "Maximum Sort Space Used": 27, + - "Sort Space Type": "Memory" + - }, + - "Plans": [ + - { + - "Node Type": "Sort", + - "Parent Relationship": "Outer", + - "Parallel Aware": false, + - "Actual Rows": 100, + - "Actual Loops": 1, + - "Sort Key": ["t.a"], + - "Sort Method": "quicksort", + - "Sort Space Used": 30, + - "Sort Space Type": "Memory", + - "Plans": [ + - { + - "Node Type": "Seq Scan", + - "Parent Relationship": "Outer", + - "Parallel Aware": false, + - "Relation Name": "t", + - "Alias": "t", + - "Actual Rows": 100, + - "Actual Loops": 1 + - } + - ] + - } + - ] + - } + - ] + - }, + - "Triggers": [ + - ] + - } + +select jsonb_pretty(explain_analyze_inc_sort_nodes_without_memory('select * from (select * from t order by a) s order by a, b limit 55')); + jsonb_pretty +-------------------------------------------------- + [ + + { + + "Sort Key": [ + + "t.a", + + "t.b" + + ], + + "Node Type": "Incremental Sort", + + "Actual Rows": 55, + + "Actual Loops": 1, + + "Presorted Key": [ + + "t.a" + + ], + + "Parallel Aware": false, + + "Full-sort Groups": { + + "Group Count": 2, + + "Sort Methods Used": [ + + "quicksort", + + "top-N heapsort" + + ], + + "Sort Space Memory": { + + "Average Sort Space Used": "NN",+ + "Maximum Sort Space Used": "NN" + + } + + }, + + "Parent Relationship": "Outer" + + } + ] (1 row) +select explain_analyze_inc_sort_nodes_verify_invariants('select * from (select * from t order by a) s order by a, b limit 55'); + explain_analyze_inc_sort_nodes_verify_invariants +-------------------------------------------------- + t +(1 row) + delete from t; -- An initial small group followed by a large group. insert into t(a, b) select (case when i < 5 then i else 9 end), i from generate_series(1, 100) n(i); @@ -630,90 +703,69 @@ select * from t left join (select * from (select * from t order by a) v order by (2 rows) rollback; --- Test EXPLAIN ANALYZE (text output) with both fullsort and presorted groups. -explain (analyze, costs off, summary off, timing off) -select * from (select * from t order by a) s order by a, b limit 70; - QUERY PLAN +-- Test EXPLAIN ANALYZE with both fullsort and presorted groups. +select explain_analyze_without_memory('select * from (select * from t order by a) s order by a, b limit 70'); + explain_analyze_without_memory --------------------------------------------------------------------------------- Limit (actual rows=70 loops=1) -> Incremental Sort (actual rows=70 loops=1) Sort Key: t.a, t.b Presorted Key: t.a - Full-sort Groups: 1 (Methods: quicksort) Memory: 28kB (avg), 28kB (max) - Presorted Groups: 5 (Methods: quicksort) Memory: 25kB (avg), 25kB (max) + Full-sort Groups: 1 (Methods: quicksort) Memory: NNkB (avg), NNkB (max) + Presorted Groups: 5 (Methods: quicksort) Memory: NNkB (avg), NNkB (max) -> Sort (actual rows=100 loops=1) Sort Key: t.a - Sort Method: quicksort Memory: 30kB + Sort Method: quicksort Memory: NNkB -> Seq Scan on t (actual rows=100 loops=1) (10 rows) -explain (analyze, costs off, summary off, timing off, format json) -select * from (select * from t order by a) s order by a, b limit 70; - QUERY PLAN ---------------------------------------------------- - [ + - { + - "Plan": { + - "Node Type": "Limit", + - "Parallel Aware": false, + - "Actual Rows": 70, + - "Actual Loops": 1, + - "Plans": [ + - { + - "Node Type": "Incremental Sort", + - "Parent Relationship": "Outer", + - "Parallel Aware": false, + - "Actual Rows": 70, + - "Actual Loops": 1, + - "Sort Key": ["t.a", "t.b"], + - "Presorted Key": ["t.a"], + - "Full-sort Groups": { + - "Group Count": 1, + - "Sort Methods Used": ["quicksort"], + - "Average Sort Space Used": 28, + - "Maximum Sort Space Used": 28, + - "Sort Space Type": "Memory" + - }, + - "Presorted Groups": { + - "Group Count": 5, + - "Sort Methods Used": ["quicksort"], + - "Average Sort Space Used": 25, + - "Maximum Sort Space Used": 25, + - "Sort Space Type": "Memory" + - }, + - "Plans": [ + - { + - "Node Type": "Sort", + - "Parent Relationship": "Outer", + - "Parallel Aware": false, + - "Actual Rows": 100, + - "Actual Loops": 1, + - "Sort Key": ["t.a"], + - "Sort Method": "quicksort", + - "Sort Space Used": 30, + - "Sort Space Type": "Memory", + - "Plans": [ + - { + - "Node Type": "Seq Scan", + - "Parent Relationship": "Outer",+ - "Parallel Aware": false, + - "Relation Name": "t", + - "Alias": "t", + - "Actual Rows": 100, + - "Actual Loops": 1 + - } + - ] + - } + - ] + - } + - ] + - }, + - "Triggers": [ + - ] + - } + +select jsonb_pretty(explain_analyze_inc_sort_nodes_without_memory('select * from (select * from t order by a) s order by a, b limit 70')); + jsonb_pretty +-------------------------------------------------- + [ + + { + + "Sort Key": [ + + "t.a", + + "t.b" + + ], + + "Node Type": "Incremental Sort", + + "Actual Rows": 70, + + "Actual Loops": 1, + + "Presorted Key": [ + + "t.a" + + ], + + "Parallel Aware": false, + + "Full-sort Groups": { + + "Group Count": 1, + + "Sort Methods Used": [ + + "quicksort" + + ], + + "Sort Space Memory": { + + "Average Sort Space Used": "NN",+ + "Maximum Sort Space Used": "NN" + + } + + }, + + "Presorted Groups": { + + "Group Count": 5, + + "Sort Methods Used": [ + + "quicksort" + + ], + + "Sort Space Memory": { + + "Average Sort Space Used": "NN",+ + "Maximum Sort Space Used": "NN" + + } + + }, + + "Parent Relationship": "Outer" + + } + ] (1 row) +select explain_analyze_inc_sort_nodes_verify_invariants('select * from (select * from t order by a) s order by a, b limit 70'); + explain_analyze_inc_sort_nodes_verify_invariants +-------------------------------------------------- + t +(1 row) + delete from t; -- Small groups of 10 tuples each tested around each mode transition point. insert into t(a, b) select i / 10, i from generate_series(1, 70) n(i); diff --git a/src/test/regress/sql/incremental_sort.sql b/src/test/regress/sql/incremental_sort.sql index 040324be32..b990b3b3de 100644 --- a/src/test/regress/sql/incremental_sort.sql +++ b/src/test/regress/sql/incremental_sort.sql @@ -18,6 +18,106 @@ reset work_mem; create table t(a integer, b integer); +create or replace function explain_analyze_without_memory(query text) +returns table (out_line text) language plpgsql +as +$$ +declare + line text; +begin + for line in + execute 'explain (analyze, costs off, summary off, timing off) ' || query + loop + out_line := regexp_replace(line, '\d+kB', 'NNkB', 'g'); + return next; + end loop; +end; +$$; + +create or replace function explain_analyze_inc_sort_nodes(query text) +returns jsonb language plpgsql +as +$$ +declare + elements jsonb; + element jsonb; + matching_nodes jsonb := '[]'::jsonb; +begin + execute 'explain (analyze, costs off, summary off, timing off, format ''json'') ' || query into strict elements; + while jsonb_array_length(elements) > 0 loop + element := elements->0; + elements := elements - 0; + case jsonb_typeof(element) + when 'array' then + if jsonb_array_length(element) > 0 then + elements := elements || element; + end if; + when 'object' then + if element ? 'Plan' then + elements := elements || jsonb_build_array(element->'Plan'); + element := element - 'Plan'; + else + if element ? 'Plans' then + elements := elements || jsonb_build_array(element->'Plans'); + element := element - 'Plans'; + end if; + if (element->>'Node Type')::text = 'Incremental Sort' then + matching_nodes := matching_nodes || element; + end if; + end if; + end case; + end loop; + return matching_nodes; +end; +$$; + +create or replace function explain_analyze_inc_sort_nodes_without_memory(query text) +returns jsonb language plpgsql +as +$$ +declare + nodes jsonb := '[]'::jsonb; + node jsonb; + group_key text; + space_key text; +begin + for node in select * from jsonb_array_elements(explain_analyze_inc_sort_nodes(query)) t loop + for group_key in select unnest(array['Full-sort Groups', 'Presorted Groups']::text[]) t loop + for space_key in select unnest(array['Sort Space Memory', 'Sort Space Disk']::text[]) t loop + node := jsonb_set(node, array[group_key, space_key, 'Average Sort Space Used'], '"NN"', false); + node := jsonb_set(node, array[group_key, space_key, 'Maximum Sort Space Used'], '"NN"', false); + end loop; + end loop; + nodes := nodes || node; + end loop; + return nodes; +end; +$$; + +create or replace function explain_analyze_inc_sort_nodes_verify_invariants(query text) +returns bool language plpgsql +as +$$ +declare + node jsonb; + group_stats jsonb; + group_key text; + space_key text; +begin + for node in select * from jsonb_array_elements(explain_analyze_inc_sort_nodes(query)) t loop + for group_key in select unnest(array['Full-sort Groups', 'Presorted Groups']::text[]) t loop + group_stats := node->group_key; + for space_key in select unnest(array['Sort Space Memory', 'Sort Space Disk']::text[]) t loop + if (group_stats->space_key->'Maximum Sort Space Used')::bigint < (group_stats->space_key->'Maximum Sort Space Used')::bigint then + raise exception '% has invalid max space < average space', group_key; + end if; + end loop; + end loop; + end loop; + return true; +end; +$$; + -- A single large group tested around each mode transition point. insert into t(a, b) select 1, i from generate_series(1, 100) n(i); explain (costs off) select * from (select * from t order by a) s order by a, b limit 31; @@ -36,11 +136,10 @@ delete from t; insert into t(a, b) select (case when i < 50 then 1 else 2 end), i from generate_series(1, 100) n(i); explain (costs off) select * from (select * from t order by a) s order by a, b limit 55; select * from (select * from t order by a) s order by a, b limit 55; --- Test EXPLAIN ANALYZE (text output) with only a fullsort group. -explain (analyze, costs off, summary off, timing off) -select * from (select * from t order by a) s order by a, b limit 55; -explain (analyze, costs off, summary off, timing off, format json) -select * from (select * from t order by a) s order by a, b limit 55; +-- Test EXPLAIN ANALYZE with only a fullsort group. +select explain_analyze_without_memory('select * from (select * from t order by a) s order by a, b limit 55'); +select jsonb_pretty(explain_analyze_inc_sort_nodes_without_memory('select * from (select * from t order by a) s order by a, b limit 55')); +select explain_analyze_inc_sort_nodes_verify_invariants('select * from (select * from t order by a) s order by a, b limit 55'); delete from t; -- An initial small group followed by a large group. @@ -58,11 +157,10 @@ set local enable_sort = off; explain (costs off) select * from t left join (select * from (select * from t order by a) v order by a, b) s on s.a = t.a where t.a in (1, 2); select * from t left join (select * from (select * from t order by a) v order by a, b) s on s.a = t.a where t.a in (1, 2); rollback; --- Test EXPLAIN ANALYZE (text output) with both fullsort and presorted groups. -explain (analyze, costs off, summary off, timing off) -select * from (select * from t order by a) s order by a, b limit 70; -explain (analyze, costs off, summary off, timing off, format json) -select * from (select * from t order by a) s order by a, b limit 70; +-- Test EXPLAIN ANALYZE with both fullsort and presorted groups. +select explain_analyze_without_memory('select * from (select * from t order by a) s order by a, b limit 70'); +select jsonb_pretty(explain_analyze_inc_sort_nodes_without_memory('select * from (select * from t order by a) s order by a, b limit 70')); +select explain_analyze_inc_sort_nodes_verify_invariants('select * from (select * from t order by a) s order by a, b limit 70'); delete from t; -- Small groups of 10 tuples each tested around each mode transition point. -- 2.17.1