Add memory/disk usage for WindowAgg nodes in EXPLAIN
1eff8279d4 added memory/disk usage for materialize nodes in EXPLAIN
ANALYZE.
In the commit message:
There are a few other executor node types that use tuplestores, so we
could also consider adding these details to the EXPLAIN ANALYZE for
those nodes too.
So I wanted to Add memory/disk usage for WindowAgg. Patch attached.
Since WindowAgg node could create multiple tuplestore for each Window
partition, we need to track each tuplestore storage usage so that the
maximum storage usage is determined. For this purpose I added new
fields to the WindowAggState.
--
Tatsuo Ishii
SRA OSS LLC
English: http://www.sraoss.co.jp/index_en/
Japanese:http://www.sraoss.co.jp
Attachments:
v1-0001-Add-memory-disk-usage-for-WindowAgg-nodes-in-EXPL.patchtext/x-patch; charset=us-asciiDownload
From 00c7546659e305be45dbeb13a14bcde5066be76f Mon Sep 17 00:00:00 2001
From: Tatsuo Ishii <ishii@postgresql.org>
Date: Sat, 6 Jul 2024 19:48:30 +0900
Subject: [PATCH v1] Add memory/disk usage for WindowAgg nodes in EXPLAIN.
---
src/backend/commands/explain.c | 37 ++++++++++++++++++++++++++++
src/backend/executor/nodeWindowAgg.c | 19 ++++++++++++++
src/include/nodes/execnodes.h | 2 ++
3 files changed, 58 insertions(+)
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 1e80fd8b68..177687ea80 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -126,6 +126,7 @@ static void show_incremental_sort_info(IncrementalSortState *incrsortstate,
ExplainState *es);
static void show_hash_info(HashState *hashstate, ExplainState *es);
static void show_material_info(MaterialState *mstate, ExplainState *es);
+static void show_windowagg_info(WindowAggState *winstate, ExplainState *es);
static void show_memoize_info(MemoizeState *mstate, List *ancestors,
ExplainState *es);
static void show_hashagg_info(AggState *aggstate, ExplainState *es);
@@ -2215,6 +2216,7 @@ ExplainNode(PlanState *planstate, List *ancestors,
planstate, es);
show_upper_qual(((WindowAgg *) plan)->runConditionOrig,
"Run Condition", planstate, ancestors, es);
+ show_windowagg_info(castNode(WindowAggState, planstate), es);
break;
case T_Group:
show_group_keys(castNode(GroupState, planstate), ancestors, es);
@@ -3362,6 +3364,41 @@ show_material_info(MaterialState *mstate, ExplainState *es)
}
}
+/*
+ * Show information on WindowAgg node, storage method and maximum memory/disk
+ * space used.
+ */
+static void
+show_windowagg_info(WindowAggState *winstate, ExplainState *es)
+{
+ const char *storageType;
+ int64 spaceUsedKB;
+
+ /*
+ * Nothing to show if ANALYZE option wasn't used or if execution didn't
+ * get as far as creating the tuplestore.
+ */
+ if (!es->analyze || winstate->storageType == NULL)
+ return;
+
+ storageType = winstate->storageType;
+ spaceUsedKB = BYTES_TO_KILOBYTES(winstate->storageSize);
+
+ if (es->format != EXPLAIN_FORMAT_TEXT)
+ {
+ ExplainPropertyText("Storage", storageType, es);
+ ExplainPropertyInteger("Maximum Storage", "kB", spaceUsedKB, es);
+ }
+ else
+ {
+ ExplainIndentText(es);
+ appendStringInfo(es->str,
+ "Storage: %s Maximum Storage: " INT64_FORMAT "kB\n",
+ storageType,
+ spaceUsedKB);
+ }
+}
+
/*
* Show information on memoize hits/misses/evictions and memory usage.
*/
diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c
index 3221fa1522..bcfe144511 100644
--- a/src/backend/executor/nodeWindowAgg.c
+++ b/src/backend/executor/nodeWindowAgg.c
@@ -1360,7 +1360,23 @@ release_partition(WindowAggState *winstate)
}
if (winstate->buffer)
+ {
+ int64 spaceUsed = tuplestore_space_used(winstate->buffer);
+
+ /*
+ * We want to track the max mem/disk usage so that we can use the info
+ * in EXPLAIN (ANALYZE).
+ */
+ if (spaceUsed > winstate->storageSize)
+ {
+ if (winstate->storageType != NULL)
+ pfree(winstate->storageType);
+ winstate->storageType =
+ pstrdup(tuplestore_storage_type_name(winstate->buffer));
+ winstate->storageSize = spaceUsed;
+ }
tuplestore_end(winstate->buffer);
+ }
winstate->buffer = NULL;
winstate->partition_spooled = false;
}
@@ -2671,6 +2687,9 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
winstate->partition_spooled = false;
winstate->more_partitions = false;
+ winstate->storageType = NULL;
+ winstate->storageSize = 0;
+
return winstate;
}
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b62c96f206..7a3dfa2bc3 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -2567,6 +2567,8 @@ typedef struct WindowAggState
ExprState *partEqfunction; /* equality funcs for partition columns */
ExprState *ordEqfunction; /* equality funcs for ordering columns */
Tuplestorestate *buffer; /* stores rows of current partition */
+ int64 storageSize; /* max storage size in buffer */
+ char *storageType; /* the storage type above */
int current_ptr; /* read pointer # for current row */
int framehead_ptr; /* read pointer # for frame head, if used */
int frametail_ptr; /* read pointer # for frame tail, if used */
--
2.25.1
On Sat, 6 Jul 2024 at 23:23, Tatsuo Ishii <ishii@postgresql.org> wrote:
So I wanted to Add memory/disk usage for WindowAgg. Patch attached.
Thanks for working on that.
Since WindowAgg node could create multiple tuplestore for each Window
partition, we need to track each tuplestore storage usage so that the
maximum storage usage is determined. For this purpose I added new
fields to the WindowAggState.
I'd recently been looking at the code that recreates the tuplestore
for each partition and thought we could do a bit better. In [1]/messages/by-id/CAHoyFK9n-QCXKTUWT_xxtXninSMEv+gbJN66-y6prM3f4WkEHw@mail.gmail.com, I
proposed a patch to make this better.
If you based your patch on [1]/messages/by-id/CAHoyFK9n-QCXKTUWT_xxtXninSMEv+gbJN66-y6prM3f4WkEHw@mail.gmail.com, maybe a better way of doing this is
having tuplestore.c track the maximum space used on disk in an extra
field which is updated with tuplestore_clear(). It's probably ok to
update a new field called maxDiskSpace in tuplestore_clear() if
state->status != TSS_INMEM. If the tuplestore went to disk then an
extra call to BufFileSize() isn't going to be noticeable, even in
cases where we only just went over work_mem. You could then adjust
tuplestore_space_used() to look at maxDiskSpace and return that value
if it's larger than BufFileSize(state->myfile) and state->maxSpace.
You could check if maxDiskSpace == 0 to determine if the tuplestore
has ever gone to disk. tuplestore_storage_type_name() would also need
to check maxDiskSpace and return "Disk" if that's non-zero.
David
[1]: /messages/by-id/CAHoyFK9n-QCXKTUWT_xxtXninSMEv+gbJN66-y6prM3f4WkEHw@mail.gmail.com
On Sat, 6 Jul 2024 at 23:23, Tatsuo Ishii <ishii@postgresql.org> wrote:
So I wanted to Add memory/disk usage for WindowAgg. Patch attached.
Thanks for working on that.
Thank you for the infrastructure you created in tuplestore.c and explain.c.
BTW, it seems these executor nodes (other than Materialize and Window
Aggregate node) use tuplestore for their own purpose.
CTE Scan
Recursive Union
Table Function Scan
I have already implemented that for CTE Scan. Do you think other two
nodes are worth to add the information? I think for consistency sake,
it will better to add the info Recursive Union and Table Function
Scan.
Since WindowAgg node could create multiple tuplestore for each Window
partition, we need to track each tuplestore storage usage so that the
maximum storage usage is determined. For this purpose I added new
fields to the WindowAggState.I'd recently been looking at the code that recreates the tuplestore
for each partition and thought we could do a bit better. In [1], I
proposed a patch to make this better.If you based your patch on [1], maybe a better way of doing this is
having tuplestore.c track the maximum space used on disk in an extra
field which is updated with tuplestore_clear(). It's probably ok to
update a new field called maxDiskSpace in tuplestore_clear() if
state->status != TSS_INMEM. If the tuplestore went to disk then an
extra call to BufFileSize() isn't going to be noticeable, even in
cases where we only just went over work_mem. You could then adjust
tuplestore_space_used() to look at maxDiskSpace and return that value
if it's larger than BufFileSize(state->myfile) and state->maxSpace.
You could check if maxDiskSpace == 0 to determine if the tuplestore
has ever gone to disk. tuplestore_storage_type_name() would also need
to check maxDiskSpace and return "Disk" if that's non-zero.
Thank you for the suggestion. Yes, I noticed [1] and once it is
committed, I will start to study tuplestore.c in this direction.
Best reagards,
--
Tatsuo Ishii
SRA OSS LLC
English: http://www.sraoss.co.jp/index_en/
Japanese:http://www.sraoss.co.jp
On Tue, 9 Jul 2024 at 14:44, Tatsuo Ishii <ishii@postgresql.org> wrote:
BTW, it seems these executor nodes (other than Materialize and Window
Aggregate node) use tuplestore for their own purpose.CTE Scan
Recursive Union
Table Function ScanI have already implemented that for CTE Scan. Do you think other two
nodes are worth to add the information?
Yes, I think so. I'd keep each as a separate patch so they can be
considered independently. Doing all of them should hopefully ensure we
strike the right balance of what code to put in explain.c and what
code to put in tuplestore.c. I think the WindowAgg's tuplestore usage
pattern might show that the API I picked isn't well suited when a
tuplestore is cleared and refilled over and over.
David
On Tue, Jul 9, 2024 at 8:20 AM David Rowley <dgrowleyml@gmail.com> wrote:
On Tue, 9 Jul 2024 at 14:44, Tatsuo Ishii <ishii@postgresql.org> wrote:
BTW, it seems these executor nodes (other than Materialize and Window
Aggregate node) use tuplestore for their own purpose.CTE Scan
Recursive Union
Table Function ScanI have already implemented that for CTE Scan. Do you think other two
nodes are worth to add the information?Yes, I think so. I'd keep each as a separate patch so they can be
considered independently. Doing all of them should hopefully ensure we
strike the right balance of what code to put in explain.c and what
code to put in tuplestore.c.
+1
+ if (es->format != EXPLAIN_FORMAT_TEXT)
+ {
+ ExplainPropertyText("Storage", storageType, es);
+ ExplainPropertyInteger("Maximum Storage", "kB", spaceUsedKB, es);
+ }
+ else
+ {
+ ExplainIndentText(es);
+ appendStringInfo(es->str,
+ "Storage: %s Maximum Storage: " INT64_FORMAT "kB\n",
+ storageType,
+ spaceUsedKB);
+ }
It will be good to move this code to a function which will be called
by show_*_info functions(). We might even convert it into a tuplestore
specific implementation hook after David's work.
--
Best Wishes,
Ashutosh Bapat
Yes, I think so. I'd keep each as a separate patch so they can be
considered independently. Doing all of them should hopefully ensure we
strike the right balance of what code to put in explain.c and what
code to put in tuplestore.c.+1
+ if (es->format != EXPLAIN_FORMAT_TEXT) + { + ExplainPropertyText("Storage", storageType, es); + ExplainPropertyInteger("Maximum Storage", "kB", spaceUsedKB, es); + } + else + { + ExplainIndentText(es); + appendStringInfo(es->str, + "Storage: %s Maximum Storage: " INT64_FORMAT "kB\n", + storageType, + spaceUsedKB); + }It will be good to move this code to a function which will be called
by show_*_info functions().
I have already implemented that in this direction in my working in
progress patch:
/*
* Show information regarding storage method and maximum memory/disk space
* used.
*/
static void
show_storage_info(Tuplestorestate *tupstore, ExplainState *es)
Which can be shared by Material and CTE scan node. I am going to post
it after I take care Recursive Union and Table Function Scan node.
We might even convert it into a tuplestore
specific implementation hook after David's work.
Best reagards,
--
Tatsuo Ishii
SRA OSS LLC
English: http://www.sraoss.co.jp/index_en/
Japanese:http://www.sraoss.co.jp
Yes, I think so. I'd keep each as a separate patch so they can be
considered independently. Doing all of them should hopefully ensure we
strike the right balance of what code to put in explain.c and what
code to put in tuplestore.c.+1
+ if (es->format != EXPLAIN_FORMAT_TEXT) + { + ExplainPropertyText("Storage", storageType, es); + ExplainPropertyInteger("Maximum Storage", "kB", spaceUsedKB, es); + } + else + { + ExplainIndentText(es); + appendStringInfo(es->str, + "Storage: %s Maximum Storage: " INT64_FORMAT "kB\n", + storageType, + spaceUsedKB); + }It will be good to move this code to a function which will be called
by show_*_info functions().I have already implemented that in this direction in my working in
progress patch:
Attached are the v2 patches. As suggested by David, I split them
into multiple patches so that each patch implements the feature for
each node. You need to apply the patches in the order of patch number
(if you want to apply all of them, "git apply v2-*.patch" should
work).
v2-0001-Refactor-show_material_info.patch:
This refactors show_material_info(). The guts are moved to new
show_storage_info() so that it can be shared by not only Materialized
node.
v2-0002-Add-memory-disk-usage-for-CTE-Scan-nodes-in-EXPLA.patch:
This adds memory/disk usage for CTE Scan nodes in EXPLAIN (ANALYZE) command.
v2-0003-Add-memory-disk-usage-for-Table-Function-Scan-nod.patch:
This adds memory/disk usage for Table Function Scan nodes in EXPLAIN (ANALYZE) command.
v2-0004-Add-memory-disk-usage-for-Recursive-Union-nodes-i.patch:
This adds memory/disk usage for Recursive Union nodes in EXPLAIN
(ANALYZE) command. Also show_storage_info() is changed so that it
accepts int64 storage_used, char *storage_type arguments. They are
used if the target node uses multiple tuplestores, in case a simple
call to tuplestore_space_used() does not work. Such executor nodes
need to collect storage_used while running the node. This type of node
includes Recursive Union and Window Aggregate.
v2-0005-Add-memory-disk-usage-for-Window-Aggregate-nodes-.patch: This
adds memory/disk usage for Window Aggregate nodes in EXPLAIN (ANALYZE)
command. Note that if David's proposal
/messages/by-id/CAHoyFK9n-QCXKTUWT_xxtXninSMEv+gbJN66-y6prM3f4WkEHw@mail.gmail.com
is committed, this will need to be adjusted.
For a demonstration, how storage/memory usage is shown in EXPLAIN
(notice "Storage: Memory Maximum Storage: 120kB" etc.). The script
used is attached (test.sql.txt). The SQLs are shamelessly copied from
David's example and the regression test (some of them were modified by
me).
EXPLAIN (ANALYZE, COSTS OFF)
SELECT count(t1.b) FROM (VALUES(1),(2)) t2(x) LEFT JOIN (SELECT * FROM t1 WHERE a <= 100) t1 ON TRUE;
QUERY PLAN
---------------------------------------------------------------------------------
Aggregate (actual time=0.345..0.346 rows=1 loops=1)
-> Nested Loop Left Join (actual time=0.015..0.330 rows=200 loops=1)
-> Values Scan on "*VALUES*" (actual time=0.001..0.003 rows=2 loops=1)
-> Materialize (actual time=0.006..0.152 rows=100 loops=2)
Storage: Memory Maximum Storage: 120kB
-> Seq Scan on t1 (actual time=0.007..0.213 rows=100 loops=1)
Filter: (a <= 100)
Rows Removed by Filter: 900
Planning Time: 0.202 ms
Execution Time: 0.377 ms
(10 rows)
-- CTE Scan node
EXPLAIN (ANALYZE, COSTS OFF)
WITH RECURSIVE t(n) AS (
VALUES (1)
UNION ALL
SELECT n+1 FROM t WHERE n < 100
)
SELECT sum(n) OVER() FROM t;
QUERY PLAN
-----------------------------------------------------------------------------------
WindowAgg (actual time=0.151..0.169 rows=100 loops=1)
Storage: Memory Maximum Storage: 20kB
CTE t
-> Recursive Union (actual time=0.001..0.105 rows=100 loops=1)
Storage: Memory Maximum Storage: 17kB
-> Result (actual time=0.001..0.001 rows=1 loops=1)
-> WorkTable Scan on t t_1 (actual time=0.000..0.000 rows=1 loops=100)
Filter: (n < 100)
Rows Removed by Filter: 0
-> CTE Scan on t (actual time=0.002..0.127 rows=100 loops=1)
Storage: Memory Maximum Storage: 20kB
Planning Time: 0.053 ms
Execution Time: 0.192 ms
(13 rows)
-- Table Function Scan node
CREATE OR REPLACE VIEW public.jsonb_table_view6 AS
SELECT js2,
jsb2w,
jsb2q,
ia,
ta,
jba
FROM JSON_TABLE(
'null'::jsonb, '$[*]' AS json_table_path_0
PASSING
1 + 2 AS a,
'"foo"'::json AS "b c"
COLUMNS (
js2 json PATH '$' WITHOUT WRAPPER KEEP QUOTES,
jsb2w jsonb PATH '$' WITH UNCONDITIONAL WRAPPER KEEP QUOTES,
jsb2q jsonb PATH '$' WITHOUT WRAPPER OMIT QUOTES,
ia integer[] PATH '$' WITHOUT WRAPPER KEEP QUOTES,
ta text[] PATH '$' WITHOUT WRAPPER KEEP QUOTES,
jba jsonb[] PATH '$' WITHOUT WRAPPER KEEP QUOTES
)
);
CREATE VIEW
EXPLAIN (ANALYZE, COSTS OFF) SELECT * FROM jsonb_table_view6;
QUERY PLAN
-------------------------------------------------------------------------------
Table Function Scan on "json_table" (actual time=0.024..0.025 rows=1 loops=1)
Storage: Memory Maximum Storage: 17kB
Planning Time: 0.100 ms
Execution Time: 0.054 ms
(4 rows)
Attachments:
v2-0001-Refactor-show_material_info.patchtext/x-patch; charset=us-asciiDownload
From 1d2f67dff48601f92b5378838c0e908d200d4493 Mon Sep 17 00:00:00 2001
From: Tatsuo Ishii <ishii@postgresql.org>
Date: Wed, 10 Jul 2024 14:02:13 +0900
Subject: [PATCH v2 1/5] Refactor show_material_info().
---
src/backend/commands/explain.c | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 118db12903..43ef33295e 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -125,6 +125,7 @@ static void show_sort_info(SortState *sortstate, ExplainState *es);
static void show_incremental_sort_info(IncrementalSortState *incrsortstate,
ExplainState *es);
static void show_hash_info(HashState *hashstate, ExplainState *es);
+static void show_storage_info(Tuplestorestate *tupstore, ExplainState *es);
static void show_material_info(MaterialState *mstate, ExplainState *es);
static void show_memoize_info(MemoizeState *mstate, List *ancestors,
ExplainState *es);
@@ -3326,13 +3327,11 @@ show_hash_info(HashState *hashstate, ExplainState *es)
}
/*
- * Show information on material node, storage method and maximum memory/disk
- * space used.
+ * Show information on storage method and maximum memory/disk space used.
*/
static void
-show_material_info(MaterialState *mstate, ExplainState *es)
+show_storage_info(Tuplestorestate *tupstore, ExplainState *es)
{
- Tuplestorestate *tupstore = mstate->tuplestorestate;
const char *storageType;
int64 spaceUsedKB;
@@ -3361,6 +3360,18 @@ show_material_info(MaterialState *mstate, ExplainState *es)
}
}
+/*
+ * Show information on material node, storage method and maximum memory/disk
+ * space used.
+ */
+static void
+show_material_info(MaterialState *mstate, ExplainState *es)
+{
+ Tuplestorestate *tupstore = mstate->tuplestorestate;
+
+ show_storage_info(tupstore, es);
+}
+
/*
* Show information on memoize hits/misses/evictions and memory usage.
*/
--
2.25.1
v2-0002-Add-memory-disk-usage-for-CTE-Scan-nodes-in-EXPLA.patchtext/x-patch; charset=us-asciiDownload
From e4eb2288c90e92a42ba85c672d477b1a93ebf6f8 Mon Sep 17 00:00:00 2001
From: Tatsuo Ishii <ishii@postgresql.org>
Date: Wed, 10 Jul 2024 14:09:46 +0900
Subject: [PATCH v2 2/5] Add memory/disk usage for CTE Scan nodes in EXPLAIN
---
src/backend/commands/explain.c | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 43ef33295e..661e9101cb 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -127,6 +127,7 @@ static void show_incremental_sort_info(IncrementalSortState *incrsortstate,
static void show_hash_info(HashState *hashstate, ExplainState *es);
static void show_storage_info(Tuplestorestate *tupstore, ExplainState *es);
static void show_material_info(MaterialState *mstate, ExplainState *es);
+static void show_ctescan_info(CteScanState *ctescanstate, ExplainState *es);
static void show_memoize_info(MemoizeState *mstate, List *ancestors,
ExplainState *es);
static void show_hashagg_info(AggState *aggstate, ExplainState *es);
@@ -2028,6 +2029,8 @@ ExplainNode(PlanState *planstate, List *ancestors,
if (plan->qual)
show_instrumentation_count("Rows Removed by Filter", 1,
planstate, es);
+ if (IsA(plan, CteScan))
+ show_ctescan_info(castNode(CteScanState, planstate), es);
break;
case T_Gather:
{
@@ -3372,6 +3375,18 @@ show_material_info(MaterialState *mstate, ExplainState *es)
show_storage_info(tupstore, es);
}
+/*
+ * Show information on CTE Scan node, storage method and maximum memory/disk
+ * space used.
+ */
+static void
+show_ctescan_info(CteScanState *ctescanstate, ExplainState *es)
+{
+ Tuplestorestate *tupstore = ctescanstate->leader->cte_table;
+
+ show_storage_info(tupstore, es);
+}
+
/*
* Show information on memoize hits/misses/evictions and memory usage.
*/
--
2.25.1
v2-0003-Add-memory-disk-usage-for-Table-Function-Scan-nod.patchtext/x-patch; charset=us-asciiDownload
From a7bfaf711b5a1ecb4fe58d3b46db1911383dbfb8 Mon Sep 17 00:00:00 2001
From: Tatsuo Ishii <ishii@postgresql.org>
Date: Wed, 10 Jul 2024 14:32:59 +0900
Subject: [PATCH v2 3/5] Add memory/disk usage for Table Function Scan nodes in
EXPLAIN
---
src/backend/commands/explain.c | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 661e9101cb..1a4e83ad38 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -128,6 +128,7 @@ static void show_hash_info(HashState *hashstate, ExplainState *es);
static void show_storage_info(Tuplestorestate *tupstore, ExplainState *es);
static void show_material_info(MaterialState *mstate, ExplainState *es);
static void show_ctescan_info(CteScanState *ctescanstate, ExplainState *es);
+static void show_table_func_can_info(TableFuncScanState *tscanstate, ExplainState *es);
static void show_memoize_info(MemoizeState *mstate, List *ancestors,
ExplainState *es);
static void show_hashagg_info(AggState *aggstate, ExplainState *es);
@@ -2112,6 +2113,7 @@ ExplainNode(PlanState *planstate, List *ancestors,
if (plan->qual)
show_instrumentation_count("Rows Removed by Filter", 1,
planstate, es);
+ show_table_func_can_info(castNode(TableFuncScanState, planstate), es);
break;
case T_TidScan:
{
@@ -3387,6 +3389,18 @@ show_ctescan_info(CteScanState *ctescanstate, ExplainState *es)
show_storage_info(tupstore, es);
}
+/*
+ * Show information on Table Function Scan node, storage method and maximum
+ * memory/disk space used.
+ */
+static void
+show_table_func_can_info(TableFuncScanState *tscanstate, ExplainState *es)
+{
+ Tuplestorestate *tupstore = tscanstate->tupstore;
+
+ show_storage_info(tupstore, es);
+}
+
/*
* Show information on memoize hits/misses/evictions and memory usage.
*/
--
2.25.1
v2-0004-Add-memory-disk-usage-for-Recursive-Union-nodes-i.patchtext/x-patch; charset=us-asciiDownload
From b03264d55d0c23885fe9ce4f9d93a6ca65d45261 Mon Sep 17 00:00:00 2001
From: Tatsuo Ishii <ishii@postgresql.org>
Date: Wed, 10 Jul 2024 16:06:59 +0900
Subject: [PATCH v2 4/5] Add memory/disk usage for Recursive Union nodes in
EXPLAIN
---
src/backend/commands/explain.c | 47 +++++++++++++++++------
src/backend/executor/nodeRecursiveunion.c | 30 +++++++++++++++
src/include/nodes/execnodes.h | 2 +
3 files changed, 68 insertions(+), 11 deletions(-)
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 1a4e83ad38..054d909093 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -125,10 +125,12 @@ static void show_sort_info(SortState *sortstate, ExplainState *es);
static void show_incremental_sort_info(IncrementalSortState *incrsortstate,
ExplainState *es);
static void show_hash_info(HashState *hashstate, ExplainState *es);
-static void show_storage_info(Tuplestorestate *tupstore, ExplainState *es);
+static void show_storage_info(Tuplestorestate *tupstore, int64 storage_used, char *storage_type,
+ ExplainState *es);
static void show_material_info(MaterialState *mstate, ExplainState *es);
static void show_ctescan_info(CteScanState *ctescanstate, ExplainState *es);
static void show_table_func_can_info(TableFuncScanState *tscanstate, ExplainState *es);
+static void show_recursive_union_info(RecursiveUnionState *rstate, ExplainState *es);
static void show_memoize_info(MemoizeState *mstate, List *ancestors,
ExplainState *es);
static void show_hashagg_info(AggState *aggstate, ExplainState *es);
@@ -2264,6 +2266,9 @@ ExplainNode(PlanState *planstate, List *ancestors,
show_memoize_info(castNode(MemoizeState, planstate), ancestors,
es);
break;
+ case T_RecursiveUnion:
+ show_recursive_union_info(castNode(RecursiveUnionState, planstate), es);
+ break;
default:
break;
}
@@ -3332,23 +3337,32 @@ show_hash_info(HashState *hashstate, ExplainState *es)
}
/*
- * Show information on storage method and maximum memory/disk space used.
+ * Show information on storage method and maximum memory/disk space used. If
+ * tupstore is NULL, then storage_used and storage_type are used instead.
*/
static void
-show_storage_info(Tuplestorestate *tupstore, ExplainState *es)
+show_storage_info(Tuplestorestate *tupstore, int64 storage_used, char *storage_type,
+ ExplainState *es)
{
const char *storageType;
int64 spaceUsedKB;
/*
- * Nothing to show if ANALYZE option wasn't used or if execution didn't
- * get as far as creating the tuplestore.
+ * Nothing to show if ANALYZE option wasn't used.
*/
- if (!es->analyze || tupstore == NULL)
+ if (!es->analyze)
return;
- storageType = tuplestore_storage_type_name(tupstore);
- spaceUsedKB = BYTES_TO_KILOBYTES(tuplestore_space_used(tupstore));
+ if (tupstore != NULL)
+ {
+ storageType = tuplestore_storage_type_name(tupstore);
+ spaceUsedKB = BYTES_TO_KILOBYTES(tuplestore_space_used(tupstore));
+ }
+ else
+ {
+ storageType = storage_type;
+ spaceUsedKB = BYTES_TO_KILOBYTES(storage_used);
+ }
if (es->format != EXPLAIN_FORMAT_TEXT)
{
@@ -3374,7 +3388,7 @@ show_material_info(MaterialState *mstate, ExplainState *es)
{
Tuplestorestate *tupstore = mstate->tuplestorestate;
- show_storage_info(tupstore, es);
+ show_storage_info(tupstore, 0, NULL, es);
}
/*
@@ -3386,7 +3400,7 @@ show_ctescan_info(CteScanState *ctescanstate, ExplainState *es)
{
Tuplestorestate *tupstore = ctescanstate->leader->cte_table;
- show_storage_info(tupstore, es);
+ show_storage_info(tupstore, 0, NULL, es);
}
/*
@@ -3398,7 +3412,18 @@ show_table_func_can_info(TableFuncScanState *tscanstate, ExplainState *es)
{
Tuplestorestate *tupstore = tscanstate->tupstore;
- show_storage_info(tupstore, es);
+ show_storage_info(tupstore, 0, NULL, es);
+}
+
+/*
+ * Show information on Recursive Union node, storage method and maximum
+ * memory/disk space used.
+ */
+
+static void
+show_recursive_union_info(RecursiveUnionState *rstate, ExplainState *es)
+{
+ show_storage_info(NULL, rstate->storageSize, rstate->storageType, es);
}
/*
diff --git a/src/backend/executor/nodeRecursiveunion.c b/src/backend/executor/nodeRecursiveunion.c
index c7f8a19fa4..8667b7ca93 100644
--- a/src/backend/executor/nodeRecursiveunion.c
+++ b/src/backend/executor/nodeRecursiveunion.c
@@ -52,6 +52,33 @@ build_hash_table(RecursiveUnionState *rustate)
false);
}
+/*
+ * Track the maximum memory/disk usage in working_table and
+ * intermediate_table. Supposed to be called just before tuplestore_end.
+ */
+static void
+track_storage_usage(RecursiveUnionState *state, Tuplestorestate *tup)
+{
+ int64 spaceUsed;
+
+ if (tup == NULL)
+ return;
+
+ spaceUsed = tuplestore_space_used(tup);
+
+ /*
+ * We want to track the maximum mem/disk usage so that we can use the info
+ * in EXPLAIN (ANALYZE).
+ */
+ if (spaceUsed > state->storageSize)
+ {
+ if (state->storageType != NULL)
+ pfree(state->storageType);
+ state->storageType =
+ pstrdup(tuplestore_storage_type_name(tup));
+ state->storageSize = spaceUsed;
+ }
+}
/* ----------------------------------------------------------------
* ExecRecursiveUnion(node)
@@ -120,6 +147,7 @@ ExecRecursiveUnion(PlanState *pstate)
break;
/* done with old working table ... */
+ track_storage_usage(node, node->working_table);
tuplestore_end(node->working_table);
/* intermediate table becomes working table */
@@ -191,6 +219,8 @@ ExecInitRecursiveUnion(RecursiveUnion *node, EState *estate, int eflags)
rustate->intermediate_empty = true;
rustate->working_table = tuplestore_begin_heap(false, false, work_mem);
rustate->intermediate_table = tuplestore_begin_heap(false, false, work_mem);
+ rustate->storageSize = 0;
+ rustate->storageType = NULL;
/*
* If hashing, we need a per-tuple memory context for comparisons, and a
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index cac684d9b3..bc2c0baed6 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1509,6 +1509,8 @@ typedef struct RecursiveUnionState
bool intermediate_empty;
Tuplestorestate *working_table;
Tuplestorestate *intermediate_table;
+ int64 storageSize; /* max storage size Tuplestore */
+ char *storageType; /* the storage type above */
/* Remaining fields are unused in UNION ALL case */
Oid *eqfuncoids; /* per-grouping-field equality fns */
FmgrInfo *hashfunctions; /* per-grouping-field hash fns */
--
2.25.1
v2-0005-Add-memory-disk-usage-for-Window-Aggregate-nodes-.patchtext/x-patch; charset=us-asciiDownload
From b61edc12210eb71c8a7a987171e8bbf39299989c Mon Sep 17 00:00:00 2001
From: Tatsuo Ishii <ishii@postgresql.org>
Date: Wed, 10 Jul 2024 16:22:41 +0900
Subject: [PATCH v2 5/5] Add memory/disk usage for Window Aggregate nodes in
EXPLAIN
---
src/backend/commands/explain.c | 12 ++++++++++++
src/backend/executor/nodeWindowAgg.c | 19 +++++++++++++++++++
src/include/nodes/execnodes.h | 2 ++
3 files changed, 33 insertions(+)
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 054d909093..5f32c967e1 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -131,6 +131,7 @@ static void show_material_info(MaterialState *mstate, ExplainState *es);
static void show_ctescan_info(CteScanState *ctescanstate, ExplainState *es);
static void show_table_func_can_info(TableFuncScanState *tscanstate, ExplainState *es);
static void show_recursive_union_info(RecursiveUnionState *rstate, ExplainState *es);
+static void show_windowagg_info(WindowAggState *winstate, ExplainState *es);
static void show_memoize_info(MemoizeState *mstate, List *ancestors,
ExplainState *es);
static void show_hashagg_info(AggState *aggstate, ExplainState *es);
@@ -2222,6 +2223,7 @@ ExplainNode(PlanState *planstate, List *ancestors,
planstate, es);
show_upper_qual(((WindowAgg *) plan)->runConditionOrig,
"Run Condition", planstate, ancestors, es);
+ show_windowagg_info(castNode(WindowAggState, planstate), es);
break;
case T_Group:
show_group_keys(castNode(GroupState, planstate), ancestors, es);
@@ -3426,6 +3428,16 @@ show_recursive_union_info(RecursiveUnionState *rstate, ExplainState *es)
show_storage_info(NULL, rstate->storageSize, rstate->storageType, es);
}
+/*
+ * Show information on WindowAgg node, storage method and maximum memory/disk
+ * space used.
+ */
+static void
+show_windowagg_info(WindowAggState *winstate, ExplainState *es)
+{
+ show_storage_info(NULL, winstate->storageSize, winstate->storageType, es);
+}
+
/*
* Show information on memoize hits/misses/evictions and memory usage.
*/
diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c
index 3221fa1522..bcfe144511 100644
--- a/src/backend/executor/nodeWindowAgg.c
+++ b/src/backend/executor/nodeWindowAgg.c
@@ -1360,7 +1360,23 @@ release_partition(WindowAggState *winstate)
}
if (winstate->buffer)
+ {
+ int64 spaceUsed = tuplestore_space_used(winstate->buffer);
+
+ /*
+ * We want to track the max mem/disk usage so that we can use the info
+ * in EXPLAIN (ANALYZE).
+ */
+ if (spaceUsed > winstate->storageSize)
+ {
+ if (winstate->storageType != NULL)
+ pfree(winstate->storageType);
+ winstate->storageType =
+ pstrdup(tuplestore_storage_type_name(winstate->buffer));
+ winstate->storageSize = spaceUsed;
+ }
tuplestore_end(winstate->buffer);
+ }
winstate->buffer = NULL;
winstate->partition_spooled = false;
}
@@ -2671,6 +2687,9 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags)
winstate->partition_spooled = false;
winstate->more_partitions = false;
+ winstate->storageType = NULL;
+ winstate->storageSize = 0;
+
return winstate;
}
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index bc2c0baed6..82a597aae9 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -2596,6 +2596,8 @@ typedef struct WindowAggState
ExprState *partEqfunction; /* equality funcs for partition columns */
ExprState *ordEqfunction; /* equality funcs for ordering columns */
Tuplestorestate *buffer; /* stores rows of current partition */
+ int64 storageSize; /* max storage size in buffer */
+ char *storageType; /* the storage type above */
int current_ptr; /* read pointer # for current row */
int framehead_ptr; /* read pointer # for frame head, if used */
int frametail_ptr; /* read pointer # for frame tail, if used */
--
2.25.1
Hi!
+1 for the idea of the patch. Consider it useful.
I looked at the patch set and don't see any obvious defects. It applies
without any problems and looks pretty good for me.
Only one thing is left to do. Add basic tests for the added functionality
to make it committable. For example, as in the
mentioned 1eff8279d494b9.
--
Best regards,
Maxim Orlov.
Hi!
+1 for the idea of the patch. Consider it useful.
I looked at the patch set and don't see any obvious defects. It applies
without any problems and looks pretty good for me.
Thank you for reviewing my patch.
Only one thing is left to do. Add basic tests for the added functionality
to make it committable. For example, as in the
mentioned 1eff8279d494b9.
Agreed. Probably add to explain.sql?
Best reagards,
--
Tatsuo Ishii
SRA OSS K.K.
English: http://www.sraoss.co.jp/index_en/
Japanese:http://www.sraoss.co.jp
On Wed, 4 Sept 2024 at 03:07, Tatsuo Ishii <ishii@postgresql.org> wrote:
Agreed. Probably add to explain.sql?
Yeah, I think this is an appropriate place.
--
Best regards,
Maxim Orlov.
On Wed, Jul 10, 2024 at 5:36 PM Tatsuo Ishii <ishii@postgresql.org> wrote:
Attached are the v2 patches. As suggested by David, I split them
into multiple patches so that each patch implements the feature for
each node. You need to apply the patches in the order of patch number
(if you want to apply all of them, "git apply v2-*.patch" should
work).v2-0001-Refactor-show_material_info.patch:
This refactors show_material_info(). The guts are moved to new
show_storage_info() so that it can be shared by not only Materialized
node.v2-0002-Add-memory-disk-usage-for-CTE-Scan-nodes-in-EXPLA.patch:
This adds memory/disk usage for CTE Scan nodes in EXPLAIN (ANALYZE) command.v2-0003-Add-memory-disk-usage-for-Table-Function-Scan-nod.patch:
This adds memory/disk usage for Table Function Scan nodes in EXPLAIN (ANALYZE) command.v2-0004-Add-memory-disk-usage-for-Recursive-Union-nodes-i.patch:
This adds memory/disk usage for Recursive Union nodes in EXPLAIN
(ANALYZE) command. Also show_storage_info() is changed so that it
accepts int64 storage_used, char *storage_type arguments. They are
used if the target node uses multiple tuplestores, in case a simple
call to tuplestore_space_used() does not work. Such executor nodes
need to collect storage_used while running the node. This type of node
includes Recursive Union and Window Aggregate.v2-0005-Add-memory-disk-usage-for-Window-Aggregate-nodes-.patch: This
adds memory/disk usage for Window Aggregate nodes in EXPLAIN (ANALYZE)
command. Note that if David's proposal
/messages/by-id/CAHoyFK9n-QCXKTUWT_xxtXninSMEv+gbJN66-y6prM3f4WkEHw@mail.gmail.com
is committed, this will need to be adjusted.For a demonstration, how storage/memory usage is shown in EXPLAIN
(notice "Storage: Memory Maximum Storage: 120kB" etc.). The script
used is attached (test.sql.txt). The SQLs are shamelessly copied from
David's example and the regression test (some of them were modified by
me).
hi. I can roughly understand it.
I have one minor issue with the comment.
typedef struct RecursiveUnionState
{
PlanState ps; /* its first field is NodeTag */
bool recursing;
bool intermediate_empty;
Tuplestorestate *working_table;
Tuplestorestate *intermediate_table;
int64 storageSize; /* max storage size Tuplestore */
char *storageType; /* the storage type above */
....
}
"/* the storage type above */"
is kind of ambiguous, since there is more than one Tuplestorestate.
i think it roughly means: the storage type of working_table
while the max storage of working_table.
typedef struct WindowAggState
{
ScanState ss; /* its first field is NodeTag */
/* these fields are filled in by ExecInitExpr: */
List *funcs; /* all WindowFunc nodes in targetlist */
int numfuncs; /* total number of window functions */
int numaggs; /* number that are plain aggregates */
WindowStatePerFunc perfunc; /* per-window-function information */
WindowStatePerAgg peragg; /* per-plain-aggregate information */
ExprState *partEqfunction; /* equality funcs for partition columns */
ExprState *ordEqfunction; /* equality funcs for ordering columns */
Tuplestorestate *buffer; /* stores rows of current partition */
int64 storageSize; /* max storage size in buffer */
char *storageType; /* the storage type above */
}
" /* the storage type above */"
I think it roughly means:
" the storage type of WindowAggState->buffer while the max storage of
WindowAggState->buffer".
On Wed, 10 Jul 2024 at 21:36, Tatsuo Ishii <ishii@postgresql.org> wrote:
v2-0005-Add-memory-disk-usage-for-Window-Aggregate-nodes-.patch: This
adds memory/disk usage for Window Aggregate nodes in EXPLAIN (ANALYZE)
command. Note that if David's proposal
/messages/by-id/CAHoyFK9n-QCXKTUWT_xxtXninSMEv+gbJN66-y6prM3f4WkEHw@mail.gmail.com
is committed, this will need to be adjusted.
Hi,
I pushed the changes to WindowAgg so as not to call tuplestore_end()
on every partition. Can you rebase this patch over that change?
It would be good to do this in a way that does not add any new state
to WindowAggState, you can see that I had to shuffle fields around in
that struct because the next_parition field would have caused the
struct to become larger. I've not looked closely, but I expect this
can be done by adding more code to tuplestore_updatemax() to also
track the disk space used if the current storage has gone to disk. I
expect the maxSpace field can be used for both, but we'd need another
bool field to track if the max used was by disk or memory.
I think the performance of this would also need to be tested as it
means doing an lseek() on every tuplestore_clear() when we've gone to
disk. Probably that will be dominated by all the other overheads of a
tuplestore going to disk (i.e. dumptuples() etc), but it would be good
to check this. I suggest setting work_mem = 64 and making a test case
that only just spills to disk. Maybe do a few thousand partitions
worth of that and see if you can measure any slowdown.
David
Hi,
On Wed, 10 Jul 2024 at 21:36, Tatsuo Ishii <ishii@postgresql.org> wrote:
v2-0005-Add-memory-disk-usage-for-Window-Aggregate-nodes-.patch: This
adds memory/disk usage for Window Aggregate nodes in EXPLAIN (ANALYZE)
command. Note that if David's proposal
/messages/by-id/CAHoyFK9n-QCXKTUWT_xxtXninSMEv+gbJN66-y6prM3f4WkEHw@mail.gmail.com
is committed, this will need to be adjusted.Hi,
I pushed the changes to WindowAgg so as not to call tuplestore_end()
on every partition. Can you rebase this patch over that change?It would be good to do this in a way that does not add any new state
to WindowAggState, you can see that I had to shuffle fields around in
that struct because the next_parition field would have caused the
struct to become larger. I've not looked closely, but I expect this
can be done by adding more code to tuplestore_updatemax() to also
track the disk space used if the current storage has gone to disk. I
expect the maxSpace field can be used for both, but we'd need another
bool field to track if the max used was by disk or memory.I think the performance of this would also need to be tested as it
means doing an lseek() on every tuplestore_clear() when we've gone to
disk. Probably that will be dominated by all the other overheads of a
tuplestore going to disk (i.e. dumptuples() etc), but it would be good
to check this. I suggest setting work_mem = 64 and making a test case
that only just spills to disk. Maybe do a few thousand partitions
worth of that and see if you can measure any slowdown.
Thank you for the suggestion. I will look into this.
Best reagards,
--
Tatsuo Ishii
SRA OSS K.K.
English: http://www.sraoss.co.jp/index_en/
Japanese:http://www.sraoss.co.jp
Hi,
hi. I can roughly understand it.
I have one minor issue with the comment.
typedef struct RecursiveUnionState
{
PlanState ps; /* its first field is NodeTag */
bool recursing;
bool intermediate_empty;
Tuplestorestate *working_table;
Tuplestorestate *intermediate_table;
int64 storageSize; /* max storage size Tuplestore */
char *storageType; /* the storage type above */
....
}"/* the storage type above */"
is kind of ambiguous, since there is more than one Tuplestorestate.i think it roughly means: the storage type of working_table
while the max storage of working_table.typedef struct WindowAggState
{
ScanState ss; /* its first field is NodeTag *//* these fields are filled in by ExecInitExpr: */
List *funcs; /* all WindowFunc nodes in targetlist */
int numfuncs; /* total number of window functions */
int numaggs; /* number that are plain aggregates */WindowStatePerFunc perfunc; /* per-window-function information */
WindowStatePerAgg peragg; /* per-plain-aggregate information */
ExprState *partEqfunction; /* equality funcs for partition columns */
ExprState *ordEqfunction; /* equality funcs for ordering columns */
Tuplestorestate *buffer; /* stores rows of current partition */
int64 storageSize; /* max storage size in buffer */
char *storageType; /* the storage type above */
}" /* the storage type above */"
I think it roughly means:
" the storage type of WindowAggState->buffer while the max storage of
WindowAggState->buffer".
Thank you for looking into my patch. Unfortunately I need to work on
other issue before adjusting the comments because the fields might go
away if I change the tuplestore infrastructure per David's suggestion:
/messages/by-id/CAApHDvoY8cibGcicLV0fNh=9JVx9PANcWvhkdjBnDCc9Quqytg@mail.gmail.com
After this I will rebase the patches. This commit requires changes.
https://git.postgresql.org/gitweb/?p=postgresql.git;a=commit;h=908a968612f9ed61911d8ca0a185b262b82f1269
Best reagards,
--
Tatsuo Ishii
SRA OSS K.K.
English: http://www.sraoss.co.jp/index_en/
Japanese:http://www.sraoss.co.jp
Hi David,
I pushed the changes to WindowAgg so as not to call tuplestore_end()
on every partition. Can you rebase this patch over that change?It would be good to do this in a way that does not add any new state
to WindowAggState, you can see that I had to shuffle fields around in
that struct because the next_parition field would have caused the
struct to become larger. I've not looked closely, but I expect this
can be done by adding more code to tuplestore_updatemax() to also
track the disk space used if the current storage has gone to disk. I
expect the maxSpace field can be used for both, but we'd need another
bool field to track if the max used was by disk or memory.
I have created a patch in the direction you suggested. See attached
patch (v1-0001-Enhance-tuplestore.txt). To not confuse CFbot, the
extension is "txt", not "patch".
I think the performance of this would also need to be tested as it
means doing an lseek() on every tuplestore_clear() when we've gone to
disk. Probably that will be dominated by all the other overheads of a
tuplestore going to disk (i.e. dumptuples() etc), but it would be good
to check this. I suggest setting work_mem = 64 and making a test case
that only just spills to disk. Maybe do a few thousand partitions
worth of that and see if you can measure any slowdown.
I copied your shell script and slightly modified it then ran pgbench
with (1 10 100 1000 5000 10000) window partitions (see attached shell
script). In the script I set work_mem to 64kB. It seems for 10000,
1000 and 100 partitions, the performance difference seems
noises. However, for 10, 2, 1 partitions. I see large performance
degradation with the patched version: patched is slower than stock
master in 1.5% (10 partitions), 41% (2 partitions) and 55.7% (1
partition). See the attached graph.
Attachments:
v1-0001-Enhance-tuplestore.txttext/plain; charset=us-asciiDownload
From 4749d2018f33e883c292eb904f3253d393a47c99 Mon Sep 17 00:00:00 2001
From: Tatsuo Ishii <ishii@postgresql.org>
Date: Thu, 5 Sep 2024 20:59:01 +0900
Subject: [PATCH v1] Enhance tuplestore.
Let tuplestore_updatemax() handle both memory and disk case.
---
src/backend/utils/sort/tuplestore.c | 27 ++++++++++++++++++---------
1 file changed, 18 insertions(+), 9 deletions(-)
diff --git a/src/backend/utils/sort/tuplestore.c b/src/backend/utils/sort/tuplestore.c
index 444c8e25c2..854121fc11 100644
--- a/src/backend/utils/sort/tuplestore.c
+++ b/src/backend/utils/sort/tuplestore.c
@@ -107,9 +107,10 @@ struct Tuplestorestate
bool backward; /* store extra length words in file? */
bool interXact; /* keep open through transactions? */
bool truncated; /* tuplestore_trim has removed tuples? */
+ bool inMem; /* true if maxSpace is for memory */
int64 availMem; /* remaining memory available, in bytes */
int64 allowedMem; /* total memory allowed, in bytes */
- int64 maxSpace; /* maximum space used in memory */
+ int64 maxSpace; /* maximum space used in memory or disk */
int64 tuples; /* number of tuples added */
BufFile *myfile; /* underlying file, or NULL if none */
MemoryContext context; /* memory context for holding tuples */
@@ -262,6 +263,7 @@ tuplestore_begin_common(int eflags, bool interXact, int maxKBytes)
state->eflags = eflags;
state->interXact = interXact;
state->truncated = false;
+ state->inMem = true;
state->allowedMem = maxKBytes * 1024L;
state->availMem = state->allowedMem;
state->maxSpace = 0;
@@ -1497,8 +1499,17 @@ static void
tuplestore_updatemax(Tuplestorestate *state)
{
if (state->status == TSS_INMEM)
+ {
state->maxSpace = Max(state->maxSpace,
state->allowedMem - state->availMem);
+ state->inMem = true;
+ }
+ else
+ {
+ state->maxSpace = Max(state->maxSpace,
+ BufFileSize(state->myfile));
+ state->inMem = false;
+ }
}
/*
@@ -1509,7 +1520,7 @@ tuplestore_updatemax(Tuplestorestate *state)
const char *
tuplestore_storage_type_name(Tuplestorestate *state)
{
- if (state->status == TSS_INMEM)
+ if (state->inMem)
return "Memory";
else
return "Disk";
@@ -1517,8 +1528,7 @@ tuplestore_storage_type_name(Tuplestorestate *state)
/*
* tuplestore_space_used
- * Return the maximum space used in memory unless the tuplestore has spilled
- * to disk, in which case, return the disk space used.
+ * Return the maximum space used in memory or disk.
*/
int64
tuplestore_space_used(Tuplestorestate *state)
@@ -1526,10 +1536,7 @@ tuplestore_space_used(Tuplestorestate *state)
/* First, update the maxSpace field */
tuplestore_updatemax(state);
- if (state->status == TSS_INMEM)
- return state->maxSpace;
- else
- return BufFileSize(state->myfile);
+ return state->maxSpace;
}
/*
@@ -1601,7 +1608,9 @@ writetup_heap(Tuplestorestate *state, void *tup)
if (state->backward) /* need trailing length word? */
BufFileWrite(state->myfile, &tuplen, sizeof(tuplen));
- /* no need to call tuplestore_updatemax() when not in TSS_INMEM */
+ /* update maxSpace */
+ tuplestore_updatemax(state);
+
FREEMEM(state, GetMemoryChunkSpace(tuple));
heap_free_minimal_tuple(tuple);
}
--
2.25.1
bench.pngimage/pngDownload
�PNG
IHDR ^ D ��� O�IDATx���x���o�!��P����i�)��w/E��RU�QA���B�RT:TzPZ� �����&$@�=�&����5;3{�?�0��2��6���� �����O��8� H�l�� �;� F� �^ `� #x �a/ 0�� �� �0� F� �^ `� #x �a/ 0�� �� �0� F� �^ `� #x���X��s7�����7�����$%�k��`m-m�V�-��7��=��s��4{��{�w���OnN��W�7�bc �� $s6����n��3���r�j���\{��~:J_�/��K�S����g� ���d�����M��.�����(E�y�����+)��-n��Mg���[�
z�����i ��; �!xH��G4��<��"��$H>��������K�������?�k����rDhI�lj��%m]YB�s6��~s�n�0
�n��E��p�^;�5�L{�z"*x��z{�rm;vY�r�S������+���Ay��-�~?��?��9�){���<�}�Z��i �Xt��<@��\�����Y��f��zw`#��r����v�<G�+zi�����d�$���u/V�lw���W{&��%�����|����o����c���T@��K��b���~��*�<eQ�
�k�8�\3��3J��SZ1���N\��a)��Z'���=�-����:���� ���i������sL��U�c��������h���{���������P]���_l�Q[U�:A�^J������Kw��o!5<S�()�������x��.��s?��X_c����_���f�����U�^o����jdH����UW���O�s|�^�d����R�j]4�>vA���^���= $#/ ��[u�n�N����yfj��J��#&���<=��u��i��6��<v��A]��W^��y�X��i��zj63��OY��*���?�V����w�%������pz�c-�YA�tLk���O�����_c�sZ���Z��G�N[�o��Q��i����/.���^����������c���������Z��e;g?���C��S
;���������t|�=���z������~A��4P�y���K�(�%m��?�����7�P�<1����w5�aoM�i�����:S��jh��
�{��1;P�i�;&�_��f�A��WV-7����/���UKz���~���G���<\F���q�tpZ+U��EO�RY���D�������K��v����m����Zg��vV����^����a��O����������7[�u{K-_����
����]������c��Pi�<��������kh���Z{���k� � xH����&�|=-���V@@���A�k��/o?H
/�I#:���l�6�s��{�k-
�}cI��y��>������_�y;M���~R����N�������bf5x��*��/�K�������+ ��|R������4��c*7t�7�vF.�0��}�*M��eo���{��=�e�7Vn���R��^�������6�mWC���!��W�����N���C��L������ ������z����L�7I'�O��}�e�su Ngl����(W�����S���!:�PE����s�K�D��;mw� ��<�&��,)�6}�R���~WdL}6](�N�����=�~����_�����j�3��yJ����we�����E�n ��?�����q�����L�����*��%i��T�]{���o����q8_��Fw-��1�3o���O������w���@�A���+[��z������7b�v��X*���
��M�j��5��?Q^^m��6�)TO
r��)��r��
�[[�KeU���]]�fsWD��E[#���
9b���T�r��r�6��?&9��1{}�b��HT-Yn���y��Y����W�j��:v���l6��!������Z�_�����jY���eJYM�|Y�������
b�����id ���[LD�;Vo��v�O��nu��~}�.�4�����J�,�[(���X���K�w����c�������(����JW�����Uw�C{{���C�b
���E����2v���= $/ H�E~i��\�eI�B>���o!Z��),��&��������(E���;iOUY�k���*2f�fN��>|���rTW��'jt�b�.����)?���{t�����:���1�w�o*_�Ye�j����]�3l�^z������f��I����������z�f
;�s6_�Jy��Z���w��nw��}�B������[�o�_���#V�^��8������]��n�x�c�&]��}h�)u��}�^�k���qk�������>� �<� <�:��)*2*����]�r2������y�!���2��~��[��J��G�|5O�+r����[�s]�����:��yD*t��������|n�����,u������^�}��.(������6#�����w����?*�hv��b��v�uJ'N�_�X����R�k?�������n_�_�]�;�Q
�����p��~������O����;��}��������D�a���R���������`��:���(e
�����+�W�>�_[�</����h��IG������u�.�(�"9����RE����':p�_e�qLn=u@��G������~���T�A ��x*]���6n�~�����vF��qk���Y��m�xX�*y��������)<]E����������P��a;sy�v4\�,i�.���7|��vD�V4VL�V�G��U��C�]�_�U\=��X�w����[~�Y��<N�w�:���`���w����Z����=��t�>�[?~��~�����7��������?u!Q}x�+���������� y����W:��
-�-�
�,���ET���>��s��BQ�J����)�������w���[@�����/����J��-=�Z7��%��^�����!F�m������&��������;)��/���JU1A���N���>��K��qY,��5�h����F�������}���������C*��g���r� 1��H��Iw�N��r�/���yj�FE����>��uRi��m���R����z�~[�Fu��9F������+zw����e?��>��u��U[�����L�,��W��+Kn{h����}�o]=�����# �=����?�|w�P���khk{��J�N�j���w������]��\���= $/ ��%��zv��NS��^��|�M��fR��&jm�Aj�c��3R���N��j�$RQW^�����w���Z��V�������oYk~�9z��M;@C�W[#�\�{@~Ul2BGt�z�[�=�0�u�:��j���.X���X
���{�^:��p5�]�/���w�T����J������F���|�kn����%�;m�}v��?i�:��^�gz+G�g4h�5<�Rk��FM����U�k���-�g����>�5��E�7�+���o�%l��M�w��r��^��UX���}��������o���&�Z��@��S?��XG*2�]��V����q��:t9��$�u�������5'�o� �#xH������a��S��>Zg���]�����Xq��w~��\����f-M>r��S�\
<���M ��*�y�����1������H`q�*��R3QKBn��Wf�P����}��R���~f���������Z�l$��gB�{����u������}���T���g��}�{B���\���������~��������<���1�e�������k�����������j~���d�� �� �0� ����K �����I�N ��� �� �0� F� �^ `� #x �a/ 0�� �� �0� F� �^ `� #x �a/ 0�� �� �0�����g}�m������r���i��*�,�. @2@����k�:��K�?�����4�g}�����\G��. @�G��M������������C��t@�W��G�Qm/g� �#x�Yb�(��n>�K-��{�/������ ��!x�"���U��5qC�^-yQ+'}�-�^ja�2 ����Po}��^u��>����=��Z�?���-�r�����:3 <�_����� x]�������vW�E����W�Tq�
���(((�!� �GU�1'�6�����O���r5�a��f>��C�P�g��� ��vnY[������bf����X����Q: �C�-bX�T��\����� H�^ `� #x �a/ 0�� �� �0� F� �^ `� #x �a/ 0�� �� �0� F� �^ `� #x �a/ 0�� �� �0� F� �^ `� #x �a/ 0�� �� �0� F� �^ `� #x �a/ 0�� �� �0� F� �^W\�����n���h�����zm�x�/���� H^v��S����j�f�z���Ni�*=����~����� $u/��;u [U��u�y�j��u�.�� �#x�y�����?��?z�p��h��V������ @r@���dh�w}��U�kb��<kh��F�`qve ���]����r��F�u\-sI���R��o���wU���r�����l�^�`���W, �$���h\�\��������L����*��_E�J���
�������� ���?�'}/�+s�|��d�V�l���-
]�T�
�i�� ������Fif��z�B��X�T%����
�sve ��W�,���\���� H�^ `� #x �a/ 0�� ��ohG��>gWW��R�:��.�� ����Ni����"�s��� .�� �� �0� F� �^ `� #x �a/ 0�� �� �0� F� �^ `� #x �a/ 0�� �� �0� F� �^ `� #x �a/ 0�� �� �0� F� �^ `� #x �a/ 0�� �� �0��l:9����X��7�Y�'4i�R��bqbm ���,J��G��p��Mg�T���T/� ��#x���f��f��L���n�. <4G�I!?;���R���7vv ���UG����������] x�vl�����U���(�H&^�Em�����lzC��*C B��%b�,}��BsJy&8?88X6�-���.
�uG�J;��[����j�3� {_�9��[����Hx
g�A��!J�.�[��*������������� �� �R�K��]D\~~�9�H�y����."���K�tm���O�^��N+d�?*��0� ����E���K�L|�] �d��u�G
9��* $C/ 0�� �� �0� F� �^ `� #x �a/ 0�� �� �0� F� �^ `� #x �a/ 0�� �� �0� F� �^ `� #x �a/ 0�� �� �0� F� �^ `� #x �a/ 0�� �� �0� F� �^ `� #x �a�k���+�h���r���^��c���Fg �#x������/ig��y~�����L��V��SV7g �#x�E��N_k�1��)�EJ�v���uvU ���]�_hO������:/����j����T*g� x����gu~�J{e�~���=S����F��oCU� �������[�E�V�'2��"��E���_�XU&��y�f��ka���` `@�!*��"nvN�9�H�'�}���"n�#�)�]\���'o~e;�3�3�=\Y�.����{mL�
2_$ 0+E����E���������7\:��"�*]��J�v��������e�Q��������:jV�����iZ�� ���[ xp������^U������.���S��K*I� p ��5)Kt����9� �� #x �a/ 0�� �� �0� F� �^ `� #x �a/ 0�� �� �0� F� �^ `� #x �a/ 0�� �� �0� F� �������������u���)]���R�����wg������ ��8>x����������z�Kg
�n�����T:�����e�kb�j���cM�VR~�W .���+r�~:�Js67V.��3R+S��j��rP���������u�� yp|����~��n
��0�/M
�y��SC�:|� ���\���N|VC�#�U�SR����P��h������ ���^k4�������/xV?�����SjH�YZ���\b ��a.x��),eN�� ���;T�UU��:�,��t�fl�p�����,pvq�$=���U ��\��,�R�!z��F�PE�&x����w�����a�L�a���?��2�x%l������2���tv �=3�����������z�U_�vM��������&Y�7�S��=�.#���/*��� �����Z��J�����['����u1O��7i���M� ��0����{�zm��"����=�U�b��>��k7+���*���k �����QF]'NU����^u�m����6/�'��"���
.��g�����#~
�}���������%>� ��a����T/�Kz:p�������th��Z�a�~��t !�j�E���E��������cW�Jw�To7���n��
.��7jY/�*4�[�Y�+��&��]�?S��O*� xD�^�;4�I#��RKe3���I.OU�[+ ��7�������/�Sc+q�s��q�Zd��}}�f�R*� 90xs�R��y���IU����!�g��|�nZ�s�jy�}y �/5�Wgw}�f�Q�������$�'4�����Bw6��;��Ti��M? `.x��Q�W���n��d��*l����z����[�!u
k����)�ToN�N�}�] �d����3�H���w$�4�����*�xQ����Q-���a*2\�\�� �+�>n��K�gRt�8l�+3{���^-�1C���9KZ���\]�UW6��{��;���*�����Z�?Z�
���}pp�l���d\�`��p����
G;u���K����#
q@ �k��3�~������Y�_j��"nvN�]�}�����]B<�S�W����EHH����pvp��������H�,�����F������6��)
��r�&�K������T�H��F��m���g�"��6&t=p
S7|+{�v) ��������+k������ 8���K�x�k��q�d?��9�p� ��|}}e?�t)�K�V���?G���e.xEl���4k�+*t��Q�J����l�.t�+b�X�|1R��MW�\��q�D��U_���n ���e��g�y]�������.�;�� R���[��U3�g��2�i��_�WqW+ @�d.xyU���^�3��+�F����p�B����^r��]WX2���s����. @rd����6�'}���Z�����E�|���7i �(1�.k��j���V�����4�N�<2G?�.&W;� ����/��M~MY�M�����������������:$/ ��g��������&y�Vj�h� xt��FU������?��s)�-T;W������
�v x�<��J�G.��e�k��}:�l���BM}��} <:/�K'u`�~�UR��zZ)~_����/��_
����e;��=�i�oE�l���)��C�vlW-���2X��x��=}AAg;��x6�L���+UMJ��* :s�+r�k�)?���f?�'x������{��\A���<TQ����_g���>�� ��������� x�^j�%/K�"m�&]>���}T��] !�jXEm+U�f[��'T����QK�H��z�� �F��WT���>
���C���_�L#��������?������v���(:��E���m0x�+�*^��+���5{u��N��_�f��*��� H�6#��3���3�U�T�D��Qf�~^%��I�7����W������R�����C������w5��R������gk������ N����:t�\��:�-q�I��/�_����k��unvKe��)���uZ�����'7d����4kK�������������$�0��X�?�����������1�)\}�G�G,��V������I�t�-{�W�l�R������y�Z���F����R]r���9j��*����?qR��~D���S�q_kD}?m�����^�Q��d
�;3�*��n�����Q��t������)���xL�����3�.k��g5�:B_u>�!��k�!xa�����|�� ��x�V�
G�����J���R����P�������*�/��?����Q���i�/k���Yj��-}pd����L������MTb�z1�e��5Q�_�G���y�t�����e���K��tj:�5���j�O%t�>ww�v���#7k���:�����8ZAKh����j�.���������F������wOQ�7��������1�hQQ]�U���$�m�!�=�)3�"�h��9�kQ�_��~(;K{W��������U��y I@
U�QJ������������N�z�EW9�5[K�f/���������en��g(YR���W'�6��Z���w��y��O�T�>������]�%]-5��N1���Q_�NN�������-U��<U�jq]������ �l:�j�����f9c���v_�P3�Tr�q�m��/�9����>�Z�t�*���T^G��������r ��"�j��e��9�Q����Z6�~u�N�>���k�w��s�W����q#77��l��cU���J�����i�.�9����J���+��0��J~>��O����K�����,��("w�[^i��Sg��&���LsO�Z������6�s����JY����u`C����S�fh����j��e �=W5U
�_����R��'�U%�~��?���U)���������U��S�(]9��K�u*<�J�v��f�G��l���d�MT����:��)'�����V�������Z��[�Km��/�(�����������D���������v�E �|�5c������^mU����G�����*�� �S�"��pvq=?B�]��U H�<J�F�}2����J�������k��K�0��=T�
^���<���wj��L5�����������W���W�T���)�������a6�I���V���E�v���b��B�W���ub�<mx���i��9�r�Yd;�I���M��O���_��,J_������f��\�����=�kL��Z��A]��f�������v�y�����
�����u������T��*_�1g�������gW����� ��TFg.\Jp^���o���
�sk���z����O��������by{
�ysI�UU��V��Q�o�~��/h����XXc��������{�h�P
h��^�\ISr�T��uT��hYc��o�j
��K��sWS�UT��:r1�:N���"k����U��e�WC�_[
�����F�C����{^�F���
si�yOe���>�WB{����M���W������Jk�s#�k�-�����o^}�/P �r���2�7g�����T����e����&��?�@r��3��ai�������nN���Z]~���������s
���������^�Q�m�����G��M�/�n]W
5��^�O4
�w��ec��p�Y���'�4����G�)-v��S�xc������u�Gqj�u����=�v�WV� �����o������ ����VH=�]|�\��o��;��8.^�T����.#��6��� ����+z���O�L�u1g��W�{��1 nq�l�Z�3��e���f�:~ @�d��W�g�n��� ��(;L[��}9gq|�r���5���� �����5F������\ �#�K
�0.5 ��/ $j�+*��]������Z��/
x���lK�Z�i��c�o���w��.��5��� x
�"��Hx^��R���n���@��JJ�..o�'o�F����!��F����)x�
3�V�x�G��i�w�F:)��vm��Y��2:|� m�:��? ������#����J?o�^+��s�[*{�(M9<G��Z�lM=�!�����Y[�(�u7}�����Fo�( ��i������f��g&�����)���Fm|U���P�'N������~j2�k������Em�����^�Q��d
�;3�*�����o����;B;>��.���%oe�3@S&u���D�`���x�
�;��kc�@�I�%�����a3�V ��YZ5*�����V(�~[sH��Z�>$B�k�k���X��B��/��G]&�������g���������#�2���S�f6Q�Q_�����^��.��U�w�f-�VgvU�G����U��gz���j�~�����^e�nd#���������S������z�:<vL3ZTT�qU��)���3����������m���J ��Bk�R���+�uf��=�:���^�G�U�k�����K!�y�%um�k�[^��3�,��N��V�=�t���f�+'l����r''h�Q��������[��hy�b���8���s2����k[��j�3f�j��5sO%����$s����Z�O����Ke���S^o����� \�E���+��
�s<�6�����l�����}L�3�P�4�5�v�N���������l6%t�[:���<R���������8����h������)��=��]��W[u��y�I#�kS�S��r(�j0�\���������;��^�X�zj�����N/ �*�sUS����eiZ+�Yy�ZU���Z����\e�r�}�����
�����$���:�Z%����W}���RZ��[�Km��/�(��[_���i~��.������=�]��)�W���W�F-8�UO�W>sk `�GI�(�OC�Z��_IyzZU1wM�uI����X���C��t6��Va]�����N��:�p�6<VI�d�����R����g���&M��7Y/>��~7��mQ��T���5���������Q]c��������f.yR�B;u>��Z ��g�I�����-�U�znm�4T����2��r���8�^,�-��������*��E+k����.�=w5e_�A���#s��� z��M�����U��e�WC�_[
�����F�C��W��o�j
������[�������V�5}���<�~�c��Hd
��]3���W*X(���xyU����j|�2 �<�:����?7Oa��2��������'�G��m��R�E����g�c�����
�u�UdQ��3��- ��XcM�h����`�k?���7�z�H��Zg�q�:�X�A�^���~K:�<���6U�o~��*��*�a����Z�_� �3p�+J�j�w�u����T5s��hD����7��]��pv�� \����{au��;�6X�T����P�Je����vj���j��};��2s� x�y������\Rf�3^�U��T��5N�wn��C'u��J��T���cd� ���o�G��?����������k�~�&�Y�Sr 7z���:
u�"��:� �����
y�e�\�M�^2��� HF���}Z�����,�tI�����'�������V,qv5 �s��vZ�wR��U�ugu��BO�Nu��z�{H���W*2�'UIuH+n�Xpp�l6[��,x�������h�N�rv �9rD!�oG:�����k�����wu��Qg��I��N���Z��D�|���]N���8��C�]B<aa�����=Wu�����[HH����pvp��<W��[���?�b�l}���^u+�&���}�r��}�&����e}��m-��F����7-&t=pS7|+{�v) @:��*���5��:�����s���.#I�\��T�����c��%�6�: M����:���s$W���}��~��]E\�{���?�t��U��������+&��}�}}}����U�U�ti�����s���\��(�PTu����^�1�cM�7�z;���`�����3`�3Z��j��e��������o�5������2 @e0x]����������U�U�Igu��7}T8��vOU�g����cU��\�b�W�9'�����mR���f��v� �\���J����������+�%�����>�;_3=��J���] `����]G�����y{��s�+7������>=G?�.��/u1���^\�6mtv �s�+�M�&��,���1��TD�~�@{�N���?P/ck �b�3^Q���W@�X��S+�g�} <:�/��jS�-��D
*�Rj[�v���U?��� �1x�+���\��������tF�T�����d� xt�/���:*���M�6�.�[�/>>���T�� �a.xE���&�47K-���s�6�Wx�����Z ���^Q;�w�AZ��sJcl% ���/�R��y���IU��� \��K
���]��Y�o��pf��~�����@U�4�f p)���{�{e���w�[z6zK p-�"�[���X� �T�=��OKg���K��[O+���t�p�Kit� �R�/�)���L�}+�g�f�H)��c�j�g���r�&