A GUC to prevent leader processes from running subplans?

Started by Thomas Munroabout 8 years ago11 messages

thomas.munro@enterprisedb.com

about 8 years ago

1 attachment(s)

Hi hackers,

While testing parallelism work I've wanted to be able to prevent
gather nodes from running the plan in the leader process, and I've
heard others say the same. One way would be to add a GUC
"multiplex_gather", like in the attached patch. If you set it to off,
Gather and Gather Merge won't run the subplan unless they have to
because no workers could be launched. I thought about adding a new
value for force_parallel_mode instead, but someone mentioned they
might want to do this on a production system too and
force_parallel_mode is not really for end users. Better ideas?

--
Thomas Munro
http://www.enterprisedb.com

Attachments:

0001-Add-a-GUC-to-control-whether-Gather-runs-subplans.patchapplication/octet-stream; name=0001-Add-a-GUC-to-control-whether-Gather-runs-subplans.patchDownload

From 3c48c388b2a9af473f9b9240edc219176479da8a Mon Sep 17 00:00:00 2001
From: Thomas Munro <thomas.munro@enterprisedb.com>
Date: Tue, 17 Oct 2017 23:40:03 +1300
Subject: [PATCH] Add a GUC to control whether Gather runs subplans.

Gather and Gather Merge nodes are responsible for gathering tuples from
worker processes, but also run the subplan directly in the leader process.
Add a new GUC multiplex_gather to enable or disable this type of
multiplexing.  If set to off, the leader process only runs the plan as a
fall back in case no workers could be launched.  It is initially intended
for testing, but might prove useful for end users.

Author: Thomas Munro
---
 doc/src/sgml/config.sgml               | 19 +++++++++++++++++++
 src/backend/executor/nodeGather.c      |  8 +++++---
 src/backend/executor/nodeGatherMerge.c |  6 ++++--
 src/backend/optimizer/path/costsize.c  |  5 +++--
 src/backend/optimizer/plan/planner.c   |  1 +
 src/backend/utils/misc/guc.c           | 10 ++++++++++
 src/include/optimizer/planmain.h       |  1 +
 7 files changed, 43 insertions(+), 7 deletions(-)

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index b012a269911..146f72eac18 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4265,6 +4265,25 @@ SELECT * FROM parent WHERE key = 2400;
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-multiplex-gather" xreflabel="multiplex_gather">
+      <term><varname>multiplex_gather</varname> (<type>boolean</type>)
+      <indexterm>
+       <primary><varname>multiplex_gather</> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Allows the leader process to execute the query plan under
+        <literal>Gather</> and <literal>Gather Merge</>.  The default is
+        <literal>on</>.  Setting this value to <literal>on</> can cause the
+        leader process to begin producing tuples sooner instead of waiting
+        for worker processes to start up, but might in some cases also cause
+        workers to become blocked waiting for the leader to clear tuple
+        queues.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-force-parallel-mode" xreflabel="force_parallel_mode">
       <term><varname>force_parallel_mode</varname> (<type>enum</type>)
       <indexterm>
diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c
index 8370037c433..9af83910aed 100644
--- a/src/backend/executor/nodeGather.c
+++ b/src/backend/executor/nodeGather.c
@@ -38,6 +38,7 @@
 #include "executor/nodeSubplan.h"
 #include "executor/tqueue.h"
 #include "miscadmin.h"
+#include "optimizer/planmain.h"
 #include "pgstat.h"
 #include "utils/memutils.h"
 #include "utils/rel.h"
@@ -73,7 +74,8 @@ ExecInitGather(Gather *node, EState *estate, int eflags)
 	gatherstate->ps.ExecProcNode = ExecGather;
 
 	gatherstate->initialized = false;
-	gatherstate->need_to_scan_locally = !node->single_copy;
+	gatherstate->need_to_scan_locally =
+		!node->single_copy && multiplex_gather;
 	gatherstate->tuples_needed = -1;
 
 	/*
@@ -193,9 +195,9 @@ ExecGather(PlanState *pstate)
 			node->nextreader = 0;
 		}
 
-		/* Run plan locally if no workers or not single-copy. */
+		/* Run plan locally if no workers or enabled and not single-copy. */
 		node->need_to_scan_locally = (node->nreaders == 0)
-			|| !gather->single_copy;
+			|| (!gather->single_copy && multiplex_gather);
 		node->initialized = true;
 	}
 
diff --git a/src/backend/executor/nodeGatherMerge.c b/src/backend/executor/nodeGatherMerge.c
index 70f33a9a28f..9c386f65bba 100644
--- a/src/backend/executor/nodeGatherMerge.c
+++ b/src/backend/executor/nodeGatherMerge.c
@@ -23,6 +23,7 @@
 #include "executor/tqueue.h"
 #include "lib/binaryheap.h"
 #include "miscadmin.h"
+#include "optimizer/planmain.h"
 #include "utils/memutils.h"
 #include "utils/rel.h"
 
@@ -233,8 +234,9 @@ ExecGatherMerge(PlanState *pstate)
 			}
 		}
 
-		/* always allow leader to participate */
-		node->need_to_scan_locally = true;
+		/* allow leader to participate if enabled or no choice */
+		if (multiplex_gather || node->nreaders == 0)
+			node->need_to_scan_locally = true;
 		node->initialized = true;
 	}
 
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index ce32b8a4b90..6ec0e860e6d 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -5093,7 +5093,7 @@ static double
 get_parallel_divisor(Path *path)
 {
 	double		parallel_divisor = path->parallel_workers;
-	double		leader_contribution;
+	double		leader_contribution = 0;
 
 	/*
 	 * Early experience with parallel query suggests that when there is only
@@ -5106,7 +5106,8 @@ get_parallel_divisor(Path *path)
 	 * its time servicing each worker, and the remainder executing the
 	 * parallel plan.
 	 */
-	leader_contribution = 1.0 - (0.3 * path->parallel_workers);
+	if (multiplex_gather)
+		leader_contribution = 1.0 - (0.3 * path->parallel_workers);
 	if (leader_contribution > 0)
 		parallel_divisor += leader_contribution;
 
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index ecdd7280eb8..9e0da467a16 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -61,6 +61,7 @@
 /* GUC parameters */
 double		cursor_tuple_fraction = DEFAULT_CURSOR_TUPLE_FRACTION;
 int			force_parallel_mode = FORCE_PARALLEL_OFF;
+bool		multiplex_gather = true;
 
 /* Hook for plugins to get control in planner() */
 planner_hook_type planner_hook = NULL;
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index ae22185fbdb..bef83f3d995 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -1676,6 +1676,16 @@ static struct config_bool ConfigureNamesBool[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"multiplex_gather", PGC_USERSET, QUERY_TUNING_OTHER,
+			gettext_noop("Controls whether Gather and Gather Merge also run subplans."),
+			gettext_noop("Should gather nodes also run subplans, or just gather tuples?")
+		},
+		&multiplex_gather,
+		true,
+		NULL, NULL, NULL
+	},
+
 	/* End-of-list marker */
 	{
 		{NULL, 0, 0, NULL, NULL}, NULL, false, NULL, NULL, NULL
diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h
index f1d16cffab0..2cdaf578812 100644
--- a/src/include/optimizer/planmain.h
+++ b/src/include/optimizer/planmain.h
@@ -29,6 +29,7 @@ typedef enum
 #define DEFAULT_CURSOR_TUPLE_FRACTION 0.1
 extern double cursor_tuple_fraction;
 extern int	force_parallel_mode;
+extern bool multiplex_gather;
 
 /* query_planner callback to compute query_pathkeys */
 typedef void (*query_pathkeys_callback) (PlannerInfo *root, void *extra);
-- 
2.14.1