Parallel Inserts in CREATE TABLE AS

From bca61cd199970ae050dc74b4a7d2d7b275ad0ce0 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Wed, 14 Oct 2020 14:25:24 +0530
Subject: [PATCH v2] Parallel Inserts in CREATE TABLE AS

The idea of this patch is to allow the leader and each worker
insert the tuples in parallel if the SELECT part of the CTAS is
parallelizable.

The design:
Let the planner know that the SELECT is from CTAS in createas.c
so that it can set the number of tuples transferred from the
workers to Gather node to 0. With this change, there are chances
that the planner may choose the parallel plan. After the planning,
check if the upper plan node is Gather in createas.c and mark a
parallelism flag in the CTAS dest receiver. Pass the into clause,
object id, command id from the leader to workers, so that each
worker can create its own CTAS dest receiver. Leader inserts it's
share of tuples if instructed to do, and so are workers. Each
worker writes atomically it's number of inserted tuples into a
shared memory variable, the leader combines this with it's own
number of inserted tuples and shares to the client.
---
 src/backend/access/heap/heapam.c      |  11 -
 src/backend/access/transam/xact.c     |  14 ++
 src/backend/commands/createas.c       | 321 ++++++++++++++++----------
 src/backend/commands/explain.c        |  19 ++
 src/backend/executor/execMain.c       |  19 ++
 src/backend/executor/execParallel.c   |  61 ++++-
 src/backend/executor/nodeGather.c     |  97 +++++++-
 src/backend/optimizer/path/costsize.c |  12 +
 src/include/access/xact.h             |   1 +
 src/include/commands/createas.h       |  18 ++
 src/include/executor/execParallel.h   |   1 +
 src/include/nodes/execnodes.h         |   5 +
 src/include/nodes/parsenodes.h        |   1 +
 13 files changed, 437 insertions(+), 143 deletions(-)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 1585861a02..1602525d4a 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2043,17 +2043,6 @@ static HeapTuple
 heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
 					CommandId cid, int options)
 {
-	/*
-	 * To allow parallel inserts, we need to ensure that they are safe to be
-	 * performed in workers. We have the infrastructure to allow parallel
-	 * inserts in general except for the cases where inserts generate a new
-	 * CommandId (eg. inserts into a table having a foreign key column).
-	 */
-	if (IsParallelWorker())
-		ereport(ERROR,
-				(errcode(ERRCODE_INVALID_TRANSACTION_STATE),
-				 errmsg("cannot insert tuples in a parallel worker")));
-
 	tup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
 	tup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
 	tup->t_data->t_infomask |= HEAP_XMAX_INVALID;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index af6afcebb1..24eb8fca38 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -775,6 +775,20 @@ GetCurrentCommandId(bool used)
 	return currentCommandId;
 }
 
+/*
+ *	SetCurrentCommandIdUsedForWorker
+ *
+ * For a parallel worker, record that the currentCommandId has been used.
+ * This must only be called at the start of a parallel operation.
+*/
+void
+SetCurrentCommandIdUsedForWorker(void)
+{
+	Assert(IsParallelWorker() && !currentCommandIdUsed && currentCommandId != InvalidCommandId);
+
+	currentCommandIdUsed = true;
+}
+
 /*
  *	SetParallelStartTimestamps
  *
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index d53ec952d0..a3ab772389 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,18 +51,6 @@
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 
-typedef struct
-{
-	DestReceiver pub;			/* publicly-known function pointers */
-	IntoClause *into;			/* target relation specification */
-	/* These fields are filled by intorel_startup: */
-	Relation	rel;			/* relation to write to */
-	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
-	CommandId	output_cid;		/* cmin to insert in output tuples */
-	int			ti_options;		/* table_tuple_insert performance options */
-	BulkInsertState bistate;	/* bulk insert state */
-} DR_intorel;
-
 /* utility functions for CTAS definition creation */
 static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into);
 static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -328,10 +316,28 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		query = linitial_node(Query, rewritten);
 		Assert(query->commandType == CMD_SELECT);
 
+		/*
+		 * Flag to let the planner know that the SELECT query is for
+		 * CREATE TABLE AS. This is used to make the number tuples
+		 * transferred from workers to gather node(in case parallelism
+		 * kicks in for the SELECT part of the CTAS), to zero as each
+		 * worker will parallelly insert it's share of tuples.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, NULL))
+			query->isForCTAS = true;
+
 		/* plan the query */
 		plan = pg_plan_query(query, pstate->p_sourcetext,
 							 CURSOR_OPT_PARALLEL_OK, params);
 
+		/*
+		 * SELECT part of the CTAS is parallelizable, so we can make
+		 * each parallel worker insert the tuples that are resulted
+		 * in it's execution into the target table.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, plan))
+			((DR_intorel *) dest)->is_parallel = true;
+
 		/*
 		 * Use a snapshot with an updated command ID to ensure this query sees
 		 * results of any previously executed queries.  (This could only
@@ -418,6 +424,9 @@ CreateIntoRelDestReceiver(IntoClause *intoClause)
 	self->pub.rDestroy = intorel_destroy;
 	self->pub.mydest = DestIntoRel;
 	self->into = intoClause;
+	self->is_parallel = false;
+	self->is_parallel_worker = false;
+	self->object_id = InvalidOid;
 	/* other private fields will be set during intorel_startup */
 
 	return (DestReceiver *) self;
@@ -430,135 +439,180 @@ static void
 intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 {
 	DR_intorel *myState = (DR_intorel *) self;
-	IntoClause *into = myState->into;
-	bool		is_matview;
-	char		relkind;
-	List	   *attrList;
 	ObjectAddress intoRelationAddr;
 	Relation	intoRelationDesc;
-	RangeTblEntry *rte;
-	ListCell   *lc;
-	int			attnum;
-
-	Assert(into != NULL);		/* else somebody forgot to set it */
 
-	/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
-	is_matview = (into->viewQuery != NULL);
-	relkind = is_matview ? RELKIND_MATVIEW : RELKIND_RELATION;
+	if (myState->is_parallel_worker)
+	{
+		/* In the worker */
+		intoRelationDesc = table_open(myState->object_id, AccessExclusiveLock);
+		myState->rel = intoRelationDesc;
+		myState->reladdr = InvalidObjectAddress;
+		myState->ti_options = 0;
+		myState->bistate = GetBulkInsertState();
 
-	/*
-	 * Build column definitions using "pre-cooked" type and collation info. If
-	 * a column name list was specified in CREATE TABLE AS, override the
-	 * column names derived from the query.  (Too few column names are OK, too
-	 * many are not.)
-	 */
-	attrList = NIL;
-	lc = list_head(into->colNames);
-	for (attnum = 0; attnum < typeinfo->natts; attnum++)
+		/*
+		 * Right after the table is created in the leader, the command id is
+		 * incremented (in create_ctas_internal()). The new command id is
+		 * marked as used in inintorel_startup(), then the parallel mode is
+		 * entered. The command id and transaction id are serialized into
+		 * parallel DSM, they are then available to all parallel workers. All
+		 * the workers need to mark the command id as used before insertion.
+		 */
+		(void) SetCurrentCommandIdUsedForWorker();
+		myState->output_cid = GetCurrentCommandId(false);
+	}
+	else
 	{
-		Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
-		ColumnDef  *col;
-		char	   *colname;
+		IntoClause *into = myState->into;
+		bool		is_matview;
+		char		relkind;
+		List	   *attrList;
+		Relation	intoRelationDesc;
+		RangeTblEntry *rte;
+		ListCell   *lc;
+		int			attnum;
 
-		if (lc)
-		{
-			colname = strVal(lfirst(lc));
-			lc = lnext(into->colNames, lc);
-		}
-		else
-			colname = NameStr(attribute->attname);
+		Assert(into != NULL);		/* else somebody forgot to set it */
 
-		col = makeColumnDef(colname,
-							attribute->atttypid,
-							attribute->atttypmod,
-							attribute->attcollation);
+		/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
+		is_matview = (into->viewQuery != NULL);
+		relkind = is_matview ? RELKIND_MATVIEW : RELKIND_RELATION;
 
 		/*
-		 * It's possible that the column is of a collatable type but the
-		 * collation could not be resolved, so double-check.  (We must check
-		 * this here because DefineRelation would adopt the type's default
-		 * collation rather than complaining.)
-		 */
-		if (!OidIsValid(col->collOid) &&
-			type_is_collatable(col->typeName->typeOid))
-			ereport(ERROR,
-					(errcode(ERRCODE_INDETERMINATE_COLLATION),
-					 errmsg("no collation was derived for column \"%s\" with collatable type %s",
-							col->colname,
-							format_type_be(col->typeName->typeOid)),
-					 errhint("Use the COLLATE clause to set the collation explicitly.")));
+		* Build column definitions using "pre-cooked" type and collation info. If
+		* a column name list was specified in CREATE TABLE AS, override the
+		* column names derived from the query.  (Too few column names are OK, too
+		* many are not.)
+		*/
+		attrList = NIL;
+		lc = list_head(into->colNames);
+		for (attnum = 0; attnum < typeinfo->natts; attnum++)
+		{
+			Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
+			ColumnDef  *col;
+			char	   *colname;
 
-		attrList = lappend(attrList, col);
-	}
+			if (lc)
+			{
+				colname = strVal(lfirst(lc));
+				lc = lnext(into->colNames, lc);
+			}
+			else
+				colname = NameStr(attribute->attname);
 
-	if (lc != NULL)
-		ereport(ERROR,
-				(errcode(ERRCODE_SYNTAX_ERROR),
-				 errmsg("too many column names were specified")));
+			col = makeColumnDef(colname,
+								attribute->atttypid,
+								attribute->atttypmod,
+								attribute->attcollation);
 
-	/*
-	 * Actually create the target table
-	 */
-	intoRelationAddr = create_ctas_internal(attrList, into);
+			/*
+			* It's possible that the column is of a collatable type but the
+			* collation could not be resolved, so double-check.  (We must check
+			* this here because DefineRelation would adopt the type's default
+			* collation rather than complaining.)
+			*/
+			if (!OidIsValid(col->collOid) &&
+				type_is_collatable(col->typeName->typeOid))
+				ereport(ERROR,
+						(errcode(ERRCODE_INDETERMINATE_COLLATION),
+						errmsg("no collation was derived for column \"%s\" with collatable type %s",
+								col->colname,
+								format_type_be(col->typeName->typeOid)),
+						errhint("Use the COLLATE clause to set the collation explicitly.")));
 
-	/*
-	 * Finally we can open the target table
-	 */
-	intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
+			attrList = lappend(attrList, col);
+		}
 
-	/*
-	 * Check INSERT permission on the constructed table.
-	 *
-	 * XXX: It would arguably make sense to skip this check if into->skipData
-	 * is true.
-	 */
-	rte = makeNode(RangeTblEntry);
-	rte->rtekind = RTE_RELATION;
-	rte->relid = intoRelationAddr.objectId;
-	rte->relkind = relkind;
-	rte->rellockmode = RowExclusiveLock;
-	rte->requiredPerms = ACL_INSERT;
+		if (lc != NULL)
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					errmsg("too many column names were specified")));
 
-	for (attnum = 1; attnum <= intoRelationDesc->rd_att->natts; attnum++)
-		rte->insertedCols = bms_add_member(rte->insertedCols,
-										   attnum - FirstLowInvalidHeapAttributeNumber);
+		/*
+		* Actually create the target table
+		*/
+		intoRelationAddr = create_ctas_internal(attrList, into);
 
-	ExecCheckRTPerms(list_make1(rte), true);
+		/*
+		* Finally we can open the target table
+		*/
+		intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
 
-	/*
-	 * Make sure the constructed table does not have RLS enabled.
-	 *
-	 * check_enable_rls() will ereport(ERROR) itself if the user has requested
-	 * something invalid, and otherwise will return RLS_ENABLED if RLS should
-	 * be enabled here.  We don't actually support that currently, so throw
-	 * our own ereport(ERROR) if that happens.
-	 */
-	if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
-		ereport(ERROR,
-				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-				 errmsg("policies not yet implemented for this command")));
+		/*
+		* Check INSERT permission on the constructed table.
+		*
+		* XXX: It would arguably make sense to skip this check if into->skipData
+		* is true.
+		*/
+		rte = makeNode(RangeTblEntry);
+		rte->rtekind = RTE_RELATION;
+		rte->relid = intoRelationAddr.objectId;
+		rte->relkind = relkind;
+		rte->rellockmode = RowExclusiveLock;
+		rte->requiredPerms = ACL_INSERT;
+
+		for (attnum = 1; attnum <= intoRelationDesc->rd_att->natts; attnum++)
+			rte->insertedCols = bms_add_member(rte->insertedCols,
+											attnum - FirstLowInvalidHeapAttributeNumber);
+
+		ExecCheckRTPerms(list_make1(rte), true);
 
-	/*
-	 * Tentatively mark the target as populated, if it's a matview and we're
-	 * going to fill it; otherwise, no change needed.
-	 */
-	if (is_matview && !into->skipData)
-		SetMatViewPopulatedState(intoRelationDesc, true);
+		/*
+		* Make sure the constructed table does not have RLS enabled.
+		*
+		* check_enable_rls() will ereport(ERROR) itself if the user has requested
+		* something invalid, and otherwise will return RLS_ENABLED if RLS should
+		* be enabled here.  We don't actually support that currently, so throw
+		* our own ereport(ERROR) if that happens.
+		*/
+		if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					errmsg("policies not yet implemented for this command")));
 
-	/*
-	 * Fill private fields of myState for use by later routines
-	 */
-	myState->rel = intoRelationDesc;
-	myState->reladdr = intoRelationAddr;
-	myState->output_cid = GetCurrentCommandId(true);
-	myState->ti_options = TABLE_INSERT_SKIP_FSM;
-	myState->bistate = GetBulkInsertState();
+		/*
+		* Tentatively mark the target as populated, if it's a matview and we're
+		* going to fill it; otherwise, no change needed.
+		*/
+		if (is_matview && !into->skipData)
+			SetMatViewPopulatedState(intoRelationDesc, true);
 
-	/*
-	 * Valid smgr_targblock implies something already wrote to the relation.
-	 * This may be harmless, but this function hasn't planned for it.
-	 */
-	Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+		/*
+		* Fill private fields of myState for use by later routines
+		*/
+		myState->rel = intoRelationDesc;
+		myState->reladdr = intoRelationAddr;
+		myState->output_cid = GetCurrentCommandId(true);
+		myState->ti_options = TABLE_INSERT_SKIP_FSM;
+		myState->bistate = GetBulkInsertState();
+
+		/*
+		* Valid smgr_targblock implies something already wrote to the relation.
+		* This may be harmless, but this function hasn't planned for it.
+		*/
+		Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+
+		if (myState->is_parallel == true)
+		{
+			myState->object_id = intoRelationAddr.objectId;
+
+			/*
+			 * We don't need to skip contacting FSM while inserting tuples
+			 * for parallel mode, while extending the relations, workers
+			 * instead of blocking on a page while another worker is inserting,
+			 * can check the FSM for another page that can accommodate the
+			 * tuples. This results in major benefit for parallel inserts.
+			 */
+			myState->ti_options = 0;
+
+			/*
+			 * rd_createSubid is marked invalid, otherwise, the table is
+			 * not allowed to extend by the workers.
+			 */
+			myState->rel->rd_createSubid = InvalidSubTransactionId;
+		}
+	}
 }
 
 /*
@@ -614,3 +668,28 @@ intorel_destroy(DestReceiver *self)
 {
 	pfree(self);
 }
+
+/*
+ * IsParallelInsertInCTASAllowed --- determine whether or not parallel
+ * insertion is possible.
+ */
+bool IsParallelInsertInCTASAllowed(IntoClause *into, PlannedStmt *plannedstmt)
+{
+	bool allowed = false;
+
+	if (into != NULL &&
+		into->type == T_IntoClause)
+	{
+		if (into->viewQuery == NULL &&
+			into->rel->relpersistence != RELPERSISTENCE_TEMP)
+			allowed = true;
+
+		if (plannedstmt != NULL && allowed)
+		{
+			if (!IsA(plannedstmt->planTree, Gather))
+				allowed = false;
+		}
+	}
+
+	return allowed;
+}
\ No newline at end of file
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index c8e292adfa..4cf6a3da44 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -371,6 +371,16 @@ ExplainOneQuery(Query *query, int cursorOptions,
 		return;
 	}
 
+	/*
+	 * Flag to let the planner know that the SELECT query is for
+	 * CREATE TABLE AS. This is used to make the number tuples
+	 * transferred from workers to gather node(in case parallelism
+	 * kicks in for the SELECT part of the CTAS), to zero as each
+	 * worker will parallelly insert it's share of tuples.
+	 */
+	if (IsParallelInsertInCTASAllowed(into, NULL))
+		query->isForCTAS = true;
+
 	/* if an advisor plugin is present, let it manage things */
 	if (ExplainOneQuery_hook)
 		(*ExplainOneQuery_hook) (query, cursorOptions, into, es,
@@ -536,7 +546,16 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	 * AS, we'd better use the appropriate tuple receiver.
 	 */
 	if (into)
+	{
 		dest = CreateIntoRelDestReceiver(into);
+		/*
+		 * SELECT part of the CTAS is parallelizable, so we can make
+		 * each parallel worker insert the tuples that are resulted
+		 * in it's execution into the target table.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, plannedstmt))
+			((DR_intorel *) dest)->is_parallel = true;
+	}
 	else
 		dest = None_Receiver;
 
diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c
index 783eecbc13..7928e53295 100644
--- a/src/backend/executor/execMain.c
+++ b/src/backend/executor/execMain.c
@@ -45,6 +45,7 @@
 #include "access/xact.h"
 #include "catalog/namespace.h"
 #include "catalog/pg_publication.h"
+#include "commands/createas.h"
 #include "commands/matview.h"
 #include "commands/trigger.h"
 #include "executor/execdebug.h"
@@ -352,6 +353,24 @@ standard_ExecutorRun(QueryDesc *queryDesc,
 	if (sendTuples)
 		dest->rStartup(dest, operation, queryDesc->tupDesc);
 
+	/*
+	 * For parallelizing inserts in CTAS i.e. making each
+	 * parallel worker inerst it's tuples, we must send
+	 * information such as intoclause(for each worker
+	 * building it's own dest receiver), object id(for each
+	 * worker to open the table).
+	 */
+	if (queryDesc->plannedstmt->parallelModeNeeded == true &&
+		dest != NULL &&
+		dest->mydest == DestIntoRel &&
+		((DR_intorel *) dest)->is_parallel == true &&
+		((DR_intorel *) dest)->is_parallel_worker != true)
+	{
+		queryDesc->planstate->intoclause = ((DR_intorel *) dest)->into;
+		queryDesc->planstate->objectid = ((DR_intorel *) dest)->object_id;
+		queryDesc->planstate->dest = dest;
+	}
+
 	/*
 	 * run plan
 	 */
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index befde52691..fd6f9429d0 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -23,6 +23,7 @@
 
 #include "postgres.h"
 
+#include "commands/createas.h"
 #include "executor/execParallel.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
@@ -65,6 +66,7 @@
 #define PARALLEL_KEY_QUERY_TEXT		UINT64CONST(0xE000000000000008)
 #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009)
 #define PARALLEL_KEY_WAL_USAGE			UINT64CONST(0xE00000000000000A)
+#define PARALLEL_KEY_INTO_CLAUSE		UINT64CONST(0xE00000000000000B)
 
 #define PARALLEL_TUPLE_QUEUE_SIZE		65536
 
@@ -77,6 +79,8 @@ typedef struct FixedParallelExecutorState
 	dsa_pointer param_exec;
 	int			eflags;
 	int			jit_flags;
+	Oid			objectid;		/* workers to open relation/table.  */
+	pg_atomic_uint64	processed; /* number tuples inserted by all the workers. */
 } FixedParallelExecutorState;
 
 /*
@@ -600,6 +604,7 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	Size		dsa_minsize = dsa_minimum_size();
 	char	   *query_string;
 	int			query_len;
+	char 		*intoclausestr = NULL;
 
 	/*
 	 * Force any initplan outputs that we're going to pass to workers to be
@@ -712,6 +717,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_estimate_chunk(&pcxt->estimator, dsa_minsize);
 	shm_toc_estimate_keys(&pcxt->estimator, 1);
 
+	/* Estimate space for into clause for CTAS. */
+	if (planstate->intoclause != NULL &&
+		planstate->intoclause->type == T_IntoClause)
+	{
+		intoclausestr = nodeToString(planstate->intoclause);
+		shm_toc_estimate_chunk(&pcxt->estimator, strlen(intoclausestr) + 1);
+		shm_toc_estimate_keys(&pcxt->estimator, 1);
+	}
+
 	/* Everyone's had a chance to ask for space, so now create the DSM. */
 	InitializeParallelDSM(pcxt);
 
@@ -729,6 +743,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	fpes->param_exec = InvalidDsaPointer;
 	fpes->eflags = estate->es_top_eflags;
 	fpes->jit_flags = estate->es_jit_flags;
+	pg_atomic_init_u64(&fpes->processed, 0);
+	pei->processed = &fpes->processed;
+
+	if (intoclausestr != NULL &&
+		planstate->objectid != InvalidOid)
+		fpes->objectid = planstate->objectid;
+	else
+		fpes->objectid = InvalidOid;
+
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, fpes);
 
 	/* Store query string */
@@ -758,8 +781,17 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_WAL_USAGE, walusage_space);
 	pei->wal_usage = walusage_space;
 
+	if (intoclausestr != NULL)
+	{
+		char *shmptr = (char *)shm_toc_allocate(pcxt->toc,
+												strlen(intoclausestr) + 1);
+		strcpy(shmptr, intoclausestr);
+		shm_toc_insert(pcxt->toc, PARALLEL_KEY_INTO_CLAUSE, shmptr);
+	}
+
 	/* Set up the tuple queues that the workers will write into. */
-	pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	if (intoclausestr == NULL)
+		pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
 
 	/* We don't need the TupleQueueReaders yet, though. */
 	pei->reader = NULL;
@@ -1387,12 +1419,28 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 	void	   *area_space;
 	dsa_area   *area;
 	ParallelWorkerContext pwcxt;
+	char		*intoclausestr = NULL;
+	IntoClause	*intoclause = NULL;
 
 	/* Get fixed-size state. */
 	fpes = shm_toc_lookup(toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
 
-	/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
-	receiver = ExecParallelGetReceiver(seg, toc);
+	intoclausestr = shm_toc_lookup(toc, PARALLEL_KEY_INTO_CLAUSE, true);
+	if (intoclausestr != NULL)
+	{
+		/*
+		 * If the worker is for parallel insert in CTAS, then use
+		 * the proper dest receiver.
+		 */
+		intoclause = (IntoClause *) stringToNode(intoclausestr);
+		receiver = CreateIntoRelDestReceiver(intoclause);
+		((DR_intorel *)receiver)->is_parallel_worker = true;
+		((DR_intorel *)receiver)->object_id = fpes->objectid;
+	}
+	else
+		/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
+		receiver = ExecParallelGetReceiver(seg, toc);
+
 	instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_INSTRUMENTATION, true);
 	if (instrumentation != NULL)
 		instrument_options = instrumentation->instrument_options;
@@ -1471,6 +1519,13 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 			queryDesc->estate->es_jit->instr;
 	}
 
+	/*
+	 * Write out the number of tuples this worker has inserted. Leader
+	 * will use it to inform to the end client.
+	 */
+	if (intoclausestr != NULL)
+		pg_atomic_add_fetch_u64(&fpes->processed, queryDesc->estate->es_processed);
+
 	/* Must do this after capturing instrumentation. */
 	ExecutorEnd(queryDesc);
 
diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c
index a01b46af14..3c41094e0a 100644
--- a/src/backend/executor/nodeGather.c
+++ b/src/backend/executor/nodeGather.c
@@ -48,7 +48,7 @@ static TupleTableSlot *ExecGather(PlanState *pstate);
 static TupleTableSlot *gather_getnext(GatherState *gatherstate);
 static MinimalTuple gather_readnext(GatherState *gatherstate);
 static void ExecShutdownGatherWorkers(GatherState *node);
-
+static void ExecParallelInsertInCTAS(GatherState *node);
 
 /* ----------------------------------------------------------------
  *		ExecInitGather
@@ -131,6 +131,66 @@ ExecInitGather(Gather *node, EState *estate, int eflags)
 	return gatherstate;
 }
 
+/* ----------------------------------------------------------------
+ *		ExecParallelInsertInCTAS(node)
+ *
+ *		Facilitates parallel inserts by parallel workers and/or
+ *		leader for Create Table AS.
+ * ----------------------------------------------------------------
+ */
+static void
+ExecParallelInsertInCTAS(GatherState *node)
+{
+	/*
+	 * By now, for parallel workers (if launched any),
+	 * would have started their work i.e. insertion to
+	 * target table.
+	 * In case the leader is chosen to participate for
+	 * parallel inserts in CTAS, then finish it's share
+	 * before going to wait for the parallel workers to
+	 * finish.
+	*/
+	if (node->need_to_scan_locally == true &&
+		node->ps.dest != NULL &&
+		node->ps.dest->mydest == DestIntoRel)
+	{
+		EState	   *estate = node->ps.state;
+		TupleTableSlot *outerTupleSlot;
+
+		for(;;)
+		{
+			/* Install our DSA area while executing the plan. */
+			estate->es_query_dsa =
+				node->pei ? node->pei->area : NULL;
+
+			outerTupleSlot = ExecProcNode(node->ps.lefttree);
+
+			estate->es_query_dsa = NULL;
+
+			if (!TupIsNull(outerTupleSlot))
+			{
+				(void) node->ps.dest->receiveSlot(outerTupleSlot, node->ps.dest);
+				node->ps.state->es_processed++;
+			}
+
+			if(TupIsNull(outerTupleSlot))
+				break;
+		}
+	}
+
+	/* Wait for the parallel workers to finish. */
+	if (node->nworkers_launched > 0)
+	{
+		ExecShutdownGatherWorkers(node);
+		/*
+		 * Add up the total tuples inserted by all workers, to
+		 * the tuples inserted by the leader(if any). This will
+		 * be shared to client.
+		*/
+		node->ps.state->es_processed += pg_atomic_read_u64(node->pei->processed);
+	}
+}
+
 /* ----------------------------------------------------------------
  *		ExecGather(node)
  *
@@ -166,6 +226,17 @@ ExecGather(PlanState *pstate)
 		{
 			ParallelContext *pcxt;
 
+			/*
+			 * Take the necessary information to be passed to workers for
+			 * parallel inserts in CTAS.
+			 */
+			if (node->ps.intoclause != NULL &&
+				node->ps.intoclause->type == T_IntoClause)
+			{
+				node->ps.lefttree->intoclause = node->ps.intoclause;
+				node->ps.lefttree->objectid = node->ps.objectid;
+			}
+
 			/* Initialize, or re-initialize, shared state needed by workers. */
 			if (!node->pei)
 				node->pei = ExecInitParallelPlan(node->ps.lefttree,
@@ -190,13 +261,17 @@ ExecGather(PlanState *pstate)
 			/* Set up tuple queue readers to read the results. */
 			if (pcxt->nworkers_launched > 0)
 			{
-				ExecParallelCreateReaders(node->pei);
-				/* Make a working array showing the active readers */
-				node->nreaders = pcxt->nworkers_launched;
-				node->reader = (TupleQueueReader **)
-					palloc(node->nreaders * sizeof(TupleQueueReader *));
-				memcpy(node->reader, node->pei->reader,
-					   node->nreaders * sizeof(TupleQueueReader *));
+				if (!(node->ps.intoclause != NULL &&
+					node->ps.intoclause->type == T_IntoClause))
+				{
+					ExecParallelCreateReaders(node->pei);
+					/* Make a working array showing the active readers */
+					node->nreaders = pcxt->nworkers_launched;
+					node->reader = (TupleQueueReader **)
+						palloc(node->nreaders * sizeof(TupleQueueReader *));
+					memcpy(node->reader, node->pei->reader,
+						node->nreaders * sizeof(TupleQueueReader *));
+				}
 			}
 			else
 			{
@@ -220,6 +295,12 @@ ExecGather(PlanState *pstate)
 	econtext = node->ps.ps_ExprContext;
 	ResetExprContext(econtext);
 
+	if (node->ps.intoclause != NULL &&
+		node->ps.intoclause->type == T_IntoClause)
+	{
+		ExecParallelInsertInCTAS(node);
+		return NULL;
+	}
 	/*
 	 * Get next tuple, either from one of our workers, or by running the plan
 	 * ourselves.
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index cd3716d494..9ed671415b 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -377,6 +377,18 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 	else
 		path->path.rows = rel->rows;
 
+	/*
+	 * Make the number of tuples that are transferred from
+	 * workers to gather/leader node zero as each worker
+	 * parallelly insert the tuples that are resulted from
+	 * it's chunk of plan execution. This change may make
+	 * the parallel plan cheap among all other plans, and
+	 * influence the planner to consdier this parallel plan.
+	 */
+	if (root->parse->isForCTAS &&
+		root->query_level == 1)
+		path->path.rows = 0;
+
 	startup_cost = path->subpath->startup_cost;
 
 	run_cost = path->subpath->total_cost - path->subpath->startup_cost;
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index df1b43a932..1db154da63 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -388,6 +388,7 @@ extern FullTransactionId GetCurrentFullTransactionIdIfAny(void);
 extern void MarkCurrentTransactionIdLoggedIfAny(void);
 extern bool SubTransactionIsActive(SubTransactionId subxid);
 extern CommandId GetCurrentCommandId(bool used);
+extern void SetCurrentCommandIdUsedForWorker(void);
 extern void SetParallelStartTimestamps(TimestampTz xact_ts, TimestampTz stmt_ts);
 extern TimestampTz GetCurrentTransactionStartTimestamp(void);
 extern TimestampTz GetCurrentStatementStartTimestamp(void);
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 7629230254..d287c6b898 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -14,12 +14,28 @@
 #ifndef CREATEAS_H
 #define CREATEAS_H
 
+#include "access/heapam.h"
 #include "catalog/objectaddress.h"
 #include "nodes/params.h"
+#include "nodes/plannodes.h"
 #include "parser/parse_node.h"
 #include "tcop/dest.h"
 #include "utils/queryenvironment.h"
 
+typedef struct
+{
+	DestReceiver pub;			/* publicly-known function pointers */
+	IntoClause *into;			/* target relation specification */
+	/* These fields are filled by intorel_startup: */
+	Relation	rel;			/* relation to write to */
+	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
+	CommandId	output_cid;		/* cmin to insert in output tuples */
+	int			ti_options;		/* table_tuple_insert performance options */
+	BulkInsertState bistate;	/* bulk insert state */
+	bool		is_parallel;		/* true if parallelism is to be considered */
+	bool		is_parallel_worker; /* true for parallel worker */
+	Oid			object_id;			/* used for table open by parallel worker */
+} DR_intorel;
 
 extern ObjectAddress ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 									   ParamListInfo params, QueryEnvironment *queryEnv,
@@ -29,4 +45,6 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
+extern bool IsParallelInsertInCTASAllowed(IntoClause *intoClause, PlannedStmt *plannedstmt);
+
 #endif							/* CREATEAS_H */
diff --git a/src/include/executor/execParallel.h b/src/include/executor/execParallel.h
index 5a39a5b29c..77f69946bf 100644
--- a/src/include/executor/execParallel.h
+++ b/src/include/executor/execParallel.h
@@ -35,6 +35,7 @@ typedef struct ParallelExecutorInfo
 	/* These two arrays have pcxt->nworkers_launched entries: */
 	shm_mq_handle **tqueue;		/* tuple queues for worker output */
 	struct TupleQueueReader **reader;	/* tuple reader/writer support */
+	volatile pg_atomic_uint64	*processed;	/* number of tuples inserted by all workers */
 } ParallelExecutorInfo;
 
 extern ParallelExecutorInfo *ExecInitParallelPlan(PlanState *planstate,
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index a926ff1711..92eb1b34f0 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -23,6 +23,7 @@
 #include "nodes/tidbitmap.h"
 #include "partitioning/partdefs.h"
 #include "storage/condition_variable.h"
+#include "tcop/dest.h"
 #include "utils/hsearch.h"
 #include "utils/queryenvironment.h"
 #include "utils/reltrigger.h"
@@ -1009,6 +1010,10 @@ typedef struct PlanState
 	bool		outeropsset;
 	bool		inneropsset;
 	bool		resultopsset;
+	/* Below is parallel inserts in CTAS related info. */
+	IntoClause	*intoclause;
+	Oid			objectid;
+	DestReceiver *dest;
 } PlanState;
 
 /* ----------------
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 60c2f45466..fe43dc941e 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -180,6 +180,7 @@ typedef struct Query
 	 */
 	int			stmt_location;	/* start location, or -1 if unknown */
 	int			stmt_len;		/* length in bytes; 0 means "rest of string" */
+	bool		isForCTAS;		/* true if the select query is for create table as */
 } Query;
 
 
-- 
2.25.1

From b80460c0390317cceecd66ce9780feafd55bd5b2 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 19 Oct 2020 22:06:41 +0530
Subject: [PATCH v3] Parallel Inserts in CREATE TABLE AS

The idea of this patch is to allow the leader and each worker
insert the tuples in parallel if the SELECT part of the CTAS is
parallelizable.

The design:
Let the planner know that the SELECT is from CTAS in createas.c
so that it can set the number of tuples transferred from the
workers to Gather node to 0. With this change, there are chances
that the planner may choose the parallel plan. After the planning,
check if the upper plan node is Gather in createas.c and mark a
parallelism flag in the CTAS dest receiver. Pass the into clause,
object id, command id from the leader to workers, so that each
worker can create its own CTAS dest receiver. Leader inserts it's
share of tuples if instructed to do, and so are workers. Each
worker writes atomically it's number of inserted tuples into a
shared memory variable, the leader combines this with it's own
number of inserted tuples and shares to the client.
---
 src/backend/access/heap/heapam.c             |  11 -
 src/backend/access/transam/xact.c            |  30 +-
 src/backend/commands/createas.c              | 341 ++++++++++++-------
 src/backend/commands/explain.c               |  36 ++
 src/backend/executor/execMain.c              |  19 ++
 src/backend/executor/execParallel.c          |  60 +++-
 src/backend/executor/nodeGather.c            | 101 +++++-
 src/backend/optimizer/path/costsize.c        |  12 +-
 src/include/access/xact.h                    |   1 +
 src/include/commands/createas.h              |  20 ++
 src/include/executor/execParallel.h          |   1 +
 src/include/nodes/execnodes.h                |   5 +
 src/include/nodes/parsenodes.h               |   1 +
 src/test/regress/expected/write_parallel.out | 143 ++++++++
 src/test/regress/sql/write_parallel.sql      |  65 ++++
 15 files changed, 694 insertions(+), 152 deletions(-)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 1585861a02..1602525d4a 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2043,17 +2043,6 @@ static HeapTuple
 heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
 					CommandId cid, int options)
 {
-	/*
-	 * To allow parallel inserts, we need to ensure that they are safe to be
-	 * performed in workers. We have the infrastructure to allow parallel
-	 * inserts in general except for the cases where inserts generate a new
-	 * CommandId (eg. inserts into a table having a foreign key column).
-	 */
-	if (IsParallelWorker())
-		ereport(ERROR,
-				(errcode(ERRCODE_INVALID_TRANSACTION_STATE),
-				 errmsg("cannot insert tuples in a parallel worker")));
-
 	tup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
 	tup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
 	tup->t_data->t_infomask |= HEAP_XMAX_INVALID;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index af6afcebb1..809774c4bb 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -763,18 +763,34 @@ GetCurrentCommandId(bool used)
 	/* this is global to a transaction, not subtransaction-local */
 	if (used)
 	{
-		/*
-		 * Forbid setting currentCommandIdUsed in a parallel worker, because
-		 * we have no provision for communicating this back to the leader.  We
-		 * could relax this restriction when currentCommandIdUsed was already
-		 * true at the start of the parallel operation.
-		 */
-		Assert(!IsParallelWorker());
+		 /*
+		  * This is a temporary hack for all common parallel insert cases i.e.
+		  * insert into, ctas, copy from. To be changed later. In a parallel
+		  * worker, set currentCommandIdUsed to true only if it was not set to
+		  * true at the start of the parallel operation (by way of
+		  * SetCurrentCommandIdUsedForWorker()). We have to do this because
+		  * GetCurrentCommandId(true) may be called from anywhere, especially
+		  * for parallel inserts, within parallel worker.
+		  */
+		Assert(!(IsParallelWorker() && !currentCommandIdUsed));
 		currentCommandIdUsed = true;
 	}
 	return currentCommandId;
 }
 
+/*
+ *	SetCurrentCommandIdUsedForWorker
+ *
+ * For a parallel worker, record that the currentCommandId has been used.
+ * This must only be called at the start of a parallel operation.
+*/
+void
+SetCurrentCommandIdUsedForWorker(void)
+{
+	Assert(IsParallelWorker() && !currentCommandIdUsed && currentCommandId != InvalidCommandId);
+	currentCommandIdUsed = true;
+}
+
 /*
  *	SetParallelStartTimestamps
  *
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index d53ec952d0..9df8a7face 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,18 +51,6 @@
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 
-typedef struct
-{
-	DestReceiver pub;			/* publicly-known function pointers */
-	IntoClause *into;			/* target relation specification */
-	/* These fields are filled by intorel_startup: */
-	Relation	rel;			/* relation to write to */
-	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
-	CommandId	output_cid;		/* cmin to insert in output tuples */
-	int			ti_options;		/* table_tuple_insert performance options */
-	BulkInsertState bistate;	/* bulk insert state */
-} DR_intorel;
-
 /* utility functions for CTAS definition creation */
 static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into);
 static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -328,10 +316,27 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		query = linitial_node(Query, rewritten);
 		Assert(query->commandType == CMD_SELECT);
 
+		/*
+		 * Flag to let the planner know that the SELECT query is for CTAS. This
+		 * is used to calculate the tuple transfer cost from workers to gather
+		 * node(in case parallelism kicks in for the SELECT part of the CTAS),
+		 * to zero as each worker will parallelly insert its share of tuples.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, NULL))
+			query->isForCTAS = true;
+
 		/* plan the query */
 		plan = pg_plan_query(query, pstate->p_sourcetext,
 							 CURSOR_OPT_PARALLEL_OK, params);
 
+		/*
+		 * SELECT part of the CTAS is parallelizable, so we can make each
+		 * parallel worker insert the tuples that are resulted in it's
+		 * execution into the target table.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, plan))
+			((DR_intorel *) dest)->is_parallel = true;
+
 		/*
 		 * Use a snapshot with an updated command ID to ensure this query sees
 		 * results of any previously executed queries.  (This could only
@@ -418,6 +423,9 @@ CreateIntoRelDestReceiver(IntoClause *intoClause)
 	self->pub.rDestroy = intorel_destroy;
 	self->pub.mydest = DestIntoRel;
 	self->into = intoClause;
+	self->is_parallel = false;
+	self->is_parallel_worker = false;
+	self->object_id = InvalidOid;
 	/* other private fields will be set during intorel_startup */
 
 	return (DestReceiver *) self;
@@ -430,135 +438,180 @@ static void
 intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 {
 	DR_intorel *myState = (DR_intorel *) self;
-	IntoClause *into = myState->into;
-	bool		is_matview;
-	char		relkind;
-	List	   *attrList;
 	ObjectAddress intoRelationAddr;
 	Relation	intoRelationDesc;
-	RangeTblEntry *rte;
-	ListCell   *lc;
-	int			attnum;
-
-	Assert(into != NULL);		/* else somebody forgot to set it */
 
-	/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
-	is_matview = (into->viewQuery != NULL);
-	relkind = is_matview ? RELKIND_MATVIEW : RELKIND_RELATION;
+	if (myState->is_parallel_worker)
+	{
+		/* In the worker */
+		intoRelationDesc = table_open(myState->object_id, AccessExclusiveLock);
+		myState->rel = intoRelationDesc;
+		myState->reladdr = InvalidObjectAddress;
+		myState->ti_options = 0;
+		myState->bistate = GetBulkInsertState();
 
-	/*
-	 * Build column definitions using "pre-cooked" type and collation info. If
-	 * a column name list was specified in CREATE TABLE AS, override the
-	 * column names derived from the query.  (Too few column names are OK, too
-	 * many are not.)
-	 */
-	attrList = NIL;
-	lc = list_head(into->colNames);
-	for (attnum = 0; attnum < typeinfo->natts; attnum++)
+		/*
+		 * Right after the table is created in the leader, the command id is
+		 * incremented (in create_ctas_internal()). The new command id is
+		 * marked as used in inintorel_startup(), then the parallel mode is
+		 * entered. The command id and transaction id are serialized into
+		 * parallel DSM, they are then available to all parallel workers. All
+		 * the workers need to mark the command id as used before insertion.
+		 */
+		(void) SetCurrentCommandIdUsedForWorker();
+		myState->output_cid = GetCurrentCommandId(false);
+	}
+	else
 	{
-		Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
-		ColumnDef  *col;
-		char	   *colname;
+		IntoClause *into = myState->into;
+		bool		is_matview;
+		char		relkind;
+		List	   *attrList;
+		Relation	intoRelationDesc;
+		RangeTblEntry *rte;
+		ListCell   *lc;
+		int			attnum;
 
-		if (lc)
-		{
-			colname = strVal(lfirst(lc));
-			lc = lnext(into->colNames, lc);
-		}
-		else
-			colname = NameStr(attribute->attname);
+		Assert(into != NULL);		/* else somebody forgot to set it */
 
-		col = makeColumnDef(colname,
-							attribute->atttypid,
-							attribute->atttypmod,
-							attribute->attcollation);
+		/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
+		is_matview = (into->viewQuery != NULL);
+		relkind = is_matview ? RELKIND_MATVIEW : RELKIND_RELATION;
 
 		/*
-		 * It's possible that the column is of a collatable type but the
-		 * collation could not be resolved, so double-check.  (We must check
-		 * this here because DefineRelation would adopt the type's default
-		 * collation rather than complaining.)
-		 */
-		if (!OidIsValid(col->collOid) &&
-			type_is_collatable(col->typeName->typeOid))
-			ereport(ERROR,
-					(errcode(ERRCODE_INDETERMINATE_COLLATION),
-					 errmsg("no collation was derived for column \"%s\" with collatable type %s",
-							col->colname,
-							format_type_be(col->typeName->typeOid)),
-					 errhint("Use the COLLATE clause to set the collation explicitly.")));
+		* Build column definitions using "pre-cooked" type and collation info. If
+		* a column name list was specified in CREATE TABLE AS, override the
+		* column names derived from the query.  (Too few column names are OK, too
+		* many are not.)
+		*/
+		attrList = NIL;
+		lc = list_head(into->colNames);
+		for (attnum = 0; attnum < typeinfo->natts; attnum++)
+		{
+			Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
+			ColumnDef  *col;
+			char	   *colname;
 
-		attrList = lappend(attrList, col);
-	}
+			if (lc)
+			{
+				colname = strVal(lfirst(lc));
+				lc = lnext(into->colNames, lc);
+			}
+			else
+				colname = NameStr(attribute->attname);
 
-	if (lc != NULL)
-		ereport(ERROR,
-				(errcode(ERRCODE_SYNTAX_ERROR),
-				 errmsg("too many column names were specified")));
+			col = makeColumnDef(colname,
+								attribute->atttypid,
+								attribute->atttypmod,
+								attribute->attcollation);
 
-	/*
-	 * Actually create the target table
-	 */
-	intoRelationAddr = create_ctas_internal(attrList, into);
+			/*
+			* It's possible that the column is of a collatable type but the
+			* collation could not be resolved, so double-check.  (We must check
+			* this here because DefineRelation would adopt the type's default
+			* collation rather than complaining.)
+			*/
+			if (!OidIsValid(col->collOid) &&
+				type_is_collatable(col->typeName->typeOid))
+				ereport(ERROR,
+						(errcode(ERRCODE_INDETERMINATE_COLLATION),
+						errmsg("no collation was derived for column \"%s\" with collatable type %s",
+								col->colname,
+								format_type_be(col->typeName->typeOid)),
+						errhint("Use the COLLATE clause to set the collation explicitly.")));
 
-	/*
-	 * Finally we can open the target table
-	 */
-	intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
+			attrList = lappend(attrList, col);
+		}
 
-	/*
-	 * Check INSERT permission on the constructed table.
-	 *
-	 * XXX: It would arguably make sense to skip this check if into->skipData
-	 * is true.
-	 */
-	rte = makeNode(RangeTblEntry);
-	rte->rtekind = RTE_RELATION;
-	rte->relid = intoRelationAddr.objectId;
-	rte->relkind = relkind;
-	rte->rellockmode = RowExclusiveLock;
-	rte->requiredPerms = ACL_INSERT;
+		if (lc != NULL)
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					errmsg("too many column names were specified")));
 
-	for (attnum = 1; attnum <= intoRelationDesc->rd_att->natts; attnum++)
-		rte->insertedCols = bms_add_member(rte->insertedCols,
-										   attnum - FirstLowInvalidHeapAttributeNumber);
+		/*
+		* Actually create the target table
+		*/
+		intoRelationAddr = create_ctas_internal(attrList, into);
 
-	ExecCheckRTPerms(list_make1(rte), true);
+		/*
+		* Finally we can open the target table
+		*/
+		intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
 
-	/*
-	 * Make sure the constructed table does not have RLS enabled.
-	 *
-	 * check_enable_rls() will ereport(ERROR) itself if the user has requested
-	 * something invalid, and otherwise will return RLS_ENABLED if RLS should
-	 * be enabled here.  We don't actually support that currently, so throw
-	 * our own ereport(ERROR) if that happens.
-	 */
-	if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
-		ereport(ERROR,
-				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-				 errmsg("policies not yet implemented for this command")));
+		/*
+		* Check INSERT permission on the constructed table.
+		*
+		* XXX: It would arguably make sense to skip this check if into->skipData
+		* is true.
+		*/
+		rte = makeNode(RangeTblEntry);
+		rte->rtekind = RTE_RELATION;
+		rte->relid = intoRelationAddr.objectId;
+		rte->relkind = relkind;
+		rte->rellockmode = RowExclusiveLock;
+		rte->requiredPerms = ACL_INSERT;
+
+		for (attnum = 1; attnum <= intoRelationDesc->rd_att->natts; attnum++)
+			rte->insertedCols = bms_add_member(rte->insertedCols,
+											attnum - FirstLowInvalidHeapAttributeNumber);
+
+		ExecCheckRTPerms(list_make1(rte), true);
 
-	/*
-	 * Tentatively mark the target as populated, if it's a matview and we're
-	 * going to fill it; otherwise, no change needed.
-	 */
-	if (is_matview && !into->skipData)
-		SetMatViewPopulatedState(intoRelationDesc, true);
+		/*
+		* Make sure the constructed table does not have RLS enabled.
+		*
+		* check_enable_rls() will ereport(ERROR) itself if the user has requested
+		* something invalid, and otherwise will return RLS_ENABLED if RLS should
+		* be enabled here.  We don't actually support that currently, so throw
+		* our own ereport(ERROR) if that happens.
+		*/
+		if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					errmsg("policies not yet implemented for this command")));
 
-	/*
-	 * Fill private fields of myState for use by later routines
-	 */
-	myState->rel = intoRelationDesc;
-	myState->reladdr = intoRelationAddr;
-	myState->output_cid = GetCurrentCommandId(true);
-	myState->ti_options = TABLE_INSERT_SKIP_FSM;
-	myState->bistate = GetBulkInsertState();
+		/*
+		* Tentatively mark the target as populated, if it's a matview and we're
+		* going to fill it; otherwise, no change needed.
+		*/
+		if (is_matview && !into->skipData)
+			SetMatViewPopulatedState(intoRelationDesc, true);
 
-	/*
-	 * Valid smgr_targblock implies something already wrote to the relation.
-	 * This may be harmless, but this function hasn't planned for it.
-	 */
-	Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+		/*
+		* Fill private fields of myState for use by later routines
+		*/
+		myState->rel = intoRelationDesc;
+		myState->reladdr = intoRelationAddr;
+		myState->output_cid = GetCurrentCommandId(true);
+		myState->ti_options = TABLE_INSERT_SKIP_FSM;
+		myState->bistate = GetBulkInsertState();
+
+		/*
+		* Valid smgr_targblock implies something already wrote to the relation.
+		* This may be harmless, but this function hasn't planned for it.
+		*/
+		Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+
+		if (myState->is_parallel == true)
+		{
+			myState->object_id = intoRelationAddr.objectId;
+
+			/*
+			 * We don't need to skip contacting FSM while inserting tuples
+			 * for parallel mode, while extending the relations, workers
+			 * instead of blocking on a page while another worker is inserting,
+			 * can check the FSM for another page that can accommodate the
+			 * tuples. This results in major benefit for parallel inserts.
+			 */
+			myState->ti_options = 0;
+
+			/*
+			 * rd_createSubid is marked invalid, otherwise, the table is
+			 * not allowed to extend by the workers.
+			 */
+			myState->rel->rd_createSubid = InvalidSubTransactionId;
+		}
+	}
 }
 
 /*
@@ -614,3 +667,49 @@ intorel_destroy(DestReceiver *self)
 {
 	pfree(self);
 }
+
+/*
+ * IsParallelInsertInCTASAllowed --- determine whether or not parallel
+ * insertion is possible.
+ */
+bool IsParallelInsertInCTASAllowed(IntoClause *into, PlannedStmt *plannedstmt)
+{
+	bool allowed = false;
+
+	if (into != NULL &&
+		IsA(into, IntoClause))
+	{
+		if (into->rel != NULL &&
+			into->rel->relpersistence != RELPERSISTENCE_TEMP)
+			allowed = true;
+
+		if (plannedstmt != NULL && allowed)
+		{
+			/*
+			 * We allow parallel inserts by the workers only if the upper node
+			 * is Gather. We can not let workers do parallel inserts when
+			 * GatherMerge node is involved as the leader backend does the
+			 * final phase(merge the results by workers).
+			 */
+			if (plannedstmt->parallelModeNeeded &&
+				plannedstmt->planTree != NULL &&
+				IsA(plannedstmt->planTree, Gather) &&
+				plannedstmt->planTree->lefttree != NULL &&
+				plannedstmt->planTree->lefttree->parallel_aware &&
+				plannedstmt->planTree->lefttree->parallel_safe)
+			{
+				/*
+				 * Since there are no rows that are transferred from workers to
+				 * Gather node, so we set it to 0 to be visible in explain
+				 * plans. Note that we would have accounted this for cost
+				 * calculations in cost_gather().
+				 */
+				plannedstmt->planTree->plan_rows = 0;
+			}
+			else
+				allowed = false;
+		}
+	}
+
+	return allowed;
+}
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 41317f1837..b559d2d6e1 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -371,6 +371,15 @@ ExplainOneQuery(Query *query, int cursorOptions,
 		return;
 	}
 
+	/*
+	 * Flag to let the planner know that the SELECT query is for CTAS. This is
+	 * used to calculate the tuple transfer cost from workers to gather node(in
+	 * case parallelism kicks in for the SELECT part of the CTAS), to zero as
+	 * each worker will parallelly insert its share of tuples.
+	 */
+	if (IsParallelInsertInCTASAllowed(into, NULL))
+		query->isForCTAS = true;
+
 	/* if an advisor plugin is present, let it manage things */
 	if (ExplainOneQuery_hook)
 		(*ExplainOneQuery_hook) (query, cursorOptions, into, es,
@@ -536,7 +545,17 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	 * AS, we'd better use the appropriate tuple receiver.
 	 */
 	if (into)
+	{
 		dest = CreateIntoRelDestReceiver(into);
+
+		/*
+		 * SELECT part of the CTAS is parallelizable, so we can make each
+		 * parallel worker insert the tuples that are resulted in it's
+		 * execution into the target table.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, plannedstmt))
+			((DR_intorel *) dest)->is_parallel = true;
+	}
 	else
 		dest = None_Receiver;
 
@@ -1753,6 +1772,23 @@ ExplainNode(PlanState *planstate, List *ancestors,
 			{
 				Gather	   *gather = (Gather *) plan;
 
+				if (IsA(planstate, GatherState) &&
+					planstate->intoclause != NULL &&
+					IsA(planstate->intoclause,IntoClause) &&
+					planstate->dest != NULL &&
+					planstate->dest->mydest == DestIntoRel &&
+					((DR_intorel *) planstate->dest)->is_parallel == true &&
+					planstate->intoclause->rel != NULL &&
+					planstate->intoclause->rel->relname != NULL)
+				{
+					ExplainIndentText(es);
+					appendStringInfoString(es->str, "->  ");
+					appendStringInfoString(es->str, "Create ");
+					appendStringInfo(es->str, "%s\n", planstate->intoclause->rel->relname);
+					es->indent++;
+					ExplainIndentText(es);
+				}
+
 				show_scan_qual(plan->qual, "Filter", planstate, ancestors, es);
 				if (plan->qual)
 					show_instrumentation_count("Rows Removed by Filter", 1,
diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c
index aea0479448..5482ba4e3a 100644
--- a/src/backend/executor/execMain.c
+++ b/src/backend/executor/execMain.c
@@ -45,6 +45,7 @@
 #include "access/xact.h"
 #include "catalog/namespace.h"
 #include "catalog/pg_publication.h"
+#include "commands/createas.h"
 #include "commands/matview.h"
 #include "commands/trigger.h"
 #include "executor/execdebug.h"
@@ -352,6 +353,24 @@ standard_ExecutorRun(QueryDesc *queryDesc,
 	if (sendTuples)
 		dest->rStartup(dest, operation, queryDesc->tupDesc);
 
+	/*
+	 * For parallelizing inserts in CTAS i.e. making each
+	 * parallel worker inerst it's tuples, we must send
+	 * information such as intoclause(for each worker
+	 * building it's own dest receiver), object id(for each
+	 * worker to open the table).
+	 */
+	if (queryDesc->plannedstmt->parallelModeNeeded == true &&
+		dest != NULL &&
+		dest->mydest == DestIntoRel &&
+		((DR_intorel *) dest)->is_parallel == true &&
+		((DR_intorel *) dest)->is_parallel_worker != true)
+	{
+		queryDesc->planstate->intoclause = ((DR_intorel *) dest)->into;
+		queryDesc->planstate->objectid = ((DR_intorel *) dest)->object_id;
+		queryDesc->planstate->dest = dest;
+	}
+
 	/*
 	 * run plan
 	 */
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index befde52691..b2aa06102f 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -23,6 +23,7 @@
 
 #include "postgres.h"
 
+#include "commands/createas.h"
 #include "executor/execParallel.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
@@ -65,6 +66,7 @@
 #define PARALLEL_KEY_QUERY_TEXT		UINT64CONST(0xE000000000000008)
 #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009)
 #define PARALLEL_KEY_WAL_USAGE			UINT64CONST(0xE00000000000000A)
+#define PARALLEL_KEY_INTO_CLAUSE		UINT64CONST(0xE00000000000000B)
 
 #define PARALLEL_TUPLE_QUEUE_SIZE		65536
 
@@ -77,6 +79,8 @@ typedef struct FixedParallelExecutorState
 	dsa_pointer param_exec;
 	int			eflags;
 	int			jit_flags;
+	Oid			objectid;		/* workers to open relation/table.  */
+	pg_atomic_uint64	processed; /* number tuples inserted by all the workers. */
 } FixedParallelExecutorState;
 
 /*
@@ -600,6 +604,7 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	Size		dsa_minsize = dsa_minimum_size();
 	char	   *query_string;
 	int			query_len;
+	char 		*intoclausestr = NULL;
 
 	/*
 	 * Force any initplan outputs that we're going to pass to workers to be
@@ -712,6 +717,14 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_estimate_chunk(&pcxt->estimator, dsa_minsize);
 	shm_toc_estimate_keys(&pcxt->estimator, 1);
 
+	/* Estimate space for into clause for CTAS. */
+	if (ISCTAS(planstate->intoclause))
+	{
+		intoclausestr = nodeToString(planstate->intoclause);
+		shm_toc_estimate_chunk(&pcxt->estimator, strlen(intoclausestr) + 1);
+		shm_toc_estimate_keys(&pcxt->estimator, 1);
+	}
+
 	/* Everyone's had a chance to ask for space, so now create the DSM. */
 	InitializeParallelDSM(pcxt);
 
@@ -729,6 +742,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	fpes->param_exec = InvalidDsaPointer;
 	fpes->eflags = estate->es_top_eflags;
 	fpes->jit_flags = estate->es_jit_flags;
+	pg_atomic_init_u64(&fpes->processed, 0);
+	pei->processed = &fpes->processed;
+
+	if (intoclausestr != NULL &&
+		planstate->objectid != InvalidOid)
+		fpes->objectid = planstate->objectid;
+	else
+		fpes->objectid = InvalidOid;
+
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, fpes);
 
 	/* Store query string */
@@ -758,8 +780,17 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_WAL_USAGE, walusage_space);
 	pei->wal_usage = walusage_space;
 
+	if (intoclausestr != NULL)
+	{
+		char *shmptr = (char *)shm_toc_allocate(pcxt->toc,
+												strlen(intoclausestr) + 1);
+		strcpy(shmptr, intoclausestr);
+		shm_toc_insert(pcxt->toc, PARALLEL_KEY_INTO_CLAUSE, shmptr);
+	}
+
 	/* Set up the tuple queues that the workers will write into. */
-	pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	if (intoclausestr == NULL)
+		pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
 
 	/* We don't need the TupleQueueReaders yet, though. */
 	pei->reader = NULL;
@@ -1387,12 +1418,28 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 	void	   *area_space;
 	dsa_area   *area;
 	ParallelWorkerContext pwcxt;
+	char		*intoclausestr = NULL;
+	IntoClause	*intoclause = NULL;
 
 	/* Get fixed-size state. */
 	fpes = shm_toc_lookup(toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
 
-	/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
-	receiver = ExecParallelGetReceiver(seg, toc);
+	intoclausestr = shm_toc_lookup(toc, PARALLEL_KEY_INTO_CLAUSE, true);
+	if (intoclausestr != NULL)
+	{
+		/*
+		 * If the worker is for parallel insert in CTAS, then use
+		 * the proper dest receiver.
+		 */
+		intoclause = (IntoClause *) stringToNode(intoclausestr);
+		receiver = CreateIntoRelDestReceiver(intoclause);
+		((DR_intorel *)receiver)->is_parallel_worker = true;
+		((DR_intorel *)receiver)->object_id = fpes->objectid;
+	}
+	else
+		/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
+		receiver = ExecParallelGetReceiver(seg, toc);
+
 	instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_INSTRUMENTATION, true);
 	if (instrumentation != NULL)
 		instrument_options = instrumentation->instrument_options;
@@ -1471,6 +1518,13 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 			queryDesc->estate->es_jit->instr;
 	}
 
+	/*
+	 * Write out the number of tuples this worker has inserted. Leader
+	 * will use it to inform to the end client.
+	 */
+	if (intoclausestr != NULL)
+		pg_atomic_add_fetch_u64(&fpes->processed, queryDesc->estate->es_processed);
+
 	/* Must do this after capturing instrumentation. */
 	ExecutorEnd(queryDesc);
 
diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c
index a01b46af14..99211e4941 100644
--- a/src/backend/executor/nodeGather.c
+++ b/src/backend/executor/nodeGather.c
@@ -32,6 +32,7 @@
 
 #include "access/relscan.h"
 #include "access/xact.h"
+#include "commands/createas.h"
 #include "executor/execdebug.h"
 #include "executor/execParallel.h"
 #include "executor/nodeGather.h"
@@ -48,7 +49,7 @@ static TupleTableSlot *ExecGather(PlanState *pstate);
 static TupleTableSlot *gather_getnext(GatherState *gatherstate);
 static MinimalTuple gather_readnext(GatherState *gatherstate);
 static void ExecShutdownGatherWorkers(GatherState *node);
-
+static void ExecParallelInsertInCTAS(GatherState *node);
 
 /* ----------------------------------------------------------------
  *		ExecInitGather
@@ -131,6 +132,69 @@ ExecInitGather(Gather *node, EState *estate, int eflags)
 	return gatherstate;
 }
 
+/* ----------------------------------------------------------------
+ *		ExecParallelInsertInCTAS(node)
+ *
+ *		Facilitates parallel inserts by parallel workers and/or
+ *		leader for Create Table AS.
+ * ----------------------------------------------------------------
+ */
+static void
+ExecParallelInsertInCTAS(GatherState *node)
+{
+	/* Enable leader to insert in case no parallel workers were launched. */
+	if (node->nworkers_launched == 0 &&
+		!node->need_to_scan_locally)
+		node->need_to_scan_locally = true;
+	/*
+	 * By now, for parallel workers (if launched any), would have started their
+	 * work i.e. insertion to target table. In case the leader is chosen to
+	 * participate for parallel inserts in CTAS, then finish it's share before
+	 * going to wait for the parallel workers to finish.
+	 */
+	if (node->need_to_scan_locally == true &&
+		node->ps.dest != NULL &&
+		node->ps.dest->mydest == DestIntoRel)
+	{
+		EState	   *estate = node->ps.state;
+		TupleTableSlot *outerTupleSlot;
+
+		for(;;)
+		{
+			/* Install our DSA area while executing the plan. */
+			estate->es_query_dsa =
+					node->pei ? node->pei->area : NULL;
+
+			outerTupleSlot = ExecProcNode(node->ps.lefttree);
+
+			estate->es_query_dsa = NULL;
+
+			if (!TupIsNull(outerTupleSlot))
+			{
+				(void) node->ps.dest->receiveSlot(outerTupleSlot, node->ps.dest);
+				node->ps.state->es_processed++;
+			}
+
+			if(TupIsNull(outerTupleSlot))
+				break;
+		}
+
+		node->need_to_scan_locally = false;
+	}
+
+	/* Wait for the parallel workers to finish. */
+	if (node->nworkers_launched > 0)
+	{
+		ExecShutdownGatherWorkers(node);
+
+		/*
+		 * Add up the total tuples inserted by all workers, to the tuples
+		 * inserted by the leader(if any). This will be shared to client.
+		 */
+		node->ps.state->es_processed += pg_atomic_read_u64(node->pei->processed);
+	}
+}
+
 /* ----------------------------------------------------------------
  *		ExecGather(node)
  *
@@ -166,6 +230,16 @@ ExecGather(PlanState *pstate)
 		{
 			ParallelContext *pcxt;
 
+			/*
+			 * Take the necessary information to be passed to workers for
+			 * parallel inserts in CTAS.
+			 */
+			if (ISCTAS(node->ps.intoclause))
+			{
+				node->ps.lefttree->intoclause = node->ps.intoclause;
+				node->ps.lefttree->objectid = node->ps.objectid;
+			}
+
 			/* Initialize, or re-initialize, shared state needed by workers. */
 			if (!node->pei)
 				node->pei = ExecInitParallelPlan(node->ps.lefttree,
@@ -190,13 +264,16 @@ ExecGather(PlanState *pstate)
 			/* Set up tuple queue readers to read the results. */
 			if (pcxt->nworkers_launched > 0)
 			{
-				ExecParallelCreateReaders(node->pei);
-				/* Make a working array showing the active readers */
-				node->nreaders = pcxt->nworkers_launched;
-				node->reader = (TupleQueueReader **)
-					palloc(node->nreaders * sizeof(TupleQueueReader *));
-				memcpy(node->reader, node->pei->reader,
-					   node->nreaders * sizeof(TupleQueueReader *));
+				if (!(ISCTAS(node->ps.intoclause)))
+				{
+					ExecParallelCreateReaders(node->pei);
+					/* Make a working array showing the active readers */
+					node->nreaders = pcxt->nworkers_launched;
+					node->reader = (TupleQueueReader **)
+						palloc(node->nreaders * sizeof(TupleQueueReader *));
+					memcpy(node->reader, node->pei->reader,
+						node->nreaders * sizeof(TupleQueueReader *));
+				}
 			}
 			else
 			{
@@ -208,7 +285,8 @@ ExecGather(PlanState *pstate)
 		}
 
 		/* Run plan locally if no workers or enabled and not single-copy. */
-		node->need_to_scan_locally = (node->nreaders == 0)
+		node->need_to_scan_locally = (node->nreaders == 0 &&
+			!(ISCTAS(node->ps.intoclause)))
 			|| (!gather->single_copy && parallel_leader_participation);
 		node->initialized = true;
 	}
@@ -220,6 +298,11 @@ ExecGather(PlanState *pstate)
 	econtext = node->ps.ps_ExprContext;
 	ResetExprContext(econtext);
 
+	if (ISCTAS(node->ps.intoclause))
+	{
+		ExecParallelInsertInCTAS(node);
+		return NULL;
+	}
 	/*
 	 * Get next tuple, either from one of our workers, or by running the plan
 	 * ourselves.
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 733f7ea543..72307e0927 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -393,7 +393,17 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 
 	/* Parallel setup and communication cost. */
 	startup_cost += parallel_setup_cost;
-	run_cost += parallel_tuple_cost * path->path.rows;
+
+	/*
+	 * Make the number of tuples that are transferred from workers to gather
+	 * node zero as each worker parallelly insert the tuples that are resulted
+	 * from its chunk of plan execution. This change may make the parallel
+	 * plan cheap among all other plans, and influence the planner to consider
+	 * this parallel plan.
+	 */
+	if (!(root->parse->isForCTAS &&
+		root->query_level == 1))
+		run_cost += parallel_tuple_cost * path->path.rows;
 
 	path->path.startup_cost = startup_cost;
 	path->path.total_cost = (startup_cost + run_cost);
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7320de345c..5beae6c617 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -389,6 +389,7 @@ extern FullTransactionId GetCurrentFullTransactionIdIfAny(void);
 extern void MarkCurrentTransactionIdLoggedIfAny(void);
 extern bool SubTransactionIsActive(SubTransactionId subxid);
 extern CommandId GetCurrentCommandId(bool used);
+extern void SetCurrentCommandIdUsedForWorker(void);
 extern void SetParallelStartTimestamps(TimestampTz xact_ts, TimestampTz stmt_ts);
 extern TimestampTz GetCurrentTransactionStartTimestamp(void);
 extern TimestampTz GetCurrentStatementStartTimestamp(void);
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 7629230254..af72ffcfe2 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -14,12 +14,30 @@
 #ifndef CREATEAS_H
 #define CREATEAS_H
 
+#include "access/heapam.h"
 #include "catalog/objectaddress.h"
 #include "nodes/params.h"
+#include "nodes/plannodes.h"
 #include "parser/parse_node.h"
 #include "tcop/dest.h"
 #include "utils/queryenvironment.h"
 
+typedef struct
+{
+	DestReceiver pub;			/* publicly-known function pointers */
+	IntoClause *into;			/* target relation specification */
+	/* These fields are filled by intorel_startup: */
+	Relation	rel;			/* relation to write to */
+	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
+	CommandId	output_cid;		/* cmin to insert in output tuples */
+	int			ti_options;		/* table_tuple_insert performance options */
+	BulkInsertState bistate;	/* bulk insert state */
+	bool		is_parallel;		/* true if parallelism is to be considered */
+	bool		is_parallel_worker; /* true for parallel worker */
+	Oid			object_id;			/* used for table open by parallel worker */
+} DR_intorel;
+
+#define ISCTAS(intoclause) (intoclause != NULL && IsA(intoclause, IntoClause))
 
 extern ObjectAddress ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 									   ParamListInfo params, QueryEnvironment *queryEnv,
@@ -29,4 +47,6 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
+extern bool IsParallelInsertInCTASAllowed(IntoClause *intoClause, PlannedStmt *plannedstmt);
+
 #endif							/* CREATEAS_H */
diff --git a/src/include/executor/execParallel.h b/src/include/executor/execParallel.h
index 5a39a5b29c..77f69946bf 100644
--- a/src/include/executor/execParallel.h
+++ b/src/include/executor/execParallel.h
@@ -35,6 +35,7 @@ typedef struct ParallelExecutorInfo
 	/* These two arrays have pcxt->nworkers_launched entries: */
 	shm_mq_handle **tqueue;		/* tuple queues for worker output */
 	struct TupleQueueReader **reader;	/* tuple reader/writer support */
+	volatile pg_atomic_uint64	*processed;	/* number of tuples inserted by all workers */
 } ParallelExecutorInfo;
 
 extern ParallelExecutorInfo *ExecInitParallelPlan(PlanState *planstate,
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 6c0a7d68d6..5083c4cfb5 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -23,6 +23,7 @@
 #include "nodes/tidbitmap.h"
 #include "partitioning/partdefs.h"
 #include "storage/condition_variable.h"
+#include "tcop/dest.h"
 #include "utils/hsearch.h"
 #include "utils/queryenvironment.h"
 #include "utils/reltrigger.h"
@@ -1020,6 +1021,10 @@ typedef struct PlanState
 	bool		outeropsset;
 	bool		inneropsset;
 	bool		resultopsset;
+	/* Below is parallel inserts in CTAS related info. */
+	IntoClause	*intoclause;
+	Oid			objectid;
+	DestReceiver *dest;
 } PlanState;
 
 /* ----------------
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 60c2f45466..fe43dc941e 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -180,6 +180,7 @@ typedef struct Query
 	 */
 	int			stmt_location;	/* start location, or -1 if unknown */
 	int			stmt_len;		/* length in bytes; 0 means "rest of string" */
+	bool		isForCTAS;		/* true if the select query is for create table as */
 } Query;
 
 
diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 0c4da2591a..11ef18b8a4 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -76,4 +76,147 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
+--
+-- Test parallel inserts in create table as/select into/create materialized view
+--
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+                     QUERY PLAN                      
+-----------------------------------------------------
+ LockRows (actual rows=10000 loops=1)
+   ->  Seq Scan on tenk1 (actual rows=10000 loops=1)
+(2 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+select length(stringu1) from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_mat_view
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop materialized view parallel_mat_view;
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+deallocate parallel_write_prep;
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+select nextval('parallel_write_sequence'), four from tenk1;
+                  QUERY PLAN                   
+-----------------------------------------------
+ Seq Scan on tenk1 (actual rows=10000 loops=1)
+(1 row)
+
+drop table parallel_write;
+drop sequence parallel_write_sequence;
 rollback;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 78b479cedf..dd4233b399 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -39,4 +39,69 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
 
+--
+-- Test parallel inserts in create table as/select into/create materialized view
+--
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+select length(stringu1) from tenk1;
+drop materialized view parallel_mat_view;
+
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+deallocate parallel_write_prep;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+select nextval('parallel_write_sequence'), four from tenk1;
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+
 rollback;
-- 
2.25.1

From 127309ffab82e372b338914ccc900463d8c63157 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 23 Nov 2020 16:27:45 +0530
Subject: [PATCH v4] Parallel Inserts in CREATE TABLE AS

The idea of this patch is to allow the leader and each worker
insert the tuples in parallel if the SELECT part of the CTAS is
parallelizable.

The design:
Let the planner know that the SELECT is from CTAS in createas.c
so that it can set the number of tuples transferred from the
workers to Gather node to 0. With this change, there are chances
that the planner may choose the parallel plan. After the planning,
check if the upper plan node is Gather in createas.c and mark a
parallelism flag in the CTAS dest receiver. Pass the into clause,
object id, command id from the leader to workers, so that each
worker can create its own CTAS dest receiver. Leader inserts it's
share of tuples if instructed to do, and so are workers. Each
worker writes atomically it's number of inserted tuples into a
shared memory variable, the leader combines this with it's own
number of inserted tuples and shares to the client.
---
 src/backend/access/heap/heapam.c             |  11 -
 src/backend/access/transam/xact.c            |  30 +-
 src/backend/commands/createas.c              | 314 ++++++++++++-------
 src/backend/commands/explain.c               |  36 +++
 src/backend/executor/execMain.c              |  19 ++
 src/backend/executor/execParallel.c          |  60 +++-
 src/backend/executor/nodeGather.c            | 101 +++++-
 src/backend/optimizer/path/costsize.c        |  12 +-
 src/include/access/xact.h                    |   1 +
 src/include/commands/createas.h              |  20 ++
 src/include/executor/execParallel.h          |   1 +
 src/include/nodes/execnodes.h                |   5 +
 src/include/nodes/parsenodes.h               |   1 +
 src/test/regress/expected/write_parallel.out | 143 +++++++++
 src/test/regress/sql/write_parallel.sql      |  65 ++++
 15 files changed, 682 insertions(+), 137 deletions(-)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 1b2f70499e..3045c0f046 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2043,17 +2043,6 @@ static HeapTuple
 heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
 					CommandId cid, int options)
 {
-	/*
-	 * To allow parallel inserts, we need to ensure that they are safe to be
-	 * performed in workers. We have the infrastructure to allow parallel
-	 * inserts in general except for the cases where inserts generate a new
-	 * CommandId (eg. inserts into a table having a foreign key column).
-	 */
-	if (IsParallelWorker())
-		ereport(ERROR,
-				(errcode(ERRCODE_INVALID_TRANSACTION_STATE),
-				 errmsg("cannot insert tuples in a parallel worker")));
-
 	tup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
 	tup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
 	tup->t_data->t_infomask |= HEAP_XMAX_INVALID;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 03c553e7ea..c85a9906f1 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -763,18 +763,34 @@ GetCurrentCommandId(bool used)
 	/* this is global to a transaction, not subtransaction-local */
 	if (used)
 	{
-		/*
-		 * Forbid setting currentCommandIdUsed in a parallel worker, because
-		 * we have no provision for communicating this back to the leader.  We
-		 * could relax this restriction when currentCommandIdUsed was already
-		 * true at the start of the parallel operation.
-		 */
-		Assert(!IsParallelWorker());
+		 /*
+		  * This is a temporary hack for all common parallel insert cases i.e.
+		  * insert into, ctas, copy from. To be changed later. In a parallel
+		  * worker, set currentCommandIdUsed to true only if it was not set to
+		  * true at the start of the parallel operation (by way of
+		  * SetCurrentCommandIdUsedForWorker()). We have to do this because
+		  * GetCurrentCommandId(true) may be called from anywhere, especially
+		  * for parallel inserts, within parallel worker.
+		  */
+		Assert(!(IsParallelWorker() && !currentCommandIdUsed));
 		currentCommandIdUsed = true;
 	}
 	return currentCommandId;
 }
 
+/*
+ *	SetCurrentCommandIdUsedForWorker
+ *
+ * For a parallel worker, record that the currentCommandId has been used.
+ * This must only be called at the start of a parallel operation.
+*/
+void
+SetCurrentCommandIdUsedForWorker(void)
+{
+	Assert(IsParallelWorker() && !currentCommandIdUsed && currentCommandId != InvalidCommandId);
+	currentCommandIdUsed = true;
+}
+
 /*
  *	SetParallelStartTimestamps
  *
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 6bf6c5a310..d5fdb8ea07 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,18 +51,6 @@
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 
-typedef struct
-{
-	DestReceiver pub;			/* publicly-known function pointers */
-	IntoClause *into;			/* target relation specification */
-	/* These fields are filled by intorel_startup: */
-	Relation	rel;			/* relation to write to */
-	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
-	CommandId	output_cid;		/* cmin to insert in output tuples */
-	int			ti_options;		/* table_tuple_insert performance options */
-	BulkInsertState bistate;	/* bulk insert state */
-} DR_intorel;
-
 /* utility functions for CTAS definition creation */
 static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into);
 static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -328,10 +316,27 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		query = linitial_node(Query, rewritten);
 		Assert(query->commandType == CMD_SELECT);
 
+		/*
+		 * Flag to let the planner know that the SELECT query is for CTAS. This
+		 * is used to calculate the tuple transfer cost from workers to gather
+		 * node(in case parallelism kicks in for the SELECT part of the CTAS),
+		 * to zero as each worker will parallelly insert its share of tuples.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, NULL))
+			query->isForCTAS = true;
+
 		/* plan the query */
 		plan = pg_plan_query(query, pstate->p_sourcetext,
 							 CURSOR_OPT_PARALLEL_OK, params);
 
+		/*
+		 * SELECT part of the CTAS is parallelizable, so we can make each
+		 * parallel worker insert the tuples that are resulted in it's
+		 * execution into the target table.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, plan))
+			((DR_intorel *) dest)->is_parallel = true;
+
 		/*
 		 * Use a snapshot with an updated command ID to ensure this query sees
 		 * results of any previously executed queries.  (This could only
@@ -418,6 +423,9 @@ CreateIntoRelDestReceiver(IntoClause *intoClause)
 	self->pub.rDestroy = intorel_destroy;
 	self->pub.mydest = DestIntoRel;
 	self->into = intoClause;
+	self->is_parallel = false;
+	self->is_parallel_worker = false;
+	self->object_id = InvalidOid;
 	/* other private fields will be set during intorel_startup */
 
 	return (DestReceiver *) self;
@@ -430,121 +438,169 @@ static void
 intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 {
 	DR_intorel *myState = (DR_intorel *) self;
-	IntoClause *into = myState->into;
-	bool		is_matview;
-	List	   *attrList;
 	ObjectAddress intoRelationAddr;
 	Relation	intoRelationDesc;
-	ListCell   *lc;
-	int			attnum;
-
-	Assert(into != NULL);		/* else somebody forgot to set it */
 
-	/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
-	is_matview = (into->viewQuery != NULL);
+	if (myState->is_parallel_worker)
+	{
+		/* In the worker */
+		intoRelationDesc = table_open(myState->object_id, AccessExclusiveLock);
+		myState->rel = intoRelationDesc;
+		myState->reladdr = InvalidObjectAddress;
+		myState->ti_options = 0;
+		myState->bistate = GetBulkInsertState();
 
-	/*
-	 * Build column definitions using "pre-cooked" type and collation info. If
-	 * a column name list was specified in CREATE TABLE AS, override the
-	 * column names derived from the query.  (Too few column names are OK, too
-	 * many are not.)
-	 */
-	attrList = NIL;
-	lc = list_head(into->colNames);
-	for (attnum = 0; attnum < typeinfo->natts; attnum++)
+		/*
+		 * Right after the table is created in the leader, the command id is
+		 * incremented (in create_ctas_internal()). The new command id is
+		 * marked as used in inintorel_startup(), then the parallel mode is
+		 * entered. The command id and transaction id are serialized into
+		 * parallel DSM, they are then available to all parallel workers. All
+		 * the workers need to mark the command id as used before insertion.
+		 */
+		(void) SetCurrentCommandIdUsedForWorker();
+		myState->output_cid = GetCurrentCommandId(false);
+	}
+	else
 	{
-		Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
-		ColumnDef  *col;
-		char	   *colname;
+		IntoClause *into = myState->into;
+		bool		is_matview;
+		List	   *attrList;
+		ListCell   *lc;
+		int			attnum;
 
-		if (lc)
+		Assert(into != NULL);		/* else somebody forgot to set it */
+
+		/*
+		 * This code supports both CREATE TABLE AS and CREATE MATERIALIZED
+		 * VIEW.
+		 */
+		is_matview = (into->viewQuery != NULL);
+
+		/*
+		 * Build column definitions using "pre-cooked" type and collation info.
+		 * If a column name list was specified in CREATE TABLE AS, override the
+		 * column names derived from the query.  (Too few column names are OK,
+		 * too many are not.)
+		 */
+		attrList = NIL;
+		lc = list_head(into->colNames);
+		for (attnum = 0; attnum < typeinfo->natts; attnum++)
 		{
-			colname = strVal(lfirst(lc));
-			lc = lnext(into->colNames, lc);
+			Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
+			ColumnDef  *col;
+			char	   *colname;
+
+			if (lc)
+			{
+				colname = strVal(lfirst(lc));
+				lc = lnext(into->colNames, lc);
+			}
+			else
+				colname = NameStr(attribute->attname);
+
+			col = makeColumnDef(colname,
+								attribute->atttypid,
+								attribute->atttypmod,
+								attribute->attcollation);
+
+			/*
+			 * It's possible that the column is of a collatable type but the
+			 * collation could not be resolved, so double-check.  (We must
+			 * check this here because DefineRelation would adopt the type's
+			 * default collation rather than complaining.)
+			 */
+			if (!OidIsValid(col->collOid) &&
+				type_is_collatable(col->typeName->typeOid))
+				ereport(ERROR,
+						(errcode(ERRCODE_INDETERMINATE_COLLATION),
+						errmsg("no collation was derived for column \"%s\" with collatable type %s",
+								col->colname,
+								format_type_be(col->typeName->typeOid)),
+						errhint("Use the COLLATE clause to set the collation explicitly.")));
+
+			attrList = lappend(attrList, col);
 		}
-		else
-			colname = NameStr(attribute->attname);
 
-		col = makeColumnDef(colname,
-							attribute->atttypid,
-							attribute->atttypmod,
-							attribute->attcollation);
+		if (lc != NULL)
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					errmsg("too many column names were specified")));
 
 		/*
-		 * It's possible that the column is of a collatable type but the
-		 * collation could not be resolved, so double-check.  (We must check
-		 * this here because DefineRelation would adopt the type's default
-		 * collation rather than complaining.)
+		 * Actually create the target table
 		 */
-		if (!OidIsValid(col->collOid) &&
-			type_is_collatable(col->typeName->typeOid))
-			ereport(ERROR,
-					(errcode(ERRCODE_INDETERMINATE_COLLATION),
-					 errmsg("no collation was derived for column \"%s\" with collatable type %s",
-							col->colname,
-							format_type_be(col->typeName->typeOid)),
-					 errhint("Use the COLLATE clause to set the collation explicitly.")));
+		intoRelationAddr = create_ctas_internal(attrList, into);
 
-		attrList = lappend(attrList, col);
-	}
+		/*
+		 * Finally we can open the target table
+		 */
+		intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
 
-	if (lc != NULL)
-		ereport(ERROR,
-				(errcode(ERRCODE_SYNTAX_ERROR),
-				 errmsg("too many column names were specified")));
+		/*
+		 * Make sure the constructed table does not have RLS enabled.
+		 *
+		 * check_enable_rls() will ereport(ERROR) itself if the user has
+		 * requested something invalid, and otherwise will return RLS_ENABLED
+		 * if RLS should be enabled here.  We don't actually support that
+		 * currently, so throw our own ereport(ERROR) if that happens.
+		 */
+		if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					errmsg("policies not yet implemented for this command")));
 
-	/*
-	 * Actually create the target table
-	 */
-	intoRelationAddr = create_ctas_internal(attrList, into);
+		/*
+		 * Tentatively mark the target as populated, if it's a matview and
+		 * we're going to fill it; otherwise, no change needed.
+		 */
+		if (is_matview && !into->skipData)
+			SetMatViewPopulatedState(intoRelationDesc, true);
 
-	/*
-	 * Finally we can open the target table
-	 */
-	intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
+		/*
+		 * Fill private fields of myState for use by later routines
+		 */
+		myState->rel = intoRelationDesc;
+		myState->reladdr = intoRelationAddr;
+		myState->output_cid = GetCurrentCommandId(true);
+		myState->ti_options = TABLE_INSERT_SKIP_FSM;
 
-	/*
-	 * Make sure the constructed table does not have RLS enabled.
-	 *
-	 * check_enable_rls() will ereport(ERROR) itself if the user has requested
-	 * something invalid, and otherwise will return RLS_ENABLED if RLS should
-	 * be enabled here.  We don't actually support that currently, so throw
-	 * our own ereport(ERROR) if that happens.
-	 */
-	if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
-		ereport(ERROR,
-				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-				 errmsg("policies not yet implemented for this command")));
+		/*
+		 * If WITH NO DATA is specified, there is no need to set up the state
+		 * for bulk inserts as there are no tuples to insert.
+		 */
+		if (!into->skipData)
+			myState->bistate = GetBulkInsertState();
+		else
+			myState->bistate = NULL;
 
-	/*
-	 * Tentatively mark the target as populated, if it's a matview and we're
-	 * going to fill it; otherwise, no change needed.
-	 */
-	if (is_matview && !into->skipData)
-		SetMatViewPopulatedState(intoRelationDesc, true);
+		if (myState->is_parallel == true)
+		{
+			myState->object_id = intoRelationAddr.objectId;
 
-	/*
-	 * Fill private fields of myState for use by later routines
-	 */
-	myState->rel = intoRelationDesc;
-	myState->reladdr = intoRelationAddr;
-	myState->output_cid = GetCurrentCommandId(true);
-	myState->ti_options = TABLE_INSERT_SKIP_FSM;
+			/*
+			 * We don't need to skip contacting FSM while inserting tuples
+			 * for parallel mode, while extending the relations, workers
+			 * instead of blocking on a page while another worker is inserting,
+			 * can check the FSM for another page that can accommodate the
+			 * tuples. This results in major benefit for parallel inserts.
+			 */
+			myState->ti_options = 0;
 
-	/*
-	 * If WITH NO DATA is specified, there is no need to set up the state for
-	 * bulk inserts as there are no tuples to insert.
-	 */
-	if (!into->skipData)
-		myState->bistate = GetBulkInsertState();
-	else
-		myState->bistate = NULL;
+			/*
+			 * rd_createSubid is marked invalid, otherwise, the table is
+			 * not allowed to extend by the workers.
+			 */
+			myState->rel->rd_createSubid = InvalidSubTransactionId;
+		}
 
-	/*
-	 * Valid smgr_targblock implies something already wrote to the relation.
-	 * This may be harmless, but this function hasn't planned for it.
-	 */
-	Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+		/*
+		 * Valid smgr_targblock implies something already wrote to the
+		 * relation. This may be harmless, but this function hasn't planned for
+		 * it.
+		 */
+		Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+	}
 }
 
 /*
@@ -606,3 +662,49 @@ intorel_destroy(DestReceiver *self)
 {
 	pfree(self);
 }
+
+/*
+ * IsParallelInsertInCTASAllowed --- determine whether or not parallel
+ * insertion is possible.
+ */
+bool IsParallelInsertInCTASAllowed(IntoClause *into, PlannedStmt *plannedstmt)
+{
+	bool allowed = false;
+
+	if (into != NULL &&
+		IsA(into, IntoClause))
+	{
+		if (into->rel != NULL &&
+			into->rel->relpersistence != RELPERSISTENCE_TEMP)
+			allowed = true;
+
+		if (plannedstmt != NULL && allowed)
+		{
+			/*
+			 * We allow parallel inserts by the workers only if the upper node
+			 * is Gather. We can not let workers do parallel inserts when
+			 * GatherMerge node is involved as the leader backend does the
+			 * final phase(merge the results by workers).
+			 */
+			if (plannedstmt->parallelModeNeeded &&
+				plannedstmt->planTree != NULL &&
+				IsA(plannedstmt->planTree, Gather) &&
+				plannedstmt->planTree->lefttree != NULL &&
+				plannedstmt->planTree->lefttree->parallel_aware &&
+				plannedstmt->planTree->lefttree->parallel_safe)
+			{
+				/*
+				 * Since there are no rows that are transferred from workers to
+				 * Gather node, so we set it to 0 to be visible in explain
+				 * plans. Note that we would have accounted this for cost
+				 * calculations in cost_gather().
+				 */
+				plannedstmt->planTree->plan_rows = 0;
+			}
+			else
+				allowed = false;
+		}
+	}
+
+	return allowed;
+}
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 43f9b01e83..bb01c8fc53 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -371,6 +371,15 @@ ExplainOneQuery(Query *query, int cursorOptions,
 		return;
 	}
 
+	/*
+	 * Flag to let the planner know that the SELECT query is for CTAS. This is
+	 * used to calculate the tuple transfer cost from workers to gather node(in
+	 * case parallelism kicks in for the SELECT part of the CTAS), to zero as
+	 * each worker will parallelly insert its share of tuples.
+	 */
+	if (IsParallelInsertInCTASAllowed(into, NULL))
+		query->isForCTAS = true;
+
 	/* if an advisor plugin is present, let it manage things */
 	if (ExplainOneQuery_hook)
 		(*ExplainOneQuery_hook) (query, cursorOptions, into, es,
@@ -536,7 +545,17 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	 * AS, we'd better use the appropriate tuple receiver.
 	 */
 	if (into)
+	{
 		dest = CreateIntoRelDestReceiver(into);
+
+		/*
+		 * SELECT part of the CTAS is parallelizable, so we can make each
+		 * parallel worker insert the tuples that are resulted in it's
+		 * execution into the target table.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, plannedstmt))
+			((DR_intorel *) dest)->is_parallel = true;
+	}
 	else
 		dest = None_Receiver;
 
@@ -1753,6 +1772,23 @@ ExplainNode(PlanState *planstate, List *ancestors,
 			{
 				Gather	   *gather = (Gather *) plan;
 
+				if (IsA(planstate, GatherState) &&
+					planstate->intoclause != NULL &&
+					IsA(planstate->intoclause,IntoClause) &&
+					planstate->dest != NULL &&
+					planstate->dest->mydest == DestIntoRel &&
+					((DR_intorel *) planstate->dest)->is_parallel == true &&
+					planstate->intoclause->rel != NULL &&
+					planstate->intoclause->rel->relname != NULL)
+				{
+					ExplainIndentText(es);
+					appendStringInfoString(es->str, "->  ");
+					appendStringInfoString(es->str, "Create ");
+					appendStringInfo(es->str, "%s\n", planstate->intoclause->rel->relname);
+					es->indent++;
+					ExplainIndentText(es);
+				}
+
 				show_scan_qual(plan->qual, "Filter", planstate, ancestors, es);
 				if (plan->qual)
 					show_instrumentation_count("Rows Removed by Filter", 1,
diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c
index 7179f589f9..ed99725779 100644
--- a/src/backend/executor/execMain.c
+++ b/src/backend/executor/execMain.c
@@ -45,6 +45,7 @@
 #include "access/xact.h"
 #include "catalog/namespace.h"
 #include "catalog/pg_publication.h"
+#include "commands/createas.h"
 #include "commands/matview.h"
 #include "commands/trigger.h"
 #include "executor/execdebug.h"
@@ -352,6 +353,24 @@ standard_ExecutorRun(QueryDesc *queryDesc,
 	if (sendTuples)
 		dest->rStartup(dest, operation, queryDesc->tupDesc);
 
+	/*
+	 * For parallelizing inserts in CTAS i.e. making each
+	 * parallel worker inerst it's tuples, we must send
+	 * information such as intoclause(for each worker
+	 * building it's own dest receiver), object id(for each
+	 * worker to open the table).
+	 */
+	if (queryDesc->plannedstmt->parallelModeNeeded == true &&
+		dest != NULL &&
+		dest->mydest == DestIntoRel &&
+		((DR_intorel *) dest)->is_parallel == true &&
+		((DR_intorel *) dest)->is_parallel_worker != true)
+	{
+		queryDesc->planstate->intoclause = ((DR_intorel *) dest)->into;
+		queryDesc->planstate->objectid = ((DR_intorel *) dest)->object_id;
+		queryDesc->planstate->dest = dest;
+	}
+
 	/*
 	 * run plan
 	 */
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index befde52691..b2aa06102f 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -23,6 +23,7 @@
 
 #include "postgres.h"
 
+#include "commands/createas.h"
 #include "executor/execParallel.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
@@ -65,6 +66,7 @@
 #define PARALLEL_KEY_QUERY_TEXT		UINT64CONST(0xE000000000000008)
 #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009)
 #define PARALLEL_KEY_WAL_USAGE			UINT64CONST(0xE00000000000000A)
+#define PARALLEL_KEY_INTO_CLAUSE		UINT64CONST(0xE00000000000000B)
 
 #define PARALLEL_TUPLE_QUEUE_SIZE		65536
 
@@ -77,6 +79,8 @@ typedef struct FixedParallelExecutorState
 	dsa_pointer param_exec;
 	int			eflags;
 	int			jit_flags;
+	Oid			objectid;		/* workers to open relation/table.  */
+	pg_atomic_uint64	processed; /* number tuples inserted by all the workers. */
 } FixedParallelExecutorState;
 
 /*
@@ -600,6 +604,7 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	Size		dsa_minsize = dsa_minimum_size();
 	char	   *query_string;
 	int			query_len;
+	char 		*intoclausestr = NULL;
 
 	/*
 	 * Force any initplan outputs that we're going to pass to workers to be
@@ -712,6 +717,14 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_estimate_chunk(&pcxt->estimator, dsa_minsize);
 	shm_toc_estimate_keys(&pcxt->estimator, 1);
 
+	/* Estimate space for into clause for CTAS. */
+	if (ISCTAS(planstate->intoclause))
+	{
+		intoclausestr = nodeToString(planstate->intoclause);
+		shm_toc_estimate_chunk(&pcxt->estimator, strlen(intoclausestr) + 1);
+		shm_toc_estimate_keys(&pcxt->estimator, 1);
+	}
+
 	/* Everyone's had a chance to ask for space, so now create the DSM. */
 	InitializeParallelDSM(pcxt);
 
@@ -729,6 +742,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	fpes->param_exec = InvalidDsaPointer;
 	fpes->eflags = estate->es_top_eflags;
 	fpes->jit_flags = estate->es_jit_flags;
+	pg_atomic_init_u64(&fpes->processed, 0);
+	pei->processed = &fpes->processed;
+
+	if (intoclausestr != NULL &&
+		planstate->objectid != InvalidOid)
+		fpes->objectid = planstate->objectid;
+	else
+		fpes->objectid = InvalidOid;
+
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, fpes);
 
 	/* Store query string */
@@ -758,8 +780,17 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_WAL_USAGE, walusage_space);
 	pei->wal_usage = walusage_space;
 
+	if (intoclausestr != NULL)
+	{
+		char *shmptr = (char *)shm_toc_allocate(pcxt->toc,
+												strlen(intoclausestr) + 1);
+		strcpy(shmptr, intoclausestr);
+		shm_toc_insert(pcxt->toc, PARALLEL_KEY_INTO_CLAUSE, shmptr);
+	}
+
 	/* Set up the tuple queues that the workers will write into. */
-	pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	if (intoclausestr == NULL)
+		pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
 
 	/* We don't need the TupleQueueReaders yet, though. */
 	pei->reader = NULL;
@@ -1387,12 +1418,28 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 	void	   *area_space;
 	dsa_area   *area;
 	ParallelWorkerContext pwcxt;
+	char		*intoclausestr = NULL;
+	IntoClause	*intoclause = NULL;
 
 	/* Get fixed-size state. */
 	fpes = shm_toc_lookup(toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
 
-	/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
-	receiver = ExecParallelGetReceiver(seg, toc);
+	intoclausestr = shm_toc_lookup(toc, PARALLEL_KEY_INTO_CLAUSE, true);
+	if (intoclausestr != NULL)
+	{
+		/*
+		 * If the worker is for parallel insert in CTAS, then use
+		 * the proper dest receiver.
+		 */
+		intoclause = (IntoClause *) stringToNode(intoclausestr);
+		receiver = CreateIntoRelDestReceiver(intoclause);
+		((DR_intorel *)receiver)->is_parallel_worker = true;
+		((DR_intorel *)receiver)->object_id = fpes->objectid;
+	}
+	else
+		/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
+		receiver = ExecParallelGetReceiver(seg, toc);
+
 	instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_INSTRUMENTATION, true);
 	if (instrumentation != NULL)
 		instrument_options = instrumentation->instrument_options;
@@ -1471,6 +1518,13 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 			queryDesc->estate->es_jit->instr;
 	}
 
+	/*
+	 * Write out the number of tuples this worker has inserted. Leader
+	 * will use it to inform to the end client.
+	 */
+	if (intoclausestr != NULL)
+		pg_atomic_add_fetch_u64(&fpes->processed, queryDesc->estate->es_processed);
+
 	/* Must do this after capturing instrumentation. */
 	ExecutorEnd(queryDesc);
 
diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c
index a01b46af14..99211e4941 100644
--- a/src/backend/executor/nodeGather.c
+++ b/src/backend/executor/nodeGather.c
@@ -32,6 +32,7 @@
 
 #include "access/relscan.h"
 #include "access/xact.h"
+#include "commands/createas.h"
 #include "executor/execdebug.h"
 #include "executor/execParallel.h"
 #include "executor/nodeGather.h"
@@ -48,7 +49,7 @@ static TupleTableSlot *ExecGather(PlanState *pstate);
 static TupleTableSlot *gather_getnext(GatherState *gatherstate);
 static MinimalTuple gather_readnext(GatherState *gatherstate);
 static void ExecShutdownGatherWorkers(GatherState *node);
-
+static void ExecParallelInsertInCTAS(GatherState *node);
 
 /* ----------------------------------------------------------------
  *		ExecInitGather
@@ -131,6 +132,69 @@ ExecInitGather(Gather *node, EState *estate, int eflags)
 	return gatherstate;
 }
 
+/* ----------------------------------------------------------------
+ *		ExecParallelInsertInCTAS(node)
+ *
+ *		Facilitates parallel inserts by parallel workers and/or
+ *		leader for Create Table AS.
+ * ----------------------------------------------------------------
+ */
+static void
+ExecParallelInsertInCTAS(GatherState *node)
+{
+	/* Enable leader to insert in case no parallel workers were launched. */
+	if (node->nworkers_launched == 0 &&
+		!node->need_to_scan_locally)
+		node->need_to_scan_locally = true;
+	/*
+	 * By now, for parallel workers (if launched any), would have started their
+	 * work i.e. insertion to target table. In case the leader is chosen to
+	 * participate for parallel inserts in CTAS, then finish it's share before
+	 * going to wait for the parallel workers to finish.
+	 */
+	if (node->need_to_scan_locally == true &&
+		node->ps.dest != NULL &&
+		node->ps.dest->mydest == DestIntoRel)
+	{
+		EState	   *estate = node->ps.state;
+		TupleTableSlot *outerTupleSlot;
+
+		for(;;)
+		{
+			/* Install our DSA area while executing the plan. */
+			estate->es_query_dsa =
+					node->pei ? node->pei->area : NULL;
+
+			outerTupleSlot = ExecProcNode(node->ps.lefttree);
+
+			estate->es_query_dsa = NULL;
+
+			if (!TupIsNull(outerTupleSlot))
+			{
+				(void) node->ps.dest->receiveSlot(outerTupleSlot, node->ps.dest);
+				node->ps.state->es_processed++;
+			}
+
+			if(TupIsNull(outerTupleSlot))
+				break;
+		}
+
+		node->need_to_scan_locally = false;
+	}
+
+	/* Wait for the parallel workers to finish. */
+	if (node->nworkers_launched > 0)
+	{
+		ExecShutdownGatherWorkers(node);
+
+		/*
+		 * Add up the total tuples inserted by all workers, to the tuples
+		 * inserted by the leader(if any). This will be shared to client.
+		 */
+		node->ps.state->es_processed += pg_atomic_read_u64(node->pei->processed);
+	}
+}
+
 /* ----------------------------------------------------------------
  *		ExecGather(node)
  *
@@ -166,6 +230,16 @@ ExecGather(PlanState *pstate)
 		{
 			ParallelContext *pcxt;
 
+			/*
+			 * Take the necessary information to be passed to workers for
+			 * parallel inserts in CTAS.
+			 */
+			if (ISCTAS(node->ps.intoclause))
+			{
+				node->ps.lefttree->intoclause = node->ps.intoclause;
+				node->ps.lefttree->objectid = node->ps.objectid;
+			}
+
 			/* Initialize, or re-initialize, shared state needed by workers. */
 			if (!node->pei)
 				node->pei = ExecInitParallelPlan(node->ps.lefttree,
@@ -190,13 +264,16 @@ ExecGather(PlanState *pstate)
 			/* Set up tuple queue readers to read the results. */
 			if (pcxt->nworkers_launched > 0)
 			{
-				ExecParallelCreateReaders(node->pei);
-				/* Make a working array showing the active readers */
-				node->nreaders = pcxt->nworkers_launched;
-				node->reader = (TupleQueueReader **)
-					palloc(node->nreaders * sizeof(TupleQueueReader *));
-				memcpy(node->reader, node->pei->reader,
-					   node->nreaders * sizeof(TupleQueueReader *));
+				if (!(ISCTAS(node->ps.intoclause)))
+				{
+					ExecParallelCreateReaders(node->pei);
+					/* Make a working array showing the active readers */
+					node->nreaders = pcxt->nworkers_launched;
+					node->reader = (TupleQueueReader **)
+						palloc(node->nreaders * sizeof(TupleQueueReader *));
+					memcpy(node->reader, node->pei->reader,
+						node->nreaders * sizeof(TupleQueueReader *));
+				}
 			}
 			else
 			{
@@ -208,7 +285,8 @@ ExecGather(PlanState *pstate)
 		}
 
 		/* Run plan locally if no workers or enabled and not single-copy. */
-		node->need_to_scan_locally = (node->nreaders == 0)
+		node->need_to_scan_locally = (node->nreaders == 0 &&
+			!(ISCTAS(node->ps.intoclause)))
 			|| (!gather->single_copy && parallel_leader_participation);
 		node->initialized = true;
 	}
@@ -220,6 +298,11 @@ ExecGather(PlanState *pstate)
 	econtext = node->ps.ps_ExprContext;
 	ResetExprContext(econtext);
 
+	if (ISCTAS(node->ps.intoclause))
+	{
+		ExecParallelInsertInCTAS(node);
+		return NULL;
+	}
 	/*
 	 * Get next tuple, either from one of our workers, or by running the plan
 	 * ourselves.
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index f1dfdc1a4a..d0111c8a5c 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -393,7 +393,17 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 
 	/* Parallel setup and communication cost. */
 	startup_cost += parallel_setup_cost;
-	run_cost += parallel_tuple_cost * path->path.rows;
+
+	/*
+	 * Make the number of tuples that are transferred from workers to gather
+	 * node zero as each worker parallelly insert the tuples that are resulted
+	 * from its chunk of plan execution. This change may make the parallel
+	 * plan cheap among all other plans, and influence the planner to consider
+	 * this parallel plan.
+	 */
+	if (!(root->parse->isForCTAS &&
+		root->query_level == 1))
+		run_cost += parallel_tuple_cost * path->path.rows;
 
 	path->path.startup_cost = startup_cost;
 	path->path.total_cost = (startup_cost + run_cost);
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7320de345c..5beae6c617 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -389,6 +389,7 @@ extern FullTransactionId GetCurrentFullTransactionIdIfAny(void);
 extern void MarkCurrentTransactionIdLoggedIfAny(void);
 extern bool SubTransactionIsActive(SubTransactionId subxid);
 extern CommandId GetCurrentCommandId(bool used);
+extern void SetCurrentCommandIdUsedForWorker(void);
 extern void SetParallelStartTimestamps(TimestampTz xact_ts, TimestampTz stmt_ts);
 extern TimestampTz GetCurrentTransactionStartTimestamp(void);
 extern TimestampTz GetCurrentStatementStartTimestamp(void);
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 7629230254..af72ffcfe2 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -14,12 +14,30 @@
 #ifndef CREATEAS_H
 #define CREATEAS_H
 
+#include "access/heapam.h"
 #include "catalog/objectaddress.h"
 #include "nodes/params.h"
+#include "nodes/plannodes.h"
 #include "parser/parse_node.h"
 #include "tcop/dest.h"
 #include "utils/queryenvironment.h"
 
+typedef struct
+{
+	DestReceiver pub;			/* publicly-known function pointers */
+	IntoClause *into;			/* target relation specification */
+	/* These fields are filled by intorel_startup: */
+	Relation	rel;			/* relation to write to */
+	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
+	CommandId	output_cid;		/* cmin to insert in output tuples */
+	int			ti_options;		/* table_tuple_insert performance options */
+	BulkInsertState bistate;	/* bulk insert state */
+	bool		is_parallel;		/* true if parallelism is to be considered */
+	bool		is_parallel_worker; /* true for parallel worker */
+	Oid			object_id;			/* used for table open by parallel worker */
+} DR_intorel;
+
+#define ISCTAS(intoclause) (intoclause != NULL && IsA(intoclause, IntoClause))
 
 extern ObjectAddress ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 									   ParamListInfo params, QueryEnvironment *queryEnv,
@@ -29,4 +47,6 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
+extern bool IsParallelInsertInCTASAllowed(IntoClause *intoClause, PlannedStmt *plannedstmt);
+
 #endif							/* CREATEAS_H */
diff --git a/src/include/executor/execParallel.h b/src/include/executor/execParallel.h
index 5a39a5b29c..77f69946bf 100644
--- a/src/include/executor/execParallel.h
+++ b/src/include/executor/execParallel.h
@@ -35,6 +35,7 @@ typedef struct ParallelExecutorInfo
 	/* These two arrays have pcxt->nworkers_launched entries: */
 	shm_mq_handle **tqueue;		/* tuple queues for worker output */
 	struct TupleQueueReader **reader;	/* tuple reader/writer support */
+	volatile pg_atomic_uint64	*processed;	/* number of tuples inserted by all workers */
 } ParallelExecutorInfo;
 
 extern ParallelExecutorInfo *ExecInitParallelPlan(PlanState *planstate,
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index f6824bf2e1..c7ce365438 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -23,6 +23,7 @@
 #include "nodes/tidbitmap.h"
 #include "partitioning/partdefs.h"
 #include "storage/condition_variable.h"
+#include "tcop/dest.h"
 #include "utils/hsearch.h"
 #include "utils/queryenvironment.h"
 #include "utils/reltrigger.h"
@@ -1020,6 +1021,10 @@ typedef struct PlanState
 	bool		outeropsset;
 	bool		inneropsset;
 	bool		resultopsset;
+	/* Below is parallel inserts in CTAS related info. */
+	IntoClause	*intoclause;
+	Oid			objectid;
+	DestReceiver *dest;
 } PlanState;
 
 /* ----------------
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index d1f9ef29ca..909313466d 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -180,6 +180,7 @@ typedef struct Query
 	 */
 	int			stmt_location;	/* start location, or -1 if unknown */
 	int			stmt_len;		/* length in bytes; 0 means "rest of string" */
+	bool		isForCTAS;		/* true if the select query is for create table as */
 } Query;
 
 
diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 0c4da2591a..11ef18b8a4 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -76,4 +76,147 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
+--
+-- Test parallel inserts in create table as/select into/create materialized view
+--
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+                     QUERY PLAN                      
+-----------------------------------------------------
+ LockRows (actual rows=10000 loops=1)
+   ->  Seq Scan on tenk1 (actual rows=10000 loops=1)
+(2 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+select length(stringu1) from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_mat_view
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop materialized view parallel_mat_view;
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+deallocate parallel_write_prep;
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+select nextval('parallel_write_sequence'), four from tenk1;
+                  QUERY PLAN                   
+-----------------------------------------------
+ Seq Scan on tenk1 (actual rows=10000 loops=1)
+(1 row)
+
+drop table parallel_write;
+drop sequence parallel_write_sequence;
 rollback;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 78b479cedf..dd4233b399 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -39,4 +39,69 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
 
+--
+-- Test parallel inserts in create table as/select into/create materialized view
+--
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+select length(stringu1) from tenk1;
+drop materialized view parallel_mat_view;
+
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+deallocate parallel_write_prep;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+select nextval('parallel_write_sequence'), four from tenk1;
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+
 rollback;
-- 
2.25.1

From 21fbbb8d4297e6daa7a7ec696a36327f592089bd Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Wed, 25 Nov 2020 04:44:29 +0530
Subject: [PATCH v5] Parallel Inserts in CREATE TABLE AS

The idea of this patch is to allow the leader and each worker
insert the tuples in parallel if the SELECT part of the CTAS is
parallelizable.

The design:
Let the planner know that the SELECT is from CTAS in createas.c
so that it can set the number of tuples transferred from the
workers to Gather node to 0. With this change, there are chances
that the planner may choose the parallel plan. After the planning,
check if the upper plan node is Gather in createas.c and mark a
parallelism flag in the CTAS dest receiver. Pass the into clause,
object id, command id from the leader to workers, so that each
worker can create its own CTAS dest receiver. Leader inserts it's
share of tuples if instructed to do, and so are workers. Each
worker writes atomically it's number of inserted tuples into a
shared memory variable, the leader combines this with it's own
number of inserted tuples and shares to the client.
---
 src/backend/access/heap/heapam.c             |  11 -
 src/backend/access/transam/xact.c            |  30 +-
 src/backend/commands/createas.c              | 313 ++++++++++++-------
 src/backend/commands/explain.c               |  36 +++
 src/backend/executor/execMain.c              |  18 ++
 src/backend/executor/execParallel.c          |  68 +++-
 src/backend/executor/nodeGather.c            |  99 +++++-
 src/backend/optimizer/path/costsize.c        |  12 +-
 src/include/access/xact.h                    |   1 +
 src/include/commands/createas.h              |  22 ++
 src/include/executor/execParallel.h          |   2 +
 src/include/nodes/execnodes.h                |   5 +
 src/include/nodes/parsenodes.h               |   1 +
 src/test/regress/expected/write_parallel.out | 143 +++++++++
 src/test/regress/sql/write_parallel.sql      |  65 ++++
 15 files changed, 688 insertions(+), 138 deletions(-)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 1b2f70499e..3045c0f046 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2043,17 +2043,6 @@ static HeapTuple
 heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
 					CommandId cid, int options)
 {
-	/*
-	 * To allow parallel inserts, we need to ensure that they are safe to be
-	 * performed in workers. We have the infrastructure to allow parallel
-	 * inserts in general except for the cases where inserts generate a new
-	 * CommandId (eg. inserts into a table having a foreign key column).
-	 */
-	if (IsParallelWorker())
-		ereport(ERROR,
-				(errcode(ERRCODE_INVALID_TRANSACTION_STATE),
-				 errmsg("cannot insert tuples in a parallel worker")));
-
 	tup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
 	tup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
 	tup->t_data->t_infomask |= HEAP_XMAX_INVALID;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 9cd0b7c11b..db6eedd635 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -763,18 +763,34 @@ GetCurrentCommandId(bool used)
 	/* this is global to a transaction, not subtransaction-local */
 	if (used)
 	{
-		/*
-		 * Forbid setting currentCommandIdUsed in a parallel worker, because
-		 * we have no provision for communicating this back to the leader.  We
-		 * could relax this restriction when currentCommandIdUsed was already
-		 * true at the start of the parallel operation.
-		 */
-		Assert(!IsParallelWorker());
+		 /*
+		  * This is a temporary hack for all common parallel insert cases i.e.
+		  * insert into, ctas, copy from. To be changed later. In a parallel
+		  * worker, set currentCommandIdUsed to true only if it was not set to
+		  * true at the start of the parallel operation (by way of
+		  * SetCurrentCommandIdUsedForWorker()). We have to do this because
+		  * GetCurrentCommandId(true) may be called from anywhere, especially
+		  * for parallel inserts, within parallel worker.
+		  */
+		Assert(!(IsParallelWorker() && !currentCommandIdUsed));
 		currentCommandIdUsed = true;
 	}
 	return currentCommandId;
 }
 
+/*
+ *	SetCurrentCommandIdUsedForWorker
+ *
+ * For a parallel worker, record that the currentCommandId has been used. This
+ * must only be called at the start of a parallel operation.
+ */
+void
+SetCurrentCommandIdUsedForWorker(void)
+{
+	Assert(IsParallelWorker() && !currentCommandIdUsed && currentCommandId != InvalidCommandId);
+	currentCommandIdUsed = true;
+}
+
 /*
  *	SetParallelStartTimestamps
  *
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 6bf6c5a310..afab904c6b 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,18 +51,6 @@
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 
-typedef struct
-{
-	DestReceiver pub;			/* publicly-known function pointers */
-	IntoClause *into;			/* target relation specification */
-	/* These fields are filled by intorel_startup: */
-	Relation	rel;			/* relation to write to */
-	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
-	CommandId	output_cid;		/* cmin to insert in output tuples */
-	int			ti_options;		/* table_tuple_insert performance options */
-	BulkInsertState bistate;	/* bulk insert state */
-} DR_intorel;
-
 /* utility functions for CTAS definition creation */
 static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into);
 static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -328,10 +316,27 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		query = linitial_node(Query, rewritten);
 		Assert(query->commandType == CMD_SELECT);
 
+		/*
+		 * Flag to let the planner know that the SELECT query is for CTAS. This
+		 * is used to calculate the tuple transfer cost from workers to gather
+		 * node(in case parallelism kicks in for the SELECT part of the CTAS),
+		 * to zero as each worker will parallelly insert its share of tuples.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, NULL))
+			query->isForCTAS = true;
+
 		/* plan the query */
 		plan = pg_plan_query(query, pstate->p_sourcetext,
 							 CURSOR_OPT_PARALLEL_OK, params);
 
+		/*
+		 * SELECT part of the CTAS is parallelizable, so we can make each
+		 * parallel worker insert the tuples that are resulted in it's
+		 * execution into the target table.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, plan))
+			((DR_intorel *) dest)->is_parallel = true;
+
 		/*
 		 * Use a snapshot with an updated command ID to ensure this query sees
 		 * results of any previously executed queries.  (This could only
@@ -418,6 +423,9 @@ CreateIntoRelDestReceiver(IntoClause *intoClause)
 	self->pub.rDestroy = intorel_destroy;
 	self->pub.mydest = DestIntoRel;
 	self->into = intoClause;
+	self->is_parallel = false;
+	self->is_parallel_worker = false;
+	self->object_id = InvalidOid;
 	/* other private fields will be set during intorel_startup */
 
 	return (DestReceiver *) self;
@@ -430,121 +438,169 @@ static void
 intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 {
 	DR_intorel *myState = (DR_intorel *) self;
-	IntoClause *into = myState->into;
-	bool		is_matview;
-	List	   *attrList;
 	ObjectAddress intoRelationAddr;
 	Relation	intoRelationDesc;
-	ListCell   *lc;
-	int			attnum;
-
-	Assert(into != NULL);		/* else somebody forgot to set it */
 
-	/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
-	is_matview = (into->viewQuery != NULL);
+	if (myState->is_parallel_worker)
+	{
+		/* In the worker */
+		intoRelationDesc = table_open(myState->object_id, AccessExclusiveLock);
+		myState->rel = intoRelationDesc;
+		myState->reladdr = InvalidObjectAddress;
+		myState->ti_options = 0;
+		myState->bistate = GetBulkInsertState();
 
-	/*
-	 * Build column definitions using "pre-cooked" type and collation info. If
-	 * a column name list was specified in CREATE TABLE AS, override the
-	 * column names derived from the query.  (Too few column names are OK, too
-	 * many are not.)
-	 */
-	attrList = NIL;
-	lc = list_head(into->colNames);
-	for (attnum = 0; attnum < typeinfo->natts; attnum++)
+		/*
+		 * Right after the table is created in the leader, the command id is
+		 * incremented (in create_ctas_internal()). The new command id is
+		 * marked as used in inintorel_startup(), then the parallel mode is
+		 * entered. The command id and transaction id are serialized into
+		 * parallel DSM, they are then available to all parallel workers. All
+		 * the workers need to mark the command id as used before insertion.
+		 */
+		(void) SetCurrentCommandIdUsedForWorker();
+		myState->output_cid = GetCurrentCommandId(false);
+	}
+	else
 	{
-		Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
-		ColumnDef  *col;
-		char	   *colname;
+		IntoClause *into = myState->into;
+		bool		is_matview;
+		List	   *attrList;
+		ListCell   *lc;
+		int			attnum;
 
-		if (lc)
+		Assert(into != NULL);		/* else somebody forgot to set it */
+
+		/*
+		 * This code supports both CREATE TABLE AS and CREATE MATERIALIZED
+		 * VIEW.
+		 */
+		is_matview = (into->viewQuery != NULL);
+
+		/*
+		 * Build column definitions using "pre-cooked" type and collation info.
+		 * If a column name list was specified in CREATE TABLE AS, override the
+		 * column names derived from the query.  (Too few column names are OK,
+		 * too many are not.)
+		 */
+		attrList = NIL;
+		lc = list_head(into->colNames);
+		for (attnum = 0; attnum < typeinfo->natts; attnum++)
 		{
-			colname = strVal(lfirst(lc));
-			lc = lnext(into->colNames, lc);
+			Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
+			ColumnDef  *col;
+			char	   *colname;
+
+			if (lc)
+			{
+				colname = strVal(lfirst(lc));
+				lc = lnext(into->colNames, lc);
+			}
+			else
+				colname = NameStr(attribute->attname);
+
+			col = makeColumnDef(colname,
+								attribute->atttypid,
+								attribute->atttypmod,
+								attribute->attcollation);
+
+			/*
+			 * It's possible that the column is of a collatable type but the
+			 * collation could not be resolved, so double-check.  (We must
+			 * check this here because DefineRelation would adopt the type's
+			 * default collation rather than complaining.)
+			 */
+			if (!OidIsValid(col->collOid) &&
+				type_is_collatable(col->typeName->typeOid))
+				ereport(ERROR,
+						(errcode(ERRCODE_INDETERMINATE_COLLATION),
+						errmsg("no collation was derived for column \"%s\" with collatable type %s",
+								col->colname,
+								format_type_be(col->typeName->typeOid)),
+						errhint("Use the COLLATE clause to set the collation explicitly.")));
+
+			attrList = lappend(attrList, col);
 		}
-		else
-			colname = NameStr(attribute->attname);
 
-		col = makeColumnDef(colname,
-							attribute->atttypid,
-							attribute->atttypmod,
-							attribute->attcollation);
+		if (lc != NULL)
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					errmsg("too many column names were specified")));
 
 		/*
-		 * It's possible that the column is of a collatable type but the
-		 * collation could not be resolved, so double-check.  (We must check
-		 * this here because DefineRelation would adopt the type's default
-		 * collation rather than complaining.)
+		 * Actually create the target table
 		 */
-		if (!OidIsValid(col->collOid) &&
-			type_is_collatable(col->typeName->typeOid))
-			ereport(ERROR,
-					(errcode(ERRCODE_INDETERMINATE_COLLATION),
-					 errmsg("no collation was derived for column \"%s\" with collatable type %s",
-							col->colname,
-							format_type_be(col->typeName->typeOid)),
-					 errhint("Use the COLLATE clause to set the collation explicitly.")));
+		intoRelationAddr = create_ctas_internal(attrList, into);
 
-		attrList = lappend(attrList, col);
-	}
+		/*
+		 * Finally we can open the target table
+		 */
+		intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
 
-	if (lc != NULL)
-		ereport(ERROR,
-				(errcode(ERRCODE_SYNTAX_ERROR),
-				 errmsg("too many column names were specified")));
+		/*
+		 * Make sure the constructed table does not have RLS enabled.
+		 *
+		 * check_enable_rls() will ereport(ERROR) itself if the user has
+		 * requested something invalid, and otherwise will return RLS_ENABLED
+		 * if RLS should be enabled here.  We don't actually support that
+		 * currently, so throw our own ereport(ERROR) if that happens.
+		 */
+		if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					errmsg("policies not yet implemented for this command")));
 
-	/*
-	 * Actually create the target table
-	 */
-	intoRelationAddr = create_ctas_internal(attrList, into);
+		/*
+		 * Tentatively mark the target as populated, if it's a matview and
+		 * we're going to fill it; otherwise, no change needed.
+		 */
+		if (is_matview && !into->skipData)
+			SetMatViewPopulatedState(intoRelationDesc, true);
 
-	/*
-	 * Finally we can open the target table
-	 */
-	intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
+		/*
+		 * Fill private fields of myState for use by later routines
+		 */
+		myState->rel = intoRelationDesc;
+		myState->reladdr = intoRelationAddr;
+		myState->output_cid = GetCurrentCommandId(true);
+		myState->ti_options = TABLE_INSERT_SKIP_FSM;
 
-	/*
-	 * Make sure the constructed table does not have RLS enabled.
-	 *
-	 * check_enable_rls() will ereport(ERROR) itself if the user has requested
-	 * something invalid, and otherwise will return RLS_ENABLED if RLS should
-	 * be enabled here.  We don't actually support that currently, so throw
-	 * our own ereport(ERROR) if that happens.
-	 */
-	if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
-		ereport(ERROR,
-				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-				 errmsg("policies not yet implemented for this command")));
+		/*
+		 * If WITH NO DATA is specified, there is no need to set up the state
+		 * for bulk inserts as there are no tuples to insert.
+		 */
+		if (!into->skipData)
+			myState->bistate = GetBulkInsertState();
+		else
+			myState->bistate = NULL;
 
-	/*
-	 * Tentatively mark the target as populated, if it's a matview and we're
-	 * going to fill it; otherwise, no change needed.
-	 */
-	if (is_matview && !into->skipData)
-		SetMatViewPopulatedState(intoRelationDesc, true);
+		if (myState->is_parallel == true)
+		{
+			myState->object_id = intoRelationAddr.objectId;
 
-	/*
-	 * Fill private fields of myState for use by later routines
-	 */
-	myState->rel = intoRelationDesc;
-	myState->reladdr = intoRelationAddr;
-	myState->output_cid = GetCurrentCommandId(true);
-	myState->ti_options = TABLE_INSERT_SKIP_FSM;
+			/*
+			 * We don't need to skip contacting FSM while inserting tuples
+			 * for parallel mode, while extending the relations, workers
+			 * instead of blocking on a page while another worker is inserting,
+			 * can check the FSM for another page that can accommodate the
+			 * tuples. This results in major benefit for parallel inserts.
+			 */
+			myState->ti_options = 0;
 
-	/*
-	 * If WITH NO DATA is specified, there is no need to set up the state for
-	 * bulk inserts as there are no tuples to insert.
-	 */
-	if (!into->skipData)
-		myState->bistate = GetBulkInsertState();
-	else
-		myState->bistate = NULL;
+			/*
+			 * rd_createSubid is marked invalid, otherwise, the table is
+			 * not allowed to extend by the workers.
+			 */
+			myState->rel->rd_createSubid = InvalidSubTransactionId;
+		}
 
-	/*
-	 * Valid smgr_targblock implies something already wrote to the relation.
-	 * This may be harmless, but this function hasn't planned for it.
-	 */
-	Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+		/*
+		 * Valid smgr_targblock implies something already wrote to the
+		 * relation. This may be harmless, but this function hasn't planned for
+		 * it.
+		 */
+		Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+	}
 }
 
 /*
@@ -606,3 +662,48 @@ intorel_destroy(DestReceiver *self)
 {
 	pfree(self);
 }
+
+/*
+ * IsParallelInsertInCTASAllowed --- determine whether or not parallel
+ * insertion is possible.
+ */
+bool IsParallelInsertInCTASAllowed(IntoClause *into, PlannedStmt *plannedstmt)
+{
+	bool allowed = false;
+
+	if (ISCTAS(into))
+	{
+		if (into->rel != NULL &&
+			into->rel->relpersistence != RELPERSISTENCE_TEMP)
+			allowed = true;
+
+		if (plannedstmt != NULL && allowed)
+		{
+			/*
+			 * We allow parallel inserts by the workers only if the upper node
+			 * is Gather. We can not let workers do parallel inserts when
+			 * GatherMerge node is involved as the leader backend does the
+			 * final phase(merge the results by workers).
+			 */
+			if (plannedstmt->parallelModeNeeded &&
+				plannedstmt->planTree != NULL &&
+				IsA(plannedstmt->planTree, Gather) &&
+				plannedstmt->planTree->lefttree != NULL &&
+				plannedstmt->planTree->lefttree->parallel_aware &&
+				plannedstmt->planTree->lefttree->parallel_safe)
+			{
+				/*
+				 * Since there are no rows that are transferred from workers to
+				 * Gather node, so we set it to 0 to be visible in explain
+				 * plans. Note that we would have accounted this for cost
+				 * calculations in cost_gather().
+				 */
+				plannedstmt->planTree->plan_rows = 0;
+			}
+			else
+				allowed = false;
+		}
+	}
+
+	return allowed;
+}
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 43f9b01e83..bb01c8fc53 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -371,6 +371,15 @@ ExplainOneQuery(Query *query, int cursorOptions,
 		return;
 	}
 
+	/*
+	 * Flag to let the planner know that the SELECT query is for CTAS. This is
+	 * used to calculate the tuple transfer cost from workers to gather node(in
+	 * case parallelism kicks in for the SELECT part of the CTAS), to zero as
+	 * each worker will parallelly insert its share of tuples.
+	 */
+	if (IsParallelInsertInCTASAllowed(into, NULL))
+		query->isForCTAS = true;
+
 	/* if an advisor plugin is present, let it manage things */
 	if (ExplainOneQuery_hook)
 		(*ExplainOneQuery_hook) (query, cursorOptions, into, es,
@@ -536,7 +545,17 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	 * AS, we'd better use the appropriate tuple receiver.
 	 */
 	if (into)
+	{
 		dest = CreateIntoRelDestReceiver(into);
+
+		/*
+		 * SELECT part of the CTAS is parallelizable, so we can make each
+		 * parallel worker insert the tuples that are resulted in it's
+		 * execution into the target table.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, plannedstmt))
+			((DR_intorel *) dest)->is_parallel = true;
+	}
 	else
 		dest = None_Receiver;
 
@@ -1753,6 +1772,23 @@ ExplainNode(PlanState *planstate, List *ancestors,
 			{
 				Gather	   *gather = (Gather *) plan;
 
+				if (IsA(planstate, GatherState) &&
+					planstate->intoclause != NULL &&
+					IsA(planstate->intoclause,IntoClause) &&
+					planstate->dest != NULL &&
+					planstate->dest->mydest == DestIntoRel &&
+					((DR_intorel *) planstate->dest)->is_parallel == true &&
+					planstate->intoclause->rel != NULL &&
+					planstate->intoclause->rel->relname != NULL)
+				{
+					ExplainIndentText(es);
+					appendStringInfoString(es->str, "->  ");
+					appendStringInfoString(es->str, "Create ");
+					appendStringInfo(es->str, "%s\n", planstate->intoclause->rel->relname);
+					es->indent++;
+					ExplainIndentText(es);
+				}
+
 				show_scan_qual(plan->qual, "Filter", planstate, ancestors, es);
 				if (plan->qual)
 					show_instrumentation_count("Rows Removed by Filter", 1,
diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c
index 7179f589f9..e4efa3ac76 100644
--- a/src/backend/executor/execMain.c
+++ b/src/backend/executor/execMain.c
@@ -45,6 +45,7 @@
 #include "access/xact.h"
 #include "catalog/namespace.h"
 #include "catalog/pg_publication.h"
+#include "commands/createas.h"
 #include "commands/matview.h"
 #include "commands/trigger.h"
 #include "executor/execdebug.h"
@@ -352,6 +353,23 @@ standard_ExecutorRun(QueryDesc *queryDesc,
 	if (sendTuples)
 		dest->rStartup(dest, operation, queryDesc->tupDesc);
 
+	/*
+	 * For parallelizing inserts in CTAS i.e. making each parallel worker
+	 * insert the tuples, we must send information such as intoclause(for each
+	 * worker to build separate dest receiver), object id(for each worker to
+	 * open the table).
+	 */
+	if (queryDesc->plannedstmt->parallelModeNeeded == true &&
+		dest != NULL &&
+		dest->mydest == DestIntoRel &&
+		((DR_intorel *) dest)->is_parallel == true &&
+		((DR_intorel *) dest)->is_parallel_worker != true)
+	{
+		queryDesc->planstate->intoclause = ((DR_intorel *) dest)->into;
+		queryDesc->planstate->objectid = ((DR_intorel *) dest)->object_id;
+		queryDesc->planstate->dest = dest;
+	}
+
 	/*
 	 * run plan
 	 */
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index befde52691..442c633232 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -23,6 +23,7 @@
 
 #include "postgres.h"
 
+#include "commands/createas.h"
 #include "executor/execParallel.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
@@ -65,6 +66,7 @@
 #define PARALLEL_KEY_QUERY_TEXT		UINT64CONST(0xE000000000000008)
 #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009)
 #define PARALLEL_KEY_WAL_USAGE			UINT64CONST(0xE00000000000000A)
+#define PARALLEL_KEY_INTO_CLAUSE		UINT64CONST(0xE00000000000000B)
 
 #define PARALLEL_TUPLE_QUEUE_SIZE		65536
 
@@ -77,6 +79,9 @@ typedef struct FixedParallelExecutorState
 	dsa_pointer param_exec;
 	int			eflags;
 	int			jit_flags;
+	Oid			objectid;		/* workers to open relation/table.  */
+	/* number tuples inserted by all the workers. */
+	pg_atomic_uint64	processed;
 } FixedParallelExecutorState;
 
 /*
@@ -600,6 +605,8 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	Size		dsa_minsize = dsa_minimum_size();
 	char	   *query_string;
 	int			query_len;
+	char 		*intoclausestr = NULL;
+	int			intoclause_len = 0;
 
 	/*
 	 * Force any initplan outputs that we're going to pass to workers to be
@@ -712,6 +719,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_estimate_chunk(&pcxt->estimator, dsa_minsize);
 	shm_toc_estimate_keys(&pcxt->estimator, 1);
 
+	/* Estimate space for into clause for CTAS. */
+	if (ISCTAS(planstate->intoclause))
+	{
+		intoclausestr = nodeToString(planstate->intoclause);
+		intoclause_len = strlen(intoclausestr);
+		shm_toc_estimate_chunk(&pcxt->estimator, intoclause_len + 1);
+		shm_toc_estimate_keys(&pcxt->estimator, 1);
+	}
+
 	/* Everyone's had a chance to ask for space, so now create the DSM. */
 	InitializeParallelDSM(pcxt);
 
@@ -729,6 +745,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	fpes->param_exec = InvalidDsaPointer;
 	fpes->eflags = estate->es_top_eflags;
 	fpes->jit_flags = estate->es_jit_flags;
+	pg_atomic_init_u64(&fpes->processed, 0);
+	pei->processed = &fpes->processed;
+
+	if (intoclausestr != NULL &&
+		planstate->objectid != InvalidOid)
+		fpes->objectid = planstate->objectid;
+	else
+		fpes->objectid = InvalidOid;
+
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, fpes);
 
 	/* Store query string */
@@ -758,8 +783,18 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_WAL_USAGE, walusage_space);
 	pei->wal_usage = walusage_space;
 
-	/* Set up the tuple queues that the workers will write into. */
-	pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	if (intoclausestr != NULL)
+	{
+		char *intoclause_space = shm_toc_allocate(pcxt->toc,
+												  intoclause_len + 1);
+		memcpy(intoclause_space, intoclausestr, intoclause_len + 1);
+		shm_toc_insert(pcxt->toc, PARALLEL_KEY_INTO_CLAUSE, intoclause_space);
+	}
+	else
+	{
+		/* Set up the tuple queues that the workers will write into. */
+		pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	}
 
 	/* We don't need the TupleQueueReaders yet, though. */
 	pei->reader = NULL;
@@ -1387,12 +1422,30 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 	void	   *area_space;
 	dsa_area   *area;
 	ParallelWorkerContext pwcxt;
+	char		*intoclausestr = NULL;
+	IntoClause	*intoclause = NULL;
 
 	/* Get fixed-size state. */
 	fpes = shm_toc_lookup(toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
 
-	/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
-	receiver = ExecParallelGetReceiver(seg, toc);
+	intoclausestr = shm_toc_lookup(toc, PARALLEL_KEY_INTO_CLAUSE, true);
+	if (intoclausestr != NULL)
+	{
+		/*
+		 * If the worker is for parallel insert in CTAS, then use the proper
+		 * dest receiver.
+		 */
+		intoclause = (IntoClause *) stringToNode(intoclausestr);
+		receiver = CreateIntoRelDestReceiver(intoclause);
+		((DR_intorel *)receiver)->is_parallel_worker = true;
+		((DR_intorel *)receiver)->object_id = fpes->objectid;
+	}
+	else
+	{
+		/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
+		receiver = ExecParallelGetReceiver(seg, toc);
+	}
+
 	instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_INSTRUMENTATION, true);
 	if (instrumentation != NULL)
 		instrument_options = instrumentation->instrument_options;
@@ -1471,6 +1524,13 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 			queryDesc->estate->es_jit->instr;
 	}
 
+	/*
+	 * Write out the number of tuples this worker has inserted. Leader will use
+	 * it to inform to the end client.
+	 */
+	if (intoclausestr != NULL)
+		pg_atomic_add_fetch_u64(&fpes->processed, queryDesc->estate->es_processed);
+
 	/* Must do this after capturing instrumentation. */
 	ExecutorEnd(queryDesc);
 
diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c
index a01b46af14..93d0d95704 100644
--- a/src/backend/executor/nodeGather.c
+++ b/src/backend/executor/nodeGather.c
@@ -32,6 +32,7 @@
 
 #include "access/relscan.h"
 #include "access/xact.h"
+#include "commands/createas.h"
 #include "executor/execdebug.h"
 #include "executor/execParallel.h"
 #include "executor/nodeGather.h"
@@ -48,7 +49,7 @@ static TupleTableSlot *ExecGather(PlanState *pstate);
 static TupleTableSlot *gather_getnext(GatherState *gatherstate);
 static MinimalTuple gather_readnext(GatherState *gatherstate);
 static void ExecShutdownGatherWorkers(GatherState *node);
-
+static void ExecParallelInsertInCTAS(GatherState *node);
 
 /* ----------------------------------------------------------------
  *		ExecInitGather
@@ -131,6 +132,67 @@ ExecInitGather(Gather *node, EState *estate, int eflags)
 	return gatherstate;
 }
 
+/* ----------------------------------------------------------------
+ *		ExecParallelInsertInCTAS(node)
+ *
+ *		Facilitates parallel inserts by parallel workers and/or
+ *		leader for Create Table AS.
+ * ----------------------------------------------------------------
+ */
+static void
+ExecParallelInsertInCTAS(GatherState *node)
+{
+	/* Enable leader to insert in case no parallel workers were launched. */
+	if (node->nworkers_launched == 0 &&
+		!node->need_to_scan_locally)
+		node->need_to_scan_locally = true;
+	/*
+	 * By now, for parallel workers (if launched any), would have started their
+	 * work i.e. insertion to target table. In case the leader is chosen to
+	 * participate for parallel inserts in CTAS, then finish it's share before
+	 * going to wait for the parallel workers to finish.
+	 */
+	if (node->need_to_scan_locally == true &&
+		node->ps.dest != NULL &&
+		node->ps.dest->mydest == DestIntoRel)
+	{
+		EState	   *estate = node->ps.state;
+		TupleTableSlot *outerTupleSlot;
+
+		for(;;)
+		{
+			/* Install our DSA area while executing the plan. */
+			estate->es_query_dsa =
+					node->pei ? node->pei->area : NULL;
+
+			outerTupleSlot = ExecProcNode(node->ps.lefttree);
+
+			estate->es_query_dsa = NULL;
+
+			if(TupIsNull(outerTupleSlot))
+				break;
+
+			(void) node->ps.dest->receiveSlot(outerTupleSlot, node->ps.dest);
+
+			node->ps.state->es_processed++;
+		}
+
+		node->need_to_scan_locally = false;
+	}
+
+	/* Wait for the parallel workers to finish. */
+	if (node->nworkers_launched > 0)
+	{
+		ExecShutdownGatherWorkers(node);
+
+		/*
+		 * Add up the total tuples inserted by all workers, to the tuples
+		 * inserted by the leader(if any). This will be shared to client.
+		 */
+		node->ps.state->es_processed += pg_atomic_read_u64(node->pei->processed);
+	}
+}
+
 /* ----------------------------------------------------------------
  *		ExecGather(node)
  *
@@ -166,6 +228,16 @@ ExecGather(PlanState *pstate)
 		{
 			ParallelContext *pcxt;
 
+			/*
+			 * Take the necessary information to be passed to workers for
+			 * parallel inserts in CTAS.
+			 */
+			if (ISCTAS(node->ps.intoclause))
+			{
+				node->ps.lefttree->intoclause = node->ps.intoclause;
+				node->ps.lefttree->objectid = node->ps.objectid;
+			}
+
 			/* Initialize, or re-initialize, shared state needed by workers. */
 			if (!node->pei)
 				node->pei = ExecInitParallelPlan(node->ps.lefttree,
@@ -190,13 +262,16 @@ ExecGather(PlanState *pstate)
 			/* Set up tuple queue readers to read the results. */
 			if (pcxt->nworkers_launched > 0)
 			{
-				ExecParallelCreateReaders(node->pei);
-				/* Make a working array showing the active readers */
-				node->nreaders = pcxt->nworkers_launched;
-				node->reader = (TupleQueueReader **)
-					palloc(node->nreaders * sizeof(TupleQueueReader *));
-				memcpy(node->reader, node->pei->reader,
-					   node->nreaders * sizeof(TupleQueueReader *));
+				if (!(ISCTAS(node->ps.intoclause)))
+				{
+					ExecParallelCreateReaders(node->pei);
+					/* Make a working array showing the active readers */
+					node->nreaders = pcxt->nworkers_launched;
+					node->reader = (TupleQueueReader **)
+						palloc(node->nreaders * sizeof(TupleQueueReader *));
+					memcpy(node->reader, node->pei->reader,
+						node->nreaders * sizeof(TupleQueueReader *));
+				}
 			}
 			else
 			{
@@ -208,7 +283,8 @@ ExecGather(PlanState *pstate)
 		}
 
 		/* Run plan locally if no workers or enabled and not single-copy. */
-		node->need_to_scan_locally = (node->nreaders == 0)
+		node->need_to_scan_locally = (node->nreaders == 0 &&
+			!(ISCTAS(node->ps.intoclause)))
 			|| (!gather->single_copy && parallel_leader_participation);
 		node->initialized = true;
 	}
@@ -220,6 +296,11 @@ ExecGather(PlanState *pstate)
 	econtext = node->ps.ps_ExprContext;
 	ResetExprContext(econtext);
 
+	if (ISCTAS(node->ps.intoclause))
+	{
+		ExecParallelInsertInCTAS(node);
+		return NULL;
+	}
 	/*
 	 * Get next tuple, either from one of our workers, or by running the plan
 	 * ourselves.
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 22d6935824..4f03db31c7 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -393,7 +393,17 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 
 	/* Parallel setup and communication cost. */
 	startup_cost += parallel_setup_cost;
-	run_cost += parallel_tuple_cost * path->path.rows;
+
+	/*
+	 * Make the number of tuples that are transferred from workers to gather
+	 * node zero as each worker parallelly insert the tuples that are resulted
+	 * from its chunk of plan execution. This change may make the parallel
+	 * plan cheap among all other plans, and influence the planner to consider
+	 * this parallel plan.
+	 */
+	if (!(root->parse->isForCTAS &&
+		root->query_level == 1))
+		run_cost += parallel_tuple_cost * path->path.rows;
 
 	path->path.startup_cost = startup_cost;
 	path->path.total_cost = (startup_cost + run_cost);
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7320de345c..5beae6c617 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -389,6 +389,7 @@ extern FullTransactionId GetCurrentFullTransactionIdIfAny(void);
 extern void MarkCurrentTransactionIdLoggedIfAny(void);
 extern bool SubTransactionIsActive(SubTransactionId subxid);
 extern CommandId GetCurrentCommandId(bool used);
+extern void SetCurrentCommandIdUsedForWorker(void);
 extern void SetParallelStartTimestamps(TimestampTz xact_ts, TimestampTz stmt_ts);
 extern TimestampTz GetCurrentTransactionStartTimestamp(void);
 extern TimestampTz GetCurrentStatementStartTimestamp(void);
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 7629230254..9271e84e4d 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -14,12 +14,31 @@
 #ifndef CREATEAS_H
 #define CREATEAS_H
 
+#include "access/heapam.h"
 #include "catalog/objectaddress.h"
 #include "nodes/params.h"
+#include "nodes/plannodes.h"
 #include "parser/parse_node.h"
 #include "tcop/dest.h"
 #include "utils/queryenvironment.h"
 
+typedef struct
+{
+	DestReceiver pub;			/* publicly-known function pointers */
+	IntoClause *into;			/* target relation specification */
+	/* These fields are filled by intorel_startup: */
+	Relation	rel;			/* relation to write to */
+	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
+	CommandId	output_cid;		/* cmin to insert in output tuples */
+	int			ti_options;		/* table_tuple_insert performance options */
+	BulkInsertState bistate;	/* bulk insert state */
+	bool		is_parallel;	/* is parallelism to be considered? */
+	bool		is_parallel_worker; /* true for parallel worker */
+	/* Used for table open by parallel worker */
+	Oid			object_id;
+} DR_intorel;
+
+#define ISCTAS(intoclause) (intoclause != NULL && IsA(intoclause, IntoClause))
 
 extern ObjectAddress ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 									   ParamListInfo params, QueryEnvironment *queryEnv,
@@ -29,4 +48,7 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
+extern bool IsParallelInsertInCTASAllowed(IntoClause *intoClause,
+										  PlannedStmt *plannedstmt);
+
 #endif							/* CREATEAS_H */
diff --git a/src/include/executor/execParallel.h b/src/include/executor/execParallel.h
index 5a39a5b29c..e475fbdd35 100644
--- a/src/include/executor/execParallel.h
+++ b/src/include/executor/execParallel.h
@@ -35,6 +35,8 @@ typedef struct ParallelExecutorInfo
 	/* These two arrays have pcxt->nworkers_launched entries: */
 	shm_mq_handle **tqueue;		/* tuple queues for worker output */
 	struct TupleQueueReader **reader;	/* tuple reader/writer support */
+	/* Number of tuples inserted by all workers */
+	volatile pg_atomic_uint64	*processed;
 } ParallelExecutorInfo;
 
 extern ParallelExecutorInfo *ExecInitParallelPlan(PlanState *planstate,
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 61ba4c3666..5277d66150 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -23,6 +23,7 @@
 #include "nodes/tidbitmap.h"
 #include "partitioning/partdefs.h"
 #include "storage/condition_variable.h"
+#include "tcop/dest.h"
 #include "utils/hsearch.h"
 #include "utils/queryenvironment.h"
 #include "utils/reltrigger.h"
@@ -1009,6 +1010,10 @@ typedef struct PlanState
 	bool		outeropsset;
 	bool		inneropsset;
 	bool		resultopsset;
+	/* Prallel inserts in CTAS related info is specified below. */
+	IntoClause	*intoclause;
+	Oid			objectid;
+	DestReceiver *dest;
 } PlanState;
 
 /* ----------------
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index d1f9ef29ca..65c393743c 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -180,6 +180,7 @@ typedef struct Query
 	 */
 	int			stmt_location;	/* start location, or -1 if unknown */
 	int			stmt_len;		/* length in bytes; 0 means "rest of string" */
+	bool		isForCTAS; 		/* Is the SELECT query for CREATE TABLE AS */
 } Query;
 
 
diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 0c4da2591a..11ef18b8a4 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -76,4 +76,147 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
+--
+-- Test parallel inserts in create table as/select into/create materialized view
+--
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+                     QUERY PLAN                      
+-----------------------------------------------------
+ LockRows (actual rows=10000 loops=1)
+   ->  Seq Scan on tenk1 (actual rows=10000 loops=1)
+(2 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+select length(stringu1) from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_mat_view
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop materialized view parallel_mat_view;
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+deallocate parallel_write_prep;
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+select nextval('parallel_write_sequence'), four from tenk1;
+                  QUERY PLAN                   
+-----------------------------------------------
+ Seq Scan on tenk1 (actual rows=10000 loops=1)
+(1 row)
+
+drop table parallel_write;
+drop sequence parallel_write_sequence;
 rollback;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 78b479cedf..dd4233b399 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -39,4 +39,69 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
 
+--
+-- Test parallel inserts in create table as/select into/create materialized view
+--
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+select length(stringu1) from tenk1;
+drop materialized view parallel_mat_view;
+
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+deallocate parallel_write_prep;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+select nextval('parallel_write_sequence'), four from tenk1;
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+
 rollback;
-- 
2.25.1

From b5f111c7dcc7efa336aa5da59b6bdbd3689a70e5 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 30 Nov 2020 09:36:03 +0530
Subject: [PATCH v6] Parallel Inserts in CREATE TABLE AS

The idea of this patch is to allow the leader and each worker
insert the tuples in parallel if the SELECT part of the CTAS is
parallelizable.

The design:
Let the planner know that the SELECT is from CTAS in createas.c
so that it can set the number of tuples transferred from the
workers to Gather node to 0. With this change, there are chances
that the planner may choose the parallel plan. After the planning,
check if the upper plan node is Gather in createas.c and mark a
parallelism flag in the CTAS dest receiver. Pass the into clause,
object id, command id from the leader to workers, so that each
worker can create its own CTAS dest receiver. Leader inserts it's
share of tuples if instructed to do, and so are workers. Each
worker writes atomically it's number of inserted tuples into a
shared memory variable, the leader combines this with it's own
number of inserted tuples and shares to the client.s
---
 src/backend/access/heap/heapam.c             |  11 -
 src/backend/access/transam/xact.c            |  30 +-
 src/backend/commands/createas.c              | 324 +++++++++++++------
 src/backend/commands/explain.c               |  39 +++
 src/backend/executor/execMain.c              |  17 +
 src/backend/executor/execParallel.c          |  67 +++-
 src/backend/executor/nodeGather.c            |  97 +++++-
 src/backend/optimizer/path/costsize.c        |  12 +-
 src/include/access/xact.h                    |   1 +
 src/include/commands/createas.h              |  23 ++
 src/include/executor/execParallel.h          |   2 +
 src/include/nodes/execnodes.h                |   5 +
 src/include/nodes/parsenodes.h               |   1 +
 src/test/regress/expected/write_parallel.out | 190 +++++++++++
 src/test/regress/sql/write_parallel.sql      |  84 +++++
 15 files changed, 765 insertions(+), 138 deletions(-)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 1b2f70499e..3045c0f046 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2043,17 +2043,6 @@ static HeapTuple
 heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
 					CommandId cid, int options)
 {
-	/*
-	 * To allow parallel inserts, we need to ensure that they are safe to be
-	 * performed in workers. We have the infrastructure to allow parallel
-	 * inserts in general except for the cases where inserts generate a new
-	 * CommandId (eg. inserts into a table having a foreign key column).
-	 */
-	if (IsParallelWorker())
-		ereport(ERROR,
-				(errcode(ERRCODE_INVALID_TRANSACTION_STATE),
-				 errmsg("cannot insert tuples in a parallel worker")));
-
 	tup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
 	tup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
 	tup->t_data->t_infomask |= HEAP_XMAX_INVALID;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 9cd0b7c11b..db6eedd635 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -763,18 +763,34 @@ GetCurrentCommandId(bool used)
 	/* this is global to a transaction, not subtransaction-local */
 	if (used)
 	{
-		/*
-		 * Forbid setting currentCommandIdUsed in a parallel worker, because
-		 * we have no provision for communicating this back to the leader.  We
-		 * could relax this restriction when currentCommandIdUsed was already
-		 * true at the start of the parallel operation.
-		 */
-		Assert(!IsParallelWorker());
+		 /*
+		  * This is a temporary hack for all common parallel insert cases i.e.
+		  * insert into, ctas, copy from. To be changed later. In a parallel
+		  * worker, set currentCommandIdUsed to true only if it was not set to
+		  * true at the start of the parallel operation (by way of
+		  * SetCurrentCommandIdUsedForWorker()). We have to do this because
+		  * GetCurrentCommandId(true) may be called from anywhere, especially
+		  * for parallel inserts, within parallel worker.
+		  */
+		Assert(!(IsParallelWorker() && !currentCommandIdUsed));
 		currentCommandIdUsed = true;
 	}
 	return currentCommandId;
 }
 
+/*
+ *	SetCurrentCommandIdUsedForWorker
+ *
+ * For a parallel worker, record that the currentCommandId has been used. This
+ * must only be called at the start of a parallel operation.
+ */
+void
+SetCurrentCommandIdUsedForWorker(void)
+{
+	Assert(IsParallelWorker() && !currentCommandIdUsed && currentCommandId != InvalidCommandId);
+	currentCommandIdUsed = true;
+}
+
 /*
  *	SetParallelStartTimestamps
  *
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 6bf6c5a310..c76790f776 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,18 +51,6 @@
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 
-typedef struct
-{
-	DestReceiver pub;			/* publicly-known function pointers */
-	IntoClause *into;			/* target relation specification */
-	/* These fields are filled by intorel_startup: */
-	Relation	rel;			/* relation to write to */
-	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
-	CommandId	output_cid;		/* cmin to insert in output tuples */
-	int			ti_options;		/* table_tuple_insert performance options */
-	BulkInsertState bistate;	/* bulk insert state */
-} DR_intorel;
-
 /* utility functions for CTAS definition creation */
 static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into);
 static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -328,6 +316,15 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		query = linitial_node(Query, rewritten);
 		Assert(query->commandType == CMD_SELECT);
 
+		/*
+		 * Flag to let the planner know that the SELECT query is for CTAS. This
+		 * is used to calculate the tuple transfer cost from workers to gather
+		 * node(in case parallelism kicks in for the SELECT part of the CTAS),
+		 * to zero as each worker will parallelly insert its share of tuples.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, NULL))
+			query->isForCTAS = true;
+
 		/* plan the query */
 		plan = pg_plan_query(query, pstate->p_sourcetext,
 							 CURSOR_OPT_PARALLEL_OK, params);
@@ -350,6 +347,16 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		/* call ExecutorStart to prepare the plan for execution */
 		ExecutorStart(queryDesc, GetIntoRelEFlags(into));
 
+		/*
+		 * SELECT part of the CTAS is parallelizable, so we can make each
+		 * parallel worker insert the tuples that are resulted in it's
+		 * execution into the target table. We need plan state to be
+		 * initialized by the executor to decide whether to allow parallel
+		 * inserts or not.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, queryDesc))
+			((DR_intorel *) dest)->is_parallel = true;
+
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
 
@@ -418,6 +425,9 @@ CreateIntoRelDestReceiver(IntoClause *intoClause)
 	self->pub.rDestroy = intorel_destroy;
 	self->pub.mydest = DestIntoRel;
 	self->into = intoClause;
+	self->is_parallel = false;
+	self->is_parallel_worker = false;
+	self->object_id = InvalidOid;
 	/* other private fields will be set during intorel_startup */
 
 	return (DestReceiver *) self;
@@ -430,121 +440,169 @@ static void
 intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 {
 	DR_intorel *myState = (DR_intorel *) self;
-	IntoClause *into = myState->into;
-	bool		is_matview;
-	List	   *attrList;
 	ObjectAddress intoRelationAddr;
 	Relation	intoRelationDesc;
-	ListCell   *lc;
-	int			attnum;
-
-	Assert(into != NULL);		/* else somebody forgot to set it */
 
-	/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
-	is_matview = (into->viewQuery != NULL);
+	if (myState->is_parallel_worker)
+	{
+		/* In the worker */
+		intoRelationDesc = table_open(myState->object_id, AccessExclusiveLock);
+		myState->rel = intoRelationDesc;
+		myState->reladdr = InvalidObjectAddress;
+		myState->ti_options = 0;
+		myState->bistate = GetBulkInsertState();
 
-	/*
-	 * Build column definitions using "pre-cooked" type and collation info. If
-	 * a column name list was specified in CREATE TABLE AS, override the
-	 * column names derived from the query.  (Too few column names are OK, too
-	 * many are not.)
-	 */
-	attrList = NIL;
-	lc = list_head(into->colNames);
-	for (attnum = 0; attnum < typeinfo->natts; attnum++)
+		/*
+		 * Right after the table is created in the leader, the command id is
+		 * incremented (in create_ctas_internal()). The new command id is
+		 * marked as used in intorel_startup(), then the parallel mode is
+		 * entered. The command id and transaction id are serialized into
+		 * parallel DSM, they are then available to all parallel workers. All
+		 * the workers need to mark the command id as used before insertion.
+		 */
+		(void) SetCurrentCommandIdUsedForWorker();
+		myState->output_cid = GetCurrentCommandId(false);
+	}
+	else
 	{
-		Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
-		ColumnDef  *col;
-		char	   *colname;
+		IntoClause *into = myState->into;
+		bool		is_matview;
+		List	   *attrList;
+		ListCell   *lc;
+		int			attnum;
 
-		if (lc)
+		Assert(into != NULL);		/* else somebody forgot to set it */
+
+		/*
+		 * This code supports both CREATE TABLE AS and CREATE MATERIALIZED
+		 * VIEW.
+		 */
+		is_matview = (into->viewQuery != NULL);
+
+		/*
+		 * Build column definitions using "pre-cooked" type and collation info.
+		 * If a column name list was specified in CREATE TABLE AS, override the
+		 * column names derived from the query.  (Too few column names are OK,
+		 * too many are not.)
+		 */
+		attrList = NIL;
+		lc = list_head(into->colNames);
+		for (attnum = 0; attnum < typeinfo->natts; attnum++)
 		{
-			colname = strVal(lfirst(lc));
-			lc = lnext(into->colNames, lc);
+			Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
+			ColumnDef  *col;
+			char	   *colname;
+
+			if (lc)
+			{
+				colname = strVal(lfirst(lc));
+				lc = lnext(into->colNames, lc);
+			}
+			else
+				colname = NameStr(attribute->attname);
+
+			col = makeColumnDef(colname,
+								attribute->atttypid,
+								attribute->atttypmod,
+								attribute->attcollation);
+
+			/*
+			 * It's possible that the column is of a collatable type but the
+			 * collation could not be resolved, so double-check.  (We must
+			 * check this here because DefineRelation would adopt the type's
+			 * default collation rather than complaining.)
+			 */
+			if (!OidIsValid(col->collOid) &&
+				type_is_collatable(col->typeName->typeOid))
+				ereport(ERROR,
+						(errcode(ERRCODE_INDETERMINATE_COLLATION),
+						errmsg("no collation was derived for column \"%s\" with collatable type %s",
+								col->colname,
+								format_type_be(col->typeName->typeOid)),
+						errhint("Use the COLLATE clause to set the collation explicitly.")));
+
+			attrList = lappend(attrList, col);
 		}
-		else
-			colname = NameStr(attribute->attname);
 
-		col = makeColumnDef(colname,
-							attribute->atttypid,
-							attribute->atttypmod,
-							attribute->attcollation);
+		if (lc != NULL)
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					errmsg("too many column names were specified")));
 
 		/*
-		 * It's possible that the column is of a collatable type but the
-		 * collation could not be resolved, so double-check.  (We must check
-		 * this here because DefineRelation would adopt the type's default
-		 * collation rather than complaining.)
+		 * Actually create the target table
 		 */
-		if (!OidIsValid(col->collOid) &&
-			type_is_collatable(col->typeName->typeOid))
-			ereport(ERROR,
-					(errcode(ERRCODE_INDETERMINATE_COLLATION),
-					 errmsg("no collation was derived for column \"%s\" with collatable type %s",
-							col->colname,
-							format_type_be(col->typeName->typeOid)),
-					 errhint("Use the COLLATE clause to set the collation explicitly.")));
+		intoRelationAddr = create_ctas_internal(attrList, into);
 
-		attrList = lappend(attrList, col);
-	}
+		/*
+		 * Finally we can open the target table
+		 */
+		intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
 
-	if (lc != NULL)
-		ereport(ERROR,
-				(errcode(ERRCODE_SYNTAX_ERROR),
-				 errmsg("too many column names were specified")));
+		/*
+		 * Make sure the constructed table does not have RLS enabled.
+		 *
+		 * check_enable_rls() will ereport(ERROR) itself if the user has
+		 * requested something invalid, and otherwise will return RLS_ENABLED
+		 * if RLS should be enabled here.  We don't actually support that
+		 * currently, so throw our own ereport(ERROR) if that happens.
+		 */
+		if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					errmsg("policies not yet implemented for this command")));
 
-	/*
-	 * Actually create the target table
-	 */
-	intoRelationAddr = create_ctas_internal(attrList, into);
+		/*
+		 * Tentatively mark the target as populated, if it's a matview and
+		 * we're going to fill it; otherwise, no change needed.
+		 */
+		if (is_matview && !into->skipData)
+			SetMatViewPopulatedState(intoRelationDesc, true);
 
-	/*
-	 * Finally we can open the target table
-	 */
-	intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
+		/*
+		 * Fill private fields of myState for use by later routines
+		 */
+		myState->rel = intoRelationDesc;
+		myState->reladdr = intoRelationAddr;
+		myState->output_cid = GetCurrentCommandId(true);
+		myState->ti_options = TABLE_INSERT_SKIP_FSM;
 
-	/*
-	 * Make sure the constructed table does not have RLS enabled.
-	 *
-	 * check_enable_rls() will ereport(ERROR) itself if the user has requested
-	 * something invalid, and otherwise will return RLS_ENABLED if RLS should
-	 * be enabled here.  We don't actually support that currently, so throw
-	 * our own ereport(ERROR) if that happens.
-	 */
-	if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
-		ereport(ERROR,
-				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-				 errmsg("policies not yet implemented for this command")));
+		/*
+		 * If WITH NO DATA is specified, there is no need to set up the state
+		 * for bulk inserts as there are no tuples to insert.
+		 */
+		if (!into->skipData)
+			myState->bistate = GetBulkInsertState();
+		else
+			myState->bistate = NULL;
 
-	/*
-	 * Tentatively mark the target as populated, if it's a matview and we're
-	 * going to fill it; otherwise, no change needed.
-	 */
-	if (is_matview && !into->skipData)
-		SetMatViewPopulatedState(intoRelationDesc, true);
+		if (myState->is_parallel)
+		{
+			myState->object_id = intoRelationAddr.objectId;
 
-	/*
-	 * Fill private fields of myState for use by later routines
-	 */
-	myState->rel = intoRelationDesc;
-	myState->reladdr = intoRelationAddr;
-	myState->output_cid = GetCurrentCommandId(true);
-	myState->ti_options = TABLE_INSERT_SKIP_FSM;
+			/*
+			 * We don't need to skip contacting FSM while inserting tuples
+			 * for parallel mode, while extending the relations, workers
+			 * instead of blocking on a page while another worker is inserting,
+			 * can check the FSM for another page that can accommodate the
+			 * tuples. This results in major benefit for parallel inserts.
+			 */
+			myState->ti_options = 0;
 
-	/*
-	 * If WITH NO DATA is specified, there is no need to set up the state for
-	 * bulk inserts as there are no tuples to insert.
-	 */
-	if (!into->skipData)
-		myState->bistate = GetBulkInsertState();
-	else
-		myState->bistate = NULL;
+			/*
+			 * rd_createSubid is marked invalid, otherwise, the table is
+			 * not allowed to extend by the workers.
+			 */
+			myState->rel->rd_createSubid = InvalidSubTransactionId;
+		}
 
-	/*
-	 * Valid smgr_targblock implies something already wrote to the relation.
-	 * This may be harmless, but this function hasn't planned for it.
-	 */
-	Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+		/*
+		 * Valid smgr_targblock implies something already wrote to the
+		 * relation. This may be harmless, but this function hasn't planned for
+		 * it.
+		 */
+		Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+	}
 }
 
 /*
@@ -606,3 +664,57 @@ intorel_destroy(DestReceiver *self)
 {
 	pfree(self);
 }
+
+/*
+ * IsParallelInsertInCTASAllowed --- determine whether or not parallel
+ * insertion is possible.
+ */
+bool IsParallelInsertInCTASAllowed(IntoClause *into, QueryDesc *queryDesc)
+{
+	bool allowed = false;
+
+	if (ISCTAS(into))
+	{
+		RangeVar   *rel = into->rel;
+
+		/* Allow parallel inserts only if the table is not temporary. */
+		if (rel && rel->relpersistence != RELPERSISTENCE_TEMP)
+			allowed = true;
+
+		if (queryDesc && allowed)
+		{
+			PlanState  *ps = queryDesc->planstate;
+			PlannedStmt *plannedstmt = queryDesc->plannedstmt;
+
+			/*
+			 * We allow parallel inserts by the workers only if the Gather node
+			 * has no projections to perform and if the upper node is Gather.
+			 * In case, the Gather node has projections, which is possible if
+			 * there are any subplans in the query, the workers can not do
+			 * those projections. And when the upper node is GatherMerge, then
+			 * the leader has to perform the final phase i.e. merge the results
+			 * by workers.
+			 */
+			if (ps && IsA(ps, GatherState) && !ps->ps_ProjInfo &&
+				plannedstmt->parallelModeNeeded &&
+				plannedstmt->planTree &&
+				IsA(plannedstmt->planTree, Gather) &&
+				plannedstmt->planTree->lefttree &&
+				plannedstmt->planTree->lefttree->parallel_aware &&
+				plannedstmt->planTree->lefttree->parallel_safe)
+			{
+				/*
+				 * Since there are no rows that are transferred from workers to
+				 * Gather node, so we set it to 0 to be visible in explain
+				 * plans. Note that we would have already accounted this for
+				 * cost calculations in cost_gather().
+				 */
+				plannedstmt->planTree->plan_rows = 0;
+			}
+			else
+				allowed = false;
+		}
+	}
+
+	return allowed;
+}
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 43f9b01e83..121c118bce 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -371,6 +371,15 @@ ExplainOneQuery(Query *query, int cursorOptions,
 		return;
 	}
 
+	/*
+	 * Flag to let the planner know that the SELECT query is for CTAS. This is
+	 * used to calculate the tuple transfer cost from workers to gather node(in
+	 * case parallelism kicks in for the SELECT part of the CTAS), to zero as
+	 * each worker will parallelly insert its share of tuples.
+	 */
+	if (IsParallelInsertInCTASAllowed(into, NULL))
+		query->isForCTAS = true;
+
 	/* if an advisor plugin is present, let it manage things */
 	if (ExplainOneQuery_hook)
 		(*ExplainOneQuery_hook) (query, cursorOptions, into, es,
@@ -556,6 +565,19 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	/* call ExecutorStart to prepare the plan for execution */
 	ExecutorStart(queryDesc, eflags);
 
+	if (into)
+	{
+		/*
+		 * SELECT part of the CTAS is parallelizable, so we can make each
+		 * parallel worker insert the tuples that are resulted in it's
+		 * execution into the target table. We need plan state to be
+		 * initialized by the executor to decide whether to allow parallel
+		 * inserts or not.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, queryDesc))
+			((DR_intorel *) dest)->is_parallel = true;
+	}
+
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
 	{
@@ -1753,6 +1775,23 @@ ExplainNode(PlanState *planstate, List *ancestors,
 			{
 				Gather	   *gather = (Gather *) plan;
 
+				if (IsA(planstate, GatherState) &&
+					planstate->intoclause &&
+					IsA(planstate->intoclause,IntoClause) &&
+					planstate->dest &&
+					planstate->dest->mydest == DestIntoRel &&
+					((DR_intorel *) planstate->dest)->is_parallel &&
+					planstate->intoclause->rel &&
+					planstate->intoclause->rel->relname)
+				{
+					ExplainIndentText(es);
+					appendStringInfoString(es->str, "->  ");
+					appendStringInfoString(es->str, "Create ");
+					appendStringInfo(es->str, "%s\n", planstate->intoclause->rel->relname);
+					es->indent++;
+					ExplainIndentText(es);
+				}
+
 				show_scan_qual(plan->qual, "Filter", planstate, ancestors, es);
 				if (plan->qual)
 					show_instrumentation_count("Rows Removed by Filter", 1,
diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c
index 7179f589f9..c91741981e 100644
--- a/src/backend/executor/execMain.c
+++ b/src/backend/executor/execMain.c
@@ -45,6 +45,7 @@
 #include "access/xact.h"
 #include "catalog/namespace.h"
 #include "catalog/pg_publication.h"
+#include "commands/createas.h"
 #include "commands/matview.h"
 #include "commands/trigger.h"
 #include "executor/execdebug.h"
@@ -352,6 +353,22 @@ standard_ExecutorRun(QueryDesc *queryDesc,
 	if (sendTuples)
 		dest->rStartup(dest, operation, queryDesc->tupDesc);
 
+	/*
+	 * For parallelizing inserts in CTAS i.e. making each parallel worker
+	 * insert the tuples, we must send information such as intoclause(for each
+	 * worker to build separate dest receiver), object id(for each worker to
+	 * open the table).
+	 */
+	if (queryDesc->plannedstmt->parallelModeNeeded &&
+		dest && dest->mydest == DestIntoRel &&
+		((DR_intorel *) dest)->is_parallel &&
+		!(((DR_intorel *) dest)->is_parallel_worker))
+	{
+		queryDesc->planstate->intoclause = ((DR_intorel *) dest)->into;
+		queryDesc->planstate->objectid = ((DR_intorel *) dest)->object_id;
+		queryDesc->planstate->dest = dest;
+	}
+
 	/*
 	 * run plan
 	 */
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index befde52691..2ea4664732 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -23,6 +23,7 @@
 
 #include "postgres.h"
 
+#include "commands/createas.h"
 #include "executor/execParallel.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
@@ -65,6 +66,7 @@
 #define PARALLEL_KEY_QUERY_TEXT		UINT64CONST(0xE000000000000008)
 #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009)
 #define PARALLEL_KEY_WAL_USAGE			UINT64CONST(0xE00000000000000A)
+#define PARALLEL_KEY_INTO_CLAUSE		UINT64CONST(0xE00000000000000B)
 
 #define PARALLEL_TUPLE_QUEUE_SIZE		65536
 
@@ -77,6 +79,9 @@ typedef struct FixedParallelExecutorState
 	dsa_pointer param_exec;
 	int			eflags;
 	int			jit_flags;
+	Oid			objectid;		/* workers to open relation/table.  */
+	/* Number tuples inserted by all the workers. */
+	pg_atomic_uint64	processed;
 } FixedParallelExecutorState;
 
 /*
@@ -600,6 +605,8 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	Size		dsa_minsize = dsa_minimum_size();
 	char	   *query_string;
 	int			query_len;
+	char 		*intoclausestr = NULL;
+	int			intoclause_len = 0;
 
 	/*
 	 * Force any initplan outputs that we're going to pass to workers to be
@@ -712,6 +719,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_estimate_chunk(&pcxt->estimator, dsa_minsize);
 	shm_toc_estimate_keys(&pcxt->estimator, 1);
 
+	/* Estimate space for into clause for CTAS. */
+	if (ISCTAS(planstate->intoclause))
+	{
+		intoclausestr = nodeToString(planstate->intoclause);
+		intoclause_len = strlen(intoclausestr);
+		shm_toc_estimate_chunk(&pcxt->estimator, intoclause_len + 1);
+		shm_toc_estimate_keys(&pcxt->estimator, 1);
+	}
+
 	/* Everyone's had a chance to ask for space, so now create the DSM. */
 	InitializeParallelDSM(pcxt);
 
@@ -729,6 +745,14 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	fpes->param_exec = InvalidDsaPointer;
 	fpes->eflags = estate->es_top_eflags;
 	fpes->jit_flags = estate->es_jit_flags;
+	pg_atomic_init_u64(&fpes->processed, 0);
+	pei->processed = &fpes->processed;
+
+	if (intoclausestr && planstate->objectid != InvalidOid)
+		fpes->objectid = planstate->objectid;
+	else
+		fpes->objectid = InvalidOid;
+
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, fpes);
 
 	/* Store query string */
@@ -758,8 +782,18 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_WAL_USAGE, walusage_space);
 	pei->wal_usage = walusage_space;
 
-	/* Set up the tuple queues that the workers will write into. */
-	pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	if (intoclausestr)
+	{
+		char *intoclause_space = shm_toc_allocate(pcxt->toc,
+												  intoclause_len + 1);
+		memcpy(intoclause_space, intoclausestr, intoclause_len + 1);
+		shm_toc_insert(pcxt->toc, PARALLEL_KEY_INTO_CLAUSE, intoclause_space);
+	}
+	else
+	{
+		/* Set up the tuple queues that the workers will write into. */
+		pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	}
 
 	/* We don't need the TupleQueueReaders yet, though. */
 	pei->reader = NULL;
@@ -1387,12 +1421,30 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 	void	   *area_space;
 	dsa_area   *area;
 	ParallelWorkerContext pwcxt;
+	char		*intoclausestr = NULL;
+	IntoClause	*intoclause = NULL;
 
 	/* Get fixed-size state. */
 	fpes = shm_toc_lookup(toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
 
-	/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
-	receiver = ExecParallelGetReceiver(seg, toc);
+	intoclausestr = shm_toc_lookup(toc, PARALLEL_KEY_INTO_CLAUSE, true);
+	if (intoclausestr)
+	{
+		/*
+		 * If the worker is for parallel insert in CTAS, then use the proper
+		 * dest receiver.
+		 */
+		intoclause = (IntoClause *) stringToNode(intoclausestr);
+		receiver = CreateIntoRelDestReceiver(intoclause);
+		((DR_intorel *)receiver)->is_parallel_worker = true;
+		((DR_intorel *)receiver)->object_id = fpes->objectid;
+	}
+	else
+	{
+		/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
+		receiver = ExecParallelGetReceiver(seg, toc);
+	}
+
 	instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_INSTRUMENTATION, true);
 	if (instrumentation != NULL)
 		instrument_options = instrumentation->instrument_options;
@@ -1471,6 +1523,13 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 			queryDesc->estate->es_jit->instr;
 	}
 
+	/*
+	 * Write out the number of tuples this worker has inserted. Leader will use
+	 * it to inform to the end client.
+	 */
+	if (intoclausestr)
+		pg_atomic_add_fetch_u64(&fpes->processed, queryDesc->estate->es_processed);
+
 	/* Must do this after capturing instrumentation. */
 	ExecutorEnd(queryDesc);
 
diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c
index a01b46af14..0c1283af44 100644
--- a/src/backend/executor/nodeGather.c
+++ b/src/backend/executor/nodeGather.c
@@ -32,6 +32,7 @@
 
 #include "access/relscan.h"
 #include "access/xact.h"
+#include "commands/createas.h"
 #include "executor/execdebug.h"
 #include "executor/execParallel.h"
 #include "executor/nodeGather.h"
@@ -48,7 +49,7 @@ static TupleTableSlot *ExecGather(PlanState *pstate);
 static TupleTableSlot *gather_getnext(GatherState *gatherstate);
 static MinimalTuple gather_readnext(GatherState *gatherstate);
 static void ExecShutdownGatherWorkers(GatherState *node);
-
+static void ExecParallelInsertInCTAS(GatherState *node);
 
 /* ----------------------------------------------------------------
  *		ExecInitGather
@@ -131,6 +132,65 @@ ExecInitGather(Gather *node, EState *estate, int eflags)
 	return gatherstate;
 }
 
+/* ----------------------------------------------------------------
+ *		ExecParallelInsertInCTAS(node)
+ *
+ *		Facilitates parallel inserts by parallel workers and/or
+ *		leader for Create Table AS.
+ * ----------------------------------------------------------------
+ */
+static void
+ExecParallelInsertInCTAS(GatherState *node)
+{
+	/* Enable leader to insert in case no parallel workers were launched. */
+	if (node->nworkers_launched == 0)
+		node->need_to_scan_locally = true;
+	/*
+	 * By now, for parallel workers (if launched any), would have started their
+	 * work i.e. insertion to target table. In case the leader is chosen to
+	 * participate for parallel inserts in CTAS, then finish it's share before
+	 * going to wait for the parallel workers to finish.
+	 */
+	if (node->need_to_scan_locally && node->ps.dest &&
+		node->ps.dest->mydest == DestIntoRel)
+	{
+		EState	   *estate = node->ps.state;
+		TupleTableSlot *outerTupleSlot;
+
+		for(;;)
+		{
+			/* Install our DSA area while executing the plan. */
+			estate->es_query_dsa =
+					node->pei ? node->pei->area : NULL;
+
+			outerTupleSlot = ExecProcNode(node->ps.lefttree);
+
+			estate->es_query_dsa = NULL;
+
+			if(TupIsNull(outerTupleSlot))
+				break;
+
+			(void) node->ps.dest->receiveSlot(outerTupleSlot, node->ps.dest);
+
+			node->ps.state->es_processed++;
+		}
+
+		node->need_to_scan_locally = false;
+	}
+
+	/* Wait for the parallel workers to finish. */
+	if (node->nworkers_launched > 0)
+	{
+		ExecShutdownGatherWorkers(node);
+
+		/*
+		 * Add up the total tuples inserted by all workers, to the tuples
+		 * inserted by the leader(if any). This will be shared to client.
+		 */
+		node->ps.state->es_processed += pg_atomic_read_u64(node->pei->processed);
+	}
+}
+
 /* ----------------------------------------------------------------
  *		ExecGather(node)
  *
@@ -166,6 +226,16 @@ ExecGather(PlanState *pstate)
 		{
 			ParallelContext *pcxt;
 
+			/*
+			 * Take the necessary information to be passed to workers for
+			 * parallel inserts in CTAS.
+			 */
+			if (ISCTAS(node->ps.intoclause))
+			{
+				node->ps.lefttree->intoclause = node->ps.intoclause;
+				node->ps.lefttree->objectid = node->ps.objectid;
+			}
+
 			/* Initialize, or re-initialize, shared state needed by workers. */
 			if (!node->pei)
 				node->pei = ExecInitParallelPlan(node->ps.lefttree,
@@ -190,13 +260,16 @@ ExecGather(PlanState *pstate)
 			/* Set up tuple queue readers to read the results. */
 			if (pcxt->nworkers_launched > 0)
 			{
-				ExecParallelCreateReaders(node->pei);
-				/* Make a working array showing the active readers */
-				node->nreaders = pcxt->nworkers_launched;
-				node->reader = (TupleQueueReader **)
-					palloc(node->nreaders * sizeof(TupleQueueReader *));
-				memcpy(node->reader, node->pei->reader,
-					   node->nreaders * sizeof(TupleQueueReader *));
+				if (!(ISCTAS(node->ps.intoclause)))
+				{
+					ExecParallelCreateReaders(node->pei);
+					/* Make a working array showing the active readers */
+					node->nreaders = pcxt->nworkers_launched;
+					node->reader = (TupleQueueReader **)
+						palloc(node->nreaders * sizeof(TupleQueueReader *));
+					memcpy(node->reader, node->pei->reader,
+						node->nreaders * sizeof(TupleQueueReader *));
+				}
 			}
 			else
 			{
@@ -208,7 +281,8 @@ ExecGather(PlanState *pstate)
 		}
 
 		/* Run plan locally if no workers or enabled and not single-copy. */
-		node->need_to_scan_locally = (node->nreaders == 0)
+		node->need_to_scan_locally = (node->nreaders == 0 &&
+			!(ISCTAS(node->ps.intoclause)))
 			|| (!gather->single_copy && parallel_leader_participation);
 		node->initialized = true;
 	}
@@ -220,6 +294,11 @@ ExecGather(PlanState *pstate)
 	econtext = node->ps.ps_ExprContext;
 	ResetExprContext(econtext);
 
+	if (ISCTAS(node->ps.intoclause))
+	{
+		ExecParallelInsertInCTAS(node);
+		return NULL;
+	}
 	/*
 	 * Get next tuple, either from one of our workers, or by running the plan
 	 * ourselves.
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 22d6935824..4f03db31c7 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -393,7 +393,17 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 
 	/* Parallel setup and communication cost. */
 	startup_cost += parallel_setup_cost;
-	run_cost += parallel_tuple_cost * path->path.rows;
+
+	/*
+	 * Make the number of tuples that are transferred from workers to gather
+	 * node zero as each worker parallelly insert the tuples that are resulted
+	 * from its chunk of plan execution. This change may make the parallel
+	 * plan cheap among all other plans, and influence the planner to consider
+	 * this parallel plan.
+	 */
+	if (!(root->parse->isForCTAS &&
+		root->query_level == 1))
+		run_cost += parallel_tuple_cost * path->path.rows;
 
 	path->path.startup_cost = startup_cost;
 	path->path.total_cost = (startup_cost + run_cost);
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7320de345c..5beae6c617 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -389,6 +389,7 @@ extern FullTransactionId GetCurrentFullTransactionIdIfAny(void);
 extern void MarkCurrentTransactionIdLoggedIfAny(void);
 extern bool SubTransactionIsActive(SubTransactionId subxid);
 extern CommandId GetCurrentCommandId(bool used);
+extern void SetCurrentCommandIdUsedForWorker(void);
 extern void SetParallelStartTimestamps(TimestampTz xact_ts, TimestampTz stmt_ts);
 extern TimestampTz GetCurrentTransactionStartTimestamp(void);
 extern TimestampTz GetCurrentStatementStartTimestamp(void);
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 7629230254..56aba06d6a 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -14,12 +14,32 @@
 #ifndef CREATEAS_H
 #define CREATEAS_H
 
+#include "access/heapam.h"
 #include "catalog/objectaddress.h"
+#include "executor/execdesc.h"
 #include "nodes/params.h"
+#include "nodes/plannodes.h"
 #include "parser/parse_node.h"
 #include "tcop/dest.h"
 #include "utils/queryenvironment.h"
 
+typedef struct
+{
+	DestReceiver pub;			/* publicly-known function pointers */
+	IntoClause *into;			/* target relation specification */
+	/* These fields are filled by intorel_startup: */
+	Relation	rel;			/* relation to write to */
+	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
+	CommandId	output_cid;		/* cmin to insert in output tuples */
+	int			ti_options;		/* table_tuple_insert performance options */
+	BulkInsertState bistate;	/* bulk insert state */
+	bool		is_parallel;	/* is parallelism to be considered? */
+	bool		is_parallel_worker; /* true for parallel worker */
+	/* Used for table open by parallel worker. */
+	Oid			object_id;
+} DR_intorel;
+
+#define ISCTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
 
 extern ObjectAddress ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 									   ParamListInfo params, QueryEnvironment *queryEnv,
@@ -29,4 +49,7 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
+extern bool IsParallelInsertInCTASAllowed(IntoClause *intoClause,
+										  QueryDesc *queryDesc);
+
 #endif							/* CREATEAS_H */
diff --git a/src/include/executor/execParallel.h b/src/include/executor/execParallel.h
index 5a39a5b29c..9c7d0edd40 100644
--- a/src/include/executor/execParallel.h
+++ b/src/include/executor/execParallel.h
@@ -35,6 +35,8 @@ typedef struct ParallelExecutorInfo
 	/* These two arrays have pcxt->nworkers_launched entries: */
 	shm_mq_handle **tqueue;		/* tuple queues for worker output */
 	struct TupleQueueReader **reader;	/* tuple reader/writer support */
+	/* Number of tuples inserted by all workers. */
+	volatile pg_atomic_uint64	*processed;
 } ParallelExecutorInfo;
 
 extern ParallelExecutorInfo *ExecInitParallelPlan(PlanState *planstate,
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 61ba4c3666..5277d66150 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -23,6 +23,7 @@
 #include "nodes/tidbitmap.h"
 #include "partitioning/partdefs.h"
 #include "storage/condition_variable.h"
+#include "tcop/dest.h"
 #include "utils/hsearch.h"
 #include "utils/queryenvironment.h"
 #include "utils/reltrigger.h"
@@ -1009,6 +1010,10 @@ typedef struct PlanState
 	bool		outeropsset;
 	bool		inneropsset;
 	bool		resultopsset;
+	/* Prallel inserts in CTAS related info is specified below. */
+	IntoClause	*intoclause;
+	Oid			objectid;
+	DestReceiver *dest;
 } PlanState;
 
 /* ----------------
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index d1f9ef29ca..c15b01252c 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -180,6 +180,7 @@ typedef struct Query
 	 */
 	int			stmt_location;	/* start location, or -1 if unknown */
 	int			stmt_len;		/* length in bytes; 0 means "rest of string" */
+	bool		isForCTAS; 		/* is the SELECT query for CREATE TABLE AS */
 } Query;
 
 
diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 0c4da2591a..a86da449fd 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -75,5 +75,195 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 (7 rows)
 
 create table parallel_write as execute prep_stmt;
+drop table parallel_write;
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+                     QUERY PLAN                      
+-----------------------------------------------------
+ LockRows (actual rows=10000 loops=1)
+   ->  Seq Scan on tenk1 (actual rows=10000 loops=1)
+(2 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+select length(stringu1) from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_mat_view
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop materialized view parallel_mat_view;
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+deallocate parallel_write_prep;
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+                          QUERY PLAN                           
+---------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Workers Launched: 4
+     ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+select nextval('parallel_write_sequence'), four from tenk1;
+                  QUERY PLAN                   
+-----------------------------------------------
+ Seq Scan on tenk1 (actual rows=10000 loops=1)
+(1 row)
+
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;
+                                QUERY PLAN                                
+--------------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   ->  Create parallel_write
+     Workers Planned: 4
+     Params Evaluated: $1
+     Workers Launched: 3
+     InitPlan 1 (returns $1)
+       ->  Limit (actual rows=1 loops=1)
+             ->  Gather (actual rows=1 loops=1)
+                   Workers Planned: 4
+                   Workers Launched: 4
+                   ->  Parallel Seq Scan on tenk2 (actual rows=1 loops=5)
+     ->  Parallel Seq Scan on tenk1 (actual rows=625 loops=4)
+           Filter: (four = 3)
+           Rows Removed by Filter: 1875
+(14 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;
+                             QUERY PLAN                              
+---------------------------------------------------------------------
+ Gather (actual rows=2500 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=500 loops=5)
+         Filter: (four = 3)
+         Rows Removed by Filter: 1500
+   SubPlan 1
+     ->  Function Scan on generate_series (actual rows=1 loops=2500)
+(8 rows)
+
 drop table parallel_write;
 rollback;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 78b479cedf..e65a56b442 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -39,4 +39,88 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
 
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+select length(stringu1) from tenk1;
+drop materialized view parallel_mat_view;
+
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+deallocate parallel_write_prep;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+select nextval('parallel_write_sequence'), four from tenk1;
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;
+drop table parallel_write;
+
 rollback;
-- 
2.25.1

From 7eff47d971c0ac5722ddc7de5a4c7e7550bb71aa Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Sun, 6 Dec 2020 08:21:42 +0530
Subject: [PATCH v7] Parallel Inserts in CREATE TABLE AS

The idea of this patch is to allow the leader and each worker
insert the tuples in parallel if the SELECT part of the CTAS is
parallelizable.

The design:
Let the planner know that the SELECT is from CTAS in createas.c
so that it can set the number of tuples transferred from the
workers to Gather node to 0. With this change, there are chances
that the planner may choose the parallel plan. After the planning,
check if the upper plan node is Gather in createas.c and mark a
parallelism flag in the CTAS dest receiver. Pass the into clause,
object id, command id from the leader to workers, so that each
worker can create its own CTAS dest receiver. Leader inserts its
share of tuples if instructed to do, and so are workers. Each
worker writes atomically its number of inserted tuples into a
shared memory variable, the leader combines this with its own
number of inserted tuples and shares to the client.
---
 src/backend/access/heap/heapam.c             |  11 -
 src/backend/access/transam/xact.c            |  30 +-
 src/backend/commands/createas.c              | 341 +++++++++++++------
 src/backend/commands/explain.c               |  41 +++
 src/backend/executor/execParallel.c          |  70 +++-
 src/backend/executor/nodeGather.c            | 114 ++++++-
 src/backend/executor/nodeGatherMerge.c       |   4 +-
 src/backend/optimizer/path/costsize.c        |  13 +-
 src/include/access/xact.h                    |   1 +
 src/include/commands/createas.h              |  28 ++
 src/include/executor/execParallel.h          |   6 +-
 src/include/nodes/execnodes.h                |   3 +
 src/include/nodes/parsenodes.h               |   1 +
 src/test/regress/expected/write_parallel.out | 190 +++++++++++
 src/test/regress/sql/write_parallel.sql      |  84 +++++
 15 files changed, 796 insertions(+), 141 deletions(-)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 1b2f70499e..3045c0f046 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2043,17 +2043,6 @@ static HeapTuple
 heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
 					CommandId cid, int options)
 {
-	/*
-	 * To allow parallel inserts, we need to ensure that they are safe to be
-	 * performed in workers. We have the infrastructure to allow parallel
-	 * inserts in general except for the cases where inserts generate a new
-	 * CommandId (eg. inserts into a table having a foreign key column).
-	 */
-	if (IsParallelWorker())
-		ereport(ERROR,
-				(errcode(ERRCODE_INVALID_TRANSACTION_STATE),
-				 errmsg("cannot insert tuples in a parallel worker")));
-
 	tup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
 	tup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
 	tup->t_data->t_infomask |= HEAP_XMAX_INVALID;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 9cd0b7c11b..db6eedd635 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -763,18 +763,34 @@ GetCurrentCommandId(bool used)
 	/* this is global to a transaction, not subtransaction-local */
 	if (used)
 	{
-		/*
-		 * Forbid setting currentCommandIdUsed in a parallel worker, because
-		 * we have no provision for communicating this back to the leader.  We
-		 * could relax this restriction when currentCommandIdUsed was already
-		 * true at the start of the parallel operation.
-		 */
-		Assert(!IsParallelWorker());
+		 /*
+		  * This is a temporary hack for all common parallel insert cases i.e.
+		  * insert into, ctas, copy from. To be changed later. In a parallel
+		  * worker, set currentCommandIdUsed to true only if it was not set to
+		  * true at the start of the parallel operation (by way of
+		  * SetCurrentCommandIdUsedForWorker()). We have to do this because
+		  * GetCurrentCommandId(true) may be called from anywhere, especially
+		  * for parallel inserts, within parallel worker.
+		  */
+		Assert(!(IsParallelWorker() && !currentCommandIdUsed));
 		currentCommandIdUsed = true;
 	}
 	return currentCommandId;
 }
 
+/*
+ *	SetCurrentCommandIdUsedForWorker
+ *
+ * For a parallel worker, record that the currentCommandId has been used. This
+ * must only be called at the start of a parallel operation.
+ */
+void
+SetCurrentCommandIdUsedForWorker(void)
+{
+	Assert(IsParallelWorker() && !currentCommandIdUsed && currentCommandId != InvalidCommandId);
+	currentCommandIdUsed = true;
+}
+
 /*
  *	SetParallelStartTimestamps
  *
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 6bf6c5a310..5d546464dc 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,18 +51,6 @@
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 
-typedef struct
-{
-	DestReceiver pub;			/* publicly-known function pointers */
-	IntoClause *into;			/* target relation specification */
-	/* These fields are filled by intorel_startup: */
-	Relation	rel;			/* relation to write to */
-	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
-	CommandId	output_cid;		/* cmin to insert in output tuples */
-	int			ti_options;		/* table_tuple_insert performance options */
-	BulkInsertState bistate;	/* bulk insert state */
-} DR_intorel;
-
 /* utility functions for CTAS definition creation */
 static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into);
 static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -328,6 +316,15 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		query = linitial_node(Query, rewritten);
 		Assert(query->commandType == CMD_SELECT);
 
+		/*
+		 * Flag to let the planner know that the SELECT query is for CTAS. This
+		 * is used to calculate the tuple transfer cost from workers to gather
+		 * node(in case parallelism kicks in for the SELECT part of the CTAS),
+		 * to zero as each worker will insert its share of tuples in parallel.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, NULL))
+			query->isForCTAS = true;
+
 		/* plan the query */
 		plan = pg_plan_query(query, pstate->p_sourcetext,
 							 CURSOR_OPT_PARALLEL_OK, params);
@@ -350,6 +347,15 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		/* call ExecutorStart to prepare the plan for execution */
 		ExecutorStart(queryDesc, GetIntoRelEFlags(into));
 
+		/*
+		 * If SELECT part of the CTAS is parallelizable, then make each
+		 * parallel worker insert the tuples that are resulted in its execution
+		 * into the target table. We need plan state to be initialized by the
+		 * executor to decide whether to allow parallel inserts or not.
+		*/
+		if (IsParallelInsertInCTASAllowed(into, queryDesc))
+			SetCTASParallelInsertState(queryDesc);
+
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
 
@@ -418,6 +424,9 @@ CreateIntoRelDestReceiver(IntoClause *intoClause)
 	self->pub.rDestroy = intorel_destroy;
 	self->pub.mydest = DestIntoRel;
 	self->into = intoClause;
+	self->is_parallel = false;
+	self->is_parallel_worker = false;
+	self->object_id = InvalidOid;
 	/* other private fields will be set during intorel_startup */
 
 	return (DestReceiver *) self;
@@ -430,121 +439,169 @@ static void
 intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 {
 	DR_intorel *myState = (DR_intorel *) self;
-	IntoClause *into = myState->into;
-	bool		is_matview;
-	List	   *attrList;
 	ObjectAddress intoRelationAddr;
 	Relation	intoRelationDesc;
-	ListCell   *lc;
-	int			attnum;
-
-	Assert(into != NULL);		/* else somebody forgot to set it */
 
-	/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
-	is_matview = (into->viewQuery != NULL);
+	if (myState->is_parallel_worker)
+	{
+		/* In the worker */
+		intoRelationDesc = table_open(myState->object_id, AccessExclusiveLock);
+		myState->rel = intoRelationDesc;
+		myState->reladdr = InvalidObjectAddress;
+		myState->ti_options = 0;
+		myState->bistate = GetBulkInsertState();
 
-	/*
-	 * Build column definitions using "pre-cooked" type and collation info. If
-	 * a column name list was specified in CREATE TABLE AS, override the
-	 * column names derived from the query.  (Too few column names are OK, too
-	 * many are not.)
-	 */
-	attrList = NIL;
-	lc = list_head(into->colNames);
-	for (attnum = 0; attnum < typeinfo->natts; attnum++)
+		/*
+		 * Right after the table is created in the leader, the command id is
+		 * incremented (in create_ctas_internal()). The new command id is
+		 * marked as used in intorel_startup(), then the parallel mode is
+		 * entered. The command id and transaction id are serialized into
+		 * parallel DSM, they are then available to all parallel workers. All
+		 * the workers need to mark the command id as used before insertion.
+		 */
+		(void) SetCurrentCommandIdUsedForWorker();
+		myState->output_cid = GetCurrentCommandId(false);
+	}
+	else
 	{
-		Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
-		ColumnDef  *col;
-		char	   *colname;
+		IntoClause *into = myState->into;
+		bool		is_matview;
+		List	   *attrList;
+		ListCell   *lc;
+		int			attnum;
+
+		Assert(into != NULL);		/* else somebody forgot to set it */
+
+		/*
+		 * This code supports both CREATE TABLE AS and CREATE MATERIALIZED
+		 * VIEW.
+		 */
+		is_matview = (into->viewQuery != NULL);
 
-		if (lc)
+		/*
+		 * Build column definitions using "pre-cooked" type and collation info.
+		 * If a column name list was specified in CREATE TABLE AS, override the
+		 * column names derived from the query.  (Too few column names are OK,
+		 * too many are not.)
+		 */
+		attrList = NIL;
+		lc = list_head(into->colNames);
+		for (attnum = 0; attnum < typeinfo->natts; attnum++)
 		{
-			colname = strVal(lfirst(lc));
-			lc = lnext(into->colNames, lc);
+			Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
+			ColumnDef  *col;
+			char	   *colname;
+
+			if (lc)
+			{
+				colname = strVal(lfirst(lc));
+				lc = lnext(into->colNames, lc);
+			}
+			else
+				colname = NameStr(attribute->attname);
+
+			col = makeColumnDef(colname,
+								attribute->atttypid,
+								attribute->atttypmod,
+								attribute->attcollation);
+
+			/*
+			 * It's possible that the column is of a collatable type but the
+			 * collation could not be resolved, so double-check.  (We must
+			 * check this here because DefineRelation would adopt the type's
+			 * default collation rather than complaining.)
+			 */
+			if (!OidIsValid(col->collOid) &&
+				type_is_collatable(col->typeName->typeOid))
+				ereport(ERROR,
+						(errcode(ERRCODE_INDETERMINATE_COLLATION),
+						errmsg("no collation was derived for column \"%s\" with collatable type %s",
+								col->colname,
+								format_type_be(col->typeName->typeOid)),
+						errhint("Use the COLLATE clause to set the collation explicitly.")));
+
+			attrList = lappend(attrList, col);
 		}
-		else
-			colname = NameStr(attribute->attname);
 
-		col = makeColumnDef(colname,
-							attribute->atttypid,
-							attribute->atttypmod,
-							attribute->attcollation);
+		if (lc != NULL)
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					errmsg("too many column names were specified")));
 
 		/*
-		 * It's possible that the column is of a collatable type but the
-		 * collation could not be resolved, so double-check.  (We must check
-		 * this here because DefineRelation would adopt the type's default
-		 * collation rather than complaining.)
+		 * Actually create the target table
 		 */
-		if (!OidIsValid(col->collOid) &&
-			type_is_collatable(col->typeName->typeOid))
-			ereport(ERROR,
-					(errcode(ERRCODE_INDETERMINATE_COLLATION),
-					 errmsg("no collation was derived for column \"%s\" with collatable type %s",
-							col->colname,
-							format_type_be(col->typeName->typeOid)),
-					 errhint("Use the COLLATE clause to set the collation explicitly.")));
+		intoRelationAddr = create_ctas_internal(attrList, into);
 
-		attrList = lappend(attrList, col);
-	}
+		/*
+		 * Finally we can open the target table
+		 */
+		intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
 
-	if (lc != NULL)
-		ereport(ERROR,
-				(errcode(ERRCODE_SYNTAX_ERROR),
-				 errmsg("too many column names were specified")));
+		/*
+		 * Make sure the constructed table does not have RLS enabled.
+		 *
+		 * check_enable_rls() will ereport(ERROR) itself if the user has
+		 * requested something invalid, and otherwise will return RLS_ENABLED
+		 * if RLS should be enabled here.  We don't actually support that
+		 * currently, so throw our own ereport(ERROR) if that happens.
+		 */
+		if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					errmsg("policies not yet implemented for this command")));
 
-	/*
-	 * Actually create the target table
-	 */
-	intoRelationAddr = create_ctas_internal(attrList, into);
+		/*
+		 * Tentatively mark the target as populated, if it's a matview and
+		 * we're going to fill it; otherwise, no change needed.
+		 */
+		if (is_matview && !into->skipData)
+			SetMatViewPopulatedState(intoRelationDesc, true);
 
-	/*
-	 * Finally we can open the target table
-	 */
-	intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
+		/*
+		 * Fill private fields of myState for use by later routines
+		 */
+		myState->rel = intoRelationDesc;
+		myState->reladdr = intoRelationAddr;
+		myState->output_cid = GetCurrentCommandId(true);
+		myState->ti_options = TABLE_INSERT_SKIP_FSM;
 
-	/*
-	 * Make sure the constructed table does not have RLS enabled.
-	 *
-	 * check_enable_rls() will ereport(ERROR) itself if the user has requested
-	 * something invalid, and otherwise will return RLS_ENABLED if RLS should
-	 * be enabled here.  We don't actually support that currently, so throw
-	 * our own ereport(ERROR) if that happens.
-	 */
-	if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
-		ereport(ERROR,
-				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-				 errmsg("policies not yet implemented for this command")));
+		/*
+		 * If WITH NO DATA is specified, there is no need to set up the state
+		 * for bulk inserts as there are no tuples to insert.
+		 */
+		if (!into->skipData)
+			myState->bistate = GetBulkInsertState();
+		else
+			myState->bistate = NULL;
 
-	/*
-	 * Tentatively mark the target as populated, if it's a matview and we're
-	 * going to fill it; otherwise, no change needed.
-	 */
-	if (is_matview && !into->skipData)
-		SetMatViewPopulatedState(intoRelationDesc, true);
+		if (myState->is_parallel)
+		{
+			myState->object_id = intoRelationAddr.objectId;
 
-	/*
-	 * Fill private fields of myState for use by later routines
-	 */
-	myState->rel = intoRelationDesc;
-	myState->reladdr = intoRelationAddr;
-	myState->output_cid = GetCurrentCommandId(true);
-	myState->ti_options = TABLE_INSERT_SKIP_FSM;
+			/*
+			 * We don't need to skip contacting FSM while inserting tuples
+			 * for parallel mode, while extending the relations, workers
+			 * instead of blocking on a page while another worker is inserting,
+			 * can check the FSM for another page that can accommodate the
+			 * tuples. This results in major benefit for parallel inserts.
+			 */
+			myState->ti_options = 0;
 
-	/*
-	 * If WITH NO DATA is specified, there is no need to set up the state for
-	 * bulk inserts as there are no tuples to insert.
-	 */
-	if (!into->skipData)
-		myState->bistate = GetBulkInsertState();
-	else
-		myState->bistate = NULL;
+			/*
+			 * rd_createSubid is marked invalid, otherwise, the table is
+			 * not allowed to extend by the workers.
+			 */
+			myState->rel->rd_createSubid = InvalidSubTransactionId;
+		}
 
-	/*
-	 * Valid smgr_targblock implies something already wrote to the relation.
-	 * This may be harmless, but this function hasn't planned for it.
-	 */
-	Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+		/*
+		 * Valid smgr_targblock implies something already wrote to the
+		 * relation. This may be harmless, but this function hasn't planned for
+		 * it.
+		 */
+		Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+	}
 }
 
 /*
@@ -606,3 +663,75 @@ intorel_destroy(DestReceiver *self)
 {
 	pfree(self);
 }
+
+/*
+ * IsParallelInsertInCTASAllowed --- determine whether or not parallel
+ * insertion is possible.
+ */
+bool IsParallelInsertInCTASAllowed(IntoClause *into, QueryDesc *queryDesc)
+{
+	bool allowed = false;
+
+	if (IS_CTAS(into))
+	{
+		RangeVar   *rel = into->rel;
+
+		/* Allow parallel inserts only if the table is not temporary. */
+		if (rel && rel->relpersistence != RELPERSISTENCE_TEMP)
+			allowed = true;
+
+		if (queryDesc && allowed)
+		{
+			PlanState  *ps = queryDesc->planstate;
+			PlannedStmt *plannedstmt = queryDesc->plannedstmt;
+
+			/*
+			 * We allow parallel inserts by the workers only if the Gather node
+			 * has no projections to perform and if the upper node is Gather.
+			 * In case, the Gather node has projections, which is possible if
+			 * there are any subplans in the query, the workers can not do
+			 * those projections. And when the upper node is GatherMerge, then
+			 * the leader has to perform the final phase i.e. merge the results
+			 * by workers.
+			 */
+			if (!(ps && IsA(ps, GatherState) && !ps->ps_ProjInfo &&
+				plannedstmt->parallelModeNeeded &&
+				plannedstmt->planTree &&
+				IsA(plannedstmt->planTree, Gather) &&
+				plannedstmt->planTree->lefttree &&
+				plannedstmt->planTree->lefttree->parallel_aware &&
+				plannedstmt->planTree->lefttree->parallel_safe))
+			{
+				allowed = false;
+			}
+		}
+	}
+
+	return allowed;
+}
+
+/*
+ * SetCTASParallelInsertState --- set the required info for the parallel
+ * inserts, that is required in the plan exection.
+ */
+void SetCTASParallelInsertState(QueryDesc *queryDesc)
+{
+	GatherState *gstate = (GatherState *) queryDesc->planstate;
+
+	/*
+	 * For parallelizing inserts in CTAS i.e. making each parallel worker
+	 * insert the tuples, we must send information such as intoclause(for
+	 * eachworker to build separate dest receiver), object id(for each worker
+	 * to open the created table).
+	 */
+	((DR_intorel *) queryDesc->dest)->is_parallel = true;
+	gstate->dest = queryDesc->dest;
+
+	/*
+	 * Since there are no rows that are transferred from workers to Gather
+	 * node, so we set it to 0 to be visible in explain plans. Note that we
+	 * would have already accounted this for cost calculations in
+	 * cost_gather().
+	 */
+	queryDesc->plannedstmt->planTree->plan_rows = 0;
+}
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 43f9b01e83..57875e1505 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -371,6 +371,15 @@ ExplainOneQuery(Query *query, int cursorOptions,
 		return;
 	}
 
+	/*
+	 * Flag to let the planner know that the SELECT query is for CTAS. This is
+	 * used to calculate the tuple transfer cost from workers to gather node(in
+	 * case parallelism kicks in for the SELECT part of the CTAS), to zero as
+	 * each worker will insert its share of tuples in parallel.
+	 */
+	if (IsParallelInsertInCTASAllowed(into, NULL))
+		query->isForCTAS = true;
+
 	/* if an advisor plugin is present, let it manage things */
 	if (ExplainOneQuery_hook)
 		(*ExplainOneQuery_hook) (query, cursorOptions, into, es,
@@ -556,6 +565,15 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	/* call ExecutorStart to prepare the plan for execution */
 	ExecutorStart(queryDesc, eflags);
 
+	/*
+	 * If SELECT part of the CTAS is parallelizable, then make each parallel
+	 * worker insert the tuples that are resulted in its execution into the
+	 * target table. We need plan state to be initialized by the executor to
+	 * decide whether to allow parallel inserts or not.
+	 */
+	if (IsParallelInsertInCTASAllowed(into, queryDesc))
+		SetCTASParallelInsertState(queryDesc);
+
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
 	{
@@ -1775,6 +1793,29 @@ ExplainNode(PlanState *planstate, List *ancestors,
 
 				if (gather->single_copy || es->format != EXPLAIN_FORMAT_TEXT)
 					ExplainPropertyBool("Single Copy", gather->single_copy, es);
+
+				/*
+				 * Show the create table information under Gather node in case
+				 * parallel workers have inserted the rows.
+				 */
+				if (IsA(planstate, GatherState))
+				{
+					GatherState *gstate = (GatherState *) planstate;
+
+					if (IS_PARALLEL_CTAS_DEST(gstate->dest) &&
+						((DR_intorel *) gstate->dest)->into->rel &&
+						((DR_intorel *) gstate->dest)->into->rel->relname)
+					{
+						es->indent--;
+						ExplainIndentText(es);
+						appendStringInfoString(es->str, "->  ");
+						appendStringInfoString(es->str, "Create ");
+						appendStringInfo(es->str, "%s\n",
+							((DR_intorel *) gstate->dest)->into->rel->relname);
+						ExplainIndentText(es);
+						es->indent++;
+					}
+				}
 			}
 			break;
 		case T_GatherMerge:
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index befde52691..330978d018 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -23,6 +23,7 @@
 
 #include "postgres.h"
 
+#include "commands/createas.h"
 #include "executor/execParallel.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
@@ -65,6 +66,7 @@
 #define PARALLEL_KEY_QUERY_TEXT		UINT64CONST(0xE000000000000008)
 #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009)
 #define PARALLEL_KEY_WAL_USAGE			UINT64CONST(0xE00000000000000A)
+#define PARALLEL_KEY_INTO_CLAUSE		UINT64CONST(0xE00000000000000B)
 
 #define PARALLEL_TUPLE_QUEUE_SIZE		65536
 
@@ -77,6 +79,9 @@ typedef struct FixedParallelExecutorState
 	dsa_pointer param_exec;
 	int			eflags;
 	int			jit_flags;
+	Oid			objectid;		/* workers to open relation/table.  */
+	/* Number tuples inserted by all the workers. */
+	pg_atomic_uint64	processed;
 } FixedParallelExecutorState;
 
 /*
@@ -578,7 +583,8 @@ ExecParallelSetupTupleQueues(ParallelContext *pcxt, bool reinitialize)
 ParallelExecutorInfo *
 ExecInitParallelPlan(PlanState *planstate, EState *estate,
 					 Bitmapset *sendParams, int nworkers,
-					 int64 tuples_needed)
+					 int64 tuples_needed, IntoClause *intoclause,
+					 Oid objectid)
 {
 	ParallelExecutorInfo *pei;
 	ParallelContext *pcxt;
@@ -600,6 +606,8 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	Size		dsa_minsize = dsa_minimum_size();
 	char	   *query_string;
 	int			query_len;
+	char 		*intoclausestr = NULL;
+	int			intoclause_len = 0;
 
 	/*
 	 * Force any initplan outputs that we're going to pass to workers to be
@@ -712,6 +720,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_estimate_chunk(&pcxt->estimator, dsa_minsize);
 	shm_toc_estimate_keys(&pcxt->estimator, 1);
 
+	/* Estimate space for into clause for CTAS. */
+	if (IS_CTAS(intoclause) && OidIsValid(objectid))
+	{
+		intoclausestr = nodeToString(intoclause);
+		intoclause_len = strlen(intoclausestr);
+		shm_toc_estimate_chunk(&pcxt->estimator, intoclause_len + 1);
+		shm_toc_estimate_keys(&pcxt->estimator, 1);
+	}
+
 	/* Everyone's had a chance to ask for space, so now create the DSM. */
 	InitializeParallelDSM(pcxt);
 
@@ -729,6 +746,14 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	fpes->param_exec = InvalidDsaPointer;
 	fpes->eflags = estate->es_top_eflags;
 	fpes->jit_flags = estate->es_jit_flags;
+	pg_atomic_init_u64(&fpes->processed, 0);
+	pei->processed = &fpes->processed;
+
+	if (intoclausestr && OidIsValid(objectid))
+		fpes->objectid = objectid;
+	else
+		fpes->objectid = InvalidOid;
+
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, fpes);
 
 	/* Store query string */
@@ -758,8 +783,18 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_WAL_USAGE, walusage_space);
 	pei->wal_usage = walusage_space;
 
-	/* Set up the tuple queues that the workers will write into. */
-	pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	if (intoclausestr)
+	{
+		char *intoclause_space = shm_toc_allocate(pcxt->toc,
+												  intoclause_len + 1);
+		memcpy(intoclause_space, intoclausestr, intoclause_len + 1);
+		shm_toc_insert(pcxt->toc, PARALLEL_KEY_INTO_CLAUSE, intoclause_space);
+	}
+	else
+	{
+		/* Set up the tuple queues that the workers will write into. */
+		pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	}
 
 	/* We don't need the TupleQueueReaders yet, though. */
 	pei->reader = NULL;
@@ -1387,12 +1422,30 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 	void	   *area_space;
 	dsa_area   *area;
 	ParallelWorkerContext pwcxt;
+	char		*intoclausestr = NULL;
+	IntoClause	*intoclause = NULL;
 
 	/* Get fixed-size state. */
 	fpes = shm_toc_lookup(toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
 
-	/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
-	receiver = ExecParallelGetReceiver(seg, toc);
+	intoclausestr = shm_toc_lookup(toc, PARALLEL_KEY_INTO_CLAUSE, true);
+	if (intoclausestr)
+	{
+		/*
+		 * If the worker is for parallel insert in CTAS, then use the proper
+		 * dest receiver.
+		 */
+		intoclause = (IntoClause *) stringToNode(intoclausestr);
+		receiver = CreateIntoRelDestReceiver(intoclause);
+		((DR_intorel *)receiver)->is_parallel_worker = true;
+		((DR_intorel *)receiver)->object_id = fpes->objectid;
+	}
+	else
+	{
+		/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
+		receiver = ExecParallelGetReceiver(seg, toc);
+	}
+
 	instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_INSTRUMENTATION, true);
 	if (instrumentation != NULL)
 		instrument_options = instrumentation->instrument_options;
@@ -1471,6 +1524,13 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 			queryDesc->estate->es_jit->instr;
 	}
 
+	/*
+	 * Write out the number of tuples this worker has inserted. Leader will use
+	 * it to inform to the end client.
+	 */
+	if (intoclausestr)
+		pg_atomic_add_fetch_u64(&fpes->processed, queryDesc->estate->es_processed);
+
 	/* Must do this after capturing instrumentation. */
 	ExecutorEnd(queryDesc);
 
diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c
index a01b46af14..9cef6bdd61 100644
--- a/src/backend/executor/nodeGather.c
+++ b/src/backend/executor/nodeGather.c
@@ -32,6 +32,7 @@
 
 #include "access/relscan.h"
 #include "access/xact.h"
+#include "commands/createas.h"
 #include "executor/execdebug.h"
 #include "executor/execParallel.h"
 #include "executor/nodeGather.h"
@@ -48,6 +49,7 @@ static TupleTableSlot *ExecGather(PlanState *pstate);
 static TupleTableSlot *gather_getnext(GatherState *gatherstate);
 static MinimalTuple gather_readnext(GatherState *gatherstate);
 static void ExecShutdownGatherWorkers(GatherState *node);
+static void ExecParallelInsertInCTAS(GatherState *node);
 
 
 /* ----------------------------------------------------------------
@@ -131,6 +133,73 @@ ExecInitGather(Gather *node, EState *estate, int eflags)
 	return gatherstate;
 }
 
+/* ----------------------------------------------------------------
+ *		ExecParallelInsertInCTAS(node)
+ *
+ *		Facilitates parallel inserts by parallel workers and/or
+ *		leader for Create Table AS.
+ * ----------------------------------------------------------------
+ */
+static void
+ExecParallelInsertInCTAS(GatherState *node)
+{
+	/* Enable leader to insert in case no parallel workers were launched. */
+	if (node->nworkers_launched == 0)
+		node->need_to_scan_locally = true;
+
+	/*
+	 * By now, for parallel workers (if launched any), would have started their
+	 * work i.e. insertion to target table. In case the leader is chosen to
+	 * participate for parallel inserts in CTAS, then finish its share before
+	 * going to wait for the parallel workers to finish.
+	 */
+	if (node->need_to_scan_locally)
+	{
+		EState	   *estate = node->ps.state;
+		TupleTableSlot *outerTupleSlot;
+
+		for(;;)
+		{
+			/* Install our DSA area while executing the plan. */
+			estate->es_query_dsa =
+					node->pei ? node->pei->area : NULL;
+
+			outerTupleSlot = ExecProcNode(node->ps.lefttree);
+
+			estate->es_query_dsa = NULL;
+
+			if(TupIsNull(outerTupleSlot))
+				break;
+
+			(void) node->dest->receiveSlot(outerTupleSlot, node->dest);
+
+			node->ps.state->es_processed++;
+		}
+
+		node->need_to_scan_locally = false;
+	}
+
+	if (node->nworkers_launched > 0)
+	{
+		int		i;
+
+		/* Wait for the parallel workers to finish. */
+		WaitForParallelWorkersToFinish(node->pei->pcxt);
+
+		for (i = 0; i < node->nworkers_launched; i++)
+		{
+			InstrAccumParallelQuery(&node->pei->buffer_usage[i],
+									&node->pei->wal_usage[i]);
+		}
+
+		/*
+		 * Add up the total tuples inserted by all workers, to the tuples
+		 * inserted by the leader(if any). This will be shared to client.
+		 */
+		node->ps.state->es_processed += pg_atomic_read_u64(node->pei->processed);
+	}
+}
+
 /* ----------------------------------------------------------------
  *		ExecGather(node)
  *
@@ -157,6 +226,7 @@ ExecGather(PlanState *pstate)
 	{
 		EState	   *estate = node->ps.state;
 		Gather	   *gather = (Gather *) node->ps.plan;
+		bool		isctas = IS_PARALLEL_CTAS_DEST(node->dest);
 
 		/*
 		 * Sometimes we might have to run without parallelism; but if parallel
@@ -165,6 +235,18 @@ ExecGather(PlanState *pstate)
 		if (gather->num_workers > 0 && estate->es_use_parallel_mode)
 		{
 			ParallelContext *pcxt;
+			IntoClause	*intoclause = NULL;
+			Oid			objectid = InvalidOid;
+
+			/*
+			 * Take the necessary information to be passed to workers for
+			 * parallel inserts in CTAS.
+			 */
+			if (isctas)
+			{
+				intoclause = ((DR_intorel *) node->dest)->into;
+				objectid = ((DR_intorel *) node->dest)->object_id;
+			}
 
 			/* Initialize, or re-initialize, shared state needed by workers. */
 			if (!node->pei)
@@ -172,7 +254,10 @@ ExecGather(PlanState *pstate)
 												 estate,
 												 gather->initParam,
 												 gather->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 /* CTAS info */
+												 intoclause,
+												 objectid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
@@ -190,13 +275,16 @@ ExecGather(PlanState *pstate)
 			/* Set up tuple queue readers to read the results. */
 			if (pcxt->nworkers_launched > 0)
 			{
-				ExecParallelCreateReaders(node->pei);
-				/* Make a working array showing the active readers */
-				node->nreaders = pcxt->nworkers_launched;
-				node->reader = (TupleQueueReader **)
-					palloc(node->nreaders * sizeof(TupleQueueReader *));
-				memcpy(node->reader, node->pei->reader,
-					   node->nreaders * sizeof(TupleQueueReader *));
+				if (!isctas)
+				{
+					ExecParallelCreateReaders(node->pei);
+					/* Make a working array showing the active readers */
+					node->nreaders = pcxt->nworkers_launched;
+					node->reader = (TupleQueueReader **)
+						palloc(node->nreaders * sizeof(TupleQueueReader *));
+					memcpy(node->reader, node->pei->reader,
+						node->nreaders * sizeof(TupleQueueReader *));
+				}
 			}
 			else
 			{
@@ -208,9 +296,17 @@ ExecGather(PlanState *pstate)
 		}
 
 		/* Run plan locally if no workers or enabled and not single-copy. */
-		node->need_to_scan_locally = (node->nreaders == 0)
+		node->need_to_scan_locally = (node->nreaders == 0 &&
+			!isctas)
 			|| (!gather->single_copy && parallel_leader_participation);
 		node->initialized = true;
+
+		/* Perform parallel inserts for CTAS. */
+		if (isctas)
+		{
+			ExecParallelInsertInCTAS(node);
+			return NULL;
+		}
 	}
 
 	/*
diff --git a/src/backend/executor/nodeGatherMerge.c b/src/backend/executor/nodeGatherMerge.c
index 47129344f3..ee45272c17 100644
--- a/src/backend/executor/nodeGatherMerge.c
+++ b/src/backend/executor/nodeGatherMerge.c
@@ -216,7 +216,9 @@ ExecGatherMerge(PlanState *pstate)
 												 estate,
 												 gm->initParam,
 												 gm->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 NULL,
+												 InvalidOid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 22d6935824..225f3cbf01 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -393,7 +393,18 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 
 	/* Parallel setup and communication cost. */
 	startup_cost += parallel_setup_cost;
-	run_cost += parallel_tuple_cost * path->path.rows;
+
+	/*
+	 * We do not compute the parallel_tuple_cost for CTAS because the number of
+	 * tuples that are transferred from workers to the gather node is zero as
+	 * each worker, in parallel, inserts the tuples that are resulted from its
+	 * chunk of plan execution. This change may make the parallel plan cheap
+	 * among all other plans, and influence the planner to consider this
+	 * parallel plan.
+	 */
+	if (!(root->parse->isForCTAS &&
+		root->query_level == 1))
+		run_cost += parallel_tuple_cost * path->path.rows;
 
 	path->path.startup_cost = startup_cost;
 	path->path.total_cost = (startup_cost + run_cost);
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7320de345c..5beae6c617 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -389,6 +389,7 @@ extern FullTransactionId GetCurrentFullTransactionIdIfAny(void);
 extern void MarkCurrentTransactionIdLoggedIfAny(void);
 extern bool SubTransactionIsActive(SubTransactionId subxid);
 extern CommandId GetCurrentCommandId(bool used);
+extern void SetCurrentCommandIdUsedForWorker(void);
 extern void SetParallelStartTimestamps(TimestampTz xact_ts, TimestampTz stmt_ts);
 extern TimestampTz GetCurrentTransactionStartTimestamp(void);
 extern TimestampTz GetCurrentStatementStartTimestamp(void);
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 7629230254..ab3aab58c5 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -14,12 +14,35 @@
 #ifndef CREATEAS_H
 #define CREATEAS_H
 
+#include "access/heapam.h"
 #include "catalog/objectaddress.h"
+#include "executor/execdesc.h"
 #include "nodes/params.h"
+#include "nodes/plannodes.h"
 #include "parser/parse_node.h"
 #include "tcop/dest.h"
 #include "utils/queryenvironment.h"
 
+typedef struct
+{
+	DestReceiver pub;			/* publicly-known function pointers */
+	IntoClause *into;			/* target relation specification */
+	/* These fields are filled by intorel_startup: */
+	Relation	rel;			/* relation to write to */
+	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
+	CommandId	output_cid;		/* cmin to insert in output tuples */
+	int			ti_options;		/* table_tuple_insert performance options */
+	BulkInsertState bistate;	/* bulk insert state */
+	bool		is_parallel;	/* is parallelism to be considered? */
+	bool		is_parallel_worker; /* true for parallel worker */
+	/* Used for table open by parallel worker. */
+	Oid			object_id;
+} DR_intorel;
+
+#define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
+#define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
+									 IS_CTAS(((DR_intorel *) dest)->into) && \
+									 ((DR_intorel *) dest)->is_parallel)
 
 extern ObjectAddress ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 									   ParamListInfo params, QueryEnvironment *queryEnv,
@@ -29,4 +52,9 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
+extern bool IsParallelInsertInCTASAllowed(IntoClause *into,
+										  QueryDesc *queryDesc);
+
+extern void SetCTASParallelInsertState(QueryDesc *queryDesc);
+
 #endif							/* CREATEAS_H */
diff --git a/src/include/executor/execParallel.h b/src/include/executor/execParallel.h
index 5a39a5b29c..9f959f741b 100644
--- a/src/include/executor/execParallel.h
+++ b/src/include/executor/execParallel.h
@@ -35,11 +35,15 @@ typedef struct ParallelExecutorInfo
 	/* These two arrays have pcxt->nworkers_launched entries: */
 	shm_mq_handle **tqueue;		/* tuple queues for worker output */
 	struct TupleQueueReader **reader;	/* tuple reader/writer support */
+	/* Number of tuples inserted by all workers. */
+	volatile pg_atomic_uint64	*processed;
 } ParallelExecutorInfo;
 
 extern ParallelExecutorInfo *ExecInitParallelPlan(PlanState *planstate,
 												  EState *estate, Bitmapset *sendParam, int nworkers,
-												  int64 tuples_needed);
+												  int64 tuples_needed,
+												  IntoClause *intoclause,
+												  Oid objectid);
 extern void ExecParallelCreateReaders(ParallelExecutorInfo *pei);
 extern void ExecParallelFinish(ParallelExecutorInfo *pei);
 extern void ExecParallelCleanup(ParallelExecutorInfo *pei);
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 61ba4c3666..e9c4442c22 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -23,6 +23,7 @@
 #include "nodes/tidbitmap.h"
 #include "partitioning/partdefs.h"
 #include "storage/condition_variable.h"
+#include "tcop/dest.h"
 #include "utils/hsearch.h"
 #include "utils/queryenvironment.h"
 #include "utils/reltrigger.h"
@@ -2326,6 +2327,8 @@ typedef struct GatherState
 	int			nreaders;		/* number of still-active workers */
 	int			nextreader;		/* next one to try to read from */
 	struct TupleQueueReader **reader;	/* array with nreaders active entries */
+	/* Parallel inserts in CTAS related info is specified below. */
+	DestReceiver *dest;
 } GatherState;
 
 /* ----------------
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index ec14fc2036..eb267b1a6c 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -180,6 +180,7 @@ typedef struct Query
 	 */
 	int			stmt_location;	/* start location, or -1 if unknown */
 	int			stmt_len;		/* length in bytes; 0 means "rest of string" */
+	bool		isForCTAS; 		/* is the SELECT query for CREATE TABLE AS */
 } Query;
 
 
diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 0c4da2591a..635d24e76d 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -75,5 +75,195 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 (7 rows)
 
 create table parallel_write as execute prep_stmt;
+drop table parallel_write;
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+                     QUERY PLAN                      
+-----------------------------------------------------
+ LockRows (actual rows=10000 loops=1)
+   ->  Seq Scan on tenk1 (actual rows=10000 loops=1)
+(2 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_mat_view
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop materialized view parallel_mat_view;
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+deallocate parallel_write_prep;
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+select nextval('parallel_write_sequence'), four from tenk1;
+                  QUERY PLAN                   
+-----------------------------------------------
+ Seq Scan on tenk1 (actual rows=10000 loops=1)
+(1 row)
+
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;
+                               QUERY PLAN                               
+------------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Params Evaluated: $1
+   Workers Launched: 3
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=1 loops=1)
+           ->  Gather (actual rows=1 loops=1)
+                 Workers Planned: 4
+                 Workers Launched: 4
+                 ->  Parallel Seq Scan on tenk2 (actual rows=1 loops=5)
+   ->  Parallel Seq Scan on tenk1 (actual rows=625 loops=4)
+         Filter: (four = 3)
+         Rows Removed by Filter: 1875
+(14 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;
+                             QUERY PLAN                              
+---------------------------------------------------------------------
+ Gather (actual rows=2500 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=500 loops=5)
+         Filter: (four = 3)
+         Rows Removed by Filter: 1500
+   SubPlan 1
+     ->  Function Scan on generate_series (actual rows=1 loops=2500)
+(8 rows)
+
 drop table parallel_write;
 rollback;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 78b479cedf..e65a56b442 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -39,4 +39,88 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
 
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+select length(stringu1) from tenk1;
+drop materialized view parallel_mat_view;
+
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+deallocate parallel_write_prep;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+select nextval('parallel_write_sequence'), four from tenk1;
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;
+drop table parallel_write;
+
 rollback;
-- 
2.25.1

From 295ba5ba91c51655c1f92c3821912d6daffd1ab1 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 7 Dec 2020 10:15:56 +0530
Subject: [PATCH v8] Parallel Inserts in CREATE TABLE AS

The idea of this patch is to allow the leader and each worker
insert the tuples in parallel if the SELECT part of the CTAS is
parallelizable.

The design:
Let the planner know that the SELECT is from CTAS in createas.c
so that it can set the number of tuples transferred from the
workers to Gather node to 0. With this change, there are chances
that the planner may choose the parallel plan. After the planning,
check if the upper plan node is Gather in createas.c and mark a
parallelism flag in the CTAS dest receiver. Pass the into clause,
object id, command id from the leader to workers, so that each
worker can create its own CTAS dest receiver. Leader inserts its
share of tuples if instructed to do, and so are workers. Each
worker writes atomically its number of inserted tuples into a
shared memory variable, the leader combines this with its own
number of inserted tuples and shares to the client.
---
 src/backend/access/heap/heapam.c             |  11 -
 src/backend/access/transam/xact.c            |  30 +-
 src/backend/commands/createas.c              | 342 +++++++++++++------
 src/backend/commands/explain.c               |  41 +++
 src/backend/executor/execParallel.c          |  70 +++-
 src/backend/executor/nodeGather.c            | 114 ++++++-
 src/backend/executor/nodeGatherMerge.c       |   4 +-
 src/backend/optimizer/path/costsize.c        |  13 +-
 src/include/access/xact.h                    |   1 +
 src/include/commands/createas.h              |  28 ++
 src/include/executor/execParallel.h          |   6 +-
 src/include/nodes/execnodes.h                |   3 +
 src/include/nodes/parsenodes.h               |   1 +
 src/test/regress/expected/write_parallel.out | 190 +++++++++++
 src/test/regress/sql/write_parallel.sql      |  84 +++++
 15 files changed, 797 insertions(+), 141 deletions(-)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 1b2f70499e..3045c0f046 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2043,17 +2043,6 @@ static HeapTuple
 heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
 					CommandId cid, int options)
 {
-	/*
-	 * To allow parallel inserts, we need to ensure that they are safe to be
-	 * performed in workers. We have the infrastructure to allow parallel
-	 * inserts in general except for the cases where inserts generate a new
-	 * CommandId (eg. inserts into a table having a foreign key column).
-	 */
-	if (IsParallelWorker())
-		ereport(ERROR,
-				(errcode(ERRCODE_INVALID_TRANSACTION_STATE),
-				 errmsg("cannot insert tuples in a parallel worker")));
-
 	tup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
 	tup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
 	tup->t_data->t_infomask |= HEAP_XMAX_INVALID;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 9cd0b7c11b..db6eedd635 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -763,18 +763,34 @@ GetCurrentCommandId(bool used)
 	/* this is global to a transaction, not subtransaction-local */
 	if (used)
 	{
-		/*
-		 * Forbid setting currentCommandIdUsed in a parallel worker, because
-		 * we have no provision for communicating this back to the leader.  We
-		 * could relax this restriction when currentCommandIdUsed was already
-		 * true at the start of the parallel operation.
-		 */
-		Assert(!IsParallelWorker());
+		 /*
+		  * This is a temporary hack for all common parallel insert cases i.e.
+		  * insert into, ctas, copy from. To be changed later. In a parallel
+		  * worker, set currentCommandIdUsed to true only if it was not set to
+		  * true at the start of the parallel operation (by way of
+		  * SetCurrentCommandIdUsedForWorker()). We have to do this because
+		  * GetCurrentCommandId(true) may be called from anywhere, especially
+		  * for parallel inserts, within parallel worker.
+		  */
+		Assert(!(IsParallelWorker() && !currentCommandIdUsed));
 		currentCommandIdUsed = true;
 	}
 	return currentCommandId;
 }
 
+/*
+ *	SetCurrentCommandIdUsedForWorker
+ *
+ * For a parallel worker, record that the currentCommandId has been used. This
+ * must only be called at the start of a parallel operation.
+ */
+void
+SetCurrentCommandIdUsedForWorker(void)
+{
+	Assert(IsParallelWorker() && !currentCommandIdUsed && currentCommandId != InvalidCommandId);
+	currentCommandIdUsed = true;
+}
+
 /*
  *	SetParallelStartTimestamps
  *
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 6bf6c5a310..093557cf86 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,18 +51,6 @@
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 
-typedef struct
-{
-	DestReceiver pub;			/* publicly-known function pointers */
-	IntoClause *into;			/* target relation specification */
-	/* These fields are filled by intorel_startup: */
-	Relation	rel;			/* relation to write to */
-	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
-	CommandId	output_cid;		/* cmin to insert in output tuples */
-	int			ti_options;		/* table_tuple_insert performance options */
-	BulkInsertState bistate;	/* bulk insert state */
-} DR_intorel;
-
 /* utility functions for CTAS definition creation */
 static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into);
 static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -328,6 +316,15 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		query = linitial_node(Query, rewritten);
 		Assert(query->commandType == CMD_SELECT);
 
+		/*
+		 * Flag to let the planner know that the SELECT query is for CTAS. This
+		 * is used to calculate the tuple transfer cost from workers to gather
+		 * node(in case parallelism kicks in for the SELECT part of the CTAS),
+		 * to zero as each worker will insert its share of tuples in parallel.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, NULL))
+			query->isForCTAS = true;
+
 		/* plan the query */
 		plan = pg_plan_query(query, pstate->p_sourcetext,
 							 CURSOR_OPT_PARALLEL_OK, params);
@@ -350,6 +347,15 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		/* call ExecutorStart to prepare the plan for execution */
 		ExecutorStart(queryDesc, GetIntoRelEFlags(into));
 
+		/*
+		 * If SELECT part of the CTAS is parallelizable, then make each
+		 * parallel worker insert the tuples that are resulted in its execution
+		 * into the target table. We need plan state to be initialized by the
+		 * executor to decide whether to allow parallel inserts or not.
+		*/
+		if (IsParallelInsertInCTASAllowed(into, queryDesc))
+			SetCTASParallelInsertState(queryDesc);
+
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
 
@@ -418,6 +424,9 @@ CreateIntoRelDestReceiver(IntoClause *intoClause)
 	self->pub.rDestroy = intorel_destroy;
 	self->pub.mydest = DestIntoRel;
 	self->into = intoClause;
+	self->is_parallel = false;
+	self->is_parallel_worker = false;
+	self->object_id = InvalidOid;
 	/* other private fields will be set during intorel_startup */
 
 	return (DestReceiver *) self;
@@ -430,121 +439,169 @@ static void
 intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 {
 	DR_intorel *myState = (DR_intorel *) self;
-	IntoClause *into = myState->into;
-	bool		is_matview;
-	List	   *attrList;
 	ObjectAddress intoRelationAddr;
 	Relation	intoRelationDesc;
-	ListCell   *lc;
-	int			attnum;
-
-	Assert(into != NULL);		/* else somebody forgot to set it */
 
-	/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
-	is_matview = (into->viewQuery != NULL);
+	if (myState->is_parallel_worker)
+	{
+		/* In the worker */
+		intoRelationDesc = table_open(myState->object_id, AccessExclusiveLock);
+		myState->rel = intoRelationDesc;
+		myState->reladdr = InvalidObjectAddress;
+		myState->ti_options = 0;
+		myState->bistate = GetBulkInsertState();
 
-	/*
-	 * Build column definitions using "pre-cooked" type and collation info. If
-	 * a column name list was specified in CREATE TABLE AS, override the
-	 * column names derived from the query.  (Too few column names are OK, too
-	 * many are not.)
-	 */
-	attrList = NIL;
-	lc = list_head(into->colNames);
-	for (attnum = 0; attnum < typeinfo->natts; attnum++)
+		/*
+		 * Right after the table is created in the leader, the command id is
+		 * incremented (in create_ctas_internal()). The new command id is
+		 * marked as used in intorel_startup(), then the parallel mode is
+		 * entered. The command id and transaction id are serialized into
+		 * parallel DSM, they are then available to all parallel workers. All
+		 * the workers need to mark the command id as used before insertion.
+		 */
+		SetCurrentCommandIdUsedForWorker();
+		myState->output_cid = GetCurrentCommandId(false);
+	}
+	else
 	{
-		Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
-		ColumnDef  *col;
-		char	   *colname;
+		IntoClause *into = myState->into;
+		bool		is_matview;
+		List	   *attrList;
+		ListCell   *lc;
+		int			attnum;
+
+		Assert(into != NULL);		/* else somebody forgot to set it */
+
+		/*
+		 * This code supports both CREATE TABLE AS and CREATE MATERIALIZED
+		 * VIEW.
+		 */
+		is_matview = (into->viewQuery != NULL);
 
-		if (lc)
+		/*
+		 * Build column definitions using "pre-cooked" type and collation info.
+		 * If a column name list was specified in CREATE TABLE AS, override the
+		 * column names derived from the query.  (Too few column names are OK,
+		 * too many are not.)
+		 */
+		attrList = NIL;
+		lc = list_head(into->colNames);
+		for (attnum = 0; attnum < typeinfo->natts; attnum++)
 		{
-			colname = strVal(lfirst(lc));
-			lc = lnext(into->colNames, lc);
+			Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
+			ColumnDef  *col;
+			char	   *colname;
+
+			if (lc)
+			{
+				colname = strVal(lfirst(lc));
+				lc = lnext(into->colNames, lc);
+			}
+			else
+				colname = NameStr(attribute->attname);
+
+			col = makeColumnDef(colname,
+								attribute->atttypid,
+								attribute->atttypmod,
+								attribute->attcollation);
+
+			/*
+			 * It's possible that the column is of a collatable type but the
+			 * collation could not be resolved, so double-check.  (We must
+			 * check this here because DefineRelation would adopt the type's
+			 * default collation rather than complaining.)
+			 */
+			if (!OidIsValid(col->collOid) &&
+				type_is_collatable(col->typeName->typeOid))
+				ereport(ERROR,
+						(errcode(ERRCODE_INDETERMINATE_COLLATION),
+						errmsg("no collation was derived for column \"%s\" with collatable type %s",
+								col->colname,
+								format_type_be(col->typeName->typeOid)),
+						errhint("Use the COLLATE clause to set the collation explicitly.")));
+
+			attrList = lappend(attrList, col);
 		}
-		else
-			colname = NameStr(attribute->attname);
 
-		col = makeColumnDef(colname,
-							attribute->atttypid,
-							attribute->atttypmod,
-							attribute->attcollation);
+		if (lc != NULL)
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					errmsg("too many column names were specified")));
 
 		/*
-		 * It's possible that the column is of a collatable type but the
-		 * collation could not be resolved, so double-check.  (We must check
-		 * this here because DefineRelation would adopt the type's default
-		 * collation rather than complaining.)
+		 * Actually create the target table
 		 */
-		if (!OidIsValid(col->collOid) &&
-			type_is_collatable(col->typeName->typeOid))
-			ereport(ERROR,
-					(errcode(ERRCODE_INDETERMINATE_COLLATION),
-					 errmsg("no collation was derived for column \"%s\" with collatable type %s",
-							col->colname,
-							format_type_be(col->typeName->typeOid)),
-					 errhint("Use the COLLATE clause to set the collation explicitly.")));
+		intoRelationAddr = create_ctas_internal(attrList, into);
 
-		attrList = lappend(attrList, col);
-	}
+		/*
+		 * Finally we can open the target table
+		 */
+		intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
 
-	if (lc != NULL)
-		ereport(ERROR,
-				(errcode(ERRCODE_SYNTAX_ERROR),
-				 errmsg("too many column names were specified")));
+		/*
+		 * Make sure the constructed table does not have RLS enabled.
+		 *
+		 * check_enable_rls() will ereport(ERROR) itself if the user has
+		 * requested something invalid, and otherwise will return RLS_ENABLED
+		 * if RLS should be enabled here.  We don't actually support that
+		 * currently, so throw our own ereport(ERROR) if that happens.
+		 */
+		if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					errmsg("policies not yet implemented for this command")));
 
-	/*
-	 * Actually create the target table
-	 */
-	intoRelationAddr = create_ctas_internal(attrList, into);
+		/*
+		 * Tentatively mark the target as populated, if it's a matview and
+		 * we're going to fill it; otherwise, no change needed.
+		 */
+		if (is_matview && !into->skipData)
+			SetMatViewPopulatedState(intoRelationDesc, true);
 
-	/*
-	 * Finally we can open the target table
-	 */
-	intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
+		/*
+		 * Fill private fields of myState for use by later routines
+		 */
+		myState->rel = intoRelationDesc;
+		myState->reladdr = intoRelationAddr;
+		myState->output_cid = GetCurrentCommandId(true);
+		myState->ti_options = TABLE_INSERT_SKIP_FSM;
 
-	/*
-	 * Make sure the constructed table does not have RLS enabled.
-	 *
-	 * check_enable_rls() will ereport(ERROR) itself if the user has requested
-	 * something invalid, and otherwise will return RLS_ENABLED if RLS should
-	 * be enabled here.  We don't actually support that currently, so throw
-	 * our own ereport(ERROR) if that happens.
-	 */
-	if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
-		ereport(ERROR,
-				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-				 errmsg("policies not yet implemented for this command")));
+		/*
+		 * If WITH NO DATA is specified, there is no need to set up the state
+		 * for bulk inserts as there are no tuples to insert.
+		 */
+		if (!into->skipData)
+			myState->bistate = GetBulkInsertState();
+		else
+			myState->bistate = NULL;
 
-	/*
-	 * Tentatively mark the target as populated, if it's a matview and we're
-	 * going to fill it; otherwise, no change needed.
-	 */
-	if (is_matview && !into->skipData)
-		SetMatViewPopulatedState(intoRelationDesc, true);
+		if (myState->is_parallel)
+		{
+			myState->object_id = intoRelationAddr.objectId;
 
-	/*
-	 * Fill private fields of myState for use by later routines
-	 */
-	myState->rel = intoRelationDesc;
-	myState->reladdr = intoRelationAddr;
-	myState->output_cid = GetCurrentCommandId(true);
-	myState->ti_options = TABLE_INSERT_SKIP_FSM;
+			/*
+			 * We don't need to skip contacting FSM while inserting tuples
+			 * for parallel mode, while extending the relations, workers
+			 * instead of blocking on a page while another worker is inserting,
+			 * can check the FSM for another page that can accommodate the
+			 * tuples. This results in major benefit for parallel inserts.
+			 */
+			myState->ti_options = 0;
 
-	/*
-	 * If WITH NO DATA is specified, there is no need to set up the state for
-	 * bulk inserts as there are no tuples to insert.
-	 */
-	if (!into->skipData)
-		myState->bistate = GetBulkInsertState();
-	else
-		myState->bistate = NULL;
+			/*
+			 * rd_createSubid is marked invalid, otherwise, the table is not
+			 * allowed to be extended by the workers.
+			 */
+			myState->rel->rd_createSubid = InvalidSubTransactionId;
+		}
 
-	/*
-	 * Valid smgr_targblock implies something already wrote to the relation.
-	 * This may be harmless, but this function hasn't planned for it.
-	 */
-	Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+		/*
+		 * Valid smgr_targblock implies something already wrote to the
+		 * relation. This may be harmless, but this function hasn't planned for
+		 * it.
+		 */
+		Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+	}
 }
 
 /*
@@ -606,3 +663,76 @@ intorel_destroy(DestReceiver *self)
 {
 	pfree(self);
 }
+
+/*
+ * IsParallelInsertInCTASAllowed --- determine whether or not parallel
+ * insertion is possible.
+ */
+bool IsParallelInsertInCTASAllowed(IntoClause *into, QueryDesc *queryDesc)
+{
+	if (!IS_CTAS(into))
+		return false;
+
+	/*
+	 * Do not allow parallel inserts if the table is temporary. As the
+	 * temporary tables are backend local, workers can not know about them.
+	 * Currently, CTAS supports creation of normal(logged), temporary and
+	 * unlogged tables. It does not support foreign or partition table
+	 * creation. Hence the check for temporary table is enough here.
+	 */
+	if (!into->rel || into->rel->relpersistence == RELPERSISTENCE_TEMP)
+		return false;
+
+	if (queryDesc)
+	{
+		PlanState  *ps = queryDesc->planstate;
+		PlannedStmt *plannedstmt = queryDesc->plannedstmt;
+		bool  allow;
+
+		/*
+		 * We allow parallel inserts by the workers only if the Gather node has
+		 * no projections to perform and if the upper node is Gather. In case,
+		 * the Gather node has projections, which is possible if there are any
+		 * subplans in the query, the workers can not do those projections. And
+		 * when the upper node is GatherMerge, then the leader has to perform
+		 * the final phase i.e. merge the results by workers.
+		 */
+		allow = ps && IsA(ps, GatherState) && !ps->ps_ProjInfo &&
+				plannedstmt->parallelModeNeeded &&
+				plannedstmt->planTree &&
+				IsA(plannedstmt->planTree, Gather) &&
+				plannedstmt->planTree->lefttree &&
+				plannedstmt->planTree->lefttree->parallel_aware &&
+				plannedstmt->planTree->lefttree->parallel_safe;
+
+		return allow;
+	}
+
+	return false;
+}
+
+/*
+ * SetCTASParallelInsertState --- set the required info for the parallel
+ * inserts, that is required in the plan exection.
+ */
+void SetCTASParallelInsertState(QueryDesc *queryDesc)
+{
+	GatherState *gstate = (GatherState *) queryDesc->planstate;
+
+	/*
+	 * For parallelizing inserts in CTAS i.e. making each parallel worker
+	 * insert the tuples, we must send information such as intoclause(for
+	 * eachworker to build separate dest receiver), object id(for each worker
+	 * to open the created table).
+	 */
+	((DR_intorel *) queryDesc->dest)->is_parallel = true;
+	gstate->dest = queryDesc->dest;
+
+	/*
+	 * Since there are no rows that are transferred from workers to Gather
+	 * node, so we set it to 0 to be visible in explain plans. Note that we
+	 * would have already accounted this for cost calculations in
+	 * cost_gather().
+	 */
+	queryDesc->plannedstmt->planTree->plan_rows = 0;
+}
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 43f9b01e83..57875e1505 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -371,6 +371,15 @@ ExplainOneQuery(Query *query, int cursorOptions,
 		return;
 	}
 
+	/*
+	 * Flag to let the planner know that the SELECT query is for CTAS. This is
+	 * used to calculate the tuple transfer cost from workers to gather node(in
+	 * case parallelism kicks in for the SELECT part of the CTAS), to zero as
+	 * each worker will insert its share of tuples in parallel.
+	 */
+	if (IsParallelInsertInCTASAllowed(into, NULL))
+		query->isForCTAS = true;
+
 	/* if an advisor plugin is present, let it manage things */
 	if (ExplainOneQuery_hook)
 		(*ExplainOneQuery_hook) (query, cursorOptions, into, es,
@@ -556,6 +565,15 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	/* call ExecutorStart to prepare the plan for execution */
 	ExecutorStart(queryDesc, eflags);
 
+	/*
+	 * If SELECT part of the CTAS is parallelizable, then make each parallel
+	 * worker insert the tuples that are resulted in its execution into the
+	 * target table. We need plan state to be initialized by the executor to
+	 * decide whether to allow parallel inserts or not.
+	 */
+	if (IsParallelInsertInCTASAllowed(into, queryDesc))
+		SetCTASParallelInsertState(queryDesc);
+
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
 	{
@@ -1775,6 +1793,29 @@ ExplainNode(PlanState *planstate, List *ancestors,
 
 				if (gather->single_copy || es->format != EXPLAIN_FORMAT_TEXT)
 					ExplainPropertyBool("Single Copy", gather->single_copy, es);
+
+				/*
+				 * Show the create table information under Gather node in case
+				 * parallel workers have inserted the rows.
+				 */
+				if (IsA(planstate, GatherState))
+				{
+					GatherState *gstate = (GatherState *) planstate;
+
+					if (IS_PARALLEL_CTAS_DEST(gstate->dest) &&
+						((DR_intorel *) gstate->dest)->into->rel &&
+						((DR_intorel *) gstate->dest)->into->rel->relname)
+					{
+						es->indent--;
+						ExplainIndentText(es);
+						appendStringInfoString(es->str, "->  ");
+						appendStringInfoString(es->str, "Create ");
+						appendStringInfo(es->str, "%s\n",
+							((DR_intorel *) gstate->dest)->into->rel->relname);
+						ExplainIndentText(es);
+						es->indent++;
+					}
+				}
 			}
 			break;
 		case T_GatherMerge:
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index befde52691..9ef33eee54 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -23,6 +23,7 @@
 
 #include "postgres.h"
 
+#include "commands/createas.h"
 #include "executor/execParallel.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
@@ -65,6 +66,7 @@
 #define PARALLEL_KEY_QUERY_TEXT		UINT64CONST(0xE000000000000008)
 #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009)
 #define PARALLEL_KEY_WAL_USAGE			UINT64CONST(0xE00000000000000A)
+#define PARALLEL_KEY_INTO_CLAUSE		UINT64CONST(0xE00000000000000B)
 
 #define PARALLEL_TUPLE_QUEUE_SIZE		65536
 
@@ -77,6 +79,9 @@ typedef struct FixedParallelExecutorState
 	dsa_pointer param_exec;
 	int			eflags;
 	int			jit_flags;
+	Oid			objectid;		/* workers to open relation/table.  */
+	/* Number tuples inserted by all the workers. */
+	pg_atomic_uint64	processed;
 } FixedParallelExecutorState;
 
 /*
@@ -578,7 +583,8 @@ ExecParallelSetupTupleQueues(ParallelContext *pcxt, bool reinitialize)
 ParallelExecutorInfo *
 ExecInitParallelPlan(PlanState *planstate, EState *estate,
 					 Bitmapset *sendParams, int nworkers,
-					 int64 tuples_needed)
+					 int64 tuples_needed, IntoClause *intoclause,
+					 Oid objectid)
 {
 	ParallelExecutorInfo *pei;
 	ParallelContext *pcxt;
@@ -600,6 +606,8 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	Size		dsa_minsize = dsa_minimum_size();
 	char	   *query_string;
 	int			query_len;
+	char 		*intoclausestr = NULL;
+	int			intoclause_len = 0;
 
 	/*
 	 * Force any initplan outputs that we're going to pass to workers to be
@@ -712,6 +720,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_estimate_chunk(&pcxt->estimator, dsa_minsize);
 	shm_toc_estimate_keys(&pcxt->estimator, 1);
 
+	/* Estimate space for into clause for CTAS. */
+	if (IS_CTAS(intoclause) && OidIsValid(objectid))
+	{
+		intoclausestr = nodeToString(intoclause);
+		intoclause_len = strlen(intoclausestr);
+		shm_toc_estimate_chunk(&pcxt->estimator, intoclause_len + 1);
+		shm_toc_estimate_keys(&pcxt->estimator, 1);
+	}
+
 	/* Everyone's had a chance to ask for space, so now create the DSM. */
 	InitializeParallelDSM(pcxt);
 
@@ -729,6 +746,14 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	fpes->param_exec = InvalidDsaPointer;
 	fpes->eflags = estate->es_top_eflags;
 	fpes->jit_flags = estate->es_jit_flags;
+	pg_atomic_init_u64(&fpes->processed, 0);
+	pei->processed = &fpes->processed;
+
+	if (intoclausestr && OidIsValid(objectid))
+		fpes->objectid = objectid;
+	else
+		fpes->objectid = InvalidOid;
+
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, fpes);
 
 	/* Store query string */
@@ -758,8 +783,18 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_WAL_USAGE, walusage_space);
 	pei->wal_usage = walusage_space;
 
-	/* Set up the tuple queues that the workers will write into. */
-	pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	if (intoclausestr)
+	{
+		char *intoclause_space = shm_toc_allocate(pcxt->toc,
+												  intoclause_len + 1);
+		memcpy(intoclause_space, intoclausestr, intoclause_len + 1);
+		shm_toc_insert(pcxt->toc, PARALLEL_KEY_INTO_CLAUSE, intoclause_space);
+	}
+	else
+	{
+		/* Set up the tuple queues that the workers will write into. */
+		pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	}
 
 	/* We don't need the TupleQueueReaders yet, though. */
 	pei->reader = NULL;
@@ -1387,12 +1422,30 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 	void	   *area_space;
 	dsa_area   *area;
 	ParallelWorkerContext pwcxt;
+	char		*intoclausestr = NULL;
+	IntoClause	*intoclause = NULL;
 
 	/* Get fixed-size state. */
 	fpes = shm_toc_lookup(toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
 
-	/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
-	receiver = ExecParallelGetReceiver(seg, toc);
+	intoclausestr = shm_toc_lookup(toc, PARALLEL_KEY_INTO_CLAUSE, true);
+	if (intoclausestr)
+	{
+		/*
+		 * If the worker is for parallel insert in CTAS, then use the proper
+		 * dest receiver.
+		 */
+		intoclause = (IntoClause *) stringToNode(intoclausestr);
+		receiver = CreateIntoRelDestReceiver(intoclause);
+		((DR_intorel *)receiver)->is_parallel_worker = true;
+		((DR_intorel *)receiver)->object_id = fpes->objectid;
+	}
+	else
+	{
+		/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
+		receiver = ExecParallelGetReceiver(seg, toc);
+	}
+
 	instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_INSTRUMENTATION, true);
 	if (instrumentation != NULL)
 		instrument_options = instrumentation->instrument_options;
@@ -1471,6 +1524,13 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 			queryDesc->estate->es_jit->instr;
 	}
 
+	/*
+	 * Write out the number of tuples this worker has inserted. Leader will use
+	 * it to inform the end client.
+	 */
+	if (intoclausestr)
+		pg_atomic_add_fetch_u64(&fpes->processed, queryDesc->estate->es_processed);
+
 	/* Must do this after capturing instrumentation. */
 	ExecutorEnd(queryDesc);
 
diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c
index a01b46af14..9cef6bdd61 100644
--- a/src/backend/executor/nodeGather.c
+++ b/src/backend/executor/nodeGather.c
@@ -32,6 +32,7 @@
 
 #include "access/relscan.h"
 #include "access/xact.h"
+#include "commands/createas.h"
 #include "executor/execdebug.h"
 #include "executor/execParallel.h"
 #include "executor/nodeGather.h"
@@ -48,6 +49,7 @@ static TupleTableSlot *ExecGather(PlanState *pstate);
 static TupleTableSlot *gather_getnext(GatherState *gatherstate);
 static MinimalTuple gather_readnext(GatherState *gatherstate);
 static void ExecShutdownGatherWorkers(GatherState *node);
+static void ExecParallelInsertInCTAS(GatherState *node);
 
 
 /* ----------------------------------------------------------------
@@ -131,6 +133,73 @@ ExecInitGather(Gather *node, EState *estate, int eflags)
 	return gatherstate;
 }
 
+/* ----------------------------------------------------------------
+ *		ExecParallelInsertInCTAS(node)
+ *
+ *		Facilitates parallel inserts by parallel workers and/or
+ *		leader for Create Table AS.
+ * ----------------------------------------------------------------
+ */
+static void
+ExecParallelInsertInCTAS(GatherState *node)
+{
+	/* Enable leader to insert in case no parallel workers were launched. */
+	if (node->nworkers_launched == 0)
+		node->need_to_scan_locally = true;
+
+	/*
+	 * By now, for parallel workers (if launched any), would have started their
+	 * work i.e. insertion to target table. In case the leader is chosen to
+	 * participate for parallel inserts in CTAS, then finish its share before
+	 * going to wait for the parallel workers to finish.
+	 */
+	if (node->need_to_scan_locally)
+	{
+		EState	   *estate = node->ps.state;
+		TupleTableSlot *outerTupleSlot;
+
+		for(;;)
+		{
+			/* Install our DSA area while executing the plan. */
+			estate->es_query_dsa =
+					node->pei ? node->pei->area : NULL;
+
+			outerTupleSlot = ExecProcNode(node->ps.lefttree);
+
+			estate->es_query_dsa = NULL;
+
+			if(TupIsNull(outerTupleSlot))
+				break;
+
+			(void) node->dest->receiveSlot(outerTupleSlot, node->dest);
+
+			node->ps.state->es_processed++;
+		}
+
+		node->need_to_scan_locally = false;
+	}
+
+	if (node->nworkers_launched > 0)
+	{
+		int		i;
+
+		/* Wait for the parallel workers to finish. */
+		WaitForParallelWorkersToFinish(node->pei->pcxt);
+
+		for (i = 0; i < node->nworkers_launched; i++)
+		{
+			InstrAccumParallelQuery(&node->pei->buffer_usage[i],
+									&node->pei->wal_usage[i]);
+		}
+
+		/*
+		 * Add up the total tuples inserted by all workers, to the tuples
+		 * inserted by the leader(if any). This will be shared to client.
+		 */
+		node->ps.state->es_processed += pg_atomic_read_u64(node->pei->processed);
+	}
+}
+
 /* ----------------------------------------------------------------
  *		ExecGather(node)
  *
@@ -157,6 +226,7 @@ ExecGather(PlanState *pstate)
 	{
 		EState	   *estate = node->ps.state;
 		Gather	   *gather = (Gather *) node->ps.plan;
+		bool		isctas = IS_PARALLEL_CTAS_DEST(node->dest);
 
 		/*
 		 * Sometimes we might have to run without parallelism; but if parallel
@@ -165,6 +235,18 @@ ExecGather(PlanState *pstate)
 		if (gather->num_workers > 0 && estate->es_use_parallel_mode)
 		{
 			ParallelContext *pcxt;
+			IntoClause	*intoclause = NULL;
+			Oid			objectid = InvalidOid;
+
+			/*
+			 * Take the necessary information to be passed to workers for
+			 * parallel inserts in CTAS.
+			 */
+			if (isctas)
+			{
+				intoclause = ((DR_intorel *) node->dest)->into;
+				objectid = ((DR_intorel *) node->dest)->object_id;
+			}
 
 			/* Initialize, or re-initialize, shared state needed by workers. */
 			if (!node->pei)
@@ -172,7 +254,10 @@ ExecGather(PlanState *pstate)
 												 estate,
 												 gather->initParam,
 												 gather->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 /* CTAS info */
+												 intoclause,
+												 objectid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
@@ -190,13 +275,16 @@ ExecGather(PlanState *pstate)
 			/* Set up tuple queue readers to read the results. */
 			if (pcxt->nworkers_launched > 0)
 			{
-				ExecParallelCreateReaders(node->pei);
-				/* Make a working array showing the active readers */
-				node->nreaders = pcxt->nworkers_launched;
-				node->reader = (TupleQueueReader **)
-					palloc(node->nreaders * sizeof(TupleQueueReader *));
-				memcpy(node->reader, node->pei->reader,
-					   node->nreaders * sizeof(TupleQueueReader *));
+				if (!isctas)
+				{
+					ExecParallelCreateReaders(node->pei);
+					/* Make a working array showing the active readers */
+					node->nreaders = pcxt->nworkers_launched;
+					node->reader = (TupleQueueReader **)
+						palloc(node->nreaders * sizeof(TupleQueueReader *));
+					memcpy(node->reader, node->pei->reader,
+						node->nreaders * sizeof(TupleQueueReader *));
+				}
 			}
 			else
 			{
@@ -208,9 +296,17 @@ ExecGather(PlanState *pstate)
 		}
 
 		/* Run plan locally if no workers or enabled and not single-copy. */
-		node->need_to_scan_locally = (node->nreaders == 0)
+		node->need_to_scan_locally = (node->nreaders == 0 &&
+			!isctas)
 			|| (!gather->single_copy && parallel_leader_participation);
 		node->initialized = true;
+
+		/* Perform parallel inserts for CTAS. */
+		if (isctas)
+		{
+			ExecParallelInsertInCTAS(node);
+			return NULL;
+		}
 	}
 
 	/*
diff --git a/src/backend/executor/nodeGatherMerge.c b/src/backend/executor/nodeGatherMerge.c
index 47129344f3..ee45272c17 100644
--- a/src/backend/executor/nodeGatherMerge.c
+++ b/src/backend/executor/nodeGatherMerge.c
@@ -216,7 +216,9 @@ ExecGatherMerge(PlanState *pstate)
 												 estate,
 												 gm->initParam,
 												 gm->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 NULL,
+												 InvalidOid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 22d6935824..225f3cbf01 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -393,7 +393,18 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 
 	/* Parallel setup and communication cost. */
 	startup_cost += parallel_setup_cost;
-	run_cost += parallel_tuple_cost * path->path.rows;
+
+	/*
+	 * We do not compute the parallel_tuple_cost for CTAS because the number of
+	 * tuples that are transferred from workers to the gather node is zero as
+	 * each worker, in parallel, inserts the tuples that are resulted from its
+	 * chunk of plan execution. This change may make the parallel plan cheap
+	 * among all other plans, and influence the planner to consider this
+	 * parallel plan.
+	 */
+	if (!(root->parse->isForCTAS &&
+		root->query_level == 1))
+		run_cost += parallel_tuple_cost * path->path.rows;
 
 	path->path.startup_cost = startup_cost;
 	path->path.total_cost = (startup_cost + run_cost);
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7320de345c..5beae6c617 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -389,6 +389,7 @@ extern FullTransactionId GetCurrentFullTransactionIdIfAny(void);
 extern void MarkCurrentTransactionIdLoggedIfAny(void);
 extern bool SubTransactionIsActive(SubTransactionId subxid);
 extern CommandId GetCurrentCommandId(bool used);
+extern void SetCurrentCommandIdUsedForWorker(void);
 extern void SetParallelStartTimestamps(TimestampTz xact_ts, TimestampTz stmt_ts);
 extern TimestampTz GetCurrentTransactionStartTimestamp(void);
 extern TimestampTz GetCurrentStatementStartTimestamp(void);
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 7629230254..ab3aab58c5 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -14,12 +14,35 @@
 #ifndef CREATEAS_H
 #define CREATEAS_H
 
+#include "access/heapam.h"
 #include "catalog/objectaddress.h"
+#include "executor/execdesc.h"
 #include "nodes/params.h"
+#include "nodes/plannodes.h"
 #include "parser/parse_node.h"
 #include "tcop/dest.h"
 #include "utils/queryenvironment.h"
 
+typedef struct
+{
+	DestReceiver pub;			/* publicly-known function pointers */
+	IntoClause *into;			/* target relation specification */
+	/* These fields are filled by intorel_startup: */
+	Relation	rel;			/* relation to write to */
+	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
+	CommandId	output_cid;		/* cmin to insert in output tuples */
+	int			ti_options;		/* table_tuple_insert performance options */
+	BulkInsertState bistate;	/* bulk insert state */
+	bool		is_parallel;	/* is parallelism to be considered? */
+	bool		is_parallel_worker; /* true for parallel worker */
+	/* Used for table open by parallel worker. */
+	Oid			object_id;
+} DR_intorel;
+
+#define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
+#define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
+									 IS_CTAS(((DR_intorel *) dest)->into) && \
+									 ((DR_intorel *) dest)->is_parallel)
 
 extern ObjectAddress ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 									   ParamListInfo params, QueryEnvironment *queryEnv,
@@ -29,4 +52,9 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
+extern bool IsParallelInsertInCTASAllowed(IntoClause *into,
+										  QueryDesc *queryDesc);
+
+extern void SetCTASParallelInsertState(QueryDesc *queryDesc);
+
 #endif							/* CREATEAS_H */
diff --git a/src/include/executor/execParallel.h b/src/include/executor/execParallel.h
index 5a39a5b29c..9f959f741b 100644
--- a/src/include/executor/execParallel.h
+++ b/src/include/executor/execParallel.h
@@ -35,11 +35,15 @@ typedef struct ParallelExecutorInfo
 	/* These two arrays have pcxt->nworkers_launched entries: */
 	shm_mq_handle **tqueue;		/* tuple queues for worker output */
 	struct TupleQueueReader **reader;	/* tuple reader/writer support */
+	/* Number of tuples inserted by all workers. */
+	volatile pg_atomic_uint64	*processed;
 } ParallelExecutorInfo;
 
 extern ParallelExecutorInfo *ExecInitParallelPlan(PlanState *planstate,
 												  EState *estate, Bitmapset *sendParam, int nworkers,
-												  int64 tuples_needed);
+												  int64 tuples_needed,
+												  IntoClause *intoclause,
+												  Oid objectid);
 extern void ExecParallelCreateReaders(ParallelExecutorInfo *pei);
 extern void ExecParallelFinish(ParallelExecutorInfo *pei);
 extern void ExecParallelCleanup(ParallelExecutorInfo *pei);
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 61ba4c3666..e9c4442c22 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -23,6 +23,7 @@
 #include "nodes/tidbitmap.h"
 #include "partitioning/partdefs.h"
 #include "storage/condition_variable.h"
+#include "tcop/dest.h"
 #include "utils/hsearch.h"
 #include "utils/queryenvironment.h"
 #include "utils/reltrigger.h"
@@ -2326,6 +2327,8 @@ typedef struct GatherState
 	int			nreaders;		/* number of still-active workers */
 	int			nextreader;		/* next one to try to read from */
 	struct TupleQueueReader **reader;	/* array with nreaders active entries */
+	/* Parallel inserts in CTAS related info is specified below. */
+	DestReceiver *dest;
 } GatherState;
 
 /* ----------------
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index ec14fc2036..eb267b1a6c 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -180,6 +180,7 @@ typedef struct Query
 	 */
 	int			stmt_location;	/* start location, or -1 if unknown */
 	int			stmt_len;		/* length in bytes; 0 means "rest of string" */
+	bool		isForCTAS; 		/* is the SELECT query for CREATE TABLE AS */
 } Query;
 
 
diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 0c4da2591a..635d24e76d 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -75,5 +75,195 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 (7 rows)
 
 create table parallel_write as execute prep_stmt;
+drop table parallel_write;
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+                     QUERY PLAN                      
+-----------------------------------------------------
+ LockRows (actual rows=10000 loops=1)
+   ->  Seq Scan on tenk1 (actual rows=10000 loops=1)
+(2 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_mat_view
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop materialized view parallel_mat_view;
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+deallocate parallel_write_prep;
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+select nextval('parallel_write_sequence'), four from tenk1;
+                  QUERY PLAN                   
+-----------------------------------------------
+ Seq Scan on tenk1 (actual rows=10000 loops=1)
+(1 row)
+
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;
+                               QUERY PLAN                               
+------------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Params Evaluated: $1
+   Workers Launched: 3
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=1 loops=1)
+           ->  Gather (actual rows=1 loops=1)
+                 Workers Planned: 4
+                 Workers Launched: 4
+                 ->  Parallel Seq Scan on tenk2 (actual rows=1 loops=5)
+   ->  Parallel Seq Scan on tenk1 (actual rows=625 loops=4)
+         Filter: (four = 3)
+         Rows Removed by Filter: 1875
+(14 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;
+                             QUERY PLAN                              
+---------------------------------------------------------------------
+ Gather (actual rows=2500 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=500 loops=5)
+         Filter: (four = 3)
+         Rows Removed by Filter: 1500
+   SubPlan 1
+     ->  Function Scan on generate_series (actual rows=1 loops=2500)
+(8 rows)
+
 drop table parallel_write;
 rollback;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 78b479cedf..e65a56b442 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -39,4 +39,88 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
 
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+select length(stringu1) from tenk1;
+drop materialized view parallel_mat_view;
+
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+deallocate parallel_write_prep;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+select nextval('parallel_write_sequence'), four from tenk1;
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;
+drop table parallel_write;
+
 rollback;
-- 
2.25.1

From 21f8da6da53ef35ec076202f1f13ee96e01f5de1 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 7 Dec 2020 18:39:13 +0530
Subject: [PATCH v9] Parallel Inserts in CREATE TABLE AS

The idea of this patch is to allow the leader and each worker
insert the tuples in parallel if the SELECT part of the CTAS is
parallelizable.

The design:
Let the planner know that the SELECT is from CTAS in createas.c
so that it can set the number of tuples transferred from the
workers to Gather node to 0. With this change, there are chances
that the planner may choose the parallel plan. After the planning,
check if the upper plan node is Gather in createas.c and mark a
parallelism flag in the CTAS dest receiver. Pass the into clause,
object id, command id from the leader to workers, so that each
worker can create its own CTAS dest receiver. Leader inserts its
share of tuples if instructed to do, and so are workers. Each
worker writes atomically its number of inserted tuples into a
shared memory variable, the leader combines this with its own
number of inserted tuples and shares to the client.
---
 src/backend/access/heap/heapam.c             |  11 -
 src/backend/access/transam/xact.c            |  30 +-
 src/backend/commands/createas.c              | 349 +++++++++++++------
 src/backend/commands/explain.c               |  41 +++
 src/backend/executor/execParallel.c          |  70 +++-
 src/backend/executor/nodeGather.c            | 114 +++++-
 src/backend/executor/nodeGatherMerge.c       |   4 +-
 src/backend/optimizer/path/costsize.c        |  13 +-
 src/include/access/xact.h                    |   1 +
 src/include/commands/createas.h              |  28 ++
 src/include/executor/execParallel.h          |   6 +-
 src/include/nodes/execnodes.h                |   3 +
 src/include/nodes/parsenodes.h               |   1 +
 src/test/regress/expected/write_parallel.out | 190 ++++++++++
 src/test/regress/sql/write_parallel.sql      |  84 +++++
 15 files changed, 804 insertions(+), 141 deletions(-)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 1b2f70499e..3045c0f046 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2043,17 +2043,6 @@ static HeapTuple
 heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
 					CommandId cid, int options)
 {
-	/*
-	 * To allow parallel inserts, we need to ensure that they are safe to be
-	 * performed in workers. We have the infrastructure to allow parallel
-	 * inserts in general except for the cases where inserts generate a new
-	 * CommandId (eg. inserts into a table having a foreign key column).
-	 */
-	if (IsParallelWorker())
-		ereport(ERROR,
-				(errcode(ERRCODE_INVALID_TRANSACTION_STATE),
-				 errmsg("cannot insert tuples in a parallel worker")));
-
 	tup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
 	tup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
 	tup->t_data->t_infomask |= HEAP_XMAX_INVALID;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 9cd0b7c11b..db6eedd635 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -763,18 +763,34 @@ GetCurrentCommandId(bool used)
 	/* this is global to a transaction, not subtransaction-local */
 	if (used)
 	{
-		/*
-		 * Forbid setting currentCommandIdUsed in a parallel worker, because
-		 * we have no provision for communicating this back to the leader.  We
-		 * could relax this restriction when currentCommandIdUsed was already
-		 * true at the start of the parallel operation.
-		 */
-		Assert(!IsParallelWorker());
+		 /*
+		  * This is a temporary hack for all common parallel insert cases i.e.
+		  * insert into, ctas, copy from. To be changed later. In a parallel
+		  * worker, set currentCommandIdUsed to true only if it was not set to
+		  * true at the start of the parallel operation (by way of
+		  * SetCurrentCommandIdUsedForWorker()). We have to do this because
+		  * GetCurrentCommandId(true) may be called from anywhere, especially
+		  * for parallel inserts, within parallel worker.
+		  */
+		Assert(!(IsParallelWorker() && !currentCommandIdUsed));
 		currentCommandIdUsed = true;
 	}
 	return currentCommandId;
 }
 
+/*
+ *	SetCurrentCommandIdUsedForWorker
+ *
+ * For a parallel worker, record that the currentCommandId has been used. This
+ * must only be called at the start of a parallel operation.
+ */
+void
+SetCurrentCommandIdUsedForWorker(void)
+{
+	Assert(IsParallelWorker() && !currentCommandIdUsed && currentCommandId != InvalidCommandId);
+	currentCommandIdUsed = true;
+}
+
 /*
  *	SetParallelStartTimestamps
  *
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 6bf6c5a310..c455567a17 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,18 +51,6 @@
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 
-typedef struct
-{
-	DestReceiver pub;			/* publicly-known function pointers */
-	IntoClause *into;			/* target relation specification */
-	/* These fields are filled by intorel_startup: */
-	Relation	rel;			/* relation to write to */
-	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
-	CommandId	output_cid;		/* cmin to insert in output tuples */
-	int			ti_options;		/* table_tuple_insert performance options */
-	BulkInsertState bistate;	/* bulk insert state */
-} DR_intorel;
-
 /* utility functions for CTAS definition creation */
 static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into);
 static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -328,6 +316,15 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		query = linitial_node(Query, rewritten);
 		Assert(query->commandType == CMD_SELECT);
 
+		/*
+		 * Flag to let the planner know that the SELECT query is for CTAS. This
+		 * is used to calculate the tuple transfer cost from workers to gather
+		 * node(in case parallelism kicks in for the SELECT part of the CTAS),
+		 * to zero as each worker will insert its share of tuples in parallel.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, NULL))
+			query->isForCTAS = true;
+
 		/* plan the query */
 		plan = pg_plan_query(query, pstate->p_sourcetext,
 							 CURSOR_OPT_PARALLEL_OK, params);
@@ -350,6 +347,15 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		/* call ExecutorStart to prepare the plan for execution */
 		ExecutorStart(queryDesc, GetIntoRelEFlags(into));
 
+		/*
+		 * If SELECT part of the CTAS is parallelizable, then make each
+		 * parallel worker insert the tuples that are resulted in its execution
+		 * into the target table. We need plan state to be initialized by the
+		 * executor to decide whether to allow parallel inserts or not.
+		*/
+		if (IsParallelInsertInCTASAllowed(into, queryDesc))
+			SetCTASParallelInsertState(queryDesc);
+
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
 
@@ -418,6 +424,9 @@ CreateIntoRelDestReceiver(IntoClause *intoClause)
 	self->pub.rDestroy = intorel_destroy;
 	self->pub.mydest = DestIntoRel;
 	self->into = intoClause;
+	self->is_parallel = false;
+	self->is_parallel_worker = false;
+	self->object_id = InvalidOid;
 	/* other private fields will be set during intorel_startup */
 
 	return (DestReceiver *) self;
@@ -430,121 +439,169 @@ static void
 intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 {
 	DR_intorel *myState = (DR_intorel *) self;
-	IntoClause *into = myState->into;
-	bool		is_matview;
-	List	   *attrList;
 	ObjectAddress intoRelationAddr;
 	Relation	intoRelationDesc;
-	ListCell   *lc;
-	int			attnum;
-
-	Assert(into != NULL);		/* else somebody forgot to set it */
 
-	/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
-	is_matview = (into->viewQuery != NULL);
+	if (myState->is_parallel_worker)
+	{
+		/* In the worker */
+		intoRelationDesc = table_open(myState->object_id, AccessExclusiveLock);
+		myState->rel = intoRelationDesc;
+		myState->reladdr = InvalidObjectAddress;
+		myState->ti_options = 0;
+		myState->bistate = GetBulkInsertState();
 
-	/*
-	 * Build column definitions using "pre-cooked" type and collation info. If
-	 * a column name list was specified in CREATE TABLE AS, override the
-	 * column names derived from the query.  (Too few column names are OK, too
-	 * many are not.)
-	 */
-	attrList = NIL;
-	lc = list_head(into->colNames);
-	for (attnum = 0; attnum < typeinfo->natts; attnum++)
+		/*
+		 * Right after the table is created in the leader, the command id is
+		 * incremented (in create_ctas_internal()). The new command id is
+		 * marked as used in intorel_startup(), then the parallel mode is
+		 * entered. The command id and transaction id are serialized into
+		 * parallel DSM, they are then available to all parallel workers. All
+		 * the workers need to mark the command id as used before insertion.
+		 */
+		SetCurrentCommandIdUsedForWorker();
+		myState->output_cid = GetCurrentCommandId(false);
+	}
+	else
 	{
-		Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
-		ColumnDef  *col;
-		char	   *colname;
+		IntoClause *into = myState->into;
+		bool		is_matview;
+		List	   *attrList;
+		ListCell   *lc;
+		int			attnum;
+
+		Assert(into != NULL);		/* else somebody forgot to set it */
+
+		/*
+		 * This code supports both CREATE TABLE AS and CREATE MATERIALIZED
+		 * VIEW.
+		 */
+		is_matview = (into->viewQuery != NULL);
 
-		if (lc)
+		/*
+		 * Build column definitions using "pre-cooked" type and collation info.
+		 * If a column name list was specified in CREATE TABLE AS, override the
+		 * column names derived from the query.  (Too few column names are OK,
+		 * too many are not.)
+		 */
+		attrList = NIL;
+		lc = list_head(into->colNames);
+		for (attnum = 0; attnum < typeinfo->natts; attnum++)
 		{
-			colname = strVal(lfirst(lc));
-			lc = lnext(into->colNames, lc);
+			Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
+			ColumnDef  *col;
+			char	   *colname;
+
+			if (lc)
+			{
+				colname = strVal(lfirst(lc));
+				lc = lnext(into->colNames, lc);
+			}
+			else
+				colname = NameStr(attribute->attname);
+
+			col = makeColumnDef(colname,
+								attribute->atttypid,
+								attribute->atttypmod,
+								attribute->attcollation);
+
+			/*
+			 * It's possible that the column is of a collatable type but the
+			 * collation could not be resolved, so double-check.  (We must
+			 * check this here because DefineRelation would adopt the type's
+			 * default collation rather than complaining.)
+			 */
+			if (!OidIsValid(col->collOid) &&
+				type_is_collatable(col->typeName->typeOid))
+				ereport(ERROR,
+						(errcode(ERRCODE_INDETERMINATE_COLLATION),
+						errmsg("no collation was derived for column \"%s\" with collatable type %s",
+								col->colname,
+								format_type_be(col->typeName->typeOid)),
+						errhint("Use the COLLATE clause to set the collation explicitly.")));
+
+			attrList = lappend(attrList, col);
 		}
-		else
-			colname = NameStr(attribute->attname);
 
-		col = makeColumnDef(colname,
-							attribute->atttypid,
-							attribute->atttypmod,
-							attribute->attcollation);
+		if (lc != NULL)
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					errmsg("too many column names were specified")));
 
 		/*
-		 * It's possible that the column is of a collatable type but the
-		 * collation could not be resolved, so double-check.  (We must check
-		 * this here because DefineRelation would adopt the type's default
-		 * collation rather than complaining.)
+		 * Actually create the target table
 		 */
-		if (!OidIsValid(col->collOid) &&
-			type_is_collatable(col->typeName->typeOid))
-			ereport(ERROR,
-					(errcode(ERRCODE_INDETERMINATE_COLLATION),
-					 errmsg("no collation was derived for column \"%s\" with collatable type %s",
-							col->colname,
-							format_type_be(col->typeName->typeOid)),
-					 errhint("Use the COLLATE clause to set the collation explicitly.")));
+		intoRelationAddr = create_ctas_internal(attrList, into);
 
-		attrList = lappend(attrList, col);
-	}
+		/*
+		 * Finally we can open the target table
+		 */
+		intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
 
-	if (lc != NULL)
-		ereport(ERROR,
-				(errcode(ERRCODE_SYNTAX_ERROR),
-				 errmsg("too many column names were specified")));
+		/*
+		 * Make sure the constructed table does not have RLS enabled.
+		 *
+		 * check_enable_rls() will ereport(ERROR) itself if the user has
+		 * requested something invalid, and otherwise will return RLS_ENABLED
+		 * if RLS should be enabled here.  We don't actually support that
+		 * currently, so throw our own ereport(ERROR) if that happens.
+		 */
+		if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					errmsg("policies not yet implemented for this command")));
 
-	/*
-	 * Actually create the target table
-	 */
-	intoRelationAddr = create_ctas_internal(attrList, into);
+		/*
+		 * Tentatively mark the target as populated, if it's a matview and
+		 * we're going to fill it; otherwise, no change needed.
+		 */
+		if (is_matview && !into->skipData)
+			SetMatViewPopulatedState(intoRelationDesc, true);
 
-	/*
-	 * Finally we can open the target table
-	 */
-	intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
+		/*
+		 * Fill private fields of myState for use by later routines
+		 */
+		myState->rel = intoRelationDesc;
+		myState->reladdr = intoRelationAddr;
+		myState->output_cid = GetCurrentCommandId(true);
+		myState->ti_options = TABLE_INSERT_SKIP_FSM;
 
-	/*
-	 * Make sure the constructed table does not have RLS enabled.
-	 *
-	 * check_enable_rls() will ereport(ERROR) itself if the user has requested
-	 * something invalid, and otherwise will return RLS_ENABLED if RLS should
-	 * be enabled here.  We don't actually support that currently, so throw
-	 * our own ereport(ERROR) if that happens.
-	 */
-	if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
-		ereport(ERROR,
-				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-				 errmsg("policies not yet implemented for this command")));
+		/*
+		 * If WITH NO DATA is specified, there is no need to set up the state
+		 * for bulk inserts as there are no tuples to insert.
+		 */
+		if (!into->skipData)
+			myState->bistate = GetBulkInsertState();
+		else
+			myState->bistate = NULL;
 
-	/*
-	 * Tentatively mark the target as populated, if it's a matview and we're
-	 * going to fill it; otherwise, no change needed.
-	 */
-	if (is_matview && !into->skipData)
-		SetMatViewPopulatedState(intoRelationDesc, true);
+		if (myState->is_parallel)
+		{
+			myState->object_id = intoRelationAddr.objectId;
 
-	/*
-	 * Fill private fields of myState for use by later routines
-	 */
-	myState->rel = intoRelationDesc;
-	myState->reladdr = intoRelationAddr;
-	myState->output_cid = GetCurrentCommandId(true);
-	myState->ti_options = TABLE_INSERT_SKIP_FSM;
+			/*
+			 * We don't need to skip contacting FSM while inserting tuples
+			 * for parallel mode, while extending the relations, workers
+			 * instead of blocking on a page while another worker is inserting,
+			 * can check the FSM for another page that can accommodate the
+			 * tuples. This results in major benefit for parallel inserts.
+			 */
+			myState->ti_options = 0;
 
-	/*
-	 * If WITH NO DATA is specified, there is no need to set up the state for
-	 * bulk inserts as there are no tuples to insert.
-	 */
-	if (!into->skipData)
-		myState->bistate = GetBulkInsertState();
-	else
-		myState->bistate = NULL;
+			/*
+			 * rd_createSubid is marked invalid, otherwise, the table is not
+			 * allowed to be extended by the workers.
+			 */
+			myState->rel->rd_createSubid = InvalidSubTransactionId;
+		}
 
-	/*
-	 * Valid smgr_targblock implies something already wrote to the relation.
-	 * This may be harmless, but this function hasn't planned for it.
-	 */
-	Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+		/*
+		 * Valid smgr_targblock implies something already wrote to the
+		 * relation. This may be harmless, but this function hasn't planned for
+		 * it.
+		 */
+		Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+	}
 }
 
 /*
@@ -606,3 +663,83 @@ intorel_destroy(DestReceiver *self)
 {
 	pfree(self);
 }
+
+/*
+ * IsParallelInsertInCTASAllowed --- determine whether or not parallel
+ * insertion is possible.
+ */
+bool IsParallelInsertInCTASAllowed(IntoClause *into, QueryDesc *queryDesc)
+{
+	if (!IS_CTAS(into))
+		return false;
+
+	/*
+	 * Do not allow parallel inserts if the table is temporary. As the
+	 * temporary tables are backend local, workers can not know about them.
+	 * Currently, CTAS supports creation of normal(logged), temporary and
+	 * unlogged tables. It does not support foreign or partition table
+	 * creation. Hence the check for temporary table is enough here.
+	 */
+	if (!into->rel || into->rel->relpersistence == RELPERSISTENCE_TEMP)
+		return false;
+
+	/*
+	 * We intend to call IsParallelInsertInCTASAllowed() before and after
+	 * planning with queryDesc NULL and non-NULL/valid respectively. Hence
+	 * return true for before planning i.e. queryDesc NULL case when we reach
+	 * here. However, for after planning case i.e. queryDesc not NULL/valid,
+	 * proceed with Gather node checks and return accordingly.
+	 */
+	if (!queryDesc)
+		return true;
+	else
+	{
+		PlanState  *ps = queryDesc->planstate;
+		PlannedStmt *plannedstmt = queryDesc->plannedstmt;
+		bool  allow;
+
+		/*
+		 * We allow parallel inserts by the workers only if the Gather node has
+		 * no projections to perform and if the upper node is Gather. In case,
+		 * the Gather node has projections, which is possible if there are any
+		 * subplans in the query, the workers can not do those projections. And
+		 * when the upper node is GatherMerge, then the leader has to perform
+		 * the final phase i.e. merge the results by workers.
+		 */
+		allow = ps && IsA(ps, GatherState) && !ps->ps_ProjInfo &&
+				plannedstmt->parallelModeNeeded &&
+				plannedstmt->planTree &&
+				IsA(plannedstmt->planTree, Gather) &&
+				plannedstmt->planTree->lefttree &&
+				plannedstmt->planTree->lefttree->parallel_aware &&
+				plannedstmt->planTree->lefttree->parallel_safe;
+
+		return allow;
+	}
+}
+
+/*
+ * SetCTASParallelInsertState --- set the required info for the parallel
+ * inserts, that is required in the plan exection.
+ */
+void SetCTASParallelInsertState(QueryDesc *queryDesc)
+{
+	GatherState *gstate = (GatherState *) queryDesc->planstate;
+
+	/*
+	 * For parallelizing inserts in CTAS i.e. making each parallel worker
+	 * insert the tuples, we must send information such as intoclause(for
+	 * eachworker to build separate dest receiver), object id(for each worker
+	 * to open the created table).
+	 */
+	((DR_intorel *) queryDesc->dest)->is_parallel = true;
+	gstate->dest = queryDesc->dest;
+
+	/*
+	 * Since there are no rows that are transferred from workers to Gather
+	 * node, so we set it to 0 to be visible in explain plans. Note that we
+	 * would have already accounted this for cost calculations in
+	 * cost_gather().
+	 */
+	queryDesc->plannedstmt->planTree->plan_rows = 0;
+}
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 43f9b01e83..57875e1505 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -371,6 +371,15 @@ ExplainOneQuery(Query *query, int cursorOptions,
 		return;
 	}
 
+	/*
+	 * Flag to let the planner know that the SELECT query is for CTAS. This is
+	 * used to calculate the tuple transfer cost from workers to gather node(in
+	 * case parallelism kicks in for the SELECT part of the CTAS), to zero as
+	 * each worker will insert its share of tuples in parallel.
+	 */
+	if (IsParallelInsertInCTASAllowed(into, NULL))
+		query->isForCTAS = true;
+
 	/* if an advisor plugin is present, let it manage things */
 	if (ExplainOneQuery_hook)
 		(*ExplainOneQuery_hook) (query, cursorOptions, into, es,
@@ -556,6 +565,15 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	/* call ExecutorStart to prepare the plan for execution */
 	ExecutorStart(queryDesc, eflags);
 
+	/*
+	 * If SELECT part of the CTAS is parallelizable, then make each parallel
+	 * worker insert the tuples that are resulted in its execution into the
+	 * target table. We need plan state to be initialized by the executor to
+	 * decide whether to allow parallel inserts or not.
+	 */
+	if (IsParallelInsertInCTASAllowed(into, queryDesc))
+		SetCTASParallelInsertState(queryDesc);
+
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
 	{
@@ -1775,6 +1793,29 @@ ExplainNode(PlanState *planstate, List *ancestors,
 
 				if (gather->single_copy || es->format != EXPLAIN_FORMAT_TEXT)
 					ExplainPropertyBool("Single Copy", gather->single_copy, es);
+
+				/*
+				 * Show the create table information under Gather node in case
+				 * parallel workers have inserted the rows.
+				 */
+				if (IsA(planstate, GatherState))
+				{
+					GatherState *gstate = (GatherState *) planstate;
+
+					if (IS_PARALLEL_CTAS_DEST(gstate->dest) &&
+						((DR_intorel *) gstate->dest)->into->rel &&
+						((DR_intorel *) gstate->dest)->into->rel->relname)
+					{
+						es->indent--;
+						ExplainIndentText(es);
+						appendStringInfoString(es->str, "->  ");
+						appendStringInfoString(es->str, "Create ");
+						appendStringInfo(es->str, "%s\n",
+							((DR_intorel *) gstate->dest)->into->rel->relname);
+						ExplainIndentText(es);
+						es->indent++;
+					}
+				}
 			}
 			break;
 		case T_GatherMerge:
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index befde52691..9ef33eee54 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -23,6 +23,7 @@
 
 #include "postgres.h"
 
+#include "commands/createas.h"
 #include "executor/execParallel.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
@@ -65,6 +66,7 @@
 #define PARALLEL_KEY_QUERY_TEXT		UINT64CONST(0xE000000000000008)
 #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009)
 #define PARALLEL_KEY_WAL_USAGE			UINT64CONST(0xE00000000000000A)
+#define PARALLEL_KEY_INTO_CLAUSE		UINT64CONST(0xE00000000000000B)
 
 #define PARALLEL_TUPLE_QUEUE_SIZE		65536
 
@@ -77,6 +79,9 @@ typedef struct FixedParallelExecutorState
 	dsa_pointer param_exec;
 	int			eflags;
 	int			jit_flags;
+	Oid			objectid;		/* workers to open relation/table.  */
+	/* Number tuples inserted by all the workers. */
+	pg_atomic_uint64	processed;
 } FixedParallelExecutorState;
 
 /*
@@ -578,7 +583,8 @@ ExecParallelSetupTupleQueues(ParallelContext *pcxt, bool reinitialize)
 ParallelExecutorInfo *
 ExecInitParallelPlan(PlanState *planstate, EState *estate,
 					 Bitmapset *sendParams, int nworkers,
-					 int64 tuples_needed)
+					 int64 tuples_needed, IntoClause *intoclause,
+					 Oid objectid)
 {
 	ParallelExecutorInfo *pei;
 	ParallelContext *pcxt;
@@ -600,6 +606,8 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	Size		dsa_minsize = dsa_minimum_size();
 	char	   *query_string;
 	int			query_len;
+	char 		*intoclausestr = NULL;
+	int			intoclause_len = 0;
 
 	/*
 	 * Force any initplan outputs that we're going to pass to workers to be
@@ -712,6 +720,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_estimate_chunk(&pcxt->estimator, dsa_minsize);
 	shm_toc_estimate_keys(&pcxt->estimator, 1);
 
+	/* Estimate space for into clause for CTAS. */
+	if (IS_CTAS(intoclause) && OidIsValid(objectid))
+	{
+		intoclausestr = nodeToString(intoclause);
+		intoclause_len = strlen(intoclausestr);
+		shm_toc_estimate_chunk(&pcxt->estimator, intoclause_len + 1);
+		shm_toc_estimate_keys(&pcxt->estimator, 1);
+	}
+
 	/* Everyone's had a chance to ask for space, so now create the DSM. */
 	InitializeParallelDSM(pcxt);
 
@@ -729,6 +746,14 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	fpes->param_exec = InvalidDsaPointer;
 	fpes->eflags = estate->es_top_eflags;
 	fpes->jit_flags = estate->es_jit_flags;
+	pg_atomic_init_u64(&fpes->processed, 0);
+	pei->processed = &fpes->processed;
+
+	if (intoclausestr && OidIsValid(objectid))
+		fpes->objectid = objectid;
+	else
+		fpes->objectid = InvalidOid;
+
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, fpes);
 
 	/* Store query string */
@@ -758,8 +783,18 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_WAL_USAGE, walusage_space);
 	pei->wal_usage = walusage_space;
 
-	/* Set up the tuple queues that the workers will write into. */
-	pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	if (intoclausestr)
+	{
+		char *intoclause_space = shm_toc_allocate(pcxt->toc,
+												  intoclause_len + 1);
+		memcpy(intoclause_space, intoclausestr, intoclause_len + 1);
+		shm_toc_insert(pcxt->toc, PARALLEL_KEY_INTO_CLAUSE, intoclause_space);
+	}
+	else
+	{
+		/* Set up the tuple queues that the workers will write into. */
+		pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	}
 
 	/* We don't need the TupleQueueReaders yet, though. */
 	pei->reader = NULL;
@@ -1387,12 +1422,30 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 	void	   *area_space;
 	dsa_area   *area;
 	ParallelWorkerContext pwcxt;
+	char		*intoclausestr = NULL;
+	IntoClause	*intoclause = NULL;
 
 	/* Get fixed-size state. */
 	fpes = shm_toc_lookup(toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
 
-	/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
-	receiver = ExecParallelGetReceiver(seg, toc);
+	intoclausestr = shm_toc_lookup(toc, PARALLEL_KEY_INTO_CLAUSE, true);
+	if (intoclausestr)
+	{
+		/*
+		 * If the worker is for parallel insert in CTAS, then use the proper
+		 * dest receiver.
+		 */
+		intoclause = (IntoClause *) stringToNode(intoclausestr);
+		receiver = CreateIntoRelDestReceiver(intoclause);
+		((DR_intorel *)receiver)->is_parallel_worker = true;
+		((DR_intorel *)receiver)->object_id = fpes->objectid;
+	}
+	else
+	{
+		/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
+		receiver = ExecParallelGetReceiver(seg, toc);
+	}
+
 	instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_INSTRUMENTATION, true);
 	if (instrumentation != NULL)
 		instrument_options = instrumentation->instrument_options;
@@ -1471,6 +1524,13 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 			queryDesc->estate->es_jit->instr;
 	}
 
+	/*
+	 * Write out the number of tuples this worker has inserted. Leader will use
+	 * it to inform the end client.
+	 */
+	if (intoclausestr)
+		pg_atomic_add_fetch_u64(&fpes->processed, queryDesc->estate->es_processed);
+
 	/* Must do this after capturing instrumentation. */
 	ExecutorEnd(queryDesc);
 
diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c
index a01b46af14..9cef6bdd61 100644
--- a/src/backend/executor/nodeGather.c
+++ b/src/backend/executor/nodeGather.c
@@ -32,6 +32,7 @@
 
 #include "access/relscan.h"
 #include "access/xact.h"
+#include "commands/createas.h"
 #include "executor/execdebug.h"
 #include "executor/execParallel.h"
 #include "executor/nodeGather.h"
@@ -48,6 +49,7 @@ static TupleTableSlot *ExecGather(PlanState *pstate);
 static TupleTableSlot *gather_getnext(GatherState *gatherstate);
 static MinimalTuple gather_readnext(GatherState *gatherstate);
 static void ExecShutdownGatherWorkers(GatherState *node);
+static void ExecParallelInsertInCTAS(GatherState *node);
 
 
 /* ----------------------------------------------------------------
@@ -131,6 +133,73 @@ ExecInitGather(Gather *node, EState *estate, int eflags)
 	return gatherstate;
 }
 
+/* ----------------------------------------------------------------
+ *		ExecParallelInsertInCTAS(node)
+ *
+ *		Facilitates parallel inserts by parallel workers and/or
+ *		leader for Create Table AS.
+ * ----------------------------------------------------------------
+ */
+static void
+ExecParallelInsertInCTAS(GatherState *node)
+{
+	/* Enable leader to insert in case no parallel workers were launched. */
+	if (node->nworkers_launched == 0)
+		node->need_to_scan_locally = true;
+
+	/*
+	 * By now, for parallel workers (if launched any), would have started their
+	 * work i.e. insertion to target table. In case the leader is chosen to
+	 * participate for parallel inserts in CTAS, then finish its share before
+	 * going to wait for the parallel workers to finish.
+	 */
+	if (node->need_to_scan_locally)
+	{
+		EState	   *estate = node->ps.state;
+		TupleTableSlot *outerTupleSlot;
+
+		for(;;)
+		{
+			/* Install our DSA area while executing the plan. */
+			estate->es_query_dsa =
+					node->pei ? node->pei->area : NULL;
+
+			outerTupleSlot = ExecProcNode(node->ps.lefttree);
+
+			estate->es_query_dsa = NULL;
+
+			if(TupIsNull(outerTupleSlot))
+				break;
+
+			(void) node->dest->receiveSlot(outerTupleSlot, node->dest);
+
+			node->ps.state->es_processed++;
+		}
+
+		node->need_to_scan_locally = false;
+	}
+
+	if (node->nworkers_launched > 0)
+	{
+		int		i;
+
+		/* Wait for the parallel workers to finish. */
+		WaitForParallelWorkersToFinish(node->pei->pcxt);
+
+		for (i = 0; i < node->nworkers_launched; i++)
+		{
+			InstrAccumParallelQuery(&node->pei->buffer_usage[i],
+									&node->pei->wal_usage[i]);
+		}
+
+		/*
+		 * Add up the total tuples inserted by all workers, to the tuples
+		 * inserted by the leader(if any). This will be shared to client.
+		 */
+		node->ps.state->es_processed += pg_atomic_read_u64(node->pei->processed);
+	}
+}
+
 /* ----------------------------------------------------------------
  *		ExecGather(node)
  *
@@ -157,6 +226,7 @@ ExecGather(PlanState *pstate)
 	{
 		EState	   *estate = node->ps.state;
 		Gather	   *gather = (Gather *) node->ps.plan;
+		bool		isctas = IS_PARALLEL_CTAS_DEST(node->dest);
 
 		/*
 		 * Sometimes we might have to run without parallelism; but if parallel
@@ -165,6 +235,18 @@ ExecGather(PlanState *pstate)
 		if (gather->num_workers > 0 && estate->es_use_parallel_mode)
 		{
 			ParallelContext *pcxt;
+			IntoClause	*intoclause = NULL;
+			Oid			objectid = InvalidOid;
+
+			/*
+			 * Take the necessary information to be passed to workers for
+			 * parallel inserts in CTAS.
+			 */
+			if (isctas)
+			{
+				intoclause = ((DR_intorel *) node->dest)->into;
+				objectid = ((DR_intorel *) node->dest)->object_id;
+			}
 
 			/* Initialize, or re-initialize, shared state needed by workers. */
 			if (!node->pei)
@@ -172,7 +254,10 @@ ExecGather(PlanState *pstate)
 												 estate,
 												 gather->initParam,
 												 gather->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 /* CTAS info */
+												 intoclause,
+												 objectid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
@@ -190,13 +275,16 @@ ExecGather(PlanState *pstate)
 			/* Set up tuple queue readers to read the results. */
 			if (pcxt->nworkers_launched > 0)
 			{
-				ExecParallelCreateReaders(node->pei);
-				/* Make a working array showing the active readers */
-				node->nreaders = pcxt->nworkers_launched;
-				node->reader = (TupleQueueReader **)
-					palloc(node->nreaders * sizeof(TupleQueueReader *));
-				memcpy(node->reader, node->pei->reader,
-					   node->nreaders * sizeof(TupleQueueReader *));
+				if (!isctas)
+				{
+					ExecParallelCreateReaders(node->pei);
+					/* Make a working array showing the active readers */
+					node->nreaders = pcxt->nworkers_launched;
+					node->reader = (TupleQueueReader **)
+						palloc(node->nreaders * sizeof(TupleQueueReader *));
+					memcpy(node->reader, node->pei->reader,
+						node->nreaders * sizeof(TupleQueueReader *));
+				}
 			}
 			else
 			{
@@ -208,9 +296,17 @@ ExecGather(PlanState *pstate)
 		}
 
 		/* Run plan locally if no workers or enabled and not single-copy. */
-		node->need_to_scan_locally = (node->nreaders == 0)
+		node->need_to_scan_locally = (node->nreaders == 0 &&
+			!isctas)
 			|| (!gather->single_copy && parallel_leader_participation);
 		node->initialized = true;
+
+		/* Perform parallel inserts for CTAS. */
+		if (isctas)
+		{
+			ExecParallelInsertInCTAS(node);
+			return NULL;
+		}
 	}
 
 	/*
diff --git a/src/backend/executor/nodeGatherMerge.c b/src/backend/executor/nodeGatherMerge.c
index 47129344f3..ee45272c17 100644
--- a/src/backend/executor/nodeGatherMerge.c
+++ b/src/backend/executor/nodeGatherMerge.c
@@ -216,7 +216,9 @@ ExecGatherMerge(PlanState *pstate)
 												 estate,
 												 gm->initParam,
 												 gm->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 NULL,
+												 InvalidOid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 22d6935824..225f3cbf01 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -393,7 +393,18 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 
 	/* Parallel setup and communication cost. */
 	startup_cost += parallel_setup_cost;
-	run_cost += parallel_tuple_cost * path->path.rows;
+
+	/*
+	 * We do not compute the parallel_tuple_cost for CTAS because the number of
+	 * tuples that are transferred from workers to the gather node is zero as
+	 * each worker, in parallel, inserts the tuples that are resulted from its
+	 * chunk of plan execution. This change may make the parallel plan cheap
+	 * among all other plans, and influence the planner to consider this
+	 * parallel plan.
+	 */
+	if (!(root->parse->isForCTAS &&
+		root->query_level == 1))
+		run_cost += parallel_tuple_cost * path->path.rows;
 
 	path->path.startup_cost = startup_cost;
 	path->path.total_cost = (startup_cost + run_cost);
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7320de345c..5beae6c617 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -389,6 +389,7 @@ extern FullTransactionId GetCurrentFullTransactionIdIfAny(void);
 extern void MarkCurrentTransactionIdLoggedIfAny(void);
 extern bool SubTransactionIsActive(SubTransactionId subxid);
 extern CommandId GetCurrentCommandId(bool used);
+extern void SetCurrentCommandIdUsedForWorker(void);
 extern void SetParallelStartTimestamps(TimestampTz xact_ts, TimestampTz stmt_ts);
 extern TimestampTz GetCurrentTransactionStartTimestamp(void);
 extern TimestampTz GetCurrentStatementStartTimestamp(void);
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 7629230254..ab3aab58c5 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -14,12 +14,35 @@
 #ifndef CREATEAS_H
 #define CREATEAS_H
 
+#include "access/heapam.h"
 #include "catalog/objectaddress.h"
+#include "executor/execdesc.h"
 #include "nodes/params.h"
+#include "nodes/plannodes.h"
 #include "parser/parse_node.h"
 #include "tcop/dest.h"
 #include "utils/queryenvironment.h"
 
+typedef struct
+{
+	DestReceiver pub;			/* publicly-known function pointers */
+	IntoClause *into;			/* target relation specification */
+	/* These fields are filled by intorel_startup: */
+	Relation	rel;			/* relation to write to */
+	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
+	CommandId	output_cid;		/* cmin to insert in output tuples */
+	int			ti_options;		/* table_tuple_insert performance options */
+	BulkInsertState bistate;	/* bulk insert state */
+	bool		is_parallel;	/* is parallelism to be considered? */
+	bool		is_parallel_worker; /* true for parallel worker */
+	/* Used for table open by parallel worker. */
+	Oid			object_id;
+} DR_intorel;
+
+#define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
+#define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
+									 IS_CTAS(((DR_intorel *) dest)->into) && \
+									 ((DR_intorel *) dest)->is_parallel)
 
 extern ObjectAddress ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 									   ParamListInfo params, QueryEnvironment *queryEnv,
@@ -29,4 +52,9 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
+extern bool IsParallelInsertInCTASAllowed(IntoClause *into,
+										  QueryDesc *queryDesc);
+
+extern void SetCTASParallelInsertState(QueryDesc *queryDesc);
+
 #endif							/* CREATEAS_H */
diff --git a/src/include/executor/execParallel.h b/src/include/executor/execParallel.h
index 5a39a5b29c..9f959f741b 100644
--- a/src/include/executor/execParallel.h
+++ b/src/include/executor/execParallel.h
@@ -35,11 +35,15 @@ typedef struct ParallelExecutorInfo
 	/* These two arrays have pcxt->nworkers_launched entries: */
 	shm_mq_handle **tqueue;		/* tuple queues for worker output */
 	struct TupleQueueReader **reader;	/* tuple reader/writer support */
+	/* Number of tuples inserted by all workers. */
+	volatile pg_atomic_uint64	*processed;
 } ParallelExecutorInfo;
 
 extern ParallelExecutorInfo *ExecInitParallelPlan(PlanState *planstate,
 												  EState *estate, Bitmapset *sendParam, int nworkers,
-												  int64 tuples_needed);
+												  int64 tuples_needed,
+												  IntoClause *intoclause,
+												  Oid objectid);
 extern void ExecParallelCreateReaders(ParallelExecutorInfo *pei);
 extern void ExecParallelFinish(ParallelExecutorInfo *pei);
 extern void ExecParallelCleanup(ParallelExecutorInfo *pei);
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 61ba4c3666..e9c4442c22 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -23,6 +23,7 @@
 #include "nodes/tidbitmap.h"
 #include "partitioning/partdefs.h"
 #include "storage/condition_variable.h"
+#include "tcop/dest.h"
 #include "utils/hsearch.h"
 #include "utils/queryenvironment.h"
 #include "utils/reltrigger.h"
@@ -2326,6 +2327,8 @@ typedef struct GatherState
 	int			nreaders;		/* number of still-active workers */
 	int			nextreader;		/* next one to try to read from */
 	struct TupleQueueReader **reader;	/* array with nreaders active entries */
+	/* Parallel inserts in CTAS related info is specified below. */
+	DestReceiver *dest;
 } GatherState;
 
 /* ----------------
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index ec14fc2036..eb267b1a6c 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -180,6 +180,7 @@ typedef struct Query
 	 */
 	int			stmt_location;	/* start location, or -1 if unknown */
 	int			stmt_len;		/* length in bytes; 0 means "rest of string" */
+	bool		isForCTAS; 		/* is the SELECT query for CREATE TABLE AS */
 } Query;
 
 
diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 0c4da2591a..635d24e76d 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -75,5 +75,195 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 (7 rows)
 
 create table parallel_write as execute prep_stmt;
+drop table parallel_write;
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+                     QUERY PLAN                      
+-----------------------------------------------------
+ LockRows (actual rows=10000 loops=1)
+   ->  Seq Scan on tenk1 (actual rows=10000 loops=1)
+(2 rows)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_mat_view
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop materialized view parallel_mat_view;
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+deallocate parallel_write_prep;
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+select nextval('parallel_write_sequence'), four from tenk1;
+                  QUERY PLAN                   
+-----------------------------------------------
+ Seq Scan on tenk1 (actual rows=10000 loops=1)
+(1 row)
+
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;
+                               QUERY PLAN                               
+------------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Params Evaluated: $1
+   Workers Launched: 3
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=1 loops=1)
+           ->  Gather (actual rows=1 loops=1)
+                 Workers Planned: 4
+                 Workers Launched: 4
+                 ->  Parallel Seq Scan on tenk2 (actual rows=1 loops=5)
+   ->  Parallel Seq Scan on tenk1 (actual rows=625 loops=4)
+         Filter: (four = 3)
+         Rows Removed by Filter: 1875
+(14 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;
+                             QUERY PLAN                              
+---------------------------------------------------------------------
+ Gather (actual rows=2500 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=500 loops=5)
+         Filter: (four = 3)
+         Rows Removed by Filter: 1500
+   SubPlan 1
+     ->  Function Scan on generate_series (actual rows=1 loops=2500)
+(8 rows)
+
 drop table parallel_write;
 rollback;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 78b479cedf..e65a56b442 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -39,4 +39,88 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
 
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+select length(stringu1) from tenk1;
+drop materialized view parallel_mat_view;
+
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+deallocate parallel_write_prep;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+select nextval('parallel_write_sequence'), four from tenk1;
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;
+drop table parallel_write;
+
 rollback;
-- 
2.25.1

From 2939b2c51bff3548ea15c9054f9e2ab661dddd9b Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Thu, 10 Dec 2020 05:38:35 +0530
Subject: [PATCH v10] Parallel Inserts in CREATE TABLE AS

The idea of this patch is to allow the leader and each worker
insert the tuples in parallel if the SELECT part of the CTAS is
parallelizable.

The design:
Let the planner know that the SELECT is from CTAS in createas.c
so that it can set the number of tuples transferred from the
workers to Gather node to 0. With this change, there are chances
that the planner may choose the parallel plan. After the planning,
check if the upper plan node is Gather in createas.c and mark a
parallelism flag in the CTAS dest receiver. Pass the into clause,
object id, command id from the leader to workers, so that each
worker can create its own CTAS dest receiver. Leader inserts its
share of tuples if instructed to do, and so are workers. Each
worker writes atomically its number of inserted tuples into a
shared memory variable, the leader combines this with its own
number of inserted tuples and shares to the client.
---
 src/backend/access/heap/heapam.c             |  11 -
 src/backend/access/transam/xact.c            |  30 +-
 src/backend/commands/createas.c              | 332 ++++++++----
 src/backend/commands/explain.c               |  32 ++
 src/backend/executor/execParallel.c          |  70 ++-
 src/backend/executor/nodeGather.c            | 113 ++++-
 src/backend/executor/nodeGatherMerge.c       |   4 +-
 src/include/access/xact.h                    |   1 +
 src/include/commands/createas.h              |  28 ++
 src/include/executor/execParallel.h          |   6 +-
 src/include/nodes/execnodes.h                |   3 +
 src/test/regress/expected/write_parallel.out | 504 +++++++++++++++++++
 src/test/regress/sql/write_parallel.sql      | 180 +++++++
 13 files changed, 1174 insertions(+), 140 deletions(-)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index a9583f3103..86347ba273 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2043,17 +2043,6 @@ static HeapTuple
 heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
 					CommandId cid, int options)
 {
-	/*
-	 * To allow parallel inserts, we need to ensure that they are safe to be
-	 * performed in workers. We have the infrastructure to allow parallel
-	 * inserts in general except for the cases where inserts generate a new
-	 * CommandId (eg. inserts into a table having a foreign key column).
-	 */
-	if (IsParallelWorker())
-		ereport(ERROR,
-				(errcode(ERRCODE_INVALID_TRANSACTION_STATE),
-				 errmsg("cannot insert tuples in a parallel worker")));
-
 	tup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
 	tup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
 	tup->t_data->t_infomask |= HEAP_XMAX_INVALID;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 9cd0b7c11b..db6eedd635 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -763,18 +763,34 @@ GetCurrentCommandId(bool used)
 	/* this is global to a transaction, not subtransaction-local */
 	if (used)
 	{
-		/*
-		 * Forbid setting currentCommandIdUsed in a parallel worker, because
-		 * we have no provision for communicating this back to the leader.  We
-		 * could relax this restriction when currentCommandIdUsed was already
-		 * true at the start of the parallel operation.
-		 */
-		Assert(!IsParallelWorker());
+		 /*
+		  * This is a temporary hack for all common parallel insert cases i.e.
+		  * insert into, ctas, copy from. To be changed later. In a parallel
+		  * worker, set currentCommandIdUsed to true only if it was not set to
+		  * true at the start of the parallel operation (by way of
+		  * SetCurrentCommandIdUsedForWorker()). We have to do this because
+		  * GetCurrentCommandId(true) may be called from anywhere, especially
+		  * for parallel inserts, within parallel worker.
+		  */
+		Assert(!(IsParallelWorker() && !currentCommandIdUsed));
 		currentCommandIdUsed = true;
 	}
 	return currentCommandId;
 }
 
+/*
+ *	SetCurrentCommandIdUsedForWorker
+ *
+ * For a parallel worker, record that the currentCommandId has been used. This
+ * must only be called at the start of a parallel operation.
+ */
+void
+SetCurrentCommandIdUsedForWorker(void)
+{
+	Assert(IsParallelWorker() && !currentCommandIdUsed && currentCommandId != InvalidCommandId);
+	currentCommandIdUsed = true;
+}
+
 /*
  *	SetParallelStartTimestamps
  *
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 6bf6c5a310..20f59cc2b8 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,18 +51,6 @@
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 
-typedef struct
-{
-	DestReceiver pub;			/* publicly-known function pointers */
-	IntoClause *into;			/* target relation specification */
-	/* These fields are filled by intorel_startup: */
-	Relation	rel;			/* relation to write to */
-	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
-	CommandId	output_cid;		/* cmin to insert in output tuples */
-	int			ti_options;		/* table_tuple_insert performance options */
-	BulkInsertState bistate;	/* bulk insert state */
-} DR_intorel;
-
 /* utility functions for CTAS definition creation */
 static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into);
 static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -350,6 +338,15 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		/* call ExecutorStart to prepare the plan for execution */
 		ExecutorStart(queryDesc, GetIntoRelEFlags(into));
 
+		/*
+		 * If SELECT part of the CTAS is parallelizable, then make each
+		 * parallel worker insert the tuples that are resulted in its execution
+		 * into the target table. We need plan state to be initialized by the
+		 * executor to decide whether to allow parallel inserts or not.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, queryDesc))
+			SetCTASParallelInsertState(queryDesc);
+
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
 
@@ -418,6 +415,9 @@ CreateIntoRelDestReceiver(IntoClause *intoClause)
 	self->pub.rDestroy = intorel_destroy;
 	self->pub.mydest = DestIntoRel;
 	self->into = intoClause;
+	self->is_parallel = false;
+	self->is_parallel_worker = false;
+	self->object_id = InvalidOid;
 	/* other private fields will be set during intorel_startup */
 
 	return (DestReceiver *) self;
@@ -430,121 +430,169 @@ static void
 intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 {
 	DR_intorel *myState = (DR_intorel *) self;
-	IntoClause *into = myState->into;
-	bool		is_matview;
-	List	   *attrList;
 	ObjectAddress intoRelationAddr;
 	Relation	intoRelationDesc;
-	ListCell   *lc;
-	int			attnum;
 
-	Assert(into != NULL);		/* else somebody forgot to set it */
-
-	/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
-	is_matview = (into->viewQuery != NULL);
+	if (myState->is_parallel_worker)
+	{
+		/* In the worker */
+		intoRelationDesc = table_open(myState->object_id, AccessExclusiveLock);
+		myState->rel = intoRelationDesc;
+		myState->reladdr = InvalidObjectAddress;
+		myState->ti_options = 0;
+		myState->bistate = GetBulkInsertState();
 
-	/*
-	 * Build column definitions using "pre-cooked" type and collation info. If
-	 * a column name list was specified in CREATE TABLE AS, override the
-	 * column names derived from the query.  (Too few column names are OK, too
-	 * many are not.)
-	 */
-	attrList = NIL;
-	lc = list_head(into->colNames);
-	for (attnum = 0; attnum < typeinfo->natts; attnum++)
+		/*
+		 * Right after the table is created in the leader, the command id is
+		 * incremented (in create_ctas_internal()). The new command id is
+		 * marked as used in intorel_startup(), then the parallel mode is
+		 * entered. The command id and transaction id are serialized into
+		 * parallel DSM, they are then available to all parallel workers. All
+		 * the workers need to mark the command id as used before insertion.
+		 */
+		SetCurrentCommandIdUsedForWorker();
+		myState->output_cid = GetCurrentCommandId(false);
+	}
+	else
 	{
-		Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
-		ColumnDef  *col;
-		char	   *colname;
+		IntoClause *into = myState->into;
+		bool		is_matview;
+		List	   *attrList;
+		ListCell   *lc;
+		int			attnum;
+
+		Assert(into != NULL);		/* else somebody forgot to set it */
+
+		/*
+		 * This code supports both CREATE TABLE AS and CREATE MATERIALIZED
+		 * VIEW.
+		 */
+		is_matview = (into->viewQuery != NULL);
 
-		if (lc)
+		/*
+		 * Build column definitions using "pre-cooked" type and collation info.
+		 * If a column name list was specified in CREATE TABLE AS, override the
+		 * column names derived from the query.  (Too few column names are OK,
+		 * too many are not.)
+		 */
+		attrList = NIL;
+		lc = list_head(into->colNames);
+		for (attnum = 0; attnum < typeinfo->natts; attnum++)
 		{
-			colname = strVal(lfirst(lc));
-			lc = lnext(into->colNames, lc);
+			Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
+			ColumnDef  *col;
+			char	   *colname;
+
+			if (lc)
+			{
+				colname = strVal(lfirst(lc));
+				lc = lnext(into->colNames, lc);
+			}
+			else
+				colname = NameStr(attribute->attname);
+
+			col = makeColumnDef(colname,
+								attribute->atttypid,
+								attribute->atttypmod,
+								attribute->attcollation);
+
+			/*
+			 * It's possible that the column is of a collatable type but the
+			 * collation could not be resolved, so double-check.  (We must
+			 * check this here because DefineRelation would adopt the type's
+			 * default collation rather than complaining.)
+			 */
+			if (!OidIsValid(col->collOid) &&
+				type_is_collatable(col->typeName->typeOid))
+				ereport(ERROR,
+						(errcode(ERRCODE_INDETERMINATE_COLLATION),
+						errmsg("no collation was derived for column \"%s\" with collatable type %s",
+								col->colname,
+								format_type_be(col->typeName->typeOid)),
+						errhint("Use the COLLATE clause to set the collation explicitly.")));
+
+			attrList = lappend(attrList, col);
 		}
-		else
-			colname = NameStr(attribute->attname);
 
-		col = makeColumnDef(colname,
-							attribute->atttypid,
-							attribute->atttypmod,
-							attribute->attcollation);
+		if (lc != NULL)
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					errmsg("too many column names were specified")));
 
 		/*
-		 * It's possible that the column is of a collatable type but the
-		 * collation could not be resolved, so double-check.  (We must check
-		 * this here because DefineRelation would adopt the type's default
-		 * collation rather than complaining.)
+		 * Actually create the target table
 		 */
-		if (!OidIsValid(col->collOid) &&
-			type_is_collatable(col->typeName->typeOid))
-			ereport(ERROR,
-					(errcode(ERRCODE_INDETERMINATE_COLLATION),
-					 errmsg("no collation was derived for column \"%s\" with collatable type %s",
-							col->colname,
-							format_type_be(col->typeName->typeOid)),
-					 errhint("Use the COLLATE clause to set the collation explicitly.")));
+		intoRelationAddr = create_ctas_internal(attrList, into);
 
-		attrList = lappend(attrList, col);
-	}
+		/*
+		 * Finally we can open the target table
+		 */
+		intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
 
-	if (lc != NULL)
-		ereport(ERROR,
-				(errcode(ERRCODE_SYNTAX_ERROR),
-				 errmsg("too many column names were specified")));
+		/*
+		 * Make sure the constructed table does not have RLS enabled.
+		 *
+		 * check_enable_rls() will ereport(ERROR) itself if the user has
+		 * requested something invalid, and otherwise will return RLS_ENABLED
+		 * if RLS should be enabled here.  We don't actually support that
+		 * currently, so throw our own ereport(ERROR) if that happens.
+		 */
+		if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					errmsg("policies not yet implemented for this command")));
 
-	/*
-	 * Actually create the target table
-	 */
-	intoRelationAddr = create_ctas_internal(attrList, into);
+		/*
+		 * Tentatively mark the target as populated, if it's a matview and
+		 * we're going to fill it; otherwise, no change needed.
+		 */
+		if (is_matview && !into->skipData)
+			SetMatViewPopulatedState(intoRelationDesc, true);
 
-	/*
-	 * Finally we can open the target table
-	 */
-	intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
+		/*
+		 * Fill private fields of myState for use by later routines
+		 */
+		myState->rel = intoRelationDesc;
+		myState->reladdr = intoRelationAddr;
+		myState->output_cid = GetCurrentCommandId(true);
+		myState->ti_options = TABLE_INSERT_SKIP_FSM;
 
-	/*
-	 * Make sure the constructed table does not have RLS enabled.
-	 *
-	 * check_enable_rls() will ereport(ERROR) itself if the user has requested
-	 * something invalid, and otherwise will return RLS_ENABLED if RLS should
-	 * be enabled here.  We don't actually support that currently, so throw
-	 * our own ereport(ERROR) if that happens.
-	 */
-	if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
-		ereport(ERROR,
-				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-				 errmsg("policies not yet implemented for this command")));
+		/*
+		 * If WITH NO DATA is specified, there is no need to set up the state
+		 * for bulk inserts as there are no tuples to insert.
+		 */
+		if (!into->skipData)
+			myState->bistate = GetBulkInsertState();
+		else
+			myState->bistate = NULL;
 
-	/*
-	 * Tentatively mark the target as populated, if it's a matview and we're
-	 * going to fill it; otherwise, no change needed.
-	 */
-	if (is_matview && !into->skipData)
-		SetMatViewPopulatedState(intoRelationDesc, true);
+		if (myState->is_parallel)
+		{
+			myState->object_id = intoRelationAddr.objectId;
 
-	/*
-	 * Fill private fields of myState for use by later routines
-	 */
-	myState->rel = intoRelationDesc;
-	myState->reladdr = intoRelationAddr;
-	myState->output_cid = GetCurrentCommandId(true);
-	myState->ti_options = TABLE_INSERT_SKIP_FSM;
+			/*
+			 * We don't need to skip contacting FSM while inserting tuples
+			 * for parallel mode, while extending the relations, workers
+			 * instead of blocking on a page while another worker is inserting,
+			 * can check the FSM for another page that can accommodate the
+			 * tuples. This results in major benefit for parallel inserts.
+			 */
+			myState->ti_options = 0;
 
-	/*
-	 * If WITH NO DATA is specified, there is no need to set up the state for
-	 * bulk inserts as there are no tuples to insert.
-	 */
-	if (!into->skipData)
-		myState->bistate = GetBulkInsertState();
-	else
-		myState->bistate = NULL;
+			/*
+			 * rd_createSubid is marked invalid, otherwise, the table is not
+			 * allowed to be extended by the workers.
+			 */
+			myState->rel->rd_createSubid = InvalidSubTransactionId;
+		}
 
-	/*
-	 * Valid smgr_targblock implies something already wrote to the relation.
-	 * This may be harmless, but this function hasn't planned for it.
-	 */
-	Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+		/*
+		 * Valid smgr_targblock implies something already wrote to the
+		 * relation. This may be harmless, but this function hasn't planned for
+		 * it.
+		 */
+		Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+	}
 }
 
 /*
@@ -606,3 +654,75 @@ intorel_destroy(DestReceiver *self)
 {
 	pfree(self);
 }
+
+/*
+ * IsParallelInsertInCTASAllowed --- determine whether or not parallel
+ * insertion is possible.
+ */
+bool IsParallelInsertInCTASAllowed(IntoClause *into, QueryDesc *queryDesc)
+{
+	if (!IS_CTAS(into))
+		return false;
+
+	/*
+	 * Do not allow parallel inserts if the table is temporary. As the
+	 * temporary tables are backend local, workers can not know about them.
+	 * Currently, CTAS supports creation of normal(logged), temporary and
+	 * unlogged tables. It does not support foreign or partition table
+	 * creation. Hence the check for temporary table is enough here.
+	 */
+	if (!into->rel || into->rel->relpersistence == RELPERSISTENCE_TEMP)
+		return false;
+
+	if (queryDesc)
+	{
+		PlanState  *ps = queryDesc->planstate;
+		PlannedStmt *plannedstmt = queryDesc->plannedstmt;
+		bool  allow;
+
+		/*
+		 * We allow parallel inserts by the workers only if the Gather node has
+		 * no projections to perform and if the upper node is Gather. In case,
+		 * the Gather node has projections, which is possible if there are any
+		 * subplans in the query, the workers can not do those projections. And
+		 * when the upper node is GatherMerge, then the leader has to perform
+		 * the final phase i.e. merge the results by workers.
+		 */
+		allow = ps && IsA(ps, GatherState) && !ps->ps_ProjInfo &&
+				plannedstmt->parallelModeNeeded &&
+				plannedstmt->planTree &&
+				IsA(plannedstmt->planTree, Gather) &&
+				plannedstmt->planTree->lefttree &&
+				plannedstmt->planTree->lefttree->parallel_aware &&
+				plannedstmt->planTree->lefttree->parallel_safe;
+
+		return allow;
+	}
+
+	return true;
+}
+
+/*
+ * SetCTASParallelInsertState --- set the required info for the parallel
+ * inserts, that is required in the plan exection.
+ */
+void SetCTASParallelInsertState(QueryDesc *queryDesc)
+{
+	GatherState *gstate = (GatherState *) queryDesc->planstate;
+
+	/*
+	 * For parallelizing inserts in CTAS i.e. making each parallel worker
+	 * insert the tuples, we must send information such as intoclause(for
+	 * eachworker to build separate dest receiver), object id(for each worker
+	 * to open the created table).
+	 */
+	((DR_intorel *) queryDesc->dest)->is_parallel = true;
+	gstate->dest = queryDesc->dest;
+
+	/*
+	 * Since there are no rows that are transferred from workers to Gather
+	 * node, so we set it to 0 to be visible in estimated row count of explain
+	 * plans.
+	 */
+	queryDesc->plannedstmt->planTree->plan_rows = 0;
+}
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 43f9b01e83..03ac29cd64 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -556,6 +556,15 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	/* call ExecutorStart to prepare the plan for execution */
 	ExecutorStart(queryDesc, eflags);
 
+	/*
+	 * If SELECT part of the CTAS is parallelizable, then make each parallel
+	 * worker insert the tuples that are resulted in its execution into the
+	 * target table. We need plan state to be initialized by the executor to
+	 * decide whether to allow parallel inserts or not.
+	 */
+	if (IsParallelInsertInCTASAllowed(into, queryDesc))
+		SetCTASParallelInsertState(queryDesc);
+
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
 	{
@@ -1775,6 +1784,29 @@ ExplainNode(PlanState *planstate, List *ancestors,
 
 				if (gather->single_copy || es->format != EXPLAIN_FORMAT_TEXT)
 					ExplainPropertyBool("Single Copy", gather->single_copy, es);
+
+				/*
+				 * Show the create table information under Gather node in case
+				 * parallel workers have inserted the rows.
+				 */
+				if (IsA(planstate, GatherState))
+				{
+					GatherState *gstate = (GatherState *) planstate;
+
+					if (IS_PARALLEL_CTAS_DEST(gstate->dest) &&
+						((DR_intorel *) gstate->dest)->into->rel &&
+						((DR_intorel *) gstate->dest)->into->rel->relname)
+					{
+						es->indent--;
+						ExplainIndentText(es);
+						appendStringInfoString(es->str, "->  ");
+						appendStringInfoString(es->str, "Create ");
+						appendStringInfo(es->str, "%s\n",
+							((DR_intorel *) gstate->dest)->into->rel->relname);
+						ExplainIndentText(es);
+						es->indent++;
+					}
+				}
 			}
 			break;
 		case T_GatherMerge:
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index befde52691..9ef33eee54 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -23,6 +23,7 @@
 
 #include "postgres.h"
 
+#include "commands/createas.h"
 #include "executor/execParallel.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
@@ -65,6 +66,7 @@
 #define PARALLEL_KEY_QUERY_TEXT		UINT64CONST(0xE000000000000008)
 #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009)
 #define PARALLEL_KEY_WAL_USAGE			UINT64CONST(0xE00000000000000A)
+#define PARALLEL_KEY_INTO_CLAUSE		UINT64CONST(0xE00000000000000B)
 
 #define PARALLEL_TUPLE_QUEUE_SIZE		65536
 
@@ -77,6 +79,9 @@ typedef struct FixedParallelExecutorState
 	dsa_pointer param_exec;
 	int			eflags;
 	int			jit_flags;
+	Oid			objectid;		/* workers to open relation/table.  */
+	/* Number tuples inserted by all the workers. */
+	pg_atomic_uint64	processed;
 } FixedParallelExecutorState;
 
 /*
@@ -578,7 +583,8 @@ ExecParallelSetupTupleQueues(ParallelContext *pcxt, bool reinitialize)
 ParallelExecutorInfo *
 ExecInitParallelPlan(PlanState *planstate, EState *estate,
 					 Bitmapset *sendParams, int nworkers,
-					 int64 tuples_needed)
+					 int64 tuples_needed, IntoClause *intoclause,
+					 Oid objectid)
 {
 	ParallelExecutorInfo *pei;
 	ParallelContext *pcxt;
@@ -600,6 +606,8 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	Size		dsa_minsize = dsa_minimum_size();
 	char	   *query_string;
 	int			query_len;
+	char 		*intoclausestr = NULL;
+	int			intoclause_len = 0;
 
 	/*
 	 * Force any initplan outputs that we're going to pass to workers to be
@@ -712,6 +720,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_estimate_chunk(&pcxt->estimator, dsa_minsize);
 	shm_toc_estimate_keys(&pcxt->estimator, 1);
 
+	/* Estimate space for into clause for CTAS. */
+	if (IS_CTAS(intoclause) && OidIsValid(objectid))
+	{
+		intoclausestr = nodeToString(intoclause);
+		intoclause_len = strlen(intoclausestr);
+		shm_toc_estimate_chunk(&pcxt->estimator, intoclause_len + 1);
+		shm_toc_estimate_keys(&pcxt->estimator, 1);
+	}
+
 	/* Everyone's had a chance to ask for space, so now create the DSM. */
 	InitializeParallelDSM(pcxt);
 
@@ -729,6 +746,14 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	fpes->param_exec = InvalidDsaPointer;
 	fpes->eflags = estate->es_top_eflags;
 	fpes->jit_flags = estate->es_jit_flags;
+	pg_atomic_init_u64(&fpes->processed, 0);
+	pei->processed = &fpes->processed;
+
+	if (intoclausestr && OidIsValid(objectid))
+		fpes->objectid = objectid;
+	else
+		fpes->objectid = InvalidOid;
+
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, fpes);
 
 	/* Store query string */
@@ -758,8 +783,18 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_WAL_USAGE, walusage_space);
 	pei->wal_usage = walusage_space;
 
-	/* Set up the tuple queues that the workers will write into. */
-	pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	if (intoclausestr)
+	{
+		char *intoclause_space = shm_toc_allocate(pcxt->toc,
+												  intoclause_len + 1);
+		memcpy(intoclause_space, intoclausestr, intoclause_len + 1);
+		shm_toc_insert(pcxt->toc, PARALLEL_KEY_INTO_CLAUSE, intoclause_space);
+	}
+	else
+	{
+		/* Set up the tuple queues that the workers will write into. */
+		pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	}
 
 	/* We don't need the TupleQueueReaders yet, though. */
 	pei->reader = NULL;
@@ -1387,12 +1422,30 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 	void	   *area_space;
 	dsa_area   *area;
 	ParallelWorkerContext pwcxt;
+	char		*intoclausestr = NULL;
+	IntoClause	*intoclause = NULL;
 
 	/* Get fixed-size state. */
 	fpes = shm_toc_lookup(toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
 
-	/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
-	receiver = ExecParallelGetReceiver(seg, toc);
+	intoclausestr = shm_toc_lookup(toc, PARALLEL_KEY_INTO_CLAUSE, true);
+	if (intoclausestr)
+	{
+		/*
+		 * If the worker is for parallel insert in CTAS, then use the proper
+		 * dest receiver.
+		 */
+		intoclause = (IntoClause *) stringToNode(intoclausestr);
+		receiver = CreateIntoRelDestReceiver(intoclause);
+		((DR_intorel *)receiver)->is_parallel_worker = true;
+		((DR_intorel *)receiver)->object_id = fpes->objectid;
+	}
+	else
+	{
+		/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
+		receiver = ExecParallelGetReceiver(seg, toc);
+	}
+
 	instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_INSTRUMENTATION, true);
 	if (instrumentation != NULL)
 		instrument_options = instrumentation->instrument_options;
@@ -1471,6 +1524,13 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 			queryDesc->estate->es_jit->instr;
 	}
 
+	/*
+	 * Write out the number of tuples this worker has inserted. Leader will use
+	 * it to inform the end client.
+	 */
+	if (intoclausestr)
+		pg_atomic_add_fetch_u64(&fpes->processed, queryDesc->estate->es_processed);
+
 	/* Must do this after capturing instrumentation. */
 	ExecutorEnd(queryDesc);
 
diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c
index a01b46af14..e7c588c66a 100644
--- a/src/backend/executor/nodeGather.c
+++ b/src/backend/executor/nodeGather.c
@@ -32,6 +32,7 @@
 
 #include "access/relscan.h"
 #include "access/xact.h"
+#include "commands/createas.h"
 #include "executor/execdebug.h"
 #include "executor/execParallel.h"
 #include "executor/nodeGather.h"
@@ -48,6 +49,7 @@ static TupleTableSlot *ExecGather(PlanState *pstate);
 static TupleTableSlot *gather_getnext(GatherState *gatherstate);
 static MinimalTuple gather_readnext(GatherState *gatherstate);
 static void ExecShutdownGatherWorkers(GatherState *node);
+static void ExecParallelInsertInCTAS(GatherState *node);
 
 
 /* ----------------------------------------------------------------
@@ -131,6 +133,72 @@ ExecInitGather(Gather *node, EState *estate, int eflags)
 	return gatherstate;
 }
 
+/* ----------------------------------------------------------------
+ *		ExecParallelInsertInCTAS(node)
+ *
+ *		Facilitates parallel inserts by parallel workers and/or
+ *		leader for Create Table AS.
+ * ----------------------------------------------------------------
+ */
+static void
+ExecParallelInsertInCTAS(GatherState *node)
+{
+	/* Enable leader to insert in case no parallel workers were launched. */
+	if (node->nworkers_launched == 0)
+		node->need_to_scan_locally = true;
+
+	/*
+	 * By now, for parallel workers (if launched any), would have started their
+	 * work i.e. insertion to target table. In case the leader is chosen to
+	 * participate for parallel inserts in CTAS, then finish its share before
+	 * going to wait for the parallel workers to finish.
+	 */
+	if (node->need_to_scan_locally)
+	{
+		EState	   *estate = node->ps.state;
+		TupleTableSlot *outerTupleSlot;
+
+		for(;;)
+		{
+			/* Install our DSA area while executing the plan. */
+			estate->es_query_dsa =
+					node->pei ? node->pei->area : NULL;
+
+			outerTupleSlot = ExecProcNode(node->ps.lefttree);
+
+			estate->es_query_dsa = NULL;
+
+			if(TupIsNull(outerTupleSlot))
+				break;
+
+			(void) node->dest->receiveSlot(outerTupleSlot, node->dest);
+
+			node->ps.state->es_processed++;
+		}
+
+		node->need_to_scan_locally = false;
+	}
+
+	if (node->nworkers_launched > 0)
+	{
+		/*
+		 * We wait here for the parallel workers to finish their work and
+		 * accumulate the tuples they inserted and also their buffer/WAL usage.
+		 * We do not destroy the parallel context here, it will be done in
+		 * ExecShutdownGather at the end of the plan. Note that the
+		 * ExecShutdownGatherWorkers call from ExecShutdownGather will be a
+		 * no-op.
+		 */
+		ExecShutdownGatherWorkers(node);
+
+		/*
+		 * Add up the total tuples inserted by all workers, to the tuples
+		 * inserted by the leader(if any). This will be shared to client.
+		 */
+		node->ps.state->es_processed += pg_atomic_read_u64(node->pei->processed);
+	}
+}
+
 /* ----------------------------------------------------------------
  *		ExecGather(node)
  *
@@ -157,6 +225,7 @@ ExecGather(PlanState *pstate)
 	{
 		EState	   *estate = node->ps.state;
 		Gather	   *gather = (Gather *) node->ps.plan;
+		bool		isctas = IS_PARALLEL_CTAS_DEST(node->dest);
 
 		/*
 		 * Sometimes we might have to run without parallelism; but if parallel
@@ -165,6 +234,18 @@ ExecGather(PlanState *pstate)
 		if (gather->num_workers > 0 && estate->es_use_parallel_mode)
 		{
 			ParallelContext *pcxt;
+			IntoClause	*intoclause = NULL;
+			Oid			objectid = InvalidOid;
+
+			/*
+			 * Take the necessary information to be passed to workers for
+			 * parallel inserts in CTAS.
+			 */
+			if (isctas)
+			{
+				intoclause = ((DR_intorel *) node->dest)->into;
+				objectid = ((DR_intorel *) node->dest)->object_id;
+			}
 
 			/* Initialize, or re-initialize, shared state needed by workers. */
 			if (!node->pei)
@@ -172,7 +253,10 @@ ExecGather(PlanState *pstate)
 												 estate,
 												 gather->initParam,
 												 gather->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 /* CTAS info */
+												 intoclause,
+												 objectid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
@@ -190,13 +274,16 @@ ExecGather(PlanState *pstate)
 			/* Set up tuple queue readers to read the results. */
 			if (pcxt->nworkers_launched > 0)
 			{
-				ExecParallelCreateReaders(node->pei);
-				/* Make a working array showing the active readers */
-				node->nreaders = pcxt->nworkers_launched;
-				node->reader = (TupleQueueReader **)
-					palloc(node->nreaders * sizeof(TupleQueueReader *));
-				memcpy(node->reader, node->pei->reader,
-					   node->nreaders * sizeof(TupleQueueReader *));
+				if (!isctas)
+				{
+					ExecParallelCreateReaders(node->pei);
+					/* Make a working array showing the active readers */
+					node->nreaders = pcxt->nworkers_launched;
+					node->reader = (TupleQueueReader **)
+						palloc(node->nreaders * sizeof(TupleQueueReader *));
+					memcpy(node->reader, node->pei->reader,
+						node->nreaders * sizeof(TupleQueueReader *));
+				}
 			}
 			else
 			{
@@ -208,9 +295,17 @@ ExecGather(PlanState *pstate)
 		}
 
 		/* Run plan locally if no workers or enabled and not single-copy. */
-		node->need_to_scan_locally = (node->nreaders == 0)
+		node->need_to_scan_locally = (node->nreaders == 0 &&
+			!isctas)
 			|| (!gather->single_copy && parallel_leader_participation);
 		node->initialized = true;
+
+		/* Perform parallel inserts for CTAS. */
+		if (isctas)
+		{
+			ExecParallelInsertInCTAS(node);
+			return NULL;
+		}
 	}
 
 	/*
diff --git a/src/backend/executor/nodeGatherMerge.c b/src/backend/executor/nodeGatherMerge.c
index 47129344f3..ee45272c17 100644
--- a/src/backend/executor/nodeGatherMerge.c
+++ b/src/backend/executor/nodeGatherMerge.c
@@ -216,7 +216,9 @@ ExecGatherMerge(PlanState *pstate)
 												 estate,
 												 gm->initParam,
 												 gm->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 NULL,
+												 InvalidOid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7320de345c..5beae6c617 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -389,6 +389,7 @@ extern FullTransactionId GetCurrentFullTransactionIdIfAny(void);
 extern void MarkCurrentTransactionIdLoggedIfAny(void);
 extern bool SubTransactionIsActive(SubTransactionId subxid);
 extern CommandId GetCurrentCommandId(bool used);
+extern void SetCurrentCommandIdUsedForWorker(void);
 extern void SetParallelStartTimestamps(TimestampTz xact_ts, TimestampTz stmt_ts);
 extern TimestampTz GetCurrentTransactionStartTimestamp(void);
 extern TimestampTz GetCurrentStatementStartTimestamp(void);
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 7629230254..ab3aab58c5 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -14,12 +14,35 @@
 #ifndef CREATEAS_H
 #define CREATEAS_H
 
+#include "access/heapam.h"
 #include "catalog/objectaddress.h"
+#include "executor/execdesc.h"
 #include "nodes/params.h"
+#include "nodes/plannodes.h"
 #include "parser/parse_node.h"
 #include "tcop/dest.h"
 #include "utils/queryenvironment.h"
 
+typedef struct
+{
+	DestReceiver pub;			/* publicly-known function pointers */
+	IntoClause *into;			/* target relation specification */
+	/* These fields are filled by intorel_startup: */
+	Relation	rel;			/* relation to write to */
+	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
+	CommandId	output_cid;		/* cmin to insert in output tuples */
+	int			ti_options;		/* table_tuple_insert performance options */
+	BulkInsertState bistate;	/* bulk insert state */
+	bool		is_parallel;	/* is parallelism to be considered? */
+	bool		is_parallel_worker; /* true for parallel worker */
+	/* Used for table open by parallel worker. */
+	Oid			object_id;
+} DR_intorel;
+
+#define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
+#define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
+									 IS_CTAS(((DR_intorel *) dest)->into) && \
+									 ((DR_intorel *) dest)->is_parallel)
 
 extern ObjectAddress ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 									   ParamListInfo params, QueryEnvironment *queryEnv,
@@ -29,4 +52,9 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
+extern bool IsParallelInsertInCTASAllowed(IntoClause *into,
+										  QueryDesc *queryDesc);
+
+extern void SetCTASParallelInsertState(QueryDesc *queryDesc);
+
 #endif							/* CREATEAS_H */
diff --git a/src/include/executor/execParallel.h b/src/include/executor/execParallel.h
index 5a39a5b29c..9f959f741b 100644
--- a/src/include/executor/execParallel.h
+++ b/src/include/executor/execParallel.h
@@ -35,11 +35,15 @@ typedef struct ParallelExecutorInfo
 	/* These two arrays have pcxt->nworkers_launched entries: */
 	shm_mq_handle **tqueue;		/* tuple queues for worker output */
 	struct TupleQueueReader **reader;	/* tuple reader/writer support */
+	/* Number of tuples inserted by all workers. */
+	volatile pg_atomic_uint64	*processed;
 } ParallelExecutorInfo;
 
 extern ParallelExecutorInfo *ExecInitParallelPlan(PlanState *planstate,
 												  EState *estate, Bitmapset *sendParam, int nworkers,
-												  int64 tuples_needed);
+												  int64 tuples_needed,
+												  IntoClause *intoclause,
+												  Oid objectid);
 extern void ExecParallelCreateReaders(ParallelExecutorInfo *pei);
 extern void ExecParallelFinish(ParallelExecutorInfo *pei);
 extern void ExecParallelCleanup(ParallelExecutorInfo *pei);
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 61ba4c3666..e9c4442c22 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -23,6 +23,7 @@
 #include "nodes/tidbitmap.h"
 #include "partitioning/partdefs.h"
 #include "storage/condition_variable.h"
+#include "tcop/dest.h"
 #include "utils/hsearch.h"
 #include "utils/queryenvironment.h"
 #include "utils/reltrigger.h"
@@ -2326,6 +2327,8 @@ typedef struct GatherState
 	int			nreaders;		/* number of still-active workers */
 	int			nextreader;		/* next one to try to read from */
 	struct TupleQueueReader **reader;	/* array with nreaders active entries */
+	/* Parallel inserts in CTAS related info is specified below. */
+	DestReceiver *dest;
 } GatherState;
 
 /* ----------------
diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 0c4da2591a..8831597d54 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -76,4 +76,508 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+                     QUERY PLAN                      
+-----------------------------------------------------
+ LockRows (actual rows=10000 loops=1)
+   ->  Seq Scan on tenk1 (actual rows=10000 loops=1)
+(2 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+    select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_mat_view
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_mat_view;
+ count 
+-------
+ 10000
+(1 row)
+
+drop materialized view parallel_mat_view;
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+deallocate parallel_write_prep;
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select nextval('parallel_write_sequence'), four from tenk1;
+                  QUERY PLAN                   
+-----------------------------------------------
+ Seq Scan on tenk1 (actual rows=10000 loops=1)
+(1 row)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;
+                               QUERY PLAN                               
+------------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Params Evaluated: $1
+   Workers Launched: 3
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=1 loops=1)
+           ->  Gather (actual rows=1 loops=1)
+                 Workers Planned: 4
+                 Workers Launched: 4
+                 ->  Parallel Seq Scan on tenk2 (actual rows=1 loops=5)
+   ->  Parallel Seq Scan on tenk1 (actual rows=625 loops=4)
+         Filter: (four = 3)
+         Rows Removed by Filter: 1875
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+  2500
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;
+                             QUERY PLAN                              
+---------------------------------------------------------------------
+ Gather (actual rows=2500 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=500 loops=5)
+         Filter: (four = 3)
+         Rows Removed by Filter: 1500
+   SubPlan 1
+     ->  Function Scan on generate_series (actual rows=1 loops=2500)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+  2500
+(1 row)
+
+drop table parallel_write;
+create table temp1(col1) as select * from generate_series(1,5);
+create table temp2(col2) as select * from temp1;
+create table temp3(col3) as select * from temp1;
+-- parallel inserts must not occur, as there is a limit clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 limit 4;
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ Limit (actual rows=4 loops=1)
+   ->  Gather (actual rows=4 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an order by clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 order by 1;
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ Gather Merge (actual rows=5 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+   ->  Sort (actual rows=1 loops=4)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         Worker 0:  Sort Method: quicksort  Memory: 25kB
+         Worker 1:  Sort Method: quicksort  Memory: 25kB
+         Worker 2:  Sort Method: quicksort  Memory: 25kB
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(10 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an order by clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 order by 1;
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ Gather Merge (actual rows=5 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+   ->  Sort (actual rows=1 loops=4)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         Worker 0:  Sort Method: quicksort  Memory: 25kB
+         Worker 1:  Sort Method: quicksort  Memory: 25kB
+         Worker 2:  Sort Method: quicksort  Memory: 25kB
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(10 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is a distinct clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select distinct * from temp1;
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ HashAggregate (actual rows=5 loops=1)
+   Group Key: col1
+   Batches: 1  Memory Usage: 40kB
+   ->  Gather (actual rows=5 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(7 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an aggregate and group clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select count(*) from temp1 group by col1;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Finalize HashAggregate (actual rows=5 loops=1)
+   Group Key: col1
+   Batches: 1  Memory Usage: 40kB
+   ->  Gather (actual rows=5 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+         ->  Partial HashAggregate (actual rows=1 loops=4)
+               Group Key: col1
+               Batches: 1  Memory Usage: 40kB
+               Worker 0:  Batches: 1  Memory Usage: 40kB
+               Worker 1:  Batches: 1  Memory Usage: 40kB
+               Worker 2:  Batches: 1  Memory Usage: 40kB
+               ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(13 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an aggregate, group and having
+-- clauses
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select count(col1), (select col3 from
+        (select * from temp3) as tt limit 1) col4 from temp1, temp2
+    where temp1.col1 = temp2.col2 group by col4 having count(col1) > 0;
+                                    QUERY PLAN                                    
+----------------------------------------------------------------------------------
+ Finalize GroupAggregate (actual rows=1 loops=1)
+   Group Key: ($1)
+   Filter: (count(temp1.col1) > 0)
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=1 loops=1)
+           ->  Gather (actual rows=1 loops=1)
+                 Workers Planned: 3
+                 Workers Launched: 3
+                 ->  Parallel Seq Scan on temp3 (actual rows=0 loops=4)
+   ->  Gather (actual rows=1 loops=1)
+         Workers Planned: 3
+         Params Evaluated: $1
+         Workers Launched: 3
+         ->  Partial GroupAggregate (actual rows=0 loops=4)
+               Group Key: $1
+               ->  Parallel Hash Join (actual rows=1 loops=4)
+                     Hash Cond: (temp1.col1 = temp2.col2)
+                     ->  Parallel Seq Scan on temp1 (actual rows=5 loops=1)
+                     ->  Parallel Hash (actual rows=1 loops=4)
+                           Buckets: 4096  Batches: 1  Memory Usage: 64kB
+                           ->  Parallel Seq Scan on temp2 (actual rows=5 loops=1)
+(21 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is a window function
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select avg(col1) OVER (PARTITION BY col1) from temp1;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ WindowAgg (actual rows=5 loops=1)
+   ->  Sort (actual rows=5 loops=1)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         ->  Gather (actual rows=5 loops=1)
+               Workers Planned: 3
+               Workers Launched: 3
+               ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(8 rows)
+
+drop table parallel_write;
+-- nested loop join is the top node under which Gather node exists, so parallel
+-- inserts must not occur
+set enable_nestloop to on;
+set enable_mergejoin to off;
+set enable_hashjoin to off;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Nested Loop (actual rows=5 loops=1)
+   Join Filter: (temp1.col1 = temp2.col2)
+   Rows Removed by Join Filter: 20
+   ->  Gather (actual rows=5 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+   ->  Materialize (actual rows=5 loops=5)
+         ->  Gather (actual rows=5 loops=1)
+               Workers Planned: 3
+               Workers Launched: 3
+               ->  Parallel Seq Scan on temp2 (actual rows=1 loops=4)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- though the top node is Gather under which there exists parallel unsafe merge
+-- join node, so parallel inserts must not occur
+set enable_nestloop to off;
+set enable_mergejoin to on;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Gather (actual rows=5 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+   ->  Merge Join (actual rows=1 loops=4)
+         Merge Cond: (temp1.col1 = temp2.col2)
+         ->  Sort (actual rows=1 loops=4)
+               Sort Key: temp1.col1
+               Sort Method: quicksort  Memory: 25kB
+               Worker 0:  Sort Method: quicksort  Memory: 25kB
+               Worker 1:  Sort Method: quicksort  Memory: 25kB
+               Worker 2:  Sort Method: quicksort  Memory: 25kB
+               ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+         ->  Sort (actual rows=5 loops=1)
+               Sort Key: temp2.col2
+               Sort Method: quicksort  Memory: 25kB
+               ->  Seq Scan on temp2 (actual rows=5 loops=1)
+(16 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel hash join happens under Gather node, so parallel inserts must occur
+set enable_mergejoin to off;
+set enable_hashjoin to on;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+ ->  Create parallel_write
+   ->  Parallel Hash Join (actual rows=1 loops=4)
+         Hash Cond: (temp1.col1 = temp2.col2)
+         ->  Parallel Seq Scan on temp1 (actual rows=5 loops=1)
+         ->  Parallel Hash (actual rows=1 loops=4)
+               Buckets: 4096  Batches: 1  Memory Usage: 64kB
+               ->  Parallel Seq Scan on temp2 (actual rows=5 loops=1)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+reset enable_nestloop;
+reset enable_mergejoin;
+reset enable_hashjoin;
+drop table temp1;
+drop table temp2;
+drop table temp3;
 rollback;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 78b479cedf..6ee80ebd78 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -39,4 +39,184 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
 
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+    select length(stringu1) from tenk1;
+select count(*) from parallel_mat_view;
+drop materialized view parallel_mat_view;
+
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+select count(*) from parallel_write;
+deallocate parallel_write_prep;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select nextval('parallel_write_sequence'), four from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+create table temp1(col1) as select * from generate_series(1,5);
+create table temp2(col2) as select * from temp1;
+create table temp3(col3) as select * from temp1;
+
+-- parallel inserts must not occur, as there is a limit clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 limit 4;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an order by clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 order by 1;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an order by clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 order by 1;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is a distinct clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select distinct * from temp1;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an aggregate and group clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select count(*) from temp1 group by col1;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an aggregate, group and having
+-- clauses
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select count(col1), (select col3 from
+        (select * from temp3) as tt limit 1) col4 from temp1, temp2
+    where temp1.col1 = temp2.col2 group by col4 having count(col1) > 0;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is a window function
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select avg(col1) OVER (PARTITION BY col1) from temp1;
+drop table parallel_write;
+
+-- nested loop join is the top node under which Gather node exists, so parallel
+-- inserts must not occur
+set enable_nestloop to on;
+set enable_mergejoin to off;
+set enable_hashjoin to off;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- though the top node is Gather under which there exists parallel unsafe merge
+-- join node, so parallel inserts must not occur
+set enable_nestloop to off;
+set enable_mergejoin to on;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel hash join happens under Gather node, so parallel inserts must occur
+set enable_mergejoin to off;
+set enable_hashjoin to on;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+reset enable_nestloop;
+reset enable_mergejoin;
+reset enable_hashjoin;
+drop table temp1;
+drop table temp2;
+drop table temp3;
 rollback;
-- 
2.25.1

From 3bf4f3aee0f6fd9d0f7b29602113e97f35ef6a6f Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Wed, 9 Dec 2020 18:31:55 +0530
Subject: [PATCH v10] Tuple Cost Adjustment for Parallel Inserts in CTAS

---
 src/backend/commands/createas.c       |  9 ++++
 src/backend/commands/explain.c        | 10 +++++
 src/backend/optimizer/path/costsize.c | 19 ++++++++-
 src/backend/optimizer/plan/planner.c  | 61 +++++++++++++++++++++++++++
 src/include/commands/createas.h       | 16 +++++++
 src/include/nodes/parsenodes.h        |  1 +
 6 files changed, 115 insertions(+), 1 deletion(-)

diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 20f59cc2b8..eee8d19259 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -316,10 +316,19 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		query = linitial_node(Query, rewritten);
 		Assert(query->commandType == CMD_SELECT);
 
+		/*
+		 * Indication to the planner that the SELECT is from CTAS so that it
+		 * can adjust the parallel tuple cost if possible.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, NULL))
+			query->CTASParallelInsInfo |= CTAS_PARALLEL_INS_SELECT;
+
 		/* plan the query */
 		plan = pg_plan_query(query, pstate->p_sourcetext,
 							 CURSOR_OPT_PARALLEL_OK, params);
 
+		query->CTASParallelInsInfo &= CTAS_PARALLEL_INS_UNDEF;
+
 		/*
 		 * Use a snapshot with an updated command ID to ensure this query sees
 		 * results of any previously executed queries.  (This could only
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 03ac29cd64..8bd231a0c3 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -387,9 +387,19 @@ ExplainOneQuery(Query *query, int cursorOptions,
 			bufusage_start = pgBufferUsage;
 		INSTR_TIME_SET_CURRENT(planstart);
 
+		/*
+		 * Indication to the planner that the SELECT is from CTAS so that it
+		 * can adjust the parallel tuple cost if possible.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, NULL))
+			query->CTASParallelInsInfo |= CTAS_PARALLEL_INS_SELECT;
+
 		/* plan the query */
 		plan = pg_plan_query(query, queryString, cursorOptions, params);
 
+		if (into)
+			query->CTASParallelInsInfo &= CTAS_PARALLEL_INS_UNDEF;
+
 		INSTR_TIME_SET_CURRENT(planduration);
 		INSTR_TIME_SUBTRACT(planduration, planstart);
 
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 22d6935824..3a316f25f1 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -76,6 +76,7 @@
 #include "access/amapi.h"
 #include "access/htup_details.h"
 #include "access/tsmapi.h"
+#include "commands/createas.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "executor/nodeHash.h"
@@ -378,6 +379,7 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 {
 	Cost		startup_cost = 0;
 	Cost		run_cost = 0;
+	bool		ignore_tuple_cost = false;
 
 	/* Mark the path with the correct row estimate */
 	if (rows)
@@ -393,7 +395,22 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 
 	/* Parallel setup and communication cost. */
 	startup_cost += parallel_setup_cost;
-	run_cost += parallel_tuple_cost * path->path.rows;
+
+	/*
+	 * Do not consider tuple cost in case of parallel inserts by workers. We
+	 * would have set ignore flag in apply_scanjoin_target_to_paths before
+	 * generating Gather path for the upper level SELECT part of the CTAS.
+	 */
+	if ((root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT) &&
+		(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_IGN_TUP_COST))
+	{
+		ignore_tuple_cost = true;
+		/* Reset the ignore flag. */
+		root->parse->CTASParallelInsInfo &= ~CTAS_PARALLEL_INS_IGN_TUP_COST;
+	}
+
+	if (!ignore_tuple_cost)
+		run_cost += parallel_tuple_cost * path->path.rows;
 
 	path->path.startup_cost = startup_cost;
 	path->path.total_cost = (startup_cost + run_cost);
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index 1a94b58f8b..1041593237 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -28,6 +28,7 @@
 #include "catalog/pg_inherits.h"
 #include "catalog/pg_proc.h"
 #include "catalog/pg_type.h"
+#include "commands/createas.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "foreign/fdwapi.h"
@@ -7338,6 +7339,45 @@ can_partial_agg(PlannerInfo *root)
 	return true;
 }
 
+/*
+ * ignore_parallel_tuple_cost
+ *
+ * Gather node will not receive any tuples from the workers in case each worker
+ * inserts them in parallel. So, we set a flag to ignore parallel tuple cost by
+ * the Gather path in cost_gather if the SELECT is for CTAS and we are
+ * generating an upper level Gather path.
+*/
+static bool
+ignore_parallel_tuple_cost(PlannerInfo *root)
+{
+	if (root->query_level == 1 &&
+		(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT))
+	{
+		/*
+		 * In each of following cases, a parent path will be generated for the
+		 * upper Gather path(in grouping_planner), in which case we can not
+		 * let parallel inserts happen. So we do not set ignore tuple cost
+		 * flag.
+		 */
+		if (root->parse->rowMarks ||
+			limit_needed(root->parse) ||
+			root->parse->sortClause ||
+			root->parse->distinctClause ||
+			root->parse->hasWindowFuncs ||
+			root->parse->groupClause ||
+			root->parse->groupingSets ||
+			root->parse->hasAggs ||
+			root->hasHavingQual)
+			return false;
+
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_IGN_TUP_COST;
+
+		return true;
+	}
+
+	return false;
+}
+
 /*
  * apply_scanjoin_target_to_paths
  *
@@ -7557,8 +7597,29 @@ apply_scanjoin_target_to_paths(PlannerInfo *root,
 	 * one of the generated paths may turn out to be the cheapest one.
 	 */
 	if (rel->consider_parallel && !IS_OTHER_REL(rel))
+	{
+		/*
+		 * Set a flag to ignore parallel tuple cost by the Gather path in
+		 * cost_gather if the SELECT is for CTAS and we are generating an upper
+		 * level Gather path.
+		 */
+		bool	ignore = ignore_parallel_tuple_cost(root);
+
 		generate_useful_gather_paths(root, rel, false);
 
+		/*
+		 * Reset the ignore flag, in case we set it but
+		 * generate_useful_gather_paths returned without reaching cost_gather.
+		 */
+		if (ignore &&
+			(root->parse->CTASParallelInsInfo &
+			 CTAS_PARALLEL_INS_IGN_TUP_COST))
+		{
+			root->parse->CTASParallelInsInfo &=
+											~CTAS_PARALLEL_INS_IGN_TUP_COST;
+		}
+	}
+
 	/*
 	 * Reassess which paths are the cheapest, now that we've potentially added
 	 * new Gather (or Gather Merge) and/or Append (or MergeAppend) paths to
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index ab3aab58c5..6e722f0ac0 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -39,6 +39,22 @@ typedef struct
 	Oid			object_id;
 } DR_intorel;
 
+/*
+ * Information sent to the planner from CTAS to account for the cost
+ * calculations in cost_gather. We need to do this because, no tuples will be
+ * received by the Gather node if the workers insert the tuples in parallel.
+ */
+typedef enum CTASParallelInsertOpt
+{
+	CTAS_PARALLEL_INS_UNDEF = 0, /* undefined */
+	CTAS_PARALLEL_INS_SELECT = 1 << 0, /* set to this before planning */
+	/*
+	 * Set to this while planning for upper Gather path to ignore parallel
+	 * tuple cost in cost_gather.
+	 */
+	CTAS_PARALLEL_INS_IGN_TUP_COST = 1 << 1
+} CTASParallelInsertOpt;
+
 #define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
 #define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
 									 IS_CTAS(((DR_intorel *) dest)->into) && \
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index ec14fc2036..b140a42551 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -180,6 +180,7 @@ typedef struct Query
 	 */
 	int			stmt_location;	/* start location, or -1 if unknown */
 	int			stmt_len;		/* length in bytes; 0 means "rest of string" */
+	uint8 		CTASParallelInsInfo; /* parallel insert in CTAS info */
 } Query;
 
 
-- 
2.25.1

From 90ee038cec103a85307711b861b431317f1cd5bf Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 14 Dec 2020 15:16:49 +0530
Subject: [PATCH v11] Tuple Cost Adjustment for Parallel Inserts in CTAS

---
 src/backend/commands/createas.c       | 42 +++++++++++++++++-
 src/backend/commands/explain.c        | 14 ++++--
 src/backend/commands/prepare.c        |  3 +-
 src/backend/optimizer/path/costsize.c | 22 +++++++++-
 src/backend/optimizer/plan/planner.c  | 61 +++++++++++++++++++++++++++
 src/include/commands/createas.h       | 21 ++++++++-
 src/include/commands/explain.h        |  3 +-
 src/include/nodes/parsenodes.h        |  1 +
 8 files changed, 158 insertions(+), 9 deletions(-)

diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 9e6c8fb2ba..3ffea41ea6 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -316,6 +316,13 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		query = linitial_node(Query, rewritten);
 		Assert(query->commandType == CMD_SELECT);
 
+		/*
+		 * Indication to the planner that the SELECT is from CTAS so that it
+		 * can adjust the parallel tuple cost if possible.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, NULL, NULL))
+			query->CTASParallelInsInfo |= CTAS_PARALLEL_INS_SELECT;
+
 		/* plan the query */
 		plan = pg_plan_query(query, pstate->p_sourcetext,
 							 CURSOR_OPT_PARALLEL_OK, params);
@@ -344,7 +351,8 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		 * into the target table. We need plan state to be initialized by the
 		 * executor to decide whether to allow parallel inserts or not.
 		 */
-		if (IsParallelInsertInCTASAllowed(into, queryDesc))
+		if (IsParallelInsertInCTASAllowed(into, queryDesc,
+										  &query->CTASParallelInsInfo))
 			SetCTASParallelInsertState(queryDesc);
 
 		/* run the plan to completion */
@@ -659,7 +667,8 @@ intorel_destroy(DestReceiver *self)
  * IsParallelInsertInCTASAllowed --- determine whether or not parallel
  * insertion is possible.
  */
-bool IsParallelInsertInCTASAllowed(IntoClause *into, QueryDesc *queryDesc)
+bool IsParallelInsertInCTASAllowed(IntoClause *into, QueryDesc *queryDesc,
+								   uint8 *tuple_cost_flags)
 {
 	if (!IS_CTAS(into))
 		return false;
@@ -678,6 +687,7 @@ bool IsParallelInsertInCTASAllowed(IntoClause *into, QueryDesc *queryDesc)
 	{
 		PlanState  *ps = queryDesc->planstate;
 		bool  		allow;
+		bool		need_to_assert = false;
 
 		/*
 		 * We allow parallel inserts by the workers only if the Gather node has
@@ -690,6 +700,34 @@ bool IsParallelInsertInCTASAllowed(IntoClause *into, QueryDesc *queryDesc)
 		 */
 		allow = ps && IsA(ps, GatherState) && !ps->ps_ProjInfo;
 
+		/*
+		 * It should not happen that in cost_gather we have ignored the
+		 * parallel tuple cost and now we are not allowing the parallel
+		 * inserts. And also we might need assertion only if the top node is
+		 * GatherState. Because the main intention of assertion is to check if
+		 * we enforced planner to ignore the parallel tuple cost (with the
+		 * intention of choosing parallel inserts) due to which
+		 * the parallel plan was chosen, but we do not allow the parallel
+		 * inserts now.
+		 */
+		if (!allow && tuple_cost_flags && ps && IsA(ps, GatherState))
+			need_to_assert = true;
+
+		if (need_to_assert)
+		{
+			/*
+			 * If we have correctly ignored parallel tuple cost in planner
+			 * while creating Gather path, then this assertion failure should
+			 * not occur. If it occurs, that means the planner may have chosen
+			 * this parallel plan because of our enforcement to ignore the
+			 * parallel tuple cost.
+			 */
+			Assert(!(*tuple_cost_flags & CTAS_PARALLEL_INS_TUP_COST_IGNORED));
+		}
+
+		if (tuple_cost_flags)
+			*tuple_cost_flags = CTAS_PARALLEL_INS_UNDEF;
+
 		return allow;
 	}
 
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 03ac29cd64..d0152deba7 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -387,6 +387,13 @@ ExplainOneQuery(Query *query, int cursorOptions,
 			bufusage_start = pgBufferUsage;
 		INSTR_TIME_SET_CURRENT(planstart);
 
+		/*
+		 * Indication to the planner that the SELECT is from CTAS so that it
+		 * can adjust the parallel tuple cost if possible.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, NULL, NULL))
+			query->CTASParallelInsInfo |= CTAS_PARALLEL_INS_SELECT;
+
 		/* plan the query */
 		plan = pg_plan_query(query, queryString, cursorOptions, params);
 
@@ -402,7 +409,8 @@ ExplainOneQuery(Query *query, int cursorOptions,
 
 		/* run it (if needed) and produce output */
 		ExplainOnePlan(plan, into, es, queryString, params, queryEnv,
-					   &planduration, (es->buffers ? &bufusage : NULL));
+					   &planduration, (es->buffers ? &bufusage : NULL),
+					   &query->CTASParallelInsInfo);
 	}
 }
 
@@ -496,7 +504,7 @@ void
 ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 			   const char *queryString, ParamListInfo params,
 			   QueryEnvironment *queryEnv, const instr_time *planduration,
-			   const BufferUsage *bufusage)
+			   const BufferUsage *bufusage, uint8 *ctas_tuple_cost_flags)
 {
 	DestReceiver *dest;
 	QueryDesc  *queryDesc;
@@ -562,7 +570,7 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	 * target table. We need plan state to be initialized by the executor to
 	 * decide whether to allow parallel inserts or not.
 	 */
-	if (IsParallelInsertInCTASAllowed(into, queryDesc))
+	if (IsParallelInsertInCTASAllowed(into, queryDesc, ctas_tuple_cost_flags))
 		SetCTASParallelInsertState(queryDesc);
 
 	/* Execute the plan for statistics if asked for */
diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c
index 4b18be5b27..12227b6e79 100644
--- a/src/backend/commands/prepare.c
+++ b/src/backend/commands/prepare.c
@@ -674,7 +674,8 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es,
 
 		if (pstmt->commandType != CMD_UTILITY)
 			ExplainOnePlan(pstmt, into, es, query_string, paramLI, queryEnv,
-						   &planduration, (es->buffers ? &bufusage : NULL));
+						   &planduration, (es->buffers ? &bufusage : NULL),
+						   NULL);
 		else
 			ExplainOneUtility(pstmt->utilityStmt, into, es, query_string,
 							  paramLI, queryEnv);
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 22d6935824..800f25903d 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -76,6 +76,7 @@
 #include "access/amapi.h"
 #include "access/htup_details.h"
 #include "access/tsmapi.h"
+#include "commands/createas.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "executor/nodeHash.h"
@@ -378,6 +379,7 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 {
 	Cost		startup_cost = 0;
 	Cost		run_cost = 0;
+	bool		ignore_tuple_cost = false;
 
 	/* Mark the path with the correct row estimate */
 	if (rows)
@@ -393,7 +395,25 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 
 	/* Parallel setup and communication cost. */
 	startup_cost += parallel_setup_cost;
-	run_cost += parallel_tuple_cost * path->path.rows;
+
+	/*
+	 * Do not consider tuple cost in case of we intend to perform parallel
+	 * inserts by workers. We would have set ignore flag in
+	 * apply_scanjoin_target_to_paths before generating Gather path for the
+	 * upper level SELECT part of the CTAS.
+	 */
+	if ((root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT) &&
+		(root->parse->CTASParallelInsInfo &
+		 CTAS_PARALLEL_INS_TUP_COST_CAN_IGN))
+	{
+		ignore_tuple_cost = true;
+		root->parse->CTASParallelInsInfo &=
+									~CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_TUP_COST_IGNORED;
+	}
+
+	if (!ignore_tuple_cost)
+		run_cost += parallel_tuple_cost * path->path.rows;
 
 	path->path.startup_cost = startup_cost;
 	path->path.total_cost = (startup_cost + run_cost);
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index 1a94b58f8b..d287b6bfbb 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -28,6 +28,7 @@
 #include "catalog/pg_inherits.h"
 #include "catalog/pg_proc.h"
 #include "catalog/pg_type.h"
+#include "commands/createas.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "foreign/fdwapi.h"
@@ -7338,6 +7339,45 @@ can_partial_agg(PlannerInfo *root)
 	return true;
 }
 
+/*
+ * ignore_parallel_tuple_cost
+ *
+ * Gather node will not receive any tuples from the workers in case each worker
+ * inserts them in parallel. So, we set a flag to ignore parallel tuple cost by
+ * the Gather path in cost_gather if the SELECT is for CTAS and we are
+ * generating an upper level Gather path.
+*/
+static bool
+ignore_parallel_tuple_cost(PlannerInfo *root)
+{
+	if (root->query_level == 1 &&
+		(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT))
+	{
+		/*
+		 * In each of following cases, a parent path will be generated for the
+		 * upper Gather path(in grouping_planner), in which case we can not
+		 * let parallel inserts happen. So we do not set ignore tuple cost
+		 * flag.
+		 */
+		if (root->parse->rowMarks ||
+			limit_needed(root->parse) ||
+			root->parse->sortClause ||
+			root->parse->distinctClause ||
+			root->parse->hasWindowFuncs ||
+			root->parse->groupClause ||
+			root->parse->groupingSets ||
+			root->parse->hasAggs ||
+			root->hasHavingQual)
+			return false;
+
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+
+		return true;
+	}
+
+	return false;
+}
+
 /*
  * apply_scanjoin_target_to_paths
  *
@@ -7557,8 +7597,29 @@ apply_scanjoin_target_to_paths(PlannerInfo *root,
 	 * one of the generated paths may turn out to be the cheapest one.
 	 */
 	if (rel->consider_parallel && !IS_OTHER_REL(rel))
+	{
+		/*
+		 * Set a flag to ignore parallel tuple cost by the Gather path in
+		 * cost_gather if the SELECT is for CTAS and we are generating an upper
+		 * level Gather path.
+		 */
+		bool	ignore = ignore_parallel_tuple_cost(root);
+
 		generate_useful_gather_paths(root, rel, false);
 
+		/*
+		 * Reset the ignore flag, in case we set it but
+		 * generate_useful_gather_paths returned without reaching cost_gather.
+		 */
+		if (ignore &&
+			(root->parse->CTASParallelInsInfo &
+			 CTAS_PARALLEL_INS_TUP_COST_CAN_IGN))
+		{
+			root->parse->CTASParallelInsInfo &=
+										~CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+		}
+	}
+
 	/*
 	 * Reassess which paths are the cheapest, now that we've potentially added
 	 * new Gather (or Gather Merge) and/or Append (or MergeAppend) paths to
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index ab3aab58c5..e01a6152ce 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -39,6 +39,24 @@ typedef struct
 	Oid			object_id;
 } DR_intorel;
 
+/*
+ * Information sent to the planner from CTAS to account for the cost
+ * calculations in cost_gather. We need to do this because, no tuples will be
+ * received by the Gather node if the workers insert the tuples in parallel.
+ */
+typedef enum CTASParallelInsertOpt
+{
+	CTAS_PARALLEL_INS_UNDEF = 0, /* undefined */
+	CTAS_PARALLEL_INS_SELECT = 1 << 0, /* set to this before planning */
+	/*
+	 * Set to this while planning for upper Gather path to ignore parallel
+	 * tuple cost in cost_gather.
+	 */
+	CTAS_PARALLEL_INS_TUP_COST_CAN_IGN = 1 << 1,
+
+	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2
+} CTASParallelInsertOpt;
+
 #define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
 #define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
 									 IS_CTAS(((DR_intorel *) dest)->into) && \
@@ -53,7 +71,8 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
 extern bool IsParallelInsertInCTASAllowed(IntoClause *into,
-										  QueryDesc *queryDesc);
+										  QueryDesc *queryDesc,
+										  uint8 *tuple_cost_flags);
 
 extern void SetCTASParallelInsertState(QueryDesc *queryDesc);
 
diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h
index ba661d32a6..1a1806dbf1 100644
--- a/src/include/commands/explain.h
+++ b/src/include/commands/explain.h
@@ -91,7 +91,8 @@ extern void ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into,
 						   ExplainState *es, const char *queryString,
 						   ParamListInfo params, QueryEnvironment *queryEnv,
 						   const instr_time *planduration,
-						   const BufferUsage *bufusage);
+						   const BufferUsage *bufusage,
+						   uint8 *ctas_tuple_cost_flags);
 
 extern void ExplainPrintPlan(ExplainState *es, QueryDesc *queryDesc);
 extern void ExplainPrintTriggers(ExplainState *es, QueryDesc *queryDesc);
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 48a79a7657..81b148c383 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -180,6 +180,7 @@ typedef struct Query
 	 */
 	int			stmt_location;	/* start location, or -1 if unknown */
 	int			stmt_len;		/* length in bytes; 0 means "rest of string" */
+	uint8 		CTASParallelInsInfo; /* parallel insert in CTAS info */
 } Query;
 
 
-- 
2.25.1

From 84fd0237cb9b3b2650d9a9d3c139d54dc86e085c Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 14 Dec 2020 14:59:26 +0530
Subject: [PATCH v11] Parallel Inserts in CREATE TABLE AS

The idea of this patch is to allow the leader and each worker
insert the tuples in parallel if the SELECT part of the CTAS is
parallelizable.

The design:
Let the planner know that the SELECT is from CTAS in createas.c
so that it can set the number of tuples transferred from the
workers to Gather node to 0. With this change, there are chances
that the planner may choose the parallel plan. After the planning,
check if the upper plan node is Gather in createas.c and mark a
parallelism flag in the CTAS dest receiver. Pass the into clause,
object id, command id from the leader to workers, so that each
worker can create its own CTAS dest receiver. Leader inserts its
share of tuples if instructed to do, and so are workers. Each
worker writes atomically its number of inserted tuples into a
shared memory variable, the leader combines this with its own
number of inserted tuples and shares to the client.
---
 src/backend/access/heap/heapam.c             |  11 -
 src/backend/access/transam/xact.c            |  30 +-
 src/backend/commands/createas.c              | 326 ++++++++----
 src/backend/commands/explain.c               |  32 ++
 src/backend/executor/execParallel.c          |  70 ++-
 src/backend/executor/nodeGather.c            | 113 ++++-
 src/backend/executor/nodeGatherMerge.c       |   4 +-
 src/include/access/xact.h                    |   1 +
 src/include/commands/createas.h              |  28 +
 src/include/executor/execParallel.h          |   6 +-
 src/include/nodes/execnodes.h                |   3 +
 src/test/regress/expected/write_parallel.out | 505 +++++++++++++++++++
 src/test/regress/sql/write_parallel.sql      | 180 +++++++
 13 files changed, 1169 insertions(+), 140 deletions(-)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index a9583f3103..86347ba273 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2043,17 +2043,6 @@ static HeapTuple
 heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
 					CommandId cid, int options)
 {
-	/*
-	 * To allow parallel inserts, we need to ensure that they are safe to be
-	 * performed in workers. We have the infrastructure to allow parallel
-	 * inserts in general except for the cases where inserts generate a new
-	 * CommandId (eg. inserts into a table having a foreign key column).
-	 */
-	if (IsParallelWorker())
-		ereport(ERROR,
-				(errcode(ERRCODE_INVALID_TRANSACTION_STATE),
-				 errmsg("cannot insert tuples in a parallel worker")));
-
 	tup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
 	tup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
 	tup->t_data->t_infomask |= HEAP_XMAX_INVALID;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 9cd0b7c11b..db6eedd635 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -763,18 +763,34 @@ GetCurrentCommandId(bool used)
 	/* this is global to a transaction, not subtransaction-local */
 	if (used)
 	{
-		/*
-		 * Forbid setting currentCommandIdUsed in a parallel worker, because
-		 * we have no provision for communicating this back to the leader.  We
-		 * could relax this restriction when currentCommandIdUsed was already
-		 * true at the start of the parallel operation.
-		 */
-		Assert(!IsParallelWorker());
+		 /*
+		  * This is a temporary hack for all common parallel insert cases i.e.
+		  * insert into, ctas, copy from. To be changed later. In a parallel
+		  * worker, set currentCommandIdUsed to true only if it was not set to
+		  * true at the start of the parallel operation (by way of
+		  * SetCurrentCommandIdUsedForWorker()). We have to do this because
+		  * GetCurrentCommandId(true) may be called from anywhere, especially
+		  * for parallel inserts, within parallel worker.
+		  */
+		Assert(!(IsParallelWorker() && !currentCommandIdUsed));
 		currentCommandIdUsed = true;
 	}
 	return currentCommandId;
 }
 
+/*
+ *	SetCurrentCommandIdUsedForWorker
+ *
+ * For a parallel worker, record that the currentCommandId has been used. This
+ * must only be called at the start of a parallel operation.
+ */
+void
+SetCurrentCommandIdUsedForWorker(void)
+{
+	Assert(IsParallelWorker() && !currentCommandIdUsed && currentCommandId != InvalidCommandId);
+	currentCommandIdUsed = true;
+}
+
 /*
  *	SetParallelStartTimestamps
  *
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 6bf6c5a310..9e6c8fb2ba 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,18 +51,6 @@
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 
-typedef struct
-{
-	DestReceiver pub;			/* publicly-known function pointers */
-	IntoClause *into;			/* target relation specification */
-	/* These fields are filled by intorel_startup: */
-	Relation	rel;			/* relation to write to */
-	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
-	CommandId	output_cid;		/* cmin to insert in output tuples */
-	int			ti_options;		/* table_tuple_insert performance options */
-	BulkInsertState bistate;	/* bulk insert state */
-} DR_intorel;
-
 /* utility functions for CTAS definition creation */
 static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into);
 static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -350,6 +338,15 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		/* call ExecutorStart to prepare the plan for execution */
 		ExecutorStart(queryDesc, GetIntoRelEFlags(into));
 
+		/*
+		 * If SELECT part of the CTAS is parallelizable, then make each
+		 * parallel worker insert the tuples that are resulted in its execution
+		 * into the target table. We need plan state to be initialized by the
+		 * executor to decide whether to allow parallel inserts or not.
+		 */
+		if (IsParallelInsertInCTASAllowed(into, queryDesc))
+			SetCTASParallelInsertState(queryDesc);
+
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
 
@@ -418,6 +415,9 @@ CreateIntoRelDestReceiver(IntoClause *intoClause)
 	self->pub.rDestroy = intorel_destroy;
 	self->pub.mydest = DestIntoRel;
 	self->into = intoClause;
+	self->is_parallel = false;
+	self->is_parallel_worker = false;
+	self->object_id = InvalidOid;
 	/* other private fields will be set during intorel_startup */
 
 	return (DestReceiver *) self;
@@ -430,121 +430,169 @@ static void
 intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 {
 	DR_intorel *myState = (DR_intorel *) self;
-	IntoClause *into = myState->into;
-	bool		is_matview;
-	List	   *attrList;
 	ObjectAddress intoRelationAddr;
 	Relation	intoRelationDesc;
-	ListCell   *lc;
-	int			attnum;
-
-	Assert(into != NULL);		/* else somebody forgot to set it */
 
-	/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
-	is_matview = (into->viewQuery != NULL);
+	if (myState->is_parallel_worker)
+	{
+		/* In the worker */
+		intoRelationDesc = table_open(myState->object_id, AccessExclusiveLock);
+		myState->rel = intoRelationDesc;
+		myState->reladdr = InvalidObjectAddress;
+		myState->ti_options = 0;
+		myState->bistate = GetBulkInsertState();
 
-	/*
-	 * Build column definitions using "pre-cooked" type and collation info. If
-	 * a column name list was specified in CREATE TABLE AS, override the
-	 * column names derived from the query.  (Too few column names are OK, too
-	 * many are not.)
-	 */
-	attrList = NIL;
-	lc = list_head(into->colNames);
-	for (attnum = 0; attnum < typeinfo->natts; attnum++)
+		/*
+		 * Right after the table is created in the leader, the command id is
+		 * incremented (in create_ctas_internal()). The new command id is
+		 * marked as used in intorel_startup(), then the parallel mode is
+		 * entered. The command id and transaction id are serialized into
+		 * parallel DSM, they are then available to all parallel workers. All
+		 * the workers need to mark the command id as used before insertion.
+		 */
+		SetCurrentCommandIdUsedForWorker();
+		myState->output_cid = GetCurrentCommandId(false);
+	}
+	else
 	{
-		Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
-		ColumnDef  *col;
-		char	   *colname;
+		IntoClause *into = myState->into;
+		bool		is_matview;
+		List	   *attrList;
+		ListCell   *lc;
+		int			attnum;
+
+		Assert(into != NULL);		/* else somebody forgot to set it */
+
+		/*
+		 * This code supports both CREATE TABLE AS and CREATE MATERIALIZED
+		 * VIEW.
+		 */
+		is_matview = (into->viewQuery != NULL);
 
-		if (lc)
+		/*
+		 * Build column definitions using "pre-cooked" type and collation info.
+		 * If a column name list was specified in CREATE TABLE AS, override the
+		 * column names derived from the query.  (Too few column names are OK,
+		 * too many are not.)
+		 */
+		attrList = NIL;
+		lc = list_head(into->colNames);
+		for (attnum = 0; attnum < typeinfo->natts; attnum++)
 		{
-			colname = strVal(lfirst(lc));
-			lc = lnext(into->colNames, lc);
+			Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
+			ColumnDef  *col;
+			char	   *colname;
+
+			if (lc)
+			{
+				colname = strVal(lfirst(lc));
+				lc = lnext(into->colNames, lc);
+			}
+			else
+				colname = NameStr(attribute->attname);
+
+			col = makeColumnDef(colname,
+								attribute->atttypid,
+								attribute->atttypmod,
+								attribute->attcollation);
+
+			/*
+			 * It's possible that the column is of a collatable type but the
+			 * collation could not be resolved, so double-check.  (We must
+			 * check this here because DefineRelation would adopt the type's
+			 * default collation rather than complaining.)
+			 */
+			if (!OidIsValid(col->collOid) &&
+				type_is_collatable(col->typeName->typeOid))
+				ereport(ERROR,
+						(errcode(ERRCODE_INDETERMINATE_COLLATION),
+						errmsg("no collation was derived for column \"%s\" with collatable type %s",
+								col->colname,
+								format_type_be(col->typeName->typeOid)),
+						errhint("Use the COLLATE clause to set the collation explicitly.")));
+
+			attrList = lappend(attrList, col);
 		}
-		else
-			colname = NameStr(attribute->attname);
 
-		col = makeColumnDef(colname,
-							attribute->atttypid,
-							attribute->atttypmod,
-							attribute->attcollation);
+		if (lc != NULL)
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					errmsg("too many column names were specified")));
 
 		/*
-		 * It's possible that the column is of a collatable type but the
-		 * collation could not be resolved, so double-check.  (We must check
-		 * this here because DefineRelation would adopt the type's default
-		 * collation rather than complaining.)
+		 * Actually create the target table
 		 */
-		if (!OidIsValid(col->collOid) &&
-			type_is_collatable(col->typeName->typeOid))
-			ereport(ERROR,
-					(errcode(ERRCODE_INDETERMINATE_COLLATION),
-					 errmsg("no collation was derived for column \"%s\" with collatable type %s",
-							col->colname,
-							format_type_be(col->typeName->typeOid)),
-					 errhint("Use the COLLATE clause to set the collation explicitly.")));
+		intoRelationAddr = create_ctas_internal(attrList, into);
 
-		attrList = lappend(attrList, col);
-	}
+		/*
+		 * Finally we can open the target table
+		 */
+		intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
 
-	if (lc != NULL)
-		ereport(ERROR,
-				(errcode(ERRCODE_SYNTAX_ERROR),
-				 errmsg("too many column names were specified")));
+		/*
+		 * Make sure the constructed table does not have RLS enabled.
+		 *
+		 * check_enable_rls() will ereport(ERROR) itself if the user has
+		 * requested something invalid, and otherwise will return RLS_ENABLED
+		 * if RLS should be enabled here.  We don't actually support that
+		 * currently, so throw our own ereport(ERROR) if that happens.
+		 */
+		if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					errmsg("policies not yet implemented for this command")));
 
-	/*
-	 * Actually create the target table
-	 */
-	intoRelationAddr = create_ctas_internal(attrList, into);
+		/*
+		 * Tentatively mark the target as populated, if it's a matview and
+		 * we're going to fill it; otherwise, no change needed.
+		 */
+		if (is_matview && !into->skipData)
+			SetMatViewPopulatedState(intoRelationDesc, true);
 
-	/*
-	 * Finally we can open the target table
-	 */
-	intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
+		/*
+		 * Fill private fields of myState for use by later routines
+		 */
+		myState->rel = intoRelationDesc;
+		myState->reladdr = intoRelationAddr;
+		myState->output_cid = GetCurrentCommandId(true);
+		myState->ti_options = TABLE_INSERT_SKIP_FSM;
 
-	/*
-	 * Make sure the constructed table does not have RLS enabled.
-	 *
-	 * check_enable_rls() will ereport(ERROR) itself if the user has requested
-	 * something invalid, and otherwise will return RLS_ENABLED if RLS should
-	 * be enabled here.  We don't actually support that currently, so throw
-	 * our own ereport(ERROR) if that happens.
-	 */
-	if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
-		ereport(ERROR,
-				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-				 errmsg("policies not yet implemented for this command")));
+		/*
+		 * If WITH NO DATA is specified, there is no need to set up the state
+		 * for bulk inserts as there are no tuples to insert.
+		 */
+		if (!into->skipData)
+			myState->bistate = GetBulkInsertState();
+		else
+			myState->bistate = NULL;
 
-	/*
-	 * Tentatively mark the target as populated, if it's a matview and we're
-	 * going to fill it; otherwise, no change needed.
-	 */
-	if (is_matview && !into->skipData)
-		SetMatViewPopulatedState(intoRelationDesc, true);
+		if (myState->is_parallel)
+		{
+			myState->object_id = intoRelationAddr.objectId;
 
-	/*
-	 * Fill private fields of myState for use by later routines
-	 */
-	myState->rel = intoRelationDesc;
-	myState->reladdr = intoRelationAddr;
-	myState->output_cid = GetCurrentCommandId(true);
-	myState->ti_options = TABLE_INSERT_SKIP_FSM;
+			/*
+			 * We don't need to skip contacting FSM while inserting tuples
+			 * for parallel mode, while extending the relations, workers
+			 * instead of blocking on a page while another worker is inserting,
+			 * can check the FSM for another page that can accommodate the
+			 * tuples. This results in major benefit for parallel inserts.
+			 */
+			myState->ti_options = 0;
 
-	/*
-	 * If WITH NO DATA is specified, there is no need to set up the state for
-	 * bulk inserts as there are no tuples to insert.
-	 */
-	if (!into->skipData)
-		myState->bistate = GetBulkInsertState();
-	else
-		myState->bistate = NULL;
+			/*
+			 * rd_createSubid is marked invalid, otherwise, the table is not
+			 * allowed to be extended by the workers.
+			 */
+			myState->rel->rd_createSubid = InvalidSubTransactionId;
+		}
 
-	/*
-	 * Valid smgr_targblock implies something already wrote to the relation.
-	 * This may be harmless, but this function hasn't planned for it.
-	 */
-	Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+		/*
+		 * Valid smgr_targblock implies something already wrote to the
+		 * relation. This may be harmless, but this function hasn't planned for
+		 * it.
+		 */
+		Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+	}
 }
 
 /*
@@ -606,3 +654,69 @@ intorel_destroy(DestReceiver *self)
 {
 	pfree(self);
 }
+
+/*
+ * IsParallelInsertInCTASAllowed --- determine whether or not parallel
+ * insertion is possible.
+ */
+bool IsParallelInsertInCTASAllowed(IntoClause *into, QueryDesc *queryDesc)
+{
+	if (!IS_CTAS(into))
+		return false;
+
+	/*
+	 * Do not allow parallel inserts if the table is temporary. As the
+	 * temporary tables are backend local, workers can not know about them.
+	 * Currently, CTAS supports creation of normal(logged), temporary and
+	 * unlogged tables. It does not support foreign or partition table
+	 * creation. Hence the check for temporary table is enough here.
+	 */
+	if (!into->rel || into->rel->relpersistence == RELPERSISTENCE_TEMP)
+		return false;
+
+	if (queryDesc)
+	{
+		PlanState  *ps = queryDesc->planstate;
+		bool  		allow;
+
+		/*
+		 * We allow parallel inserts by the workers only if the Gather node has
+		 * no projections to perform and if the upper node is Gather. In case,
+		 * the Gather node has projections, which is possible if there are any
+		 * subplans in the query, the workers can not do those projections.
+		 * If the upper node is GatherMerge, then the leader has to perform the
+		 * final phase i.e. merge the results by workers, so we do not allow
+		 * parallel inserts.
+		 */
+		allow = ps && IsA(ps, GatherState) && !ps->ps_ProjInfo;
+
+		return allow;
+	}
+
+	return true;
+}
+
+/*
+ * SetCTASParallelInsertState --- set the required info for the parallel
+ * inserts, that is required in the plan exection.
+ */
+void SetCTASParallelInsertState(QueryDesc *queryDesc)
+{
+	GatherState *gstate = (GatherState *) queryDesc->planstate;
+
+	/*
+	 * For parallelizing inserts in CTAS i.e. making each parallel worker
+	 * insert the tuples, we must send information such as intoclause(for
+	 * eachworker to build separate dest receiver), object id(for each worker
+	 * to open the created table).
+	 */
+	((DR_intorel *) queryDesc->dest)->is_parallel = true;
+	gstate->dest = queryDesc->dest;
+
+	/*
+	 * Since there are no rows that are transferred from workers to Gather
+	 * node, so we set it to 0 to be visible in estimated row count of explain
+	 * plans.
+	 */
+	queryDesc->plannedstmt->planTree->plan_rows = 0;
+}
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 43f9b01e83..03ac29cd64 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -556,6 +556,15 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	/* call ExecutorStart to prepare the plan for execution */
 	ExecutorStart(queryDesc, eflags);
 
+	/*
+	 * If SELECT part of the CTAS is parallelizable, then make each parallel
+	 * worker insert the tuples that are resulted in its execution into the
+	 * target table. We need plan state to be initialized by the executor to
+	 * decide whether to allow parallel inserts or not.
+	 */
+	if (IsParallelInsertInCTASAllowed(into, queryDesc))
+		SetCTASParallelInsertState(queryDesc);
+
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
 	{
@@ -1775,6 +1784,29 @@ ExplainNode(PlanState *planstate, List *ancestors,
 
 				if (gather->single_copy || es->format != EXPLAIN_FORMAT_TEXT)
 					ExplainPropertyBool("Single Copy", gather->single_copy, es);
+
+				/*
+				 * Show the create table information under Gather node in case
+				 * parallel workers have inserted the rows.
+				 */
+				if (IsA(planstate, GatherState))
+				{
+					GatherState *gstate = (GatherState *) planstate;
+
+					if (IS_PARALLEL_CTAS_DEST(gstate->dest) &&
+						((DR_intorel *) gstate->dest)->into->rel &&
+						((DR_intorel *) gstate->dest)->into->rel->relname)
+					{
+						es->indent--;
+						ExplainIndentText(es);
+						appendStringInfoString(es->str, "->  ");
+						appendStringInfoString(es->str, "Create ");
+						appendStringInfo(es->str, "%s\n",
+							((DR_intorel *) gstate->dest)->into->rel->relname);
+						ExplainIndentText(es);
+						es->indent++;
+					}
+				}
 			}
 			break;
 		case T_GatherMerge:
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index befde52691..9ef33eee54 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -23,6 +23,7 @@
 
 #include "postgres.h"
 
+#include "commands/createas.h"
 #include "executor/execParallel.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
@@ -65,6 +66,7 @@
 #define PARALLEL_KEY_QUERY_TEXT		UINT64CONST(0xE000000000000008)
 #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009)
 #define PARALLEL_KEY_WAL_USAGE			UINT64CONST(0xE00000000000000A)
+#define PARALLEL_KEY_INTO_CLAUSE		UINT64CONST(0xE00000000000000B)
 
 #define PARALLEL_TUPLE_QUEUE_SIZE		65536
 
@@ -77,6 +79,9 @@ typedef struct FixedParallelExecutorState
 	dsa_pointer param_exec;
 	int			eflags;
 	int			jit_flags;
+	Oid			objectid;		/* workers to open relation/table.  */
+	/* Number tuples inserted by all the workers. */
+	pg_atomic_uint64	processed;
 } FixedParallelExecutorState;
 
 /*
@@ -578,7 +583,8 @@ ExecParallelSetupTupleQueues(ParallelContext *pcxt, bool reinitialize)
 ParallelExecutorInfo *
 ExecInitParallelPlan(PlanState *planstate, EState *estate,
 					 Bitmapset *sendParams, int nworkers,
-					 int64 tuples_needed)
+					 int64 tuples_needed, IntoClause *intoclause,
+					 Oid objectid)
 {
 	ParallelExecutorInfo *pei;
 	ParallelContext *pcxt;
@@ -600,6 +606,8 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	Size		dsa_minsize = dsa_minimum_size();
 	char	   *query_string;
 	int			query_len;
+	char 		*intoclausestr = NULL;
+	int			intoclause_len = 0;
 
 	/*
 	 * Force any initplan outputs that we're going to pass to workers to be
@@ -712,6 +720,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_estimate_chunk(&pcxt->estimator, dsa_minsize);
 	shm_toc_estimate_keys(&pcxt->estimator, 1);
 
+	/* Estimate space for into clause for CTAS. */
+	if (IS_CTAS(intoclause) && OidIsValid(objectid))
+	{
+		intoclausestr = nodeToString(intoclause);
+		intoclause_len = strlen(intoclausestr);
+		shm_toc_estimate_chunk(&pcxt->estimator, intoclause_len + 1);
+		shm_toc_estimate_keys(&pcxt->estimator, 1);
+	}
+
 	/* Everyone's had a chance to ask for space, so now create the DSM. */
 	InitializeParallelDSM(pcxt);
 
@@ -729,6 +746,14 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	fpes->param_exec = InvalidDsaPointer;
 	fpes->eflags = estate->es_top_eflags;
 	fpes->jit_flags = estate->es_jit_flags;
+	pg_atomic_init_u64(&fpes->processed, 0);
+	pei->processed = &fpes->processed;
+
+	if (intoclausestr && OidIsValid(objectid))
+		fpes->objectid = objectid;
+	else
+		fpes->objectid = InvalidOid;
+
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, fpes);
 
 	/* Store query string */
@@ -758,8 +783,18 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_WAL_USAGE, walusage_space);
 	pei->wal_usage = walusage_space;
 
-	/* Set up the tuple queues that the workers will write into. */
-	pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	if (intoclausestr)
+	{
+		char *intoclause_space = shm_toc_allocate(pcxt->toc,
+												  intoclause_len + 1);
+		memcpy(intoclause_space, intoclausestr, intoclause_len + 1);
+		shm_toc_insert(pcxt->toc, PARALLEL_KEY_INTO_CLAUSE, intoclause_space);
+	}
+	else
+	{
+		/* Set up the tuple queues that the workers will write into. */
+		pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	}
 
 	/* We don't need the TupleQueueReaders yet, though. */
 	pei->reader = NULL;
@@ -1387,12 +1422,30 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 	void	   *area_space;
 	dsa_area   *area;
 	ParallelWorkerContext pwcxt;
+	char		*intoclausestr = NULL;
+	IntoClause	*intoclause = NULL;
 
 	/* Get fixed-size state. */
 	fpes = shm_toc_lookup(toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
 
-	/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
-	receiver = ExecParallelGetReceiver(seg, toc);
+	intoclausestr = shm_toc_lookup(toc, PARALLEL_KEY_INTO_CLAUSE, true);
+	if (intoclausestr)
+	{
+		/*
+		 * If the worker is for parallel insert in CTAS, then use the proper
+		 * dest receiver.
+		 */
+		intoclause = (IntoClause *) stringToNode(intoclausestr);
+		receiver = CreateIntoRelDestReceiver(intoclause);
+		((DR_intorel *)receiver)->is_parallel_worker = true;
+		((DR_intorel *)receiver)->object_id = fpes->objectid;
+	}
+	else
+	{
+		/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
+		receiver = ExecParallelGetReceiver(seg, toc);
+	}
+
 	instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_INSTRUMENTATION, true);
 	if (instrumentation != NULL)
 		instrument_options = instrumentation->instrument_options;
@@ -1471,6 +1524,13 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 			queryDesc->estate->es_jit->instr;
 	}
 
+	/*
+	 * Write out the number of tuples this worker has inserted. Leader will use
+	 * it to inform the end client.
+	 */
+	if (intoclausestr)
+		pg_atomic_add_fetch_u64(&fpes->processed, queryDesc->estate->es_processed);
+
 	/* Must do this after capturing instrumentation. */
 	ExecutorEnd(queryDesc);
 
diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c
index a01b46af14..e7c588c66a 100644
--- a/src/backend/executor/nodeGather.c
+++ b/src/backend/executor/nodeGather.c
@@ -32,6 +32,7 @@
 
 #include "access/relscan.h"
 #include "access/xact.h"
+#include "commands/createas.h"
 #include "executor/execdebug.h"
 #include "executor/execParallel.h"
 #include "executor/nodeGather.h"
@@ -48,6 +49,7 @@ static TupleTableSlot *ExecGather(PlanState *pstate);
 static TupleTableSlot *gather_getnext(GatherState *gatherstate);
 static MinimalTuple gather_readnext(GatherState *gatherstate);
 static void ExecShutdownGatherWorkers(GatherState *node);
+static void ExecParallelInsertInCTAS(GatherState *node);
 
 
 /* ----------------------------------------------------------------
@@ -131,6 +133,72 @@ ExecInitGather(Gather *node, EState *estate, int eflags)
 	return gatherstate;
 }
 
+/* ----------------------------------------------------------------
+ *		ExecParallelInsertInCTAS(node)
+ *
+ *		Facilitates parallel inserts by parallel workers and/or
+ *		leader for Create Table AS.
+ * ----------------------------------------------------------------
+ */
+static void
+ExecParallelInsertInCTAS(GatherState *node)
+{
+	/* Enable leader to insert in case no parallel workers were launched. */
+	if (node->nworkers_launched == 0)
+		node->need_to_scan_locally = true;
+
+	/*
+	 * By now, for parallel workers (if launched any), would have started their
+	 * work i.e. insertion to target table. In case the leader is chosen to
+	 * participate for parallel inserts in CTAS, then finish its share before
+	 * going to wait for the parallel workers to finish.
+	 */
+	if (node->need_to_scan_locally)
+	{
+		EState	   *estate = node->ps.state;
+		TupleTableSlot *outerTupleSlot;
+
+		for(;;)
+		{
+			/* Install our DSA area while executing the plan. */
+			estate->es_query_dsa =
+					node->pei ? node->pei->area : NULL;
+
+			outerTupleSlot = ExecProcNode(node->ps.lefttree);
+
+			estate->es_query_dsa = NULL;
+
+			if(TupIsNull(outerTupleSlot))
+				break;
+
+			(void) node->dest->receiveSlot(outerTupleSlot, node->dest);
+
+			node->ps.state->es_processed++;
+		}
+
+		node->need_to_scan_locally = false;
+	}
+
+	if (node->nworkers_launched > 0)
+	{
+		/*
+		 * We wait here for the parallel workers to finish their work and
+		 * accumulate the tuples they inserted and also their buffer/WAL usage.
+		 * We do not destroy the parallel context here, it will be done in
+		 * ExecShutdownGather at the end of the plan. Note that the
+		 * ExecShutdownGatherWorkers call from ExecShutdownGather will be a
+		 * no-op.
+		 */
+		ExecShutdownGatherWorkers(node);
+
+		/*
+		 * Add up the total tuples inserted by all workers, to the tuples
+		 * inserted by the leader(if any). This will be shared to client.
+		 */
+		node->ps.state->es_processed += pg_atomic_read_u64(node->pei->processed);
+	}
+}
+
 /* ----------------------------------------------------------------
  *		ExecGather(node)
  *
@@ -157,6 +225,7 @@ ExecGather(PlanState *pstate)
 	{
 		EState	   *estate = node->ps.state;
 		Gather	   *gather = (Gather *) node->ps.plan;
+		bool		isctas = IS_PARALLEL_CTAS_DEST(node->dest);
 
 		/*
 		 * Sometimes we might have to run without parallelism; but if parallel
@@ -165,6 +234,18 @@ ExecGather(PlanState *pstate)
 		if (gather->num_workers > 0 && estate->es_use_parallel_mode)
 		{
 			ParallelContext *pcxt;
+			IntoClause	*intoclause = NULL;
+			Oid			objectid = InvalidOid;
+
+			/*
+			 * Take the necessary information to be passed to workers for
+			 * parallel inserts in CTAS.
+			 */
+			if (isctas)
+			{
+				intoclause = ((DR_intorel *) node->dest)->into;
+				objectid = ((DR_intorel *) node->dest)->object_id;
+			}
 
 			/* Initialize, or re-initialize, shared state needed by workers. */
 			if (!node->pei)
@@ -172,7 +253,10 @@ ExecGather(PlanState *pstate)
 												 estate,
 												 gather->initParam,
 												 gather->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 /* CTAS info */
+												 intoclause,
+												 objectid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
@@ -190,13 +274,16 @@ ExecGather(PlanState *pstate)
 			/* Set up tuple queue readers to read the results. */
 			if (pcxt->nworkers_launched > 0)
 			{
-				ExecParallelCreateReaders(node->pei);
-				/* Make a working array showing the active readers */
-				node->nreaders = pcxt->nworkers_launched;
-				node->reader = (TupleQueueReader **)
-					palloc(node->nreaders * sizeof(TupleQueueReader *));
-				memcpy(node->reader, node->pei->reader,
-					   node->nreaders * sizeof(TupleQueueReader *));
+				if (!isctas)
+				{
+					ExecParallelCreateReaders(node->pei);
+					/* Make a working array showing the active readers */
+					node->nreaders = pcxt->nworkers_launched;
+					node->reader = (TupleQueueReader **)
+						palloc(node->nreaders * sizeof(TupleQueueReader *));
+					memcpy(node->reader, node->pei->reader,
+						node->nreaders * sizeof(TupleQueueReader *));
+				}
 			}
 			else
 			{
@@ -208,9 +295,17 @@ ExecGather(PlanState *pstate)
 		}
 
 		/* Run plan locally if no workers or enabled and not single-copy. */
-		node->need_to_scan_locally = (node->nreaders == 0)
+		node->need_to_scan_locally = (node->nreaders == 0 &&
+			!isctas)
 			|| (!gather->single_copy && parallel_leader_participation);
 		node->initialized = true;
+
+		/* Perform parallel inserts for CTAS. */
+		if (isctas)
+		{
+			ExecParallelInsertInCTAS(node);
+			return NULL;
+		}
 	}
 
 	/*
diff --git a/src/backend/executor/nodeGatherMerge.c b/src/backend/executor/nodeGatherMerge.c
index 47129344f3..ee45272c17 100644
--- a/src/backend/executor/nodeGatherMerge.c
+++ b/src/backend/executor/nodeGatherMerge.c
@@ -216,7 +216,9 @@ ExecGatherMerge(PlanState *pstate)
 												 estate,
 												 gm->initParam,
 												 gm->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 NULL,
+												 InvalidOid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7320de345c..5beae6c617 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -389,6 +389,7 @@ extern FullTransactionId GetCurrentFullTransactionIdIfAny(void);
 extern void MarkCurrentTransactionIdLoggedIfAny(void);
 extern bool SubTransactionIsActive(SubTransactionId subxid);
 extern CommandId GetCurrentCommandId(bool used);
+extern void SetCurrentCommandIdUsedForWorker(void);
 extern void SetParallelStartTimestamps(TimestampTz xact_ts, TimestampTz stmt_ts);
 extern TimestampTz GetCurrentTransactionStartTimestamp(void);
 extern TimestampTz GetCurrentStatementStartTimestamp(void);
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 7629230254..ab3aab58c5 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -14,12 +14,35 @@
 #ifndef CREATEAS_H
 #define CREATEAS_H
 
+#include "access/heapam.h"
 #include "catalog/objectaddress.h"
+#include "executor/execdesc.h"
 #include "nodes/params.h"
+#include "nodes/plannodes.h"
 #include "parser/parse_node.h"
 #include "tcop/dest.h"
 #include "utils/queryenvironment.h"
 
+typedef struct
+{
+	DestReceiver pub;			/* publicly-known function pointers */
+	IntoClause *into;			/* target relation specification */
+	/* These fields are filled by intorel_startup: */
+	Relation	rel;			/* relation to write to */
+	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
+	CommandId	output_cid;		/* cmin to insert in output tuples */
+	int			ti_options;		/* table_tuple_insert performance options */
+	BulkInsertState bistate;	/* bulk insert state */
+	bool		is_parallel;	/* is parallelism to be considered? */
+	bool		is_parallel_worker; /* true for parallel worker */
+	/* Used for table open by parallel worker. */
+	Oid			object_id;
+} DR_intorel;
+
+#define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
+#define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
+									 IS_CTAS(((DR_intorel *) dest)->into) && \
+									 ((DR_intorel *) dest)->is_parallel)
 
 extern ObjectAddress ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 									   ParamListInfo params, QueryEnvironment *queryEnv,
@@ -29,4 +52,9 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
+extern bool IsParallelInsertInCTASAllowed(IntoClause *into,
+										  QueryDesc *queryDesc);
+
+extern void SetCTASParallelInsertState(QueryDesc *queryDesc);
+
 #endif							/* CREATEAS_H */
diff --git a/src/include/executor/execParallel.h b/src/include/executor/execParallel.h
index 5a39a5b29c..9f959f741b 100644
--- a/src/include/executor/execParallel.h
+++ b/src/include/executor/execParallel.h
@@ -35,11 +35,15 @@ typedef struct ParallelExecutorInfo
 	/* These two arrays have pcxt->nworkers_launched entries: */
 	shm_mq_handle **tqueue;		/* tuple queues for worker output */
 	struct TupleQueueReader **reader;	/* tuple reader/writer support */
+	/* Number of tuples inserted by all workers. */
+	volatile pg_atomic_uint64	*processed;
 } ParallelExecutorInfo;
 
 extern ParallelExecutorInfo *ExecInitParallelPlan(PlanState *planstate,
 												  EState *estate, Bitmapset *sendParam, int nworkers,
-												  int64 tuples_needed);
+												  int64 tuples_needed,
+												  IntoClause *intoclause,
+												  Oid objectid);
 extern void ExecParallelCreateReaders(ParallelExecutorInfo *pei);
 extern void ExecParallelFinish(ParallelExecutorInfo *pei);
 extern void ExecParallelCleanup(ParallelExecutorInfo *pei);
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 61ba4c3666..e9c4442c22 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -23,6 +23,7 @@
 #include "nodes/tidbitmap.h"
 #include "partitioning/partdefs.h"
 #include "storage/condition_variable.h"
+#include "tcop/dest.h"
 #include "utils/hsearch.h"
 #include "utils/queryenvironment.h"
 #include "utils/reltrigger.h"
@@ -2326,6 +2327,8 @@ typedef struct GatherState
 	int			nreaders;		/* number of still-active workers */
 	int			nextreader;		/* next one to try to read from */
 	struct TupleQueueReader **reader;	/* array with nreaders active entries */
+	/* Parallel inserts in CTAS related info is specified below. */
+	DestReceiver *dest;
 } GatherState;
 
 /* ----------------
diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 0c4da2591a..84e8f981e1 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -76,4 +76,509 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+                     QUERY PLAN                      
+-----------------------------------------------------
+ LockRows (actual rows=10000 loops=1)
+   ->  Seq Scan on tenk1 (actual rows=10000 loops=1)
+(2 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+    select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_mat_view
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_mat_view;
+ count 
+-------
+ 10000
+(1 row)
+
+drop materialized view parallel_mat_view;
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+deallocate parallel_write_prep;
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select nextval('parallel_write_sequence'), four from tenk1;
+                  QUERY PLAN                   
+-----------------------------------------------
+ Seq Scan on tenk1 (actual rows=10000 loops=1)
+(1 row)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;
+                               QUERY PLAN                               
+------------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Params Evaluated: $1
+   Workers Launched: 3
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=1 loops=1)
+           ->  Gather (actual rows=1 loops=1)
+                 Workers Planned: 4
+                 Workers Launched: 4
+                 ->  Parallel Seq Scan on tenk2 (actual rows=1 loops=5)
+   ->  Parallel Seq Scan on tenk1 (actual rows=625 loops=4)
+         Filter: (four = 3)
+         Rows Removed by Filter: 1875
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+  2500
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;
+                             QUERY PLAN                              
+---------------------------------------------------------------------
+ Gather (actual rows=2500 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=500 loops=5)
+         Filter: (four = 3)
+         Rows Removed by Filter: 1500
+   SubPlan 1
+     ->  Function Scan on generate_series (actual rows=1 loops=2500)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+  2500
+(1 row)
+
+drop table parallel_write;
+create table temp1(col1) as select * from generate_series(1,5);
+create table temp2(col2) as select * from temp1;
+create table temp3(col3) as select * from temp1;
+-- parallel inserts must not occur, as there is a limit clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 limit 4;
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ Limit (actual rows=4 loops=1)
+   ->  Gather (actual rows=4 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an order by clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 order by 1;
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ Gather Merge (actual rows=5 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+   ->  Sort (actual rows=1 loops=4)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         Worker 0:  Sort Method: quicksort  Memory: 25kB
+         Worker 1:  Sort Method: quicksort  Memory: 25kB
+         Worker 2:  Sort Method: quicksort  Memory: 25kB
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(10 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an order by clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 order by 1;
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ Gather Merge (actual rows=5 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+   ->  Sort (actual rows=1 loops=4)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         Worker 0:  Sort Method: quicksort  Memory: 25kB
+         Worker 1:  Sort Method: quicksort  Memory: 25kB
+         Worker 2:  Sort Method: quicksort  Memory: 25kB
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(10 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is a distinct clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select distinct * from temp1;
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ HashAggregate (actual rows=5 loops=1)
+   Group Key: col1
+   Batches: 1  Memory Usage: 40kB
+   ->  Gather (actual rows=5 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(7 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an aggregate and group clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select count(*) from temp1 group by col1;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Finalize HashAggregate (actual rows=5 loops=1)
+   Group Key: col1
+   Batches: 1  Memory Usage: 40kB
+   ->  Gather (actual rows=5 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+         ->  Partial HashAggregate (actual rows=1 loops=4)
+               Group Key: col1
+               Batches: 1  Memory Usage: 40kB
+               Worker 0:  Batches: 1  Memory Usage: 40kB
+               Worker 1:  Batches: 1  Memory Usage: 40kB
+               Worker 2:  Batches: 1  Memory Usage: 40kB
+               ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(13 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an aggregate, group and having
+-- clauses
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select count(col1), (select col3 from
+        (select * from temp3) as tt limit 1) col4 from temp1, temp2
+    where temp1.col1 = temp2.col2 group by col4 having count(col1) > 0;
+                                    QUERY PLAN                                    
+----------------------------------------------------------------------------------
+ Finalize GroupAggregate (actual rows=1 loops=1)
+   Group Key: ($1)
+   Filter: (count(temp1.col1) > 0)
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=1 loops=1)
+           ->  Gather (actual rows=1 loops=1)
+                 Workers Planned: 3
+                 Workers Launched: 3
+                 ->  Parallel Seq Scan on temp3 (actual rows=0 loops=4)
+   ->  Gather (actual rows=1 loops=1)
+         Workers Planned: 3
+         Params Evaluated: $1
+         Workers Launched: 3
+         ->  Partial GroupAggregate (actual rows=0 loops=4)
+               Group Key: $1
+               ->  Parallel Hash Join (actual rows=1 loops=4)
+                     Hash Cond: (temp1.col1 = temp2.col2)
+                     ->  Parallel Seq Scan on temp1 (actual rows=5 loops=1)
+                     ->  Parallel Hash (actual rows=1 loops=4)
+                           Buckets: 4096  Batches: 1  Memory Usage: 64kB
+                           ->  Parallel Seq Scan on temp2 (actual rows=5 loops=1)
+(21 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is a window function
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select avg(col1) OVER (PARTITION BY col1) from temp1;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ WindowAgg (actual rows=5 loops=1)
+   ->  Sort (actual rows=5 loops=1)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         ->  Gather (actual rows=5 loops=1)
+               Workers Planned: 3
+               Workers Launched: 3
+               ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(8 rows)
+
+drop table parallel_write;
+-- nested loop join is the top node under which Gather node exists, so parallel
+-- inserts must not occur
+set enable_nestloop to on;
+set enable_mergejoin to off;
+set enable_hashjoin to off;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Nested Loop (actual rows=5 loops=1)
+   Join Filter: (temp1.col1 = temp2.col2)
+   Rows Removed by Join Filter: 20
+   ->  Gather (actual rows=5 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+   ->  Materialize (actual rows=5 loops=5)
+         ->  Gather (actual rows=5 loops=1)
+               Workers Planned: 3
+               Workers Launched: 3
+               ->  Parallel Seq Scan on temp2 (actual rows=1 loops=4)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- the top node is Gather under which merge join happens, so parallel inserts
+-- must occur
+set enable_nestloop to off;
+set enable_mergejoin to on;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+ ->  Create parallel_write
+   ->  Merge Join (actual rows=1 loops=4)
+         Merge Cond: (temp1.col1 = temp2.col2)
+         ->  Sort (actual rows=1 loops=4)
+               Sort Key: temp1.col1
+               Sort Method: quicksort  Memory: 25kB
+               Worker 0:  Sort Method: quicksort  Memory: 25kB
+               Worker 1:  Sort Method: quicksort  Memory: 25kB
+               Worker 2:  Sort Method: quicksort  Memory: 25kB
+               ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+         ->  Sort (actual rows=5 loops=1)
+               Sort Key: temp2.col2
+               Sort Method: quicksort  Memory: 25kB
+               ->  Seq Scan on temp2 (actual rows=5 loops=1)
+(17 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel hash join happens under Gather node, so parallel inserts must occur
+set enable_mergejoin to off;
+set enable_hashjoin to on;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+ ->  Create parallel_write
+   ->  Parallel Hash Join (actual rows=1 loops=4)
+         Hash Cond: (temp1.col1 = temp2.col2)
+         ->  Parallel Seq Scan on temp1 (actual rows=5 loops=1)
+         ->  Parallel Hash (actual rows=1 loops=4)
+               Buckets: 4096  Batches: 1  Memory Usage: 64kB
+               ->  Parallel Seq Scan on temp2 (actual rows=5 loops=1)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+reset enable_nestloop;
+reset enable_mergejoin;
+reset enable_hashjoin;
+drop table temp1;
+drop table temp2;
+drop table temp3;
 rollback;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 78b479cedf..bff3fcc6b5 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -39,4 +39,184 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
 
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+    select length(stringu1) from tenk1;
+select count(*) from parallel_mat_view;
+drop materialized view parallel_mat_view;
+
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+select count(*) from parallel_write;
+deallocate parallel_write_prep;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select nextval('parallel_write_sequence'), four from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+create table temp1(col1) as select * from generate_series(1,5);
+create table temp2(col2) as select * from temp1;
+create table temp3(col3) as select * from temp1;
+
+-- parallel inserts must not occur, as there is a limit clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 limit 4;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an order by clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 order by 1;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an order by clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 order by 1;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is a distinct clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select distinct * from temp1;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an aggregate and group clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select count(*) from temp1 group by col1;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an aggregate, group and having
+-- clauses
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select count(col1), (select col3 from
+        (select * from temp3) as tt limit 1) col4 from temp1, temp2
+    where temp1.col1 = temp2.col2 group by col4 having count(col1) > 0;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is a window function
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select avg(col1) OVER (PARTITION BY col1) from temp1;
+drop table parallel_write;
+
+-- nested loop join is the top node under which Gather node exists, so parallel
+-- inserts must not occur
+set enable_nestloop to on;
+set enable_mergejoin to off;
+set enable_hashjoin to off;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- the top node is Gather under which merge join happens, so parallel inserts
+-- must occur
+set enable_nestloop to off;
+set enable_mergejoin to on;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel hash join happens under Gather node, so parallel inserts must occur
+set enable_mergejoin to off;
+set enable_hashjoin to on;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+reset enable_nestloop;
+reset enable_mergejoin;
+reset enable_hashjoin;
+drop table temp1;
+drop table temp2;
+drop table temp3;
 rollback;
-- 
2.25.1

From 23a6b88d2d42913d559ca61b0050588cdd0db0d4 Mon Sep 17 00:00:00 2001
From: root <root@localhost.localdomain>
Date: Mon, 14 Dec 2020 06:31:12 -0500
Subject: [PATCH] patch for pinsert in append

---
 src/backend/commands/createas.c       | 35 +++++++++++++++++++++++++++++++----
 src/backend/commands/explain.c        |  3 +--
 src/backend/optimizer/path/allpaths.c | 34 ++++++++++++++++++++++++++++++++++
 src/backend/optimizer/plan/planner.c  |  2 +-
 src/include/commands/createas.h       |  3 ++-
 5 files changed, 69 insertions(+), 8 deletions(-)

diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 3ffea41..6e6c467 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -351,9 +351,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		 * into the target table. We need plan state to be initialized by the
 		 * executor to decide whether to allow parallel inserts or not.
 		 */
-		if (IsParallelInsertInCTASAllowed(into, queryDesc,
-										  &query->CTASParallelInsInfo))
-			SetCTASParallelInsertState(queryDesc);
+		IsParallelInsertInCTASAllowed(into, queryDesc, &query->CTASParallelInsInfo);
 
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
@@ -663,6 +661,35 @@ intorel_destroy(DestReceiver *self)
 	pfree(self);
 }
 
+static bool
+PushDownCTASParallelInsertState(DestReceiver *dest, PlanState *ps)
+{
+	bool parallel = false;
+	
+	if(ps == NULL)
+		return parallel;
+
+	if(IsA(ps, AppendState))
+	{
+		AppendState *aps = (AppendState *) ps;
+		for(int i = 0; i < aps->as_nplans; i++)
+		{
+			parallel |= PushDownCTASParallelInsertState(dest, aps->appendplans[i]);
+		}
+	}
+	else if(IsA(ps, GatherState) && !ps->ps_ProjInfo)
+	{
+		GatherState *gstate = (GatherState *) ps;
+		parallel = true;
+
+		((DR_intorel *) dest)->is_parallel = true;
+		gstate->dest = dest;
+		ps->plan->plan_rows = 0;
+	}
+
+	return parallel;
+}
+
 /*
  * IsParallelInsertInCTASAllowed --- determine whether or not parallel
  * insertion is possible.
@@ -698,7 +725,7 @@ bool IsParallelInsertInCTASAllowed(IntoClause *into, QueryDesc *queryDesc,
 		 * final phase i.e. merge the results by workers, so we do not allow
 		 * parallel inserts.
 		 */
-		allow = ps && IsA(ps, GatherState) && !ps->ps_ProjInfo;
+		allow = PushDownCTASParallelInsertState(queryDesc->dest, ps); 
 
 		/*
 		 * It should not happen that in cost_gather we have ignored the
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index d0152de..136f6f4 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -570,8 +570,7 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	 * target table. We need plan state to be initialized by the executor to
 	 * decide whether to allow parallel inserts or not.
 	 */
-	if (IsParallelInsertInCTASAllowed(into, queryDesc, ctas_tuple_cost_flags))
-		SetCTASParallelInsertState(queryDesc);
+	IsParallelInsertInCTASAllowed(into, queryDesc, ctas_tuple_cost_flags);
 
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c
index 84a69b0..fe3332e 100644
--- a/src/backend/optimizer/path/allpaths.c
+++ b/src/backend/optimizer/path/allpaths.c
@@ -23,6 +23,7 @@
 #include "catalog/pg_class.h"
 #include "catalog/pg_operator.h"
 #include "catalog/pg_proc.h"
+#include "commands/createas.h"
 #include "foreign/fdwapi.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1103,11 +1104,44 @@ set_append_rel_size(PlannerInfo *root, RelOptInfo *rel,
 		if (root->glob->parallelModeOK && rel->consider_parallel)
 			set_rel_consider_parallel(root, childrel, childRTE);
 
+		if(childrel->rtekind == RTE_SUBQUERY)
+		{
+			if(root->query_level != 1)
+			{
+				if (root->parent_root &&
+					(root->parent_root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND) &&
+					!(root->parse->rowMarks ||
+					limit_needed(root->parse) ||
+					root->parse->sortClause ||
+					root->parse->distinctClause ||
+					root->parse->hasWindowFuncs ||
+					root->parse->groupClause ||
+					root->parse->groupingSets ||
+					root->parse->hasAggs ||
+					root->hasHavingQual))
+					root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+			}
+			else
+			{
+				if (!(root->parse->rowMarks ||
+					limit_needed(root->parse) ||
+					root->parse->sortClause ||
+					root->parse->distinctClause ||
+					root->parse->hasWindowFuncs ||
+					root->parse->groupClause ||
+					root->parse->groupingSets ||
+					root->parse->hasAggs ||
+					root->hasHavingQual))
+					root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+			}
+		}
 		/*
 		 * Compute the child's size.
 		 */
 		set_rel_size(root, childrel, childRTindex, childRTE);
 
+		if(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND)
+			root->parse->CTASParallelInsInfo &= ~CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
 		/*
 		 * It is possible that constraint exclusion detected a contradiction
 		 * within a child subquery, even though we didn't prove one above. If
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index d287b6b..da5ce1f 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -7350,7 +7350,7 @@ can_partial_agg(PlannerInfo *root)
 static bool
 ignore_parallel_tuple_cost(PlannerInfo *root)
 {
-	if (root->query_level == 1 &&
+	if ((root->query_level == 1 || (root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND)) &&
 		(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT))
 	{
 		/*
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index e01a615..7a50b18 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -54,7 +54,8 @@ typedef enum CTASParallelInsertOpt
 	 */
 	CTAS_PARALLEL_INS_TUP_COST_CAN_IGN = 1 << 1,
 
-	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2
+	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2,
+	CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND = 1 << 3
 } CTASParallelInsertOpt;
 
 #define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
-- 
1.8.3.1

From 0b411712b8b931d32e3d836ccda173e6077692c5 Mon Sep 17 00:00:00 2001
From: root <root@localhost.localdomain>
Date: Tue, 15 Dec 2020 05:58:43 -0500
Subject: [PATCH 1/2] support pctas in append parallel inserts

---
 src/backend/commands/createas.c | 35 +++++++++++++++++++++++++++++++----
 src/backend/commands/explain.c  |  3 +--
 2 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 3ffea41..6e6c467 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -351,9 +351,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		 * into the target table. We need plan state to be initialized by the
 		 * executor to decide whether to allow parallel inserts or not.
 		 */
-		if (IsParallelInsertInCTASAllowed(into, queryDesc,
-										  &query->CTASParallelInsInfo))
-			SetCTASParallelInsertState(queryDesc);
+		IsParallelInsertInCTASAllowed(into, queryDesc, &query->CTASParallelInsInfo);
 
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
@@ -663,6 +661,35 @@ intorel_destroy(DestReceiver *self)
 	pfree(self);
 }
 
+static bool
+PushDownCTASParallelInsertState(DestReceiver *dest, PlanState *ps)
+{
+	bool parallel = false;
+	
+	if(ps == NULL)
+		return parallel;
+
+	if(IsA(ps, AppendState))
+	{
+		AppendState *aps = (AppendState *) ps;
+		for(int i = 0; i < aps->as_nplans; i++)
+		{
+			parallel |= PushDownCTASParallelInsertState(dest, aps->appendplans[i]);
+		}
+	}
+	else if(IsA(ps, GatherState) && !ps->ps_ProjInfo)
+	{
+		GatherState *gstate = (GatherState *) ps;
+		parallel = true;
+
+		((DR_intorel *) dest)->is_parallel = true;
+		gstate->dest = dest;
+		ps->plan->plan_rows = 0;
+	}
+
+	return parallel;
+}
+
 /*
  * IsParallelInsertInCTASAllowed --- determine whether or not parallel
  * insertion is possible.
@@ -698,7 +725,7 @@ bool IsParallelInsertInCTASAllowed(IntoClause *into, QueryDesc *queryDesc,
 		 * final phase i.e. merge the results by workers, so we do not allow
 		 * parallel inserts.
 		 */
-		allow = ps && IsA(ps, GatherState) && !ps->ps_ProjInfo;
+		allow = PushDownCTASParallelInsertState(queryDesc->dest, ps); 
 
 		/*
 		 * It should not happen that in cost_gather we have ignored the
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index d0152de..136f6f4 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -570,8 +570,7 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	 * target table. We need plan state to be initialized by the executor to
 	 * decide whether to allow parallel inserts or not.
 	 */
-	if (IsParallelInsertInCTASAllowed(into, queryDesc, ctas_tuple_cost_flags))
-		SetCTASParallelInsertState(queryDesc);
+	IsParallelInsertInCTASAllowed(into, queryDesc, ctas_tuple_cost_flags);
 
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
-- 
1.8.3.1

From 6daa1cf2c4f724603f64c0c89e915769c9334857 Mon Sep 17 00:00:00 2001
From: root <root@localhost.localdomain>
Date: Tue, 15 Dec 2020 06:01:10 -0500
Subject: [PATCH 2/2] support pctas in append tuple cost adjustment

---
 src/backend/optimizer/path/allpaths.c | 29 +++++++++++++++++++++++++++++
 src/backend/optimizer/plan/planner.c  |  5 +++--
 src/include/commands/createas.h       |  3 ++-
 3 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c
index 84a69b0..82f85b8 100644
--- a/src/backend/optimizer/path/allpaths.c
+++ b/src/backend/optimizer/path/allpaths.c
@@ -23,6 +23,7 @@
 #include "catalog/pg_class.h"
 #include "catalog/pg_operator.h"
 #include "catalog/pg_proc.h"
+#include "commands/createas.h"
 #include "foreign/fdwapi.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1104,10 +1105,38 @@ set_append_rel_size(PlannerInfo *root, RelOptInfo *rel,
 			set_rel_consider_parallel(root, childrel, childRTE);
 
 		/*
+		 * When subplan is subquery, It's possible to do parallel insert
+		 * if top-node of subquery is Gather, so we set the flag to
+		 * ignore parallel tuple cost by the Gather path in cost_gather
+		 * if the SELECT is for CTAS.
+		 */ 
+		if(childrel->rtekind == RTE_SUBQUERY)
+		{
+			if((root->query_level == 1 ||
+				root->parent_root->parse->CTASParallelInsInfo &
+				CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND)
+				&&
+				!(root->parse->rowMarks ||
+				limit_needed(root->parse) ||
+				root->parse->sortClause ||
+				root->parse->distinctClause ||
+				root->parse->hasWindowFuncs ||
+				root->parse->groupClause ||
+				root->parse->groupingSets ||
+				root->parse->hasAggs ||
+				root->hasHavingQual))
+				root->parse->CTASParallelInsInfo |=
+										CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+		}
+
+		/*
 		 * Compute the child's size.
 		 */
 		set_rel_size(root, childrel, childRTindex, childRTE);
 
+		if(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND)
+			root->parse->CTASParallelInsInfo &= ~CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+
 		/*
 		 * It is possible that constraint exclusion detected a contradiction
 		 * within a child subquery, even though we didn't prove one above. If
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index d287b6b..7842d71 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -7350,8 +7350,9 @@ can_partial_agg(PlannerInfo *root)
 static bool
 ignore_parallel_tuple_cost(PlannerInfo *root)
 {
-	if (root->query_level == 1 &&
-		(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT))
+	if ((root->query_level == 1 ||
+	   (root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND)) &&
+	   (root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT))
 	{
 		/*
 		 * In each of following cases, a parent path will be generated for the
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index e01a615..7a50b18 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -54,7 +54,8 @@ typedef enum CTASParallelInsertOpt
 	 */
 	CTAS_PARALLEL_INS_TUP_COST_CAN_IGN = 1 << 1,
 
-	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2
+	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2,
+	CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND = 1 << 3
 } CTASParallelInsertOpt;
 
 #define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
-- 
1.8.3.1

From e5cb1c34706ef21c8942087511726d9c06e1fd65 Mon Sep 17 00:00:00 2001
From: root <root@localhost.localdomain>
Date: Tue, 15 Dec 2020 07:14:21 -0500
Subject: [PATCH 1/2] support pctas in append parallel inserts

---
 src/backend/commands/createas.c | 35 +++++++++++++++++++++++++++++++----
 src/backend/commands/explain.c  |  3 +--
 2 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 3ffea41..6e6c467 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -351,9 +351,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		 * into the target table. We need plan state to be initialized by the
 		 * executor to decide whether to allow parallel inserts or not.
 		 */
-		if (IsParallelInsertInCTASAllowed(into, queryDesc,
-										  &query->CTASParallelInsInfo))
-			SetCTASParallelInsertState(queryDesc);
+		IsParallelInsertInCTASAllowed(into, queryDesc, &query->CTASParallelInsInfo);
 
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
@@ -663,6 +661,35 @@ intorel_destroy(DestReceiver *self)
 	pfree(self);
 }
 
+static bool
+PushDownCTASParallelInsertState(DestReceiver *dest, PlanState *ps)
+{
+	bool parallel = false;
+	
+	if(ps == NULL)
+		return parallel;
+
+	if(IsA(ps, AppendState))
+	{
+		AppendState *aps = (AppendState *) ps;
+		for(int i = 0; i < aps->as_nplans; i++)
+		{
+			parallel |= PushDownCTASParallelInsertState(dest, aps->appendplans[i]);
+		}
+	}
+	else if(IsA(ps, GatherState) && !ps->ps_ProjInfo)
+	{
+		GatherState *gstate = (GatherState *) ps;
+		parallel = true;
+
+		((DR_intorel *) dest)->is_parallel = true;
+		gstate->dest = dest;
+		ps->plan->plan_rows = 0;
+	}
+
+	return parallel;
+}
+
 /*
  * IsParallelInsertInCTASAllowed --- determine whether or not parallel
  * insertion is possible.
@@ -698,7 +725,7 @@ bool IsParallelInsertInCTASAllowed(IntoClause *into, QueryDesc *queryDesc,
 		 * final phase i.e. merge the results by workers, so we do not allow
 		 * parallel inserts.
 		 */
-		allow = ps && IsA(ps, GatherState) && !ps->ps_ProjInfo;
+		allow = PushDownCTASParallelInsertState(queryDesc->dest, ps); 
 
 		/*
 		 * It should not happen that in cost_gather we have ignored the
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index d0152de..136f6f4 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -570,8 +570,7 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	 * target table. We need plan state to be initialized by the executor to
 	 * decide whether to allow parallel inserts or not.
 	 */
-	if (IsParallelInsertInCTASAllowed(into, queryDesc, ctas_tuple_cost_flags))
-		SetCTASParallelInsertState(queryDesc);
+	IsParallelInsertInCTASAllowed(into, queryDesc, ctas_tuple_cost_flags);
 
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
-- 
1.8.3.1

From 30d1e78ce73e07c463b5c946b571ee964caeb3eb Mon Sep 17 00:00:00 2001
From: root <root@localhost.localdomain>
Date: Tue, 15 Dec 2020 07:14:58 -0500
Subject: [PATCH 2/2] support pctas in append tuple cost adjustment

---
 src/backend/optimizer/path/allpaths.c | 29 +++++++++++++++++++++++++++++
 src/backend/optimizer/plan/planner.c  |  9 +++++++--
 src/include/commands/createas.h       |  3 ++-
 3 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c
index 84a69b0..82f85b8 100644
--- a/src/backend/optimizer/path/allpaths.c
+++ b/src/backend/optimizer/path/allpaths.c
@@ -23,6 +23,7 @@
 #include "catalog/pg_class.h"
 #include "catalog/pg_operator.h"
 #include "catalog/pg_proc.h"
+#include "commands/createas.h"
 #include "foreign/fdwapi.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1104,10 +1105,38 @@ set_append_rel_size(PlannerInfo *root, RelOptInfo *rel,
 			set_rel_consider_parallel(root, childrel, childRTE);
 
 		/*
+		 * When subplan is subquery, It's possible to do parallel insert
+		 * if top-node of subquery is Gather, so we set the flag to
+		 * ignore parallel tuple cost by the Gather path in cost_gather
+		 * if the SELECT is for CTAS.
+		 */ 
+		if(childrel->rtekind == RTE_SUBQUERY)
+		{
+			if((root->query_level == 1 ||
+				root->parent_root->parse->CTASParallelInsInfo &
+				CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND)
+				&&
+				!(root->parse->rowMarks ||
+				limit_needed(root->parse) ||
+				root->parse->sortClause ||
+				root->parse->distinctClause ||
+				root->parse->hasWindowFuncs ||
+				root->parse->groupClause ||
+				root->parse->groupingSets ||
+				root->parse->hasAggs ||
+				root->hasHavingQual))
+				root->parse->CTASParallelInsInfo |=
+										CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+		}
+
+		/*
 		 * Compute the child's size.
 		 */
 		set_rel_size(root, childrel, childRTindex, childRTE);
 
+		if(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND)
+			root->parse->CTASParallelInsInfo &= ~CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+
 		/*
 		 * It is possible that constraint exclusion detected a contradiction
 		 * within a child subquery, even though we didn't prove one above. If
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index d287b6b..86d42e1 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -7350,8 +7350,13 @@ can_partial_agg(PlannerInfo *root)
 static bool
 ignore_parallel_tuple_cost(PlannerInfo *root)
 {
-	if (root->query_level == 1 &&
-		(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT))
+	if(root->query_level != 1 &&
+	  (root->parent_root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND))
+	{
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_SELECT;
+	}
+
+	if (root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT)
 	{
 		/*
 		 * In each of following cases, a parent path will be generated for the
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index e01a615..7a50b18 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -54,7 +54,8 @@ typedef enum CTASParallelInsertOpt
 	 */
 	CTAS_PARALLEL_INS_TUP_COST_CAN_IGN = 1 << 1,
 
-	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2
+	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2,
+	CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND = 1 << 3
 } CTASParallelInsertOpt;
 
 #define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
-- 
1.8.3.1

From c55b3e3ca342253a4d6b881a16c410bbb1a41678 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Wed, 16 Dec 2020 11:41:37 +0530
Subject: [PATCH v12 1/3] Parallel Inserts in CREATE TABLE AS

Allow the leader and each worker insert the tuples in parallel
if the SELECT part of the CTAS is parallelizable.

The design:
The main idea is to push the CTAS dest receiver down to Gather node
and from there the required information will be shared to workers
so that they can perform parallel insertions. Leader will also
participate in insertions. After the planning, check if the upper
plan node is Gather or Append in createas.c and mark a parallelism
flag in the CTAS dest receiver and push it down to Gather node. Each
worker can create its own CTAS dest receiver with the information
passed from the leader. Leader inserts its share of tuples if
instructed to do, and so are workers. Each worker writes atomically
its number of inserted tuples into a shared memory variable, the
leader combines this with its own number of inserted tuples and
shares to the client.

Authors: Bharath Rupireddy, Hou, Zhijie
---
 src/backend/access/heap/heapam.c       |  11 -
 src/backend/access/transam/xact.c      |  30 ++-
 src/backend/commands/createas.c        | 345 +++++++++++++++++--------
 src/backend/commands/explain.c         |  31 +++
 src/backend/executor/execParallel.c    |  70 ++++-
 src/backend/executor/nodeGather.c      | 113 +++++++-
 src/backend/executor/nodeGatherMerge.c |   4 +-
 src/include/access/xact.h              |   1 +
 src/include/commands/createas.h        |  25 ++
 src/include/executor/execParallel.h    |   6 +-
 src/include/nodes/execnodes.h          |   3 +
 11 files changed, 499 insertions(+), 140 deletions(-)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index a9583f3103..86347ba273 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2043,17 +2043,6 @@ static HeapTuple
 heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
 					CommandId cid, int options)
 {
-	/*
-	 * To allow parallel inserts, we need to ensure that they are safe to be
-	 * performed in workers. We have the infrastructure to allow parallel
-	 * inserts in general except for the cases where inserts generate a new
-	 * CommandId (eg. inserts into a table having a foreign key column).
-	 */
-	if (IsParallelWorker())
-		ereport(ERROR,
-				(errcode(ERRCODE_INVALID_TRANSACTION_STATE),
-				 errmsg("cannot insert tuples in a parallel worker")));
-
 	tup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
 	tup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
 	tup->t_data->t_infomask |= HEAP_XMAX_INVALID;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 9cd0b7c11b..db6eedd635 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -763,18 +763,34 @@ GetCurrentCommandId(bool used)
 	/* this is global to a transaction, not subtransaction-local */
 	if (used)
 	{
-		/*
-		 * Forbid setting currentCommandIdUsed in a parallel worker, because
-		 * we have no provision for communicating this back to the leader.  We
-		 * could relax this restriction when currentCommandIdUsed was already
-		 * true at the start of the parallel operation.
-		 */
-		Assert(!IsParallelWorker());
+		 /*
+		  * This is a temporary hack for all common parallel insert cases i.e.
+		  * insert into, ctas, copy from. To be changed later. In a parallel
+		  * worker, set currentCommandIdUsed to true only if it was not set to
+		  * true at the start of the parallel operation (by way of
+		  * SetCurrentCommandIdUsedForWorker()). We have to do this because
+		  * GetCurrentCommandId(true) may be called from anywhere, especially
+		  * for parallel inserts, within parallel worker.
+		  */
+		Assert(!(IsParallelWorker() && !currentCommandIdUsed));
 		currentCommandIdUsed = true;
 	}
 	return currentCommandId;
 }
 
+/*
+ *	SetCurrentCommandIdUsedForWorker
+ *
+ * For a parallel worker, record that the currentCommandId has been used. This
+ * must only be called at the start of a parallel operation.
+ */
+void
+SetCurrentCommandIdUsedForWorker(void)
+{
+	Assert(IsParallelWorker() && !currentCommandIdUsed && currentCommandId != InvalidCommandId);
+	currentCommandIdUsed = true;
+}
+
 /*
  *	SetParallelStartTimestamps
  *
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 6bf6c5a310..be381f9748 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,18 +51,6 @@
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 
-typedef struct
-{
-	DestReceiver pub;			/* publicly-known function pointers */
-	IntoClause *into;			/* target relation specification */
-	/* These fields are filled by intorel_startup: */
-	Relation	rel;			/* relation to write to */
-	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
-	CommandId	output_cid;		/* cmin to insert in output tuples */
-	int			ti_options;		/* table_tuple_insert performance options */
-	BulkInsertState bistate;	/* bulk insert state */
-} DR_intorel;
-
 /* utility functions for CTAS definition creation */
 static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into);
 static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -350,6 +338,14 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		/* call ExecutorStart to prepare the plan for execution */
 		ExecutorStart(queryDesc, GetIntoRelEFlags(into));
 
+		/*
+		 * If SELECT part of the CTAS is parallelizable, then make each
+		 * parallel worker insert the tuples that are resulted in its execution
+		 * into the target table. We need plan state to be initialized by the
+		 * executor to decide whether to allow parallel inserts or not.
+		 */
+		ChooseParallelInsertsInCTAS(into, queryDesc);
+
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
 
@@ -418,6 +414,9 @@ CreateIntoRelDestReceiver(IntoClause *intoClause)
 	self->pub.rDestroy = intorel_destroy;
 	self->pub.mydest = DestIntoRel;
 	self->into = intoClause;
+	self->is_parallel = false;
+	self->is_parallel_worker = false;
+	self->object_id = InvalidOid;
 	/* other private fields will be set during intorel_startup */
 
 	return (DestReceiver *) self;
@@ -430,121 +429,169 @@ static void
 intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 {
 	DR_intorel *myState = (DR_intorel *) self;
-	IntoClause *into = myState->into;
-	bool		is_matview;
-	List	   *attrList;
 	ObjectAddress intoRelationAddr;
 	Relation	intoRelationDesc;
-	ListCell   *lc;
-	int			attnum;
-
-	Assert(into != NULL);		/* else somebody forgot to set it */
 
-	/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
-	is_matview = (into->viewQuery != NULL);
+	if (myState->is_parallel_worker)
+	{
+		/* In the worker */
+		intoRelationDesc = table_open(myState->object_id, AccessExclusiveLock);
+		myState->rel = intoRelationDesc;
+		myState->reladdr = InvalidObjectAddress;
+		myState->ti_options = 0;
+		myState->bistate = GetBulkInsertState();
 
-	/*
-	 * Build column definitions using "pre-cooked" type and collation info. If
-	 * a column name list was specified in CREATE TABLE AS, override the
-	 * column names derived from the query.  (Too few column names are OK, too
-	 * many are not.)
-	 */
-	attrList = NIL;
-	lc = list_head(into->colNames);
-	for (attnum = 0; attnum < typeinfo->natts; attnum++)
+		/*
+		 * Right after the table is created in the leader, the command id is
+		 * incremented (in create_ctas_internal()). The new command id is
+		 * marked as used in intorel_startup(), then the parallel mode is
+		 * entered. The command id and transaction id are serialized into
+		 * parallel DSM, they are then available to all parallel workers. All
+		 * the workers need to mark the command id as used before insertion.
+		 */
+		SetCurrentCommandIdUsedForWorker();
+		myState->output_cid = GetCurrentCommandId(false);
+	}
+	else
 	{
-		Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
-		ColumnDef  *col;
-		char	   *colname;
+		IntoClause *into = myState->into;
+		bool		is_matview;
+		List	   *attrList;
+		ListCell   *lc;
+		int			attnum;
+
+		Assert(into != NULL);		/* else somebody forgot to set it */
 
-		if (lc)
+		/*
+		 * This code supports both CREATE TABLE AS and CREATE MATERIALIZED
+		 * VIEW.
+		 */
+		is_matview = (into->viewQuery != NULL);
+
+		/*
+		 * Build column definitions using "pre-cooked" type and collation info.
+		 * If a column name list was specified in CREATE TABLE AS, override the
+		 * column names derived from the query.  (Too few column names are OK,
+		 * too many are not.)
+		 */
+		attrList = NIL;
+		lc = list_head(into->colNames);
+		for (attnum = 0; attnum < typeinfo->natts; attnum++)
 		{
-			colname = strVal(lfirst(lc));
-			lc = lnext(into->colNames, lc);
+			Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
+			ColumnDef  *col;
+			char	   *colname;
+
+			if (lc)
+			{
+				colname = strVal(lfirst(lc));
+				lc = lnext(into->colNames, lc);
+			}
+			else
+				colname = NameStr(attribute->attname);
+
+			col = makeColumnDef(colname,
+								attribute->atttypid,
+								attribute->atttypmod,
+								attribute->attcollation);
+
+			/*
+			 * It's possible that the column is of a collatable type but the
+			 * collation could not be resolved, so double-check.  (We must
+			 * check this here because DefineRelation would adopt the type's
+			 * default collation rather than complaining.)
+			 */
+			if (!OidIsValid(col->collOid) &&
+				type_is_collatable(col->typeName->typeOid))
+				ereport(ERROR,
+						(errcode(ERRCODE_INDETERMINATE_COLLATION),
+						errmsg("no collation was derived for column \"%s\" with collatable type %s",
+								col->colname,
+								format_type_be(col->typeName->typeOid)),
+						errhint("Use the COLLATE clause to set the collation explicitly.")));
+
+			attrList = lappend(attrList, col);
 		}
-		else
-			colname = NameStr(attribute->attname);
 
-		col = makeColumnDef(colname,
-							attribute->atttypid,
-							attribute->atttypmod,
-							attribute->attcollation);
+		if (lc != NULL)
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					errmsg("too many column names were specified")));
 
 		/*
-		 * It's possible that the column is of a collatable type but the
-		 * collation could not be resolved, so double-check.  (We must check
-		 * this here because DefineRelation would adopt the type's default
-		 * collation rather than complaining.)
+		 * Actually create the target table
 		 */
-		if (!OidIsValid(col->collOid) &&
-			type_is_collatable(col->typeName->typeOid))
-			ereport(ERROR,
-					(errcode(ERRCODE_INDETERMINATE_COLLATION),
-					 errmsg("no collation was derived for column \"%s\" with collatable type %s",
-							col->colname,
-							format_type_be(col->typeName->typeOid)),
-					 errhint("Use the COLLATE clause to set the collation explicitly.")));
+		intoRelationAddr = create_ctas_internal(attrList, into);
 
-		attrList = lappend(attrList, col);
-	}
+		/*
+		 * Finally we can open the target table
+		 */
+		intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
 
-	if (lc != NULL)
-		ereport(ERROR,
-				(errcode(ERRCODE_SYNTAX_ERROR),
-				 errmsg("too many column names were specified")));
+		/*
+		 * Make sure the constructed table does not have RLS enabled.
+		 *
+		 * check_enable_rls() will ereport(ERROR) itself if the user has
+		 * requested something invalid, and otherwise will return RLS_ENABLED
+		 * if RLS should be enabled here.  We don't actually support that
+		 * currently, so throw our own ereport(ERROR) if that happens.
+		 */
+		if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					errmsg("policies not yet implemented for this command")));
 
-	/*
-	 * Actually create the target table
-	 */
-	intoRelationAddr = create_ctas_internal(attrList, into);
+		/*
+		 * Tentatively mark the target as populated, if it's a matview and
+		 * we're going to fill it; otherwise, no change needed.
+		 */
+		if (is_matview && !into->skipData)
+			SetMatViewPopulatedState(intoRelationDesc, true);
 
-	/*
-	 * Finally we can open the target table
-	 */
-	intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
+		/*
+		 * Fill private fields of myState for use by later routines
+		 */
+		myState->rel = intoRelationDesc;
+		myState->reladdr = intoRelationAddr;
+		myState->output_cid = GetCurrentCommandId(true);
+		myState->ti_options = TABLE_INSERT_SKIP_FSM;
 
-	/*
-	 * Make sure the constructed table does not have RLS enabled.
-	 *
-	 * check_enable_rls() will ereport(ERROR) itself if the user has requested
-	 * something invalid, and otherwise will return RLS_ENABLED if RLS should
-	 * be enabled here.  We don't actually support that currently, so throw
-	 * our own ereport(ERROR) if that happens.
-	 */
-	if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
-		ereport(ERROR,
-				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-				 errmsg("policies not yet implemented for this command")));
+		/*
+		 * If WITH NO DATA is specified, there is no need to set up the state
+		 * for bulk inserts as there are no tuples to insert.
+		 */
+		if (!into->skipData)
+			myState->bistate = GetBulkInsertState();
+		else
+			myState->bistate = NULL;
 
-	/*
-	 * Tentatively mark the target as populated, if it's a matview and we're
-	 * going to fill it; otherwise, no change needed.
-	 */
-	if (is_matview && !into->skipData)
-		SetMatViewPopulatedState(intoRelationDesc, true);
+		if (myState->is_parallel)
+		{
+			myState->object_id = intoRelationAddr.objectId;
 
-	/*
-	 * Fill private fields of myState for use by later routines
-	 */
-	myState->rel = intoRelationDesc;
-	myState->reladdr = intoRelationAddr;
-	myState->output_cid = GetCurrentCommandId(true);
-	myState->ti_options = TABLE_INSERT_SKIP_FSM;
+			/*
+			 * We don't need to skip contacting FSM while inserting tuples
+			 * for parallel mode, while extending the relations, workers
+			 * instead of blocking on a page while another worker is inserting,
+			 * can check the FSM for another page that can accommodate the
+			 * tuples. This results in major benefit for parallel inserts.
+			 */
+			myState->ti_options = 0;
 
-	/*
-	 * If WITH NO DATA is specified, there is no need to set up the state for
-	 * bulk inserts as there are no tuples to insert.
-	 */
-	if (!into->skipData)
-		myState->bistate = GetBulkInsertState();
-	else
-		myState->bistate = NULL;
+			/*
+			 * rd_createSubid is marked invalid, otherwise, the table is not
+			 * allowed to be extended by the workers.
+			 */
+			myState->rel->rd_createSubid = InvalidSubTransactionId;
+		}
 
-	/*
-	 * Valid smgr_targblock implies something already wrote to the relation.
-	 * This may be harmless, but this function hasn't planned for it.
-	 */
-	Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+		/*
+		 * Valid smgr_targblock implies something already wrote to the
+		 * relation. This may be harmless, but this function hasn't planned for
+		 * it.
+		 */
+		Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+	}
 }
 
 /*
@@ -606,3 +653,89 @@ intorel_destroy(DestReceiver *self)
 {
 	pfree(self);
 }
+
+/*
+ * PushDownCTASParallelInsertState --- push the dest receiver down to the
+ * Gather nodes.
+ *
+ * In this function we only care Append and Gather nodes.
+ *
+ * Push the dest receiver to Gather node when it is either at the top of the
+ * plan or under top Append node unless it does not have any projections to do.
+ * Required information from the pushed dest receiver is sent to workers so
+ * that they can perform parallel insertions into the target table.
+ *
+ * If the top node is Append, then this function recursively checks the sub
+ * plans for Gather nodes, when found one(and if it does not have projections),
+ * then sets the dest receiver information.
+ *
+ * In any case, this function returns true if at least one Gather node can allow
+ * parallel insertions by the workers. Otherwise returns false.
+ */
+static bool
+PushDownCTASParallelInsertState(DestReceiver *dest, PlanState *ps)
+{
+	bool parallel = false;
+
+	if (ps == NULL)
+		return parallel;
+
+	if (IsA(ps, AppendState))
+	{
+		AppendState *aps = (AppendState *) ps;
+
+		for (int i = 0; i < aps->as_nplans; i++)
+		{
+			parallel |= PushDownCTASParallelInsertState(dest,
+														aps->appendplans[i]);
+		}
+	}
+	else if (IsA(ps, GatherState) && !ps->ps_ProjInfo)
+	{
+		GatherState *gstate = (GatherState *) ps;
+		parallel = true;
+
+		/*
+		 * For parallelizing inserts in CTAS i.e. making each parallel worker
+		 * insert the tuples, we must send information such as into clause (for
+		 * each worker to build separate dest receiver), object id (for each
+		 * worker to open the created table).
+		 */
+		((DR_intorel *) dest)->is_parallel = true;
+		gstate->dest = dest;
+
+		/*
+		 * Since there are no rows that are transferred from workers to Gather
+		 * node, so we set it to 0 to be visible in estimated row count of
+		 * explain plans.
+		 */
+		ps->plan->plan_rows = 0;
+	}
+
+	return parallel;
+}
+
+/*
+ * ChooseParallelInsertsInCTAS --- determine whether or not parallel
+ * insertion is possible, if yes set the parallel insert state i.e. push down
+ * the dest receiver to the Gather nodes.
+ */
+void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc)
+{
+	if (!IS_CTAS(into))
+		return;
+
+	/*
+	 * Do not allow parallel inserts if the table is temporary. As the
+	 * temporary tables are backend local, workers can not know about them.
+	 * Currently, CTAS supports creation of normal(logged), temporary and
+	 * unlogged tables. It does not support foreign or partition table
+	 * creation. Hence the check for temporary table is enough here.
+	 */
+	if (!into->rel || into->rel->relpersistence == RELPERSISTENCE_TEMP)
+		return;
+
+	if (queryDesc)
+		(void) PushDownCTASParallelInsertState(queryDesc->dest,
+											   queryDesc->planstate);
+}
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 43f9b01e83..fbd0bc5a81 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -556,6 +556,14 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	/* call ExecutorStart to prepare the plan for execution */
 	ExecutorStart(queryDesc, eflags);
 
+	/*
+	 * If SELECT part of the CTAS is parallelizable, then make each parallel
+	 * worker insert the tuples that are resulted in its execution into the
+	 * target table. We need plan state to be initialized by the executor to
+	 * decide whether to allow parallel inserts or not.
+	 */
+	ChooseParallelInsertsInCTAS(into, queryDesc);
+
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
 	{
@@ -1775,6 +1783,29 @@ ExplainNode(PlanState *planstate, List *ancestors,
 
 				if (gather->single_copy || es->format != EXPLAIN_FORMAT_TEXT)
 					ExplainPropertyBool("Single Copy", gather->single_copy, es);
+
+				/*
+				 * Show the create table information under Gather node in case
+				 * parallel workers have inserted the rows.
+				 */
+				if (IsA(planstate, GatherState))
+				{
+					GatherState *gstate = (GatherState *) planstate;
+
+					if (IS_PARALLEL_CTAS_DEST(gstate->dest) &&
+						((DR_intorel *) gstate->dest)->into->rel &&
+						((DR_intorel *) gstate->dest)->into->rel->relname)
+					{
+						es->indent--;
+						ExplainIndentText(es);
+						appendStringInfoString(es->str, "->  ");
+						appendStringInfoString(es->str, "Create ");
+						appendStringInfo(es->str, "%s\n",
+							((DR_intorel *) gstate->dest)->into->rel->relname);
+						ExplainIndentText(es);
+						es->indent++;
+					}
+				}
 			}
 			break;
 		case T_GatherMerge:
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index befde52691..9ef33eee54 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -23,6 +23,7 @@
 
 #include "postgres.h"
 
+#include "commands/createas.h"
 #include "executor/execParallel.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
@@ -65,6 +66,7 @@
 #define PARALLEL_KEY_QUERY_TEXT		UINT64CONST(0xE000000000000008)
 #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009)
 #define PARALLEL_KEY_WAL_USAGE			UINT64CONST(0xE00000000000000A)
+#define PARALLEL_KEY_INTO_CLAUSE		UINT64CONST(0xE00000000000000B)
 
 #define PARALLEL_TUPLE_QUEUE_SIZE		65536
 
@@ -77,6 +79,9 @@ typedef struct FixedParallelExecutorState
 	dsa_pointer param_exec;
 	int			eflags;
 	int			jit_flags;
+	Oid			objectid;		/* workers to open relation/table.  */
+	/* Number tuples inserted by all the workers. */
+	pg_atomic_uint64	processed;
 } FixedParallelExecutorState;
 
 /*
@@ -578,7 +583,8 @@ ExecParallelSetupTupleQueues(ParallelContext *pcxt, bool reinitialize)
 ParallelExecutorInfo *
 ExecInitParallelPlan(PlanState *planstate, EState *estate,
 					 Bitmapset *sendParams, int nworkers,
-					 int64 tuples_needed)
+					 int64 tuples_needed, IntoClause *intoclause,
+					 Oid objectid)
 {
 	ParallelExecutorInfo *pei;
 	ParallelContext *pcxt;
@@ -600,6 +606,8 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	Size		dsa_minsize = dsa_minimum_size();
 	char	   *query_string;
 	int			query_len;
+	char 		*intoclausestr = NULL;
+	int			intoclause_len = 0;
 
 	/*
 	 * Force any initplan outputs that we're going to pass to workers to be
@@ -712,6 +720,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_estimate_chunk(&pcxt->estimator, dsa_minsize);
 	shm_toc_estimate_keys(&pcxt->estimator, 1);
 
+	/* Estimate space for into clause for CTAS. */
+	if (IS_CTAS(intoclause) && OidIsValid(objectid))
+	{
+		intoclausestr = nodeToString(intoclause);
+		intoclause_len = strlen(intoclausestr);
+		shm_toc_estimate_chunk(&pcxt->estimator, intoclause_len + 1);
+		shm_toc_estimate_keys(&pcxt->estimator, 1);
+	}
+
 	/* Everyone's had a chance to ask for space, so now create the DSM. */
 	InitializeParallelDSM(pcxt);
 
@@ -729,6 +746,14 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	fpes->param_exec = InvalidDsaPointer;
 	fpes->eflags = estate->es_top_eflags;
 	fpes->jit_flags = estate->es_jit_flags;
+	pg_atomic_init_u64(&fpes->processed, 0);
+	pei->processed = &fpes->processed;
+
+	if (intoclausestr && OidIsValid(objectid))
+		fpes->objectid = objectid;
+	else
+		fpes->objectid = InvalidOid;
+
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, fpes);
 
 	/* Store query string */
@@ -758,8 +783,18 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_WAL_USAGE, walusage_space);
 	pei->wal_usage = walusage_space;
 
-	/* Set up the tuple queues that the workers will write into. */
-	pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	if (intoclausestr)
+	{
+		char *intoclause_space = shm_toc_allocate(pcxt->toc,
+												  intoclause_len + 1);
+		memcpy(intoclause_space, intoclausestr, intoclause_len + 1);
+		shm_toc_insert(pcxt->toc, PARALLEL_KEY_INTO_CLAUSE, intoclause_space);
+	}
+	else
+	{
+		/* Set up the tuple queues that the workers will write into. */
+		pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	}
 
 	/* We don't need the TupleQueueReaders yet, though. */
 	pei->reader = NULL;
@@ -1387,12 +1422,30 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 	void	   *area_space;
 	dsa_area   *area;
 	ParallelWorkerContext pwcxt;
+	char		*intoclausestr = NULL;
+	IntoClause	*intoclause = NULL;
 
 	/* Get fixed-size state. */
 	fpes = shm_toc_lookup(toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
 
-	/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
-	receiver = ExecParallelGetReceiver(seg, toc);
+	intoclausestr = shm_toc_lookup(toc, PARALLEL_KEY_INTO_CLAUSE, true);
+	if (intoclausestr)
+	{
+		/*
+		 * If the worker is for parallel insert in CTAS, then use the proper
+		 * dest receiver.
+		 */
+		intoclause = (IntoClause *) stringToNode(intoclausestr);
+		receiver = CreateIntoRelDestReceiver(intoclause);
+		((DR_intorel *)receiver)->is_parallel_worker = true;
+		((DR_intorel *)receiver)->object_id = fpes->objectid;
+	}
+	else
+	{
+		/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
+		receiver = ExecParallelGetReceiver(seg, toc);
+	}
+
 	instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_INSTRUMENTATION, true);
 	if (instrumentation != NULL)
 		instrument_options = instrumentation->instrument_options;
@@ -1471,6 +1524,13 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 			queryDesc->estate->es_jit->instr;
 	}
 
+	/*
+	 * Write out the number of tuples this worker has inserted. Leader will use
+	 * it to inform the end client.
+	 */
+	if (intoclausestr)
+		pg_atomic_add_fetch_u64(&fpes->processed, queryDesc->estate->es_processed);
+
 	/* Must do this after capturing instrumentation. */
 	ExecutorEnd(queryDesc);
 
diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c
index a01b46af14..e7c588c66a 100644
--- a/src/backend/executor/nodeGather.c
+++ b/src/backend/executor/nodeGather.c
@@ -32,6 +32,7 @@
 
 #include "access/relscan.h"
 #include "access/xact.h"
+#include "commands/createas.h"
 #include "executor/execdebug.h"
 #include "executor/execParallel.h"
 #include "executor/nodeGather.h"
@@ -48,6 +49,7 @@ static TupleTableSlot *ExecGather(PlanState *pstate);
 static TupleTableSlot *gather_getnext(GatherState *gatherstate);
 static MinimalTuple gather_readnext(GatherState *gatherstate);
 static void ExecShutdownGatherWorkers(GatherState *node);
+static void ExecParallelInsertInCTAS(GatherState *node);
 
 
 /* ----------------------------------------------------------------
@@ -131,6 +133,72 @@ ExecInitGather(Gather *node, EState *estate, int eflags)
 	return gatherstate;
 }
 
+/* ----------------------------------------------------------------
+ *		ExecParallelInsertInCTAS(node)
+ *
+ *		Facilitates parallel inserts by parallel workers and/or
+ *		leader for Create Table AS.
+ * ----------------------------------------------------------------
+ */
+static void
+ExecParallelInsertInCTAS(GatherState *node)
+{
+	/* Enable leader to insert in case no parallel workers were launched. */
+	if (node->nworkers_launched == 0)
+		node->need_to_scan_locally = true;
+
+	/*
+	 * By now, for parallel workers (if launched any), would have started their
+	 * work i.e. insertion to target table. In case the leader is chosen to
+	 * participate for parallel inserts in CTAS, then finish its share before
+	 * going to wait for the parallel workers to finish.
+	 */
+	if (node->need_to_scan_locally)
+	{
+		EState	   *estate = node->ps.state;
+		TupleTableSlot *outerTupleSlot;
+
+		for(;;)
+		{
+			/* Install our DSA area while executing the plan. */
+			estate->es_query_dsa =
+					node->pei ? node->pei->area : NULL;
+
+			outerTupleSlot = ExecProcNode(node->ps.lefttree);
+
+			estate->es_query_dsa = NULL;
+
+			if(TupIsNull(outerTupleSlot))
+				break;
+
+			(void) node->dest->receiveSlot(outerTupleSlot, node->dest);
+
+			node->ps.state->es_processed++;
+		}
+
+		node->need_to_scan_locally = false;
+	}
+
+	if (node->nworkers_launched > 0)
+	{
+		/*
+		 * We wait here for the parallel workers to finish their work and
+		 * accumulate the tuples they inserted and also their buffer/WAL usage.
+		 * We do not destroy the parallel context here, it will be done in
+		 * ExecShutdownGather at the end of the plan. Note that the
+		 * ExecShutdownGatherWorkers call from ExecShutdownGather will be a
+		 * no-op.
+		 */
+		ExecShutdownGatherWorkers(node);
+
+		/*
+		 * Add up the total tuples inserted by all workers, to the tuples
+		 * inserted by the leader(if any). This will be shared to client.
+		 */
+		node->ps.state->es_processed += pg_atomic_read_u64(node->pei->processed);
+	}
+}
+
 /* ----------------------------------------------------------------
  *		ExecGather(node)
  *
@@ -157,6 +225,7 @@ ExecGather(PlanState *pstate)
 	{
 		EState	   *estate = node->ps.state;
 		Gather	   *gather = (Gather *) node->ps.plan;
+		bool		isctas = IS_PARALLEL_CTAS_DEST(node->dest);
 
 		/*
 		 * Sometimes we might have to run without parallelism; but if parallel
@@ -165,6 +234,18 @@ ExecGather(PlanState *pstate)
 		if (gather->num_workers > 0 && estate->es_use_parallel_mode)
 		{
 			ParallelContext *pcxt;
+			IntoClause	*intoclause = NULL;
+			Oid			objectid = InvalidOid;
+
+			/*
+			 * Take the necessary information to be passed to workers for
+			 * parallel inserts in CTAS.
+			 */
+			if (isctas)
+			{
+				intoclause = ((DR_intorel *) node->dest)->into;
+				objectid = ((DR_intorel *) node->dest)->object_id;
+			}
 
 			/* Initialize, or re-initialize, shared state needed by workers. */
 			if (!node->pei)
@@ -172,7 +253,10 @@ ExecGather(PlanState *pstate)
 												 estate,
 												 gather->initParam,
 												 gather->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 /* CTAS info */
+												 intoclause,
+												 objectid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
@@ -190,13 +274,16 @@ ExecGather(PlanState *pstate)
 			/* Set up tuple queue readers to read the results. */
 			if (pcxt->nworkers_launched > 0)
 			{
-				ExecParallelCreateReaders(node->pei);
-				/* Make a working array showing the active readers */
-				node->nreaders = pcxt->nworkers_launched;
-				node->reader = (TupleQueueReader **)
-					palloc(node->nreaders * sizeof(TupleQueueReader *));
-				memcpy(node->reader, node->pei->reader,
-					   node->nreaders * sizeof(TupleQueueReader *));
+				if (!isctas)
+				{
+					ExecParallelCreateReaders(node->pei);
+					/* Make a working array showing the active readers */
+					node->nreaders = pcxt->nworkers_launched;
+					node->reader = (TupleQueueReader **)
+						palloc(node->nreaders * sizeof(TupleQueueReader *));
+					memcpy(node->reader, node->pei->reader,
+						node->nreaders * sizeof(TupleQueueReader *));
+				}
 			}
 			else
 			{
@@ -208,9 +295,17 @@ ExecGather(PlanState *pstate)
 		}
 
 		/* Run plan locally if no workers or enabled and not single-copy. */
-		node->need_to_scan_locally = (node->nreaders == 0)
+		node->need_to_scan_locally = (node->nreaders == 0 &&
+			!isctas)
 			|| (!gather->single_copy && parallel_leader_participation);
 		node->initialized = true;
+
+		/* Perform parallel inserts for CTAS. */
+		if (isctas)
+		{
+			ExecParallelInsertInCTAS(node);
+			return NULL;
+		}
 	}
 
 	/*
diff --git a/src/backend/executor/nodeGatherMerge.c b/src/backend/executor/nodeGatherMerge.c
index 47129344f3..ee45272c17 100644
--- a/src/backend/executor/nodeGatherMerge.c
+++ b/src/backend/executor/nodeGatherMerge.c
@@ -216,7 +216,9 @@ ExecGatherMerge(PlanState *pstate)
 												 estate,
 												 gm->initParam,
 												 gm->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 NULL,
+												 InvalidOid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7320de345c..5beae6c617 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -389,6 +389,7 @@ extern FullTransactionId GetCurrentFullTransactionIdIfAny(void);
 extern void MarkCurrentTransactionIdLoggedIfAny(void);
 extern bool SubTransactionIsActive(SubTransactionId subxid);
 extern CommandId GetCurrentCommandId(bool used);
+extern void SetCurrentCommandIdUsedForWorker(void);
 extern void SetParallelStartTimestamps(TimestampTz xact_ts, TimestampTz stmt_ts);
 extern TimestampTz GetCurrentTransactionStartTimestamp(void);
 extern TimestampTz GetCurrentStatementStartTimestamp(void);
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 7629230254..ed4690305b 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -14,12 +14,35 @@
 #ifndef CREATEAS_H
 #define CREATEAS_H
 
+#include "access/heapam.h"
 #include "catalog/objectaddress.h"
+#include "executor/execdesc.h"
 #include "nodes/params.h"
+#include "nodes/plannodes.h"
 #include "parser/parse_node.h"
 #include "tcop/dest.h"
 #include "utils/queryenvironment.h"
 
+typedef struct
+{
+	DestReceiver pub;			/* publicly-known function pointers */
+	IntoClause *into;			/* target relation specification */
+	/* These fields are filled by intorel_startup: */
+	Relation	rel;			/* relation to write to */
+	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
+	CommandId	output_cid;		/* cmin to insert in output tuples */
+	int			ti_options;		/* table_tuple_insert performance options */
+	BulkInsertState bistate;	/* bulk insert state */
+	bool		is_parallel;	/* is parallelism to be considered? */
+	bool		is_parallel_worker; /* true for parallel worker */
+	/* Used for table open by parallel worker. */
+	Oid			object_id;
+} DR_intorel;
+
+#define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
+#define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
+									 IS_CTAS(((DR_intorel *) dest)->into) && \
+									 ((DR_intorel *) dest)->is_parallel)
 
 extern ObjectAddress ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 									   ParamListInfo params, QueryEnvironment *queryEnv,
@@ -29,4 +52,6 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
+extern void ChooseParallelInsertsInCTAS(IntoClause *into,
+										QueryDesc *queryDesc);
 #endif							/* CREATEAS_H */
diff --git a/src/include/executor/execParallel.h b/src/include/executor/execParallel.h
index 5a39a5b29c..9f959f741b 100644
--- a/src/include/executor/execParallel.h
+++ b/src/include/executor/execParallel.h
@@ -35,11 +35,15 @@ typedef struct ParallelExecutorInfo
 	/* These two arrays have pcxt->nworkers_launched entries: */
 	shm_mq_handle **tqueue;		/* tuple queues for worker output */
 	struct TupleQueueReader **reader;	/* tuple reader/writer support */
+	/* Number of tuples inserted by all workers. */
+	volatile pg_atomic_uint64	*processed;
 } ParallelExecutorInfo;
 
 extern ParallelExecutorInfo *ExecInitParallelPlan(PlanState *planstate,
 												  EState *estate, Bitmapset *sendParam, int nworkers,
-												  int64 tuples_needed);
+												  int64 tuples_needed,
+												  IntoClause *intoclause,
+												  Oid objectid);
 extern void ExecParallelCreateReaders(ParallelExecutorInfo *pei);
 extern void ExecParallelFinish(ParallelExecutorInfo *pei);
 extern void ExecParallelCleanup(ParallelExecutorInfo *pei);
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 61ba4c3666..e9c4442c22 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -23,6 +23,7 @@
 #include "nodes/tidbitmap.h"
 #include "partitioning/partdefs.h"
 #include "storage/condition_variable.h"
+#include "tcop/dest.h"
 #include "utils/hsearch.h"
 #include "utils/queryenvironment.h"
 #include "utils/reltrigger.h"
@@ -2326,6 +2327,8 @@ typedef struct GatherState
 	int			nreaders;		/* number of still-active workers */
 	int			nextreader;		/* next one to try to read from */
 	struct TupleQueueReader **reader;	/* array with nreaders active entries */
+	/* Parallel inserts in CTAS related info is specified below. */
+	DestReceiver *dest;
 } GatherState;
 
 /* ----------------
-- 
2.25.1

From 20aa2cede64793a9f9b2b7f9b6c7574279df9c73 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Wed, 16 Dec 2020 11:45:58 +0530
Subject: [PATCH v12 2/3] Tuple Cost Adjustment for Parallel Inserts in CTAS

Let the planner know that the SELECT is from CTAS in createas.c
so that it can set the number of tuples transferred from the
workers to Gather node to 0. With this change, there are chances
that the planner may choose the parallel plan.

Authors: Bharath Rupireddy, Hou, Zhijie
---
 src/backend/commands/createas.c       | 88 ++++++++++++++++++++-------
 src/backend/commands/explain.c        |  7 ++-
 src/backend/commands/prepare.c        |  3 +-
 src/backend/optimizer/path/allpaths.c | 39 ++++++++++++
 src/backend/optimizer/path/costsize.c | 22 ++++++-
 src/backend/optimizer/plan/planner.c  | 59 ++++++++++++++++++
 src/include/commands/createas.h       | 23 ++++++-
 src/include/commands/explain.h        |  3 +-
 src/include/nodes/parsenodes.h        |  1 +
 src/include/optimizer/planner.h       | 10 +++
 10 files changed, 225 insertions(+), 30 deletions(-)

diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index be381f9748..8401e80185 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -344,7 +344,8 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		 * into the target table. We need plan state to be initialized by the
 		 * executor to decide whether to allow parallel inserts or not.
 		 */
-		ChooseParallelInsertsInCTAS(into, queryDesc);
+		ChooseParallelInsertsInCTAS(into, queryDesc,
+									&query->CTASParallelInsInfo);
 
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
@@ -673,7 +674,8 @@ intorel_destroy(DestReceiver *self)
  * parallel insertions by the workers. Otherwise returns false.
  */
 static bool
-PushDownCTASParallelInsertState(DestReceiver *dest, PlanState *ps)
+PushDownCTASParallelInsertState(DestReceiver *dest, PlanState *ps,
+								bool *gather_exists)
 {
 	bool parallel = false;
 
@@ -687,29 +689,39 @@ PushDownCTASParallelInsertState(DestReceiver *dest, PlanState *ps)
 		for (int i = 0; i < aps->as_nplans; i++)
 		{
 			parallel |= PushDownCTASParallelInsertState(dest,
-														aps->appendplans[i]);
+														aps->appendplans[i],
+														gather_exists);
 		}
 	}
-	else if (IsA(ps, GatherState) && !ps->ps_ProjInfo)
+	else if (IsA(ps, GatherState))
 	{
-		GatherState *gstate = (GatherState *) ps;
-		parallel = true;
-
 		/*
-		 * For parallelizing inserts in CTAS i.e. making each parallel worker
-		 * insert the tuples, we must send information such as into clause (for
-		 * each worker to build separate dest receiver), object id (for each
-		 * worker to open the created table).
+		 * Set to true if there exists at least one Gather node either at the
+		 * top of the plan or as a direct sub node under Append node.
 		 */
-		((DR_intorel *) dest)->is_parallel = true;
-		gstate->dest = dest;
+		*gather_exists |= true;
 
-		/*
-		 * Since there are no rows that are transferred from workers to Gather
-		 * node, so we set it to 0 to be visible in estimated row count of
-		 * explain plans.
-		 */
-		ps->plan->plan_rows = 0;
+		if (!ps->ps_ProjInfo)
+		{
+			GatherState *gstate = (GatherState *) ps;
+			parallel = true;
+
+			/*
+			 * For parallelizing inserts in CTAS i.e. making each parallel
+			 * worker insert the tuples, we must send information such as into
+			 * clause (for each worker to build separate dest receiver), object
+			 * id (for each worker to open the created table).
+			 */
+			((DR_intorel *) dest)->is_parallel = true;
+			gstate->dest = dest;
+
+			/*
+			 * Since there are no rows that are transferred from workers to
+			 * Gather node, so we set it to 0 to be visible in estimated row
+			 * count of explain plans.
+			 */
+			ps->plan->plan_rows = 0;
+		}
 	}
 
 	return parallel;
@@ -720,8 +732,12 @@ PushDownCTASParallelInsertState(DestReceiver *dest, PlanState *ps)
  * insertion is possible, if yes set the parallel insert state i.e. push down
  * the dest receiver to the Gather nodes.
  */
-void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc)
+void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc,
+								 uint8 *tuple_cost_flags)
 {
+	bool gather_exists = false;
+	bool allow = false;
+
 	if (!IS_CTAS(into))
 		return;
 
@@ -735,7 +751,33 @@ void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc)
 	if (!into->rel || into->rel->relpersistence == RELPERSISTENCE_TEMP)
 		return;
 
-	if (queryDesc)
-		(void) PushDownCTASParallelInsertState(queryDesc->dest,
-											   queryDesc->planstate);
+	if (!queryDesc)
+		return;
+
+	allow = PushDownCTASParallelInsertState(queryDesc->dest,
+											queryDesc->planstate,
+											&gather_exists);
+
+	/*
+	 * It should not happen that in cost_gather we have ignored the parallel
+	 * tuple cost and now we are not allowing the parallel inserts. And also we
+	 * might need assertion only if the top node is Gather or Append under it
+	 * we have Gather. The main intention of assertion is to check if we
+	 * enforced planner to ignore the parallel tuple cost (with the intention
+	 * of choosing parallel inserts) due to which the parallel plan may have
+	 * been chosen, but we do not allow the parallel inserts now.
+	 */
+	if (!allow && tuple_cost_flags && gather_exists)
+	{
+		/*
+		 * If we have correctly ignored parallel tuple cost in planner while
+		 * creating Gather path, then this assertion failure should not occur.
+		 * If it occurs, that means the planner may have chosen this parallel
+		 * plan because of our enforcement to ignore the parallel tuple cost.
+		 */
+		Assert(!(*tuple_cost_flags & CTAS_PARALLEL_INS_TUP_COST_IGNORED));
+	}
+
+	if (tuple_cost_flags)
+		*tuple_cost_flags = CTAS_PARALLEL_INS_UNDEF;
 }
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index fbd0bc5a81..efdb34d1f0 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -402,7 +402,8 @@ ExplainOneQuery(Query *query, int cursorOptions,
 
 		/* run it (if needed) and produce output */
 		ExplainOnePlan(plan, into, es, queryString, params, queryEnv,
-					   &planduration, (es->buffers ? &bufusage : NULL));
+					   &planduration, (es->buffers ? &bufusage : NULL),
+					   &query->CTASParallelInsInfo);
 	}
 }
 
@@ -496,7 +497,7 @@ void
 ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 			   const char *queryString, ParamListInfo params,
 			   QueryEnvironment *queryEnv, const instr_time *planduration,
-			   const BufferUsage *bufusage)
+			   const BufferUsage *bufusage, uint8 *ctas_tuple_cost_flags)
 {
 	DestReceiver *dest;
 	QueryDesc  *queryDesc;
@@ -562,7 +563,7 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	 * target table. We need plan state to be initialized by the executor to
 	 * decide whether to allow parallel inserts or not.
 	 */
-	ChooseParallelInsertsInCTAS(into, queryDesc);
+	ChooseParallelInsertsInCTAS(into, queryDesc, ctas_tuple_cost_flags);
 
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c
index 89087a7be3..07166479e7 100644
--- a/src/backend/commands/prepare.c
+++ b/src/backend/commands/prepare.c
@@ -672,7 +672,8 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es,
 
 		if (pstmt->commandType != CMD_UTILITY)
 			ExplainOnePlan(pstmt, into, es, query_string, paramLI, queryEnv,
-						   &planduration, (es->buffers ? &bufusage : NULL));
+						   &planduration, (es->buffers ? &bufusage : NULL),
+						   NULL);
 		else
 			ExplainOneUtility(pstmt->utilityStmt, into, es, query_string,
 							  paramLI, queryEnv);
diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c
index 84a69b064a..07c9d0f3d7 100644
--- a/src/backend/optimizer/path/allpaths.c
+++ b/src/backend/optimizer/path/allpaths.c
@@ -23,6 +23,7 @@
 #include "catalog/pg_class.h"
 #include "catalog/pg_operator.h"
 #include "catalog/pg_proc.h"
+#include "commands/createas.h"
 #include "foreign/fdwapi.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1103,11 +1104,49 @@ set_append_rel_size(PlannerInfo *root, RelOptInfo *rel,
 		if (root->glob->parallelModeOK && rel->consider_parallel)
 			set_rel_consider_parallel(root, childrel, childRTE);
 
+		/*
+		 * When subplan is subquery, It's possible to do parallel insert if top
+		 * node of subquery is Gather, so we set the flag to ignore parallel
+		 * tuple cost by the Gather path in cost_gather if the SELECT is for
+		 * CTAS.
+		 */
+		if (childrel->rtekind == RTE_SUBQUERY)
+		{
+			/*
+			 * We set the flag for two cases when there is no parent path will
+			 * be created(such as : limit,sort,distinct...):
+			 * i) query_level is 1
+			 * ii) query_level > 1 then set the flag in the parent_root.
+			 * The case ii) is to check append under append:
+			 * Append
+			 *	->Append
+			 *	->Gather
+			 *	->Other plan
+			 */
+			if (root->parse->CTASParallelInsInfo &
+				CTAS_PARALLEL_INS_SELECT &&
+				(root->query_level == 1 ||
+				 root->parent_root->parse->CTASParallelInsInfo &
+				 CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND) &&
+				!(HAS_PARENT_PATH_GENERATING_CLAUSE(root)))
+			{
+				root->parse->CTASParallelInsInfo |=
+										CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+			}
+		}
+
 		/*
 		 * Compute the child's size.
 		 */
 		set_rel_size(root, childrel, childRTindex, childRTE);
 
+		if (root->parse->CTASParallelInsInfo &
+			CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND)
+		{
+			root->parse->CTASParallelInsInfo &=
+										~CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+		}
+
 		/*
 		 * It is possible that constraint exclusion detected a contradiction
 		 * within a child subquery, even though we didn't prove one above. If
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 22d6935824..800f25903d 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -76,6 +76,7 @@
 #include "access/amapi.h"
 #include "access/htup_details.h"
 #include "access/tsmapi.h"
+#include "commands/createas.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "executor/nodeHash.h"
@@ -378,6 +379,7 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 {
 	Cost		startup_cost = 0;
 	Cost		run_cost = 0;
+	bool		ignore_tuple_cost = false;
 
 	/* Mark the path with the correct row estimate */
 	if (rows)
@@ -393,7 +395,25 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 
 	/* Parallel setup and communication cost. */
 	startup_cost += parallel_setup_cost;
-	run_cost += parallel_tuple_cost * path->path.rows;
+
+	/*
+	 * Do not consider tuple cost in case of we intend to perform parallel
+	 * inserts by workers. We would have set ignore flag in
+	 * apply_scanjoin_target_to_paths before generating Gather path for the
+	 * upper level SELECT part of the CTAS.
+	 */
+	if ((root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT) &&
+		(root->parse->CTASParallelInsInfo &
+		 CTAS_PARALLEL_INS_TUP_COST_CAN_IGN))
+	{
+		ignore_tuple_cost = true;
+		root->parse->CTASParallelInsInfo &=
+									~CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_TUP_COST_IGNORED;
+	}
+
+	if (!ignore_tuple_cost)
+		run_cost += parallel_tuple_cost * path->path.rows;
 
 	path->path.startup_cost = startup_cost;
 	path->path.total_cost = (startup_cost + run_cost);
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index 1a94b58f8b..7555cde61a 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -28,6 +28,7 @@
 #include "catalog/pg_inherits.h"
 #include "catalog/pg_proc.h"
 #include "catalog/pg_type.h"
+#include "commands/createas.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "foreign/fdwapi.h"
@@ -7338,6 +7339,43 @@ can_partial_agg(PlannerInfo *root)
 	return true;
 }
 
+/*
+ * ignore_parallel_tuple_cost
+ *
+ * Gather node will not receive any tuples from the workers in case each worker
+ * inserts them in parallel. So, we set a flag to ignore parallel tuple cost by
+ * the Gather path in cost_gather if the SELECT is for CTAS and we are
+ * generating an upper level Gather path.
+*/
+static bool
+ignore_parallel_tuple_cost(PlannerInfo *root)
+{
+	if (root->query_level != 1 &&
+		(root->parent_root->parse->CTASParallelInsInfo &
+		 CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND))
+	{
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_SELECT;
+	}
+
+	if (root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT)
+	{
+		/*
+		 * In each of the HAS_PARENT_PATH_GENERATING_CLAUSE cases, a parent
+		 * path will be generated for the upper Gather path(in
+		 * grouping_planner), in which case we can not let parallel inserts
+		 * happen. So we do not set ignore tuple cost flag.
+		 */
+		if (HAS_PARENT_PATH_GENERATING_CLAUSE(root))
+			return false;
+
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+
+		return true;
+	}
+
+	return false;
+}
+
 /*
  * apply_scanjoin_target_to_paths
  *
@@ -7557,8 +7595,29 @@ apply_scanjoin_target_to_paths(PlannerInfo *root,
 	 * one of the generated paths may turn out to be the cheapest one.
 	 */
 	if (rel->consider_parallel && !IS_OTHER_REL(rel))
+	{
+		/*
+		 * Set a flag to ignore parallel tuple cost by the Gather path in
+		 * cost_gather if the SELECT is for CTAS and we are generating an upper
+		 * level Gather path.
+		 */
+		bool	ignore = ignore_parallel_tuple_cost(root);
+
 		generate_useful_gather_paths(root, rel, false);
 
+		/*
+		 * Reset the ignore flag, in case we set it but
+		 * generate_useful_gather_paths returned without reaching cost_gather.
+		 */
+		if (ignore &&
+			(root->parse->CTASParallelInsInfo &
+			 CTAS_PARALLEL_INS_TUP_COST_CAN_IGN))
+		{
+			root->parse->CTASParallelInsInfo &=
+										~CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+		}
+	}
+
 	/*
 	 * Reassess which paths are the cheapest, now that we've potentially added
 	 * new Gather (or Gather Merge) and/or Append (or MergeAppend) paths to
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index ed4690305b..4103ac65f0 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -39,6 +39,26 @@ typedef struct
 	Oid			object_id;
 } DR_intorel;
 
+/*
+ * Information sent to the planner from CTAS to account for the cost
+ * calculations in cost_gather. We need to do this because, no tuples will be
+ * received by the Gather node if the workers insert the tuples in parallel.
+ */
+typedef enum CTASParallelInsertOpt
+{
+	CTAS_PARALLEL_INS_UNDEF = 0, /* undefined */
+	CTAS_PARALLEL_INS_SELECT = 1 << 0, /* set to this before planning */
+	/*
+	 * Set to this while planning for upper Gather path to ignore parallel
+	 * tuple cost in cost_gather.
+	 */
+	CTAS_PARALLEL_INS_TUP_COST_CAN_IGN = 1 << 1,
+	/* Set to this after the cost is ignored. */
+	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2,
+	/* Set to this in case tuple cost needs to be ignored for Append cases. */
+	CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND = 1 << 3
+} CTASParallelInsertOpt;
+
 #define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
 #define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
 									 IS_CTAS(((DR_intorel *) dest)->into) && \
@@ -53,5 +73,6 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
 extern void ChooseParallelInsertsInCTAS(IntoClause *into,
-										QueryDesc *queryDesc);
+										QueryDesc *queryDesc,
+										uint8 *tuple_cost_flags);
 #endif							/* CREATEAS_H */
diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h
index ba661d32a6..1a1806dbf1 100644
--- a/src/include/commands/explain.h
+++ b/src/include/commands/explain.h
@@ -91,7 +91,8 @@ extern void ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into,
 						   ExplainState *es, const char *queryString,
 						   ParamListInfo params, QueryEnvironment *queryEnv,
 						   const instr_time *planduration,
-						   const BufferUsage *bufusage);
+						   const BufferUsage *bufusage,
+						   uint8 *ctas_tuple_cost_flags);
 
 extern void ExplainPrintPlan(ExplainState *es, QueryDesc *queryDesc);
 extern void ExplainPrintTriggers(ExplainState *es, QueryDesc *queryDesc);
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 48a79a7657..81b148c383 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -180,6 +180,7 @@ typedef struct Query
 	 */
 	int			stmt_location;	/* start location, or -1 if unknown */
 	int			stmt_len;		/* length in bytes; 0 means "rest of string" */
+	uint8 		CTASParallelInsInfo; /* parallel insert in CTAS info */
 } Query;
 
 
diff --git a/src/include/optimizer/planner.h b/src/include/optimizer/planner.h
index beb7dbbcbe..74b2563828 100644
--- a/src/include/optimizer/planner.h
+++ b/src/include/optimizer/planner.h
@@ -21,6 +21,16 @@
 #include "nodes/pathnodes.h"
 #include "nodes/plannodes.h"
 
+#define HAS_PARENT_PATH_GENERATING_CLAUSE(root) \
+											(root->parse->rowMarks || \
+											 limit_needed(root->parse) || \
+											 root->parse->sortClause ||  \
+											 root->parse->distinctClause ||  \
+											 root->parse->hasWindowFuncs ||  \
+											 root->parse->groupClause ||  \
+											 root->parse->groupingSets ||  \
+											 root->parse->hasAggs ||  \
+											 root->hasHavingQual)
 
 /* Hook for plugins to get control in planner() */
 typedef PlannedStmt *(*planner_hook_type) (Query *parse,
-- 
2.25.1

From 07aa36fb45d34ac170df2048c1ad224813b7f659 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Wed, 16 Dec 2020 09:27:59 +0530
Subject: [PATCH v12 3/3] Tests For Parallel Inserts in CTAS

---
 src/test/regress/expected/write_parallel.out | 1226 ++++++++++++++++++
 src/test/regress/sql/write_parallel.sql      |  402 ++++++
 2 files changed, 1628 insertions(+)

diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 0c4da2591a..f8f1890228 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -76,4 +76,1230 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+                     QUERY PLAN                      
+-----------------------------------------------------
+ LockRows (actual rows=10000 loops=1)
+   ->  Seq Scan on tenk1 (actual rows=10000 loops=1)
+(2 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+    select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_mat_view
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_mat_view;
+ count 
+-------
+ 10000
+(1 row)
+
+drop materialized view parallel_mat_view;
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+deallocate parallel_write_prep;
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select nextval('parallel_write_sequence'), four from tenk1;
+                  QUERY PLAN                   
+-----------------------------------------------
+ Seq Scan on tenk1 (actual rows=10000 loops=1)
+(1 row)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;
+                               QUERY PLAN                               
+------------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Params Evaluated: $1
+   Workers Launched: 3
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=1 loops=1)
+           ->  Gather (actual rows=1 loops=1)
+                 Workers Planned: 4
+                 Workers Launched: 4
+                 ->  Parallel Seq Scan on tenk2 (actual rows=1 loops=5)
+   ->  Parallel Seq Scan on tenk1 (actual rows=625 loops=4)
+         Filter: (four = 3)
+         Rows Removed by Filter: 1875
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+  2500
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;
+                             QUERY PLAN                              
+---------------------------------------------------------------------
+ Gather (actual rows=2500 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=500 loops=5)
+         Filter: (four = 3)
+         Rows Removed by Filter: 1500
+   SubPlan 1
+     ->  Function Scan on generate_series (actual rows=1 loops=2500)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+  2500
+(1 row)
+
+drop table parallel_write;
+create table temp1(col1) as select * from generate_series(1,5);
+create table temp2(col2) as select * from temp1;
+create table temp3(col3) as select * from temp1;
+-- parallel inserts must not occur, as there is a limit clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 limit 4;
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ Limit (actual rows=4 loops=1)
+   ->  Gather (actual rows=4 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an order by clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 order by 1;
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ Gather Merge (actual rows=5 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+   ->  Sort (actual rows=1 loops=4)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         Worker 0:  Sort Method: quicksort  Memory: 25kB
+         Worker 1:  Sort Method: quicksort  Memory: 25kB
+         Worker 2:  Sort Method: quicksort  Memory: 25kB
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(10 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an order by clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 order by 1;
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ Gather Merge (actual rows=5 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+   ->  Sort (actual rows=1 loops=4)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         Worker 0:  Sort Method: quicksort  Memory: 25kB
+         Worker 1:  Sort Method: quicksort  Memory: 25kB
+         Worker 2:  Sort Method: quicksort  Memory: 25kB
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(10 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is a distinct clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select distinct * from temp1;
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ HashAggregate (actual rows=5 loops=1)
+   Group Key: col1
+   Batches: 1  Memory Usage: 40kB
+   ->  Gather (actual rows=5 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(7 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an aggregate and group clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select count(*) from temp1 group by col1;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Finalize HashAggregate (actual rows=5 loops=1)
+   Group Key: col1
+   Batches: 1  Memory Usage: 40kB
+   ->  Gather (actual rows=5 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+         ->  Partial HashAggregate (actual rows=1 loops=4)
+               Group Key: col1
+               Batches: 1  Memory Usage: 40kB
+               Worker 0:  Batches: 1  Memory Usage: 40kB
+               Worker 1:  Batches: 1  Memory Usage: 40kB
+               Worker 2:  Batches: 1  Memory Usage: 40kB
+               ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(13 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an aggregate, group and having
+-- clauses
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select count(col1), (select col3 from
+        (select * from temp3) as tt limit 1) col4 from temp1, temp2
+    where temp1.col1 = temp2.col2 group by col4 having count(col1) > 0;
+                                    QUERY PLAN                                    
+----------------------------------------------------------------------------------
+ Finalize GroupAggregate (actual rows=1 loops=1)
+   Group Key: ($1)
+   Filter: (count(temp1.col1) > 0)
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=1 loops=1)
+           ->  Gather (actual rows=1 loops=1)
+                 Workers Planned: 3
+                 Workers Launched: 3
+                 ->  Parallel Seq Scan on temp3 (actual rows=0 loops=4)
+   ->  Gather (actual rows=1 loops=1)
+         Workers Planned: 3
+         Params Evaluated: $1
+         Workers Launched: 3
+         ->  Partial GroupAggregate (actual rows=0 loops=4)
+               Group Key: $1
+               ->  Parallel Hash Join (actual rows=1 loops=4)
+                     Hash Cond: (temp1.col1 = temp2.col2)
+                     ->  Parallel Seq Scan on temp1 (actual rows=5 loops=1)
+                     ->  Parallel Hash (actual rows=1 loops=4)
+                           Buckets: 4096  Batches: 1  Memory Usage: 64kB
+                           ->  Parallel Seq Scan on temp2 (actual rows=5 loops=1)
+(21 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is a window function
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select avg(col1) OVER (PARTITION BY col1) from temp1;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ WindowAgg (actual rows=5 loops=1)
+   ->  Sort (actual rows=5 loops=1)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         ->  Gather (actual rows=5 loops=1)
+               Workers Planned: 3
+               Workers Launched: 3
+               ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(8 rows)
+
+drop table parallel_write;
+-- nested loop join is the top node under which Gather node exists, so parallel
+-- inserts must not occur
+set enable_nestloop to on;
+set enable_mergejoin to off;
+set enable_hashjoin to off;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Nested Loop (actual rows=5 loops=1)
+   Join Filter: (temp1.col1 = temp2.col2)
+   Rows Removed by Join Filter: 20
+   ->  Gather (actual rows=5 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+   ->  Materialize (actual rows=5 loops=5)
+         ->  Gather (actual rows=5 loops=1)
+               Workers Planned: 3
+               Workers Launched: 3
+               ->  Parallel Seq Scan on temp2 (actual rows=1 loops=4)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- the top node is Gather under which merge join happens, so parallel inserts
+-- must occur
+set enable_nestloop to off;
+set enable_mergejoin to on;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+ ->  Create parallel_write
+   ->  Merge Join (actual rows=1 loops=4)
+         Merge Cond: (temp1.col1 = temp2.col2)
+         ->  Sort (actual rows=1 loops=4)
+               Sort Key: temp1.col1
+               Sort Method: quicksort  Memory: 25kB
+               Worker 0:  Sort Method: quicksort  Memory: 25kB
+               Worker 1:  Sort Method: quicksort  Memory: 25kB
+               Worker 2:  Sort Method: quicksort  Memory: 25kB
+               ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+         ->  Sort (actual rows=5 loops=1)
+               Sort Key: temp2.col2
+               Sort Method: quicksort  Memory: 25kB
+               ->  Seq Scan on temp2 (actual rows=5 loops=1)
+(17 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel hash join happens under Gather node, so parallel inserts must occur
+set enable_mergejoin to off;
+set enable_hashjoin to on;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+ ->  Create parallel_write
+   ->  Parallel Hash Join (actual rows=1 loops=4)
+         Hash Cond: (temp1.col1 = temp2.col2)
+         ->  Parallel Seq Scan on temp1 (actual rows=5 loops=1)
+         ->  Parallel Hash (actual rows=1 loops=4)
+               Buckets: 4096  Batches: 1  Memory Usage: 64kB
+               ->  Parallel Seq Scan on temp2 (actual rows=5 loops=1)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+reset enable_nestloop;
+reset enable_mergejoin;
+reset enable_hashjoin;
+-- test cases for performing parallel inserts when Append node is at the top
+-- and Gather node is in one of its direct sub plans.
+-- case 1: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5 union all
+    select * from temp2 where col2 = 5;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Append (actual rows=0 loops=1)
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 (actual rows=0 loops=4)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: 1
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 (actual rows=0 loops=4)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: 1
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=0 loops=4)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: 1
+(22 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+-- case 2: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp2;
+                               QUERY PLAN                               
+------------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+ ->  Create parallel_write
+   ->  Parallel Append (actual rows=4 loops=4)
+         ->  Parallel Seq Scan on temp1 (actual rows=5 loops=1)
+         ->  Parallel Seq Scan on temp2 (actual rows=5 loops=1)
+         ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=5 loops=1)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select * from temp2;
+                                   QUERY PLAN                                   
+--------------------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 3
+   Params Evaluated: $1, $3
+   Workers Launched: 1
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=1 loops=1)
+           ->  Gather (actual rows=1 loops=1)
+                 Workers Planned: 3
+                 Workers Launched: 3
+                 ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=0 loops=4)
+   InitPlan 2 (returns $3)
+     ->  Limit (actual rows=1 loops=1)
+           ->  Gather (actual rows=1 loops=1)
+                 Workers Planned: 3
+                 Workers Launched: 3
+                 ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=0 loops=4)
+   ->  Parallel Append (actual rows=8 loops=2)
+         ->  Parallel Seq Scan on temp1 (actual rows=5 loops=1)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=5 loops=1)
+         ->  Parallel Seq Scan on temp2 (actual rows=5 loops=1)
+(21 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+-- case 3: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples to
+-- Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Gather
+-- 		->Parallel Seq Scan
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1 union all
+    select * from temp1 where col1 = 5;
+                                  QUERY PLAN                                   
+-------------------------------------------------------------------------------
+ Append (actual rows=5 loops=1)
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=0 loops=4)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: 1
+   ->  Seq Scan on temp1 (actual rows=5 loops=1)
+         SubPlan 1
+           ->  Limit (actual rows=1 loops=5)
+                 ->  Gather (actual rows=1 loops=5)
+                       Workers Planned: 3
+                       Workers Launched: 3
+                       ->  Parallel Seq Scan on temp2 (actual rows=0 loops=20)
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=0 loops=4)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: 1
+(22 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     7
+(1 row)
+
+drop table parallel_write;
+alter table temp2 set (parallel_workers = 0);
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 where col1 = (select 1) union all  select * from temp1 where col1 = (select 1) union all
+ select * from temp2 where col2 = (select 2);
+                               QUERY PLAN                               
+------------------------------------------------------------------------
+ Append (actual rows=1 loops=1)
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Params Evaluated: $0
+         Workers Launched: 3
+       ->  Create parallel_write
+       InitPlan 1 (returns $0)
+           ->  Result (actual rows=1 loops=1)
+         ->  Parallel Seq Scan on temp1 (actual rows=0 loops=4)
+               Filter: (col1 = $0)
+               Rows Removed by Filter: 1
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Params Evaluated: $1
+         Workers Launched: 3
+       ->  Create parallel_write
+       InitPlan 2 (returns $1)
+           ->  Result (actual rows=1 loops=1)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=0 loops=4)
+               Filter: (col1 = $1)
+               Rows Removed by Filter: 1
+   ->  Seq Scan on temp2 (actual rows=1 loops=1)
+         Filter: (col2 = $2)
+         Rows Removed by Filter: 4
+         InitPlan 3 (returns $2)
+           ->  Result (actual rows=1 loops=1)
+(26 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+alter table temp2 reset (parallel_workers);
+drop table parallel_write;
+-- case 4: parallel inserts must not occur as there will be no direct Gather
+-- node under Append node. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1;
+                                      QUERY PLAN                                       
+---------------------------------------------------------------------------------------
+ Append (actual rows=15 loops=1)
+   ->  Seq Scan on temp1 (actual rows=5 loops=1)
+   ->  Seq Scan on temp2 (actual rows=5 loops=1)
+   ->  Seq Scan on temp1 temp1_1 (actual rows=5 loops=1)
+         SubPlan 1
+           ->  Limit (actual rows=1 loops=5)
+                 ->  Gather (actual rows=1 loops=5)
+                       Workers Planned: 3
+                       Workers Launched: 3
+                       ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=0 loops=20)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+-- case 5: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Seq Scan / Join / any other non-Gather node
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+alter table temp2 set (parallel_workers = 0);
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp1;
+                               QUERY PLAN                               
+------------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+ ->  Create parallel_write
+   ->  Parallel Append (actual rows=4 loops=4)
+         ->  Seq Scan on temp2 (actual rows=5 loops=1)
+         ->  Parallel Seq Scan on temp1 (actual rows=5 loops=1)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=5 loops=1)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+alter table temp2 reset (parallel_workers);
+-- case 6: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Gather
+--  ->Gather
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp2 where col2 = (select 2);
+                                 QUERY PLAN                                 
+----------------------------------------------------------------------------
+ Append (actual rows=0 loops=1)
+   ->  Append (actual rows=0 loops=1)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=1 loops=1)
+         ->  Gather (actual rows=0 loops=1)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: 3
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Result (actual rows=0 loops=4)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=0 loops=4)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: 1
+         ->  Gather (actual rows=0 loops=1)
+               Workers Planned: 3
+               Params Evaluated: $0, $2
+               Workers Launched: 3
+             ->  Create parallel_write
+             InitPlan 3 (returns $2)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Result (actual rows=0 loops=4)
+                     One-Time Filter: ($2 = $0)
+                     ->  Parallel Seq Scan on temp2 (never executed)
+                           Filter: (col2 = $0)
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Params Evaluated: $3
+         Workers Launched: 3
+       ->  Create parallel_write
+       InitPlan 4 (returns $3)
+           ->  Result (actual rows=1 loops=1)
+         ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=0 loops=4)
+               Filter: (col2 = $3)
+               Rows Removed by Filter: 1
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Params Evaluated: $4
+         Workers Launched: 3
+       ->  Create parallel_write
+       InitPlan 5 (returns $4)
+           ->  Result (actual rows=1 loops=1)
+         ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=0 loops=4)
+               Filter: (col2 = $4)
+               Rows Removed by Filter: 1
+(47 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;
+                                     QUERY PLAN                                     
+------------------------------------------------------------------------------------
+ Append (actual rows=0 loops=1)
+   ->  Append (actual rows=0 loops=1)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=1 loops=1)
+         ->  Gather (actual rows=0 loops=1)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: 3
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Result (actual rows=0 loops=4)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=0 loops=4)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: 1
+         ->  Gather (actual rows=0 loops=1)
+               Workers Planned: 3
+               Params Evaluated: $0, $2
+               Workers Launched: 3
+             ->  Create parallel_write
+             InitPlan 3 (returns $2)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Result (actual rows=0 loops=4)
+                     One-Time Filter: ($2 = $0)
+                     ->  Parallel Seq Scan on temp2 (never executed)
+                           Filter: (col2 = $0)
+   ->  Append (actual rows=0 loops=1)
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=1 loops=1)
+         ->  Gather (actual rows=0 loops=1)
+               Workers Planned: 3
+               Params Evaluated: $3, $4
+               Workers Launched: 3
+             ->  Create parallel_write
+             InitPlan 5 (returns $4)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Result (actual rows=0 loops=4)
+                     One-Time Filter: ($4 = $3)
+                     ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=0 loops=4)
+                           Filter: (col1 = $3)
+                           Rows Removed by Filter: 1
+         ->  Gather (actual rows=0 loops=1)
+               Workers Planned: 3
+               Params Evaluated: $3, $5
+               Workers Launched: 3
+             ->  Create parallel_write
+             InitPlan 6 (returns $5)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Result (actual rows=0 loops=4)
+                     One-Time Filter: ($5 = $3)
+                     ->  Parallel Seq Scan on temp2 temp2_1 (never executed)
+                           Filter: (col2 = $3)
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=0 loops=4)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: 1
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=0 loops=4)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: 1
+(67 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+-- case 7: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Seq Scan / Join / any other non-Gather node
+--  ->Gather
+alter table temp2 set (parallel_workers = 0);
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp1 where col1 = (select 2);
+                                 QUERY PLAN                                 
+----------------------------------------------------------------------------
+ Append (actual rows=1 loops=1)
+   ->  Append (actual rows=0 loops=1)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=1 loops=1)
+         ->  Gather (actual rows=0 loops=1)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: 3
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Result (actual rows=0 loops=4)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=0 loops=4)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: 1
+         ->  Result (actual rows=0 loops=1)
+               One-Time Filter: ($2 = $0)
+               InitPlan 3 (returns $2)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Seq Scan on temp2 (never executed)
+                     Filter: (col2 = $0)
+   ->  Seq Scan on temp2 temp2_1 (actual rows=1 loops=1)
+         Filter: (col2 = $3)
+         Rows Removed by Filter: 4
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=1 loops=1)
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Params Evaluated: $4
+         Workers Launched: 3
+       ->  Create parallel_write
+       InitPlan 5 (returns $4)
+           ->  Result (actual rows=1 loops=1)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=0 loops=4)
+               Filter: (col1 = $4)
+               Rows Removed by Filter: 1
+(37 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;
+                                     QUERY PLAN                                     
+------------------------------------------------------------------------------------
+ Append (actual rows=1 loops=1)
+   ->  Append (actual rows=0 loops=1)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=1 loops=1)
+         ->  Gather (actual rows=0 loops=1)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: 3
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Result (actual rows=0 loops=4)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=0 loops=4)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: 1
+         ->  Result (actual rows=0 loops=1)
+               One-Time Filter: ($2 = $0)
+               InitPlan 3 (returns $2)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Seq Scan on temp2 (never executed)
+                     Filter: (col2 = $0)
+   ->  Append (actual rows=0 loops=1)
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=1 loops=1)
+         ->  Gather (actual rows=0 loops=1)
+               Workers Planned: 3
+               Params Evaluated: $3, $4
+               Workers Launched: 3
+             ->  Create parallel_write
+             InitPlan 5 (returns $4)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Result (actual rows=0 loops=4)
+                     One-Time Filter: ($4 = $3)
+                     ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=0 loops=4)
+                           Filter: (col1 = $3)
+                           Rows Removed by Filter: 1
+         ->  Result (actual rows=0 loops=1)
+               One-Time Filter: ($5 = $3)
+               InitPlan 6 (returns $5)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Seq Scan on temp2 temp2_1 (never executed)
+                     Filter: (col2 = $3)
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=0 loops=4)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: 1
+   ->  Seq Scan on temp2 temp2_2 (actual rows=1 loops=1)
+         Filter: (col2 = 5)
+         Rows Removed by Filter: 4
+(53 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+alter table temp2 reset (parallel_workers);
+-- case 8: parallel inserts must not occur because there is no Gather or Append
+-- node at the top for union, except/except all, intersect/intersect all
+-- cases.
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 union
+    select * from temp2;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ HashAggregate (actual rows=5 loops=1)
+   Group Key: temp1.col1
+   Batches: 1  Memory Usage: 217kB
+   ->  Append (actual rows=10 loops=1)
+         ->  Gather (actual rows=5 loops=1)
+               Workers Planned: 3
+               Workers Launched: 3
+               ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+         ->  Gather (actual rows=5 loops=1)
+               Workers Planned: 3
+               Workers Launched: 3
+               ->  Parallel Seq Scan on temp2 (actual rows=1 loops=4)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 except
+    select * from temp2 where col2 < 3;
+                                 QUERY PLAN                                 
+----------------------------------------------------------------------------
+ HashSetOp Except (actual rows=3 loops=1)
+   ->  Append (actual rows=7 loops=1)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=5 loops=1)
+               ->  Gather (actual rows=5 loops=1)
+                     Workers Planned: 3
+                     Workers Launched: 3
+                     ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=2 loops=1)
+               ->  Gather (actual rows=2 loops=1)
+                     Workers Planned: 3
+                     Workers Launched: 3
+                     ->  Parallel Seq Scan on temp2 (actual rows=0 loops=4)
+                           Filter: (col2 < 3)
+                           Rows Removed by Filter: 1
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 except all
+    select * from temp2 where col2 < 3;
+                                 QUERY PLAN                                 
+----------------------------------------------------------------------------
+ HashSetOp Except All (actual rows=3 loops=1)
+   ->  Append (actual rows=7 loops=1)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=5 loops=1)
+               ->  Gather (actual rows=5 loops=1)
+                     Workers Planned: 3
+                     Workers Launched: 3
+                     ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=2 loops=1)
+               ->  Gather (actual rows=2 loops=1)
+                     Workers Planned: 3
+                     Workers Launched: 3
+                     ->  Parallel Seq Scan on temp2 (actual rows=0 loops=4)
+                           Filter: (col2 < 3)
+                           Rows Removed by Filter: 1
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 intersect
+    select * from temp2;
+                                 QUERY PLAN                                 
+----------------------------------------------------------------------------
+ HashSetOp Intersect (actual rows=5 loops=1)
+   ->  Append (actual rows=10 loops=1)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=5 loops=1)
+               ->  Gather (actual rows=5 loops=1)
+                     Workers Planned: 3
+                     Workers Launched: 3
+                     ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=5 loops=1)
+               ->  Gather (actual rows=5 loops=1)
+                     Workers Planned: 3
+                     Workers Launched: 3
+                     ->  Parallel Seq Scan on temp2 (actual rows=1 loops=4)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 intersect all
+    select * from temp2;
+                                 QUERY PLAN                                 
+----------------------------------------------------------------------------
+ HashSetOp Intersect All (actual rows=5 loops=1)
+   ->  Append (actual rows=10 loops=1)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=5 loops=1)
+               ->  Gather (actual rows=5 loops=1)
+                     Workers Planned: 3
+                     Workers Launched: 3
+                     ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=5 loops=1)
+               ->  Gather (actual rows=5 loops=1)
+                     Workers Planned: 3
+                     Workers Launched: 3
+                     ->  Parallel Seq Scan on temp2 (actual rows=1 loops=4)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+drop table temp1;
+drop table temp2;
+drop table temp3;
 rollback;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 78b479cedf..67de31fb91 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -39,4 +39,406 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
 
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+    select length(stringu1) from tenk1;
+select count(*) from parallel_mat_view;
+drop materialized view parallel_mat_view;
+
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+select count(*) from parallel_write;
+deallocate parallel_write_prep;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select nextval('parallel_write_sequence'), four from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+create table temp1(col1) as select * from generate_series(1,5);
+create table temp2(col2) as select * from temp1;
+create table temp3(col3) as select * from temp1;
+
+-- parallel inserts must not occur, as there is a limit clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 limit 4;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an order by clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 order by 1;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an order by clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 order by 1;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is a distinct clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select distinct * from temp1;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an aggregate and group clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select count(*) from temp1 group by col1;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an aggregate, group and having
+-- clauses
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select count(col1), (select col3 from
+        (select * from temp3) as tt limit 1) col4 from temp1, temp2
+    where temp1.col1 = temp2.col2 group by col4 having count(col1) > 0;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is a window function
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select avg(col1) OVER (PARTITION BY col1) from temp1;
+drop table parallel_write;
+
+-- nested loop join is the top node under which Gather node exists, so parallel
+-- inserts must not occur
+set enable_nestloop to on;
+set enable_mergejoin to off;
+set enable_hashjoin to off;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- the top node is Gather under which merge join happens, so parallel inserts
+-- must occur
+set enable_nestloop to off;
+set enable_mergejoin to on;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel hash join happens under Gather node, so parallel inserts must occur
+set enable_mergejoin to off;
+set enable_hashjoin to on;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+reset enable_nestloop;
+reset enable_mergejoin;
+reset enable_hashjoin;
+
+-- test cases for performing parallel inserts when Append node is at the top
+-- and Gather node is in one of its direct sub plans.
+
+-- case 1: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5 union all
+    select * from temp2 where col2 = 5;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 2: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select * from temp2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 3: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples to
+-- Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Gather
+-- 		->Parallel Seq Scan
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1 union all
+    select * from temp1 where col1 = 5;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 set (parallel_workers = 0);
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 where col1 = (select 1) union all  select * from temp1 where col1 = (select 1) union all
+ select * from temp2 where col2 = (select 2);
+select count(*) from parallel_write;
+alter table temp2 reset (parallel_workers);
+drop table parallel_write;
+
+-- case 4: parallel inserts must not occur as there will be no direct Gather
+-- node under Append node. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 5: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Seq Scan / Join / any other non-Gather node
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+
+alter table temp2 set (parallel_workers = 0);
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 reset (parallel_workers);
+
+-- case 6: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Gather
+--  ->Gather
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp2 where col2 = (select 2);
+select count(*) from parallel_write;
+drop table parallel_write;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 7: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Seq Scan / Join / any other non-Gather node
+--  ->Gather
+
+alter table temp2 set (parallel_workers = 0);
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp1 where col1 = (select 2);
+select count(*) from parallel_write;
+drop table parallel_write;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 reset (parallel_workers);
+
+-- case 8: parallel inserts must not occur because there is no Gather or Append
+-- node at the top for union, except/except all, intersect/intersect all
+-- cases.
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 union
+    select * from temp2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 except
+    select * from temp2 where col2 < 3;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 except all
+    select * from temp2 where col2 < 3;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 intersect
+    select * from temp2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 intersect all
+    select * from temp2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+drop table temp1;
+drop table temp2;
+drop table temp3;
 rollback;
-- 
2.25.1

From e2c7dea3f1f0171b1d42b99f493860ccf1f6ccaa Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 21 Dec 2020 14:35:24 +0530
Subject: [PATCH v13 1/4] Parallel Inserts in CREATE TABLE AS

Allow the leader and each worker insert the tuples in parallel
if the SELECT part of the CTAS is parallelizable.

The design:
The main idea is to push the CTAS dest receiver down to Gather node
and from there the required information will be shared to workers
so that they can perform parallel insertions. Leader will also
participate in insertions. After the planning, check if the upper
plan node is Gather in createas.c and mark a parallelism flag in
the CTAS dest receiver and push it down to Gather node. Each worker
can create its own CTAS dest receiver with the information passed
from the leader. Leader inserts its share of tuples if instructed
to do, and so are workers. Each worker writes atomically its number
of inserted tuples into a shared memory variable, the leader combines
this with its own number of inserted tuples and shares to the client.
---
 src/backend/access/heap/heapam.c       |  11 -
 src/backend/access/transam/xact.c      |  30 ++-
 src/backend/commands/createas.c        | 303 ++++++++++++++++---------
 src/backend/commands/explain.c         |  31 +++
 src/backend/executor/execParallel.c    |  70 +++++-
 src/backend/executor/nodeGather.c      | 113 ++++++++-
 src/backend/executor/nodeGatherMerge.c |   4 +-
 src/include/access/xact.h              |   1 +
 src/include/commands/createas.h        |  25 ++
 src/include/executor/execParallel.h    |   6 +-
 src/include/nodes/execnodes.h          |   3 +
 11 files changed, 457 insertions(+), 140 deletions(-)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index a9583f3103..86347ba273 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2043,17 +2043,6 @@ static HeapTuple
 heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
 					CommandId cid, int options)
 {
-	/*
-	 * To allow parallel inserts, we need to ensure that they are safe to be
-	 * performed in workers. We have the infrastructure to allow parallel
-	 * inserts in general except for the cases where inserts generate a new
-	 * CommandId (eg. inserts into a table having a foreign key column).
-	 */
-	if (IsParallelWorker())
-		ereport(ERROR,
-				(errcode(ERRCODE_INVALID_TRANSACTION_STATE),
-				 errmsg("cannot insert tuples in a parallel worker")));
-
 	tup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
 	tup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
 	tup->t_data->t_infomask |= HEAP_XMAX_INVALID;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 9cd0b7c11b..db6eedd635 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -763,18 +763,34 @@ GetCurrentCommandId(bool used)
 	/* this is global to a transaction, not subtransaction-local */
 	if (used)
 	{
-		/*
-		 * Forbid setting currentCommandIdUsed in a parallel worker, because
-		 * we have no provision for communicating this back to the leader.  We
-		 * could relax this restriction when currentCommandIdUsed was already
-		 * true at the start of the parallel operation.
-		 */
-		Assert(!IsParallelWorker());
+		 /*
+		  * This is a temporary hack for all common parallel insert cases i.e.
+		  * insert into, ctas, copy from. To be changed later. In a parallel
+		  * worker, set currentCommandIdUsed to true only if it was not set to
+		  * true at the start of the parallel operation (by way of
+		  * SetCurrentCommandIdUsedForWorker()). We have to do this because
+		  * GetCurrentCommandId(true) may be called from anywhere, especially
+		  * for parallel inserts, within parallel worker.
+		  */
+		Assert(!(IsParallelWorker() && !currentCommandIdUsed));
 		currentCommandIdUsed = true;
 	}
 	return currentCommandId;
 }
 
+/*
+ *	SetCurrentCommandIdUsedForWorker
+ *
+ * For a parallel worker, record that the currentCommandId has been used. This
+ * must only be called at the start of a parallel operation.
+ */
+void
+SetCurrentCommandIdUsedForWorker(void)
+{
+	Assert(IsParallelWorker() && !currentCommandIdUsed && currentCommandId != InvalidCommandId);
+	currentCommandIdUsed = true;
+}
+
 /*
  *	SetParallelStartTimestamps
  *
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 6bf6c5a310..10f4f2b4d7 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,18 +51,6 @@
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 
-typedef struct
-{
-	DestReceiver pub;			/* publicly-known function pointers */
-	IntoClause *into;			/* target relation specification */
-	/* These fields are filled by intorel_startup: */
-	Relation	rel;			/* relation to write to */
-	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
-	CommandId	output_cid;		/* cmin to insert in output tuples */
-	int			ti_options;		/* table_tuple_insert performance options */
-	BulkInsertState bistate;	/* bulk insert state */
-} DR_intorel;
-
 /* utility functions for CTAS definition creation */
 static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into);
 static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -350,6 +338,14 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		/* call ExecutorStart to prepare the plan for execution */
 		ExecutorStart(queryDesc, GetIntoRelEFlags(into));
 
+		/*
+		 * If SELECT part of the CTAS is parallelizable, then make each
+		 * parallel worker insert the tuples that are resulted in its execution
+		 * into the target table. We need plan state to be initialized by the
+		 * executor to decide whether to allow parallel inserts or not.
+		 */
+		ChooseParallelInsertsInCTAS(into, queryDesc);
+
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
 
@@ -418,6 +414,9 @@ CreateIntoRelDestReceiver(IntoClause *intoClause)
 	self->pub.rDestroy = intorel_destroy;
 	self->pub.mydest = DestIntoRel;
 	self->into = intoClause;
+	self->is_parallel = false;
+	self->is_parallel_worker = false;
+	self->object_id = InvalidOid;
 	/* other private fields will be set during intorel_startup */
 
 	return (DestReceiver *) self;
@@ -430,121 +429,169 @@ static void
 intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 {
 	DR_intorel *myState = (DR_intorel *) self;
-	IntoClause *into = myState->into;
-	bool		is_matview;
-	List	   *attrList;
 	ObjectAddress intoRelationAddr;
 	Relation	intoRelationDesc;
-	ListCell   *lc;
-	int			attnum;
 
-	Assert(into != NULL);		/* else somebody forgot to set it */
-
-	/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
-	is_matview = (into->viewQuery != NULL);
+	if (myState->is_parallel_worker)
+	{
+		/* In the worker */
+		intoRelationDesc = table_open(myState->object_id, AccessExclusiveLock);
+		myState->rel = intoRelationDesc;
+		myState->reladdr = InvalidObjectAddress;
+		myState->ti_options = 0;
+		myState->bistate = GetBulkInsertState();
 
-	/*
-	 * Build column definitions using "pre-cooked" type and collation info. If
-	 * a column name list was specified in CREATE TABLE AS, override the
-	 * column names derived from the query.  (Too few column names are OK, too
-	 * many are not.)
-	 */
-	attrList = NIL;
-	lc = list_head(into->colNames);
-	for (attnum = 0; attnum < typeinfo->natts; attnum++)
+		/*
+		 * Right after the table is created in the leader, the command id is
+		 * incremented (in create_ctas_internal()). The new command id is
+		 * marked as used in intorel_startup(), then the parallel mode is
+		 * entered. The command id and transaction id are serialized into
+		 * parallel DSM, they are then available to all parallel workers. All
+		 * the workers need to mark the command id as used before insertion.
+		 */
+		SetCurrentCommandIdUsedForWorker();
+		myState->output_cid = GetCurrentCommandId(false);
+	}
+	else
 	{
-		Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
-		ColumnDef  *col;
-		char	   *colname;
+		IntoClause *into = myState->into;
+		bool		is_matview;
+		List	   *attrList;
+		ListCell   *lc;
+		int			attnum;
 
-		if (lc)
+		Assert(into != NULL);		/* else somebody forgot to set it */
+
+		/*
+		 * This code supports both CREATE TABLE AS and CREATE MATERIALIZED
+		 * VIEW.
+		 */
+		is_matview = (into->viewQuery != NULL);
+
+		/*
+		 * Build column definitions using "pre-cooked" type and collation info.
+		 * If a column name list was specified in CREATE TABLE AS, override the
+		 * column names derived from the query.  (Too few column names are OK,
+		 * too many are not.)
+		 */
+		attrList = NIL;
+		lc = list_head(into->colNames);
+		for (attnum = 0; attnum < typeinfo->natts; attnum++)
 		{
-			colname = strVal(lfirst(lc));
-			lc = lnext(into->colNames, lc);
+			Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
+			ColumnDef  *col;
+			char	   *colname;
+
+			if (lc)
+			{
+				colname = strVal(lfirst(lc));
+				lc = lnext(into->colNames, lc);
+			}
+			else
+				colname = NameStr(attribute->attname);
+
+			col = makeColumnDef(colname,
+								attribute->atttypid,
+								attribute->atttypmod,
+								attribute->attcollation);
+
+			/*
+			 * It's possible that the column is of a collatable type but the
+			 * collation could not be resolved, so double-check.  (We must
+			 * check this here because DefineRelation would adopt the type's
+			 * default collation rather than complaining.)
+			 */
+			if (!OidIsValid(col->collOid) &&
+				type_is_collatable(col->typeName->typeOid))
+				ereport(ERROR,
+						(errcode(ERRCODE_INDETERMINATE_COLLATION),
+						errmsg("no collation was derived for column \"%s\" with collatable type %s",
+								col->colname,
+								format_type_be(col->typeName->typeOid)),
+						errhint("Use the COLLATE clause to set the collation explicitly.")));
+
+			attrList = lappend(attrList, col);
 		}
-		else
-			colname = NameStr(attribute->attname);
 
-		col = makeColumnDef(colname,
-							attribute->atttypid,
-							attribute->atttypmod,
-							attribute->attcollation);
+		if (lc != NULL)
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					errmsg("too many column names were specified")));
 
 		/*
-		 * It's possible that the column is of a collatable type but the
-		 * collation could not be resolved, so double-check.  (We must check
-		 * this here because DefineRelation would adopt the type's default
-		 * collation rather than complaining.)
+		 * Actually create the target table
 		 */
-		if (!OidIsValid(col->collOid) &&
-			type_is_collatable(col->typeName->typeOid))
-			ereport(ERROR,
-					(errcode(ERRCODE_INDETERMINATE_COLLATION),
-					 errmsg("no collation was derived for column \"%s\" with collatable type %s",
-							col->colname,
-							format_type_be(col->typeName->typeOid)),
-					 errhint("Use the COLLATE clause to set the collation explicitly.")));
+		intoRelationAddr = create_ctas_internal(attrList, into);
 
-		attrList = lappend(attrList, col);
-	}
+		/*
+		 * Finally we can open the target table
+		 */
+		intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
 
-	if (lc != NULL)
-		ereport(ERROR,
-				(errcode(ERRCODE_SYNTAX_ERROR),
-				 errmsg("too many column names were specified")));
+		/*
+		 * Make sure the constructed table does not have RLS enabled.
+		 *
+		 * check_enable_rls() will ereport(ERROR) itself if the user has
+		 * requested something invalid, and otherwise will return RLS_ENABLED
+		 * if RLS should be enabled here.  We don't actually support that
+		 * currently, so throw our own ereport(ERROR) if that happens.
+		 */
+		if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					errmsg("policies not yet implemented for this command")));
 
-	/*
-	 * Actually create the target table
-	 */
-	intoRelationAddr = create_ctas_internal(attrList, into);
+		/*
+		 * Tentatively mark the target as populated, if it's a matview and
+		 * we're going to fill it; otherwise, no change needed.
+		 */
+		if (is_matview && !into->skipData)
+			SetMatViewPopulatedState(intoRelationDesc, true);
 
-	/*
-	 * Finally we can open the target table
-	 */
-	intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
+		/*
+		 * Fill private fields of myState for use by later routines
+		 */
+		myState->rel = intoRelationDesc;
+		myState->reladdr = intoRelationAddr;
+		myState->output_cid = GetCurrentCommandId(true);
+		myState->ti_options = TABLE_INSERT_SKIP_FSM;
 
-	/*
-	 * Make sure the constructed table does not have RLS enabled.
-	 *
-	 * check_enable_rls() will ereport(ERROR) itself if the user has requested
-	 * something invalid, and otherwise will return RLS_ENABLED if RLS should
-	 * be enabled here.  We don't actually support that currently, so throw
-	 * our own ereport(ERROR) if that happens.
-	 */
-	if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
-		ereport(ERROR,
-				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-				 errmsg("policies not yet implemented for this command")));
+		/*
+		 * If WITH NO DATA is specified, there is no need to set up the state
+		 * for bulk inserts as there are no tuples to insert.
+		 */
+		if (!into->skipData)
+			myState->bistate = GetBulkInsertState();
+		else
+			myState->bistate = NULL;
 
-	/*
-	 * Tentatively mark the target as populated, if it's a matview and we're
-	 * going to fill it; otherwise, no change needed.
-	 */
-	if (is_matview && !into->skipData)
-		SetMatViewPopulatedState(intoRelationDesc, true);
+		if (myState->is_parallel)
+		{
+			myState->object_id = intoRelationAddr.objectId;
 
-	/*
-	 * Fill private fields of myState for use by later routines
-	 */
-	myState->rel = intoRelationDesc;
-	myState->reladdr = intoRelationAddr;
-	myState->output_cid = GetCurrentCommandId(true);
-	myState->ti_options = TABLE_INSERT_SKIP_FSM;
+			/*
+			 * We don't need to skip contacting FSM while inserting tuples
+			 * for parallel mode, while extending the relations, workers
+			 * instead of blocking on a page while another worker is inserting,
+			 * can check the FSM for another page that can accommodate the
+			 * tuples. This results in major benefit for parallel inserts.
+			 */
+			myState->ti_options = 0;
 
-	/*
-	 * If WITH NO DATA is specified, there is no need to set up the state for
-	 * bulk inserts as there are no tuples to insert.
-	 */
-	if (!into->skipData)
-		myState->bistate = GetBulkInsertState();
-	else
-		myState->bistate = NULL;
+			/*
+			 * rd_createSubid is marked invalid, otherwise, the table is not
+			 * allowed to be extended by the workers.
+			 */
+			myState->rel->rd_createSubid = InvalidSubTransactionId;
+		}
 
-	/*
-	 * Valid smgr_targblock implies something already wrote to the relation.
-	 * This may be harmless, but this function hasn't planned for it.
-	 */
-	Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+		/*
+		 * Valid smgr_targblock implies something already wrote to the
+		 * relation. This may be harmless, but this function hasn't planned for
+		 * it.
+		 */
+		Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+	}
 }
 
 /*
@@ -606,3 +653,47 @@ intorel_destroy(DestReceiver *self)
 {
 	pfree(self);
 }
+
+/*
+ * ChooseParallelInsertsInCTAS --- determine whether or not parallel
+ * insertion is possible, if yes set the parallel insert state i.e. push down
+ * the dest receiver to the Gather nodes.
+ */
+void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc)
+{
+	if (!IS_CTAS(into))
+		return;
+
+	/*
+	 * Do not allow parallel inserts if the table is temporary. As the
+	 * temporary tables are backend local, workers can not know about them.
+	 * Currently, CTAS supports creation of normal(logged), temporary and
+	 * unlogged tables. It does not support foreign or partition table
+	 * creation. Hence the check for temporary table is enough here.
+	 */
+	if (!into->rel || into->rel->relpersistence == RELPERSISTENCE_TEMP)
+		return;
+
+	if (queryDesc && IsA(queryDesc->planstate, GatherState) &&
+		!queryDesc->planstate->ps_ProjInfo)
+	{
+		GatherState *gstate = (GatherState *) queryDesc->planstate;
+		DestReceiver *dest	= queryDesc->dest;
+
+		/*
+		 * For parallelizing inserts in CTAS i.e. making each parallel worker
+		 * insert the tuples, we must send information such as into clause (for
+		 * each worker to build separate dest receiver), object id (for each
+		 * worker to open the created table).
+		 */
+		((DR_intorel *) dest)->is_parallel = true;
+		gstate->dest = dest;
+
+		/*
+		 * Since there are no rows that are transferred from workers to Gather
+		 * node, so we set it to 0 to be visible in estimated row count of
+		 * explain plans.
+		 */
+		queryDesc->planstate->plan->plan_rows = 0;
+	}
+}
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 43f9b01e83..fbd0bc5a81 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -556,6 +556,14 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	/* call ExecutorStart to prepare the plan for execution */
 	ExecutorStart(queryDesc, eflags);
 
+	/*
+	 * If SELECT part of the CTAS is parallelizable, then make each parallel
+	 * worker insert the tuples that are resulted in its execution into the
+	 * target table. We need plan state to be initialized by the executor to
+	 * decide whether to allow parallel inserts or not.
+	 */
+	ChooseParallelInsertsInCTAS(into, queryDesc);
+
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
 	{
@@ -1775,6 +1783,29 @@ ExplainNode(PlanState *planstate, List *ancestors,
 
 				if (gather->single_copy || es->format != EXPLAIN_FORMAT_TEXT)
 					ExplainPropertyBool("Single Copy", gather->single_copy, es);
+
+				/*
+				 * Show the create table information under Gather node in case
+				 * parallel workers have inserted the rows.
+				 */
+				if (IsA(planstate, GatherState))
+				{
+					GatherState *gstate = (GatherState *) planstate;
+
+					if (IS_PARALLEL_CTAS_DEST(gstate->dest) &&
+						((DR_intorel *) gstate->dest)->into->rel &&
+						((DR_intorel *) gstate->dest)->into->rel->relname)
+					{
+						es->indent--;
+						ExplainIndentText(es);
+						appendStringInfoString(es->str, "->  ");
+						appendStringInfoString(es->str, "Create ");
+						appendStringInfo(es->str, "%s\n",
+							((DR_intorel *) gstate->dest)->into->rel->relname);
+						ExplainIndentText(es);
+						es->indent++;
+					}
+				}
 			}
 			break;
 		case T_GatherMerge:
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index befde52691..9ef33eee54 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -23,6 +23,7 @@
 
 #include "postgres.h"
 
+#include "commands/createas.h"
 #include "executor/execParallel.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
@@ -65,6 +66,7 @@
 #define PARALLEL_KEY_QUERY_TEXT		UINT64CONST(0xE000000000000008)
 #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009)
 #define PARALLEL_KEY_WAL_USAGE			UINT64CONST(0xE00000000000000A)
+#define PARALLEL_KEY_INTO_CLAUSE		UINT64CONST(0xE00000000000000B)
 
 #define PARALLEL_TUPLE_QUEUE_SIZE		65536
 
@@ -77,6 +79,9 @@ typedef struct FixedParallelExecutorState
 	dsa_pointer param_exec;
 	int			eflags;
 	int			jit_flags;
+	Oid			objectid;		/* workers to open relation/table.  */
+	/* Number tuples inserted by all the workers. */
+	pg_atomic_uint64	processed;
 } FixedParallelExecutorState;
 
 /*
@@ -578,7 +583,8 @@ ExecParallelSetupTupleQueues(ParallelContext *pcxt, bool reinitialize)
 ParallelExecutorInfo *
 ExecInitParallelPlan(PlanState *planstate, EState *estate,
 					 Bitmapset *sendParams, int nworkers,
-					 int64 tuples_needed)
+					 int64 tuples_needed, IntoClause *intoclause,
+					 Oid objectid)
 {
 	ParallelExecutorInfo *pei;
 	ParallelContext *pcxt;
@@ -600,6 +606,8 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	Size		dsa_minsize = dsa_minimum_size();
 	char	   *query_string;
 	int			query_len;
+	char 		*intoclausestr = NULL;
+	int			intoclause_len = 0;
 
 	/*
 	 * Force any initplan outputs that we're going to pass to workers to be
@@ -712,6 +720,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_estimate_chunk(&pcxt->estimator, dsa_minsize);
 	shm_toc_estimate_keys(&pcxt->estimator, 1);
 
+	/* Estimate space for into clause for CTAS. */
+	if (IS_CTAS(intoclause) && OidIsValid(objectid))
+	{
+		intoclausestr = nodeToString(intoclause);
+		intoclause_len = strlen(intoclausestr);
+		shm_toc_estimate_chunk(&pcxt->estimator, intoclause_len + 1);
+		shm_toc_estimate_keys(&pcxt->estimator, 1);
+	}
+
 	/* Everyone's had a chance to ask for space, so now create the DSM. */
 	InitializeParallelDSM(pcxt);
 
@@ -729,6 +746,14 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	fpes->param_exec = InvalidDsaPointer;
 	fpes->eflags = estate->es_top_eflags;
 	fpes->jit_flags = estate->es_jit_flags;
+	pg_atomic_init_u64(&fpes->processed, 0);
+	pei->processed = &fpes->processed;
+
+	if (intoclausestr && OidIsValid(objectid))
+		fpes->objectid = objectid;
+	else
+		fpes->objectid = InvalidOid;
+
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, fpes);
 
 	/* Store query string */
@@ -758,8 +783,18 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_WAL_USAGE, walusage_space);
 	pei->wal_usage = walusage_space;
 
-	/* Set up the tuple queues that the workers will write into. */
-	pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	if (intoclausestr)
+	{
+		char *intoclause_space = shm_toc_allocate(pcxt->toc,
+												  intoclause_len + 1);
+		memcpy(intoclause_space, intoclausestr, intoclause_len + 1);
+		shm_toc_insert(pcxt->toc, PARALLEL_KEY_INTO_CLAUSE, intoclause_space);
+	}
+	else
+	{
+		/* Set up the tuple queues that the workers will write into. */
+		pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	}
 
 	/* We don't need the TupleQueueReaders yet, though. */
 	pei->reader = NULL;
@@ -1387,12 +1422,30 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 	void	   *area_space;
 	dsa_area   *area;
 	ParallelWorkerContext pwcxt;
+	char		*intoclausestr = NULL;
+	IntoClause	*intoclause = NULL;
 
 	/* Get fixed-size state. */
 	fpes = shm_toc_lookup(toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
 
-	/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
-	receiver = ExecParallelGetReceiver(seg, toc);
+	intoclausestr = shm_toc_lookup(toc, PARALLEL_KEY_INTO_CLAUSE, true);
+	if (intoclausestr)
+	{
+		/*
+		 * If the worker is for parallel insert in CTAS, then use the proper
+		 * dest receiver.
+		 */
+		intoclause = (IntoClause *) stringToNode(intoclausestr);
+		receiver = CreateIntoRelDestReceiver(intoclause);
+		((DR_intorel *)receiver)->is_parallel_worker = true;
+		((DR_intorel *)receiver)->object_id = fpes->objectid;
+	}
+	else
+	{
+		/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
+		receiver = ExecParallelGetReceiver(seg, toc);
+	}
+
 	instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_INSTRUMENTATION, true);
 	if (instrumentation != NULL)
 		instrument_options = instrumentation->instrument_options;
@@ -1471,6 +1524,13 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 			queryDesc->estate->es_jit->instr;
 	}
 
+	/*
+	 * Write out the number of tuples this worker has inserted. Leader will use
+	 * it to inform the end client.
+	 */
+	if (intoclausestr)
+		pg_atomic_add_fetch_u64(&fpes->processed, queryDesc->estate->es_processed);
+
 	/* Must do this after capturing instrumentation. */
 	ExecutorEnd(queryDesc);
 
diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c
index a01b46af14..e7c588c66a 100644
--- a/src/backend/executor/nodeGather.c
+++ b/src/backend/executor/nodeGather.c
@@ -32,6 +32,7 @@
 
 #include "access/relscan.h"
 #include "access/xact.h"
+#include "commands/createas.h"
 #include "executor/execdebug.h"
 #include "executor/execParallel.h"
 #include "executor/nodeGather.h"
@@ -48,6 +49,7 @@ static TupleTableSlot *ExecGather(PlanState *pstate);
 static TupleTableSlot *gather_getnext(GatherState *gatherstate);
 static MinimalTuple gather_readnext(GatherState *gatherstate);
 static void ExecShutdownGatherWorkers(GatherState *node);
+static void ExecParallelInsertInCTAS(GatherState *node);
 
 
 /* ----------------------------------------------------------------
@@ -131,6 +133,72 @@ ExecInitGather(Gather *node, EState *estate, int eflags)
 	return gatherstate;
 }
 
+/* ----------------------------------------------------------------
+ *		ExecParallelInsertInCTAS(node)
+ *
+ *		Facilitates parallel inserts by parallel workers and/or
+ *		leader for Create Table AS.
+ * ----------------------------------------------------------------
+ */
+static void
+ExecParallelInsertInCTAS(GatherState *node)
+{
+	/* Enable leader to insert in case no parallel workers were launched. */
+	if (node->nworkers_launched == 0)
+		node->need_to_scan_locally = true;
+
+	/*
+	 * By now, for parallel workers (if launched any), would have started their
+	 * work i.e. insertion to target table. In case the leader is chosen to
+	 * participate for parallel inserts in CTAS, then finish its share before
+	 * going to wait for the parallel workers to finish.
+	 */
+	if (node->need_to_scan_locally)
+	{
+		EState	   *estate = node->ps.state;
+		TupleTableSlot *outerTupleSlot;
+
+		for(;;)
+		{
+			/* Install our DSA area while executing the plan. */
+			estate->es_query_dsa =
+					node->pei ? node->pei->area : NULL;
+
+			outerTupleSlot = ExecProcNode(node->ps.lefttree);
+
+			estate->es_query_dsa = NULL;
+
+			if(TupIsNull(outerTupleSlot))
+				break;
+
+			(void) node->dest->receiveSlot(outerTupleSlot, node->dest);
+
+			node->ps.state->es_processed++;
+		}
+
+		node->need_to_scan_locally = false;
+	}
+
+	if (node->nworkers_launched > 0)
+	{
+		/*
+		 * We wait here for the parallel workers to finish their work and
+		 * accumulate the tuples they inserted and also their buffer/WAL usage.
+		 * We do not destroy the parallel context here, it will be done in
+		 * ExecShutdownGather at the end of the plan. Note that the
+		 * ExecShutdownGatherWorkers call from ExecShutdownGather will be a
+		 * no-op.
+		 */
+		ExecShutdownGatherWorkers(node);
+
+		/*
+		 * Add up the total tuples inserted by all workers, to the tuples
+		 * inserted by the leader(if any). This will be shared to client.
+		 */
+		node->ps.state->es_processed += pg_atomic_read_u64(node->pei->processed);
+	}
+}
+
 /* ----------------------------------------------------------------
  *		ExecGather(node)
  *
@@ -157,6 +225,7 @@ ExecGather(PlanState *pstate)
 	{
 		EState	   *estate = node->ps.state;
 		Gather	   *gather = (Gather *) node->ps.plan;
+		bool		isctas = IS_PARALLEL_CTAS_DEST(node->dest);
 
 		/*
 		 * Sometimes we might have to run without parallelism; but if parallel
@@ -165,6 +234,18 @@ ExecGather(PlanState *pstate)
 		if (gather->num_workers > 0 && estate->es_use_parallel_mode)
 		{
 			ParallelContext *pcxt;
+			IntoClause	*intoclause = NULL;
+			Oid			objectid = InvalidOid;
+
+			/*
+			 * Take the necessary information to be passed to workers for
+			 * parallel inserts in CTAS.
+			 */
+			if (isctas)
+			{
+				intoclause = ((DR_intorel *) node->dest)->into;
+				objectid = ((DR_intorel *) node->dest)->object_id;
+			}
 
 			/* Initialize, or re-initialize, shared state needed by workers. */
 			if (!node->pei)
@@ -172,7 +253,10 @@ ExecGather(PlanState *pstate)
 												 estate,
 												 gather->initParam,
 												 gather->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 /* CTAS info */
+												 intoclause,
+												 objectid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
@@ -190,13 +274,16 @@ ExecGather(PlanState *pstate)
 			/* Set up tuple queue readers to read the results. */
 			if (pcxt->nworkers_launched > 0)
 			{
-				ExecParallelCreateReaders(node->pei);
-				/* Make a working array showing the active readers */
-				node->nreaders = pcxt->nworkers_launched;
-				node->reader = (TupleQueueReader **)
-					palloc(node->nreaders * sizeof(TupleQueueReader *));
-				memcpy(node->reader, node->pei->reader,
-					   node->nreaders * sizeof(TupleQueueReader *));
+				if (!isctas)
+				{
+					ExecParallelCreateReaders(node->pei);
+					/* Make a working array showing the active readers */
+					node->nreaders = pcxt->nworkers_launched;
+					node->reader = (TupleQueueReader **)
+						palloc(node->nreaders * sizeof(TupleQueueReader *));
+					memcpy(node->reader, node->pei->reader,
+						node->nreaders * sizeof(TupleQueueReader *));
+				}
 			}
 			else
 			{
@@ -208,9 +295,17 @@ ExecGather(PlanState *pstate)
 		}
 
 		/* Run plan locally if no workers or enabled and not single-copy. */
-		node->need_to_scan_locally = (node->nreaders == 0)
+		node->need_to_scan_locally = (node->nreaders == 0 &&
+			!isctas)
 			|| (!gather->single_copy && parallel_leader_participation);
 		node->initialized = true;
+
+		/* Perform parallel inserts for CTAS. */
+		if (isctas)
+		{
+			ExecParallelInsertInCTAS(node);
+			return NULL;
+		}
 	}
 
 	/*
diff --git a/src/backend/executor/nodeGatherMerge.c b/src/backend/executor/nodeGatherMerge.c
index 47129344f3..ee45272c17 100644
--- a/src/backend/executor/nodeGatherMerge.c
+++ b/src/backend/executor/nodeGatherMerge.c
@@ -216,7 +216,9 @@ ExecGatherMerge(PlanState *pstate)
 												 estate,
 												 gm->initParam,
 												 gm->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 NULL,
+												 InvalidOid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7320de345c..5beae6c617 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -389,6 +389,7 @@ extern FullTransactionId GetCurrentFullTransactionIdIfAny(void);
 extern void MarkCurrentTransactionIdLoggedIfAny(void);
 extern bool SubTransactionIsActive(SubTransactionId subxid);
 extern CommandId GetCurrentCommandId(bool used);
+extern void SetCurrentCommandIdUsedForWorker(void);
 extern void SetParallelStartTimestamps(TimestampTz xact_ts, TimestampTz stmt_ts);
 extern TimestampTz GetCurrentTransactionStartTimestamp(void);
 extern TimestampTz GetCurrentStatementStartTimestamp(void);
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 7629230254..ed4690305b 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -14,12 +14,35 @@
 #ifndef CREATEAS_H
 #define CREATEAS_H
 
+#include "access/heapam.h"
 #include "catalog/objectaddress.h"
+#include "executor/execdesc.h"
 #include "nodes/params.h"
+#include "nodes/plannodes.h"
 #include "parser/parse_node.h"
 #include "tcop/dest.h"
 #include "utils/queryenvironment.h"
 
+typedef struct
+{
+	DestReceiver pub;			/* publicly-known function pointers */
+	IntoClause *into;			/* target relation specification */
+	/* These fields are filled by intorel_startup: */
+	Relation	rel;			/* relation to write to */
+	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
+	CommandId	output_cid;		/* cmin to insert in output tuples */
+	int			ti_options;		/* table_tuple_insert performance options */
+	BulkInsertState bistate;	/* bulk insert state */
+	bool		is_parallel;	/* is parallelism to be considered? */
+	bool		is_parallel_worker; /* true for parallel worker */
+	/* Used for table open by parallel worker. */
+	Oid			object_id;
+} DR_intorel;
+
+#define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
+#define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
+									 IS_CTAS(((DR_intorel *) dest)->into) && \
+									 ((DR_intorel *) dest)->is_parallel)
 
 extern ObjectAddress ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 									   ParamListInfo params, QueryEnvironment *queryEnv,
@@ -29,4 +52,6 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
+extern void ChooseParallelInsertsInCTAS(IntoClause *into,
+										QueryDesc *queryDesc);
 #endif							/* CREATEAS_H */
diff --git a/src/include/executor/execParallel.h b/src/include/executor/execParallel.h
index 5a39a5b29c..9f959f741b 100644
--- a/src/include/executor/execParallel.h
+++ b/src/include/executor/execParallel.h
@@ -35,11 +35,15 @@ typedef struct ParallelExecutorInfo
 	/* These two arrays have pcxt->nworkers_launched entries: */
 	shm_mq_handle **tqueue;		/* tuple queues for worker output */
 	struct TupleQueueReader **reader;	/* tuple reader/writer support */
+	/* Number of tuples inserted by all workers. */
+	volatile pg_atomic_uint64	*processed;
 } ParallelExecutorInfo;
 
 extern ParallelExecutorInfo *ExecInitParallelPlan(PlanState *planstate,
 												  EState *estate, Bitmapset *sendParam, int nworkers,
-												  int64 tuples_needed);
+												  int64 tuples_needed,
+												  IntoClause *intoclause,
+												  Oid objectid);
 extern void ExecParallelCreateReaders(ParallelExecutorInfo *pei);
 extern void ExecParallelFinish(ParallelExecutorInfo *pei);
 extern void ExecParallelCleanup(ParallelExecutorInfo *pei);
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 61ba4c3666..e9c4442c22 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -23,6 +23,7 @@
 #include "nodes/tidbitmap.h"
 #include "partitioning/partdefs.h"
 #include "storage/condition_variable.h"
+#include "tcop/dest.h"
 #include "utils/hsearch.h"
 #include "utils/queryenvironment.h"
 #include "utils/reltrigger.h"
@@ -2326,6 +2327,8 @@ typedef struct GatherState
 	int			nreaders;		/* number of still-active workers */
 	int			nextreader;		/* next one to try to read from */
 	struct TupleQueueReader **reader;	/* array with nreaders active entries */
+	/* Parallel inserts in CTAS related info is specified below. */
+	DestReceiver *dest;
 } GatherState;
 
 /* ----------------
-- 
2.25.1

From 2484cfb3e57c79788448b04bfc1760b59fd20c4d Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 21 Dec 2020 15:19:23 +0530
Subject: [PATCH v13 2/4] Tuple Cost Adjustment for Parallel Inserts in CTAS

Let the planner know that the SELECT is from CTAS in createas.c
so that it can set the number of tuples transferred from the
workers to Gather node to 0. With this change, there are chances
that the planner may choose the parallel plan.
---
 src/backend/commands/createas.c       | 35 +++++++++++++++++-
 src/backend/commands/explain.c        |  7 ++--
 src/backend/commands/prepare.c        |  3 +-
 src/backend/optimizer/path/costsize.c | 22 ++++++++++-
 src/backend/optimizer/plan/planner.c  | 53 +++++++++++++++++++++++++++
 src/include/commands/createas.h       | 21 ++++++++++-
 src/include/commands/explain.h        |  3 +-
 src/include/nodes/parsenodes.h        |  1 +
 src/include/optimizer/planner.h       | 10 +++++
 9 files changed, 146 insertions(+), 9 deletions(-)

diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 10f4f2b4d7..210927d4f4 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -344,7 +344,8 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		 * into the target table. We need plan state to be initialized by the
 		 * executor to decide whether to allow parallel inserts or not.
 		 */
-		ChooseParallelInsertsInCTAS(into, queryDesc);
+		ChooseParallelInsertsInCTAS(into, queryDesc,
+									&query->CTASParallelInsInfo);
 
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
@@ -659,8 +660,11 @@ intorel_destroy(DestReceiver *self)
  * insertion is possible, if yes set the parallel insert state i.e. push down
  * the dest receiver to the Gather nodes.
  */
-void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc)
+void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc,
+								 uint8 *tuple_cost_flags)
 {
+	bool allow = false;
+
 	if (!IS_CTAS(into))
 		return;
 
@@ -695,5 +699,32 @@ void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc)
 		 * explain plans.
 		 */
 		queryDesc->planstate->plan->plan_rows = 0;
+
+		allow = true;
 	}
+
+	/*
+	 * It should not happen that in cost_gather we have ignored the parallel
+	 * tuple cost and now we are not allowing the parallel inserts. And also we
+	 * might need assertion only if the top node is Gather. The main intention
+	 * of assertion is to check if we enforced planner to ignore the parallel
+	 * tuple cost (with the intention of choosing parallel inserts) due to
+	 * which the parallel plan may have been chosen, but we do not allow the
+	 * parallel inserts now.
+	 */
+	if (!allow && tuple_cost_flags && queryDesc &&
+		IsA(queryDesc->planstate, GatherState))
+	{
+		/*
+		 * If we have correctly ignored parallel tuple cost in planner while
+		 * creating Gather path, then this assertion failure should not occur.
+		 * If it occurs, that means the planner may have chosen this parallel
+		 * plan because of our enforcement to ignore the parallel tuple cost.
+		 */
+		Assert(!(*tuple_cost_flags & CTAS_PARALLEL_INS_TUP_COST_IGNORED));
+	}
+
+	if (tuple_cost_flags)
+		*tuple_cost_flags = CTAS_PARALLEL_INS_UNDEF;
+
 }
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index fbd0bc5a81..efdb34d1f0 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -402,7 +402,8 @@ ExplainOneQuery(Query *query, int cursorOptions,
 
 		/* run it (if needed) and produce output */
 		ExplainOnePlan(plan, into, es, queryString, params, queryEnv,
-					   &planduration, (es->buffers ? &bufusage : NULL));
+					   &planduration, (es->buffers ? &bufusage : NULL),
+					   &query->CTASParallelInsInfo);
 	}
 }
 
@@ -496,7 +497,7 @@ void
 ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 			   const char *queryString, ParamListInfo params,
 			   QueryEnvironment *queryEnv, const instr_time *planduration,
-			   const BufferUsage *bufusage)
+			   const BufferUsage *bufusage, uint8 *ctas_tuple_cost_flags)
 {
 	DestReceiver *dest;
 	QueryDesc  *queryDesc;
@@ -562,7 +563,7 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	 * target table. We need plan state to be initialized by the executor to
 	 * decide whether to allow parallel inserts or not.
 	 */
-	ChooseParallelInsertsInCTAS(into, queryDesc);
+	ChooseParallelInsertsInCTAS(into, queryDesc, ctas_tuple_cost_flags);
 
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c
index 89087a7be3..07166479e7 100644
--- a/src/backend/commands/prepare.c
+++ b/src/backend/commands/prepare.c
@@ -672,7 +672,8 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es,
 
 		if (pstmt->commandType != CMD_UTILITY)
 			ExplainOnePlan(pstmt, into, es, query_string, paramLI, queryEnv,
-						   &planduration, (es->buffers ? &bufusage : NULL));
+						   &planduration, (es->buffers ? &bufusage : NULL),
+						   NULL);
 		else
 			ExplainOneUtility(pstmt->utilityStmt, into, es, query_string,
 							  paramLI, queryEnv);
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 22d6935824..800f25903d 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -76,6 +76,7 @@
 #include "access/amapi.h"
 #include "access/htup_details.h"
 #include "access/tsmapi.h"
+#include "commands/createas.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "executor/nodeHash.h"
@@ -378,6 +379,7 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 {
 	Cost		startup_cost = 0;
 	Cost		run_cost = 0;
+	bool		ignore_tuple_cost = false;
 
 	/* Mark the path with the correct row estimate */
 	if (rows)
@@ -393,7 +395,25 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 
 	/* Parallel setup and communication cost. */
 	startup_cost += parallel_setup_cost;
-	run_cost += parallel_tuple_cost * path->path.rows;
+
+	/*
+	 * Do not consider tuple cost in case of we intend to perform parallel
+	 * inserts by workers. We would have set ignore flag in
+	 * apply_scanjoin_target_to_paths before generating Gather path for the
+	 * upper level SELECT part of the CTAS.
+	 */
+	if ((root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT) &&
+		(root->parse->CTASParallelInsInfo &
+		 CTAS_PARALLEL_INS_TUP_COST_CAN_IGN))
+	{
+		ignore_tuple_cost = true;
+		root->parse->CTASParallelInsInfo &=
+									~CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_TUP_COST_IGNORED;
+	}
+
+	if (!ignore_tuple_cost)
+		run_cost += parallel_tuple_cost * path->path.rows;
 
 	path->path.startup_cost = startup_cost;
 	path->path.total_cost = (startup_cost + run_cost);
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index 1a94b58f8b..f1134711b0 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -28,6 +28,7 @@
 #include "catalog/pg_inherits.h"
 #include "catalog/pg_proc.h"
 #include "catalog/pg_type.h"
+#include "commands/createas.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "foreign/fdwapi.h"
@@ -7338,6 +7339,37 @@ can_partial_agg(PlannerInfo *root)
 	return true;
 }
 
+/*
+ * ignore_parallel_tuple_cost
+ *
+ * Gather node will not receive any tuples from the workers in case each worker
+ * inserts them in parallel. So, we set a flag to ignore parallel tuple cost by
+ * the Gather path in cost_gather if the SELECT is for CTAS and we are
+ * generating an upper level Gather path.
+*/
+static bool
+ignore_parallel_tuple_cost(PlannerInfo *root)
+{
+	if (root->query_level == 1 &&
+		(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT))
+	{
+		/*
+		 * In each of the HAS_PARENT_PATH_GENERATING_CLAUSE cases, a parent
+		 * path will be generated for the upper Gather path(in
+		 * grouping_planner), in which case we can not let parallel inserts
+		 * happen. So we do not set ignore tuple cost flag.
+		 */
+		if (HAS_PARENT_PATH_GENERATING_CLAUSE(root))
+			return false;
+
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+
+		return true;
+	}
+
+	return false;
+}
+
 /*
  * apply_scanjoin_target_to_paths
  *
@@ -7557,8 +7589,29 @@ apply_scanjoin_target_to_paths(PlannerInfo *root,
 	 * one of the generated paths may turn out to be the cheapest one.
 	 */
 	if (rel->consider_parallel && !IS_OTHER_REL(rel))
+	{
+		/*
+		 * Set a flag to ignore parallel tuple cost by the Gather path in
+		 * cost_gather if the SELECT is for CTAS and we are generating an upper
+		 * level Gather path.
+		 */
+		bool	ignore = ignore_parallel_tuple_cost(root);
+
 		generate_useful_gather_paths(root, rel, false);
 
+		/*
+		 * Reset the ignore flag, in case we set it but
+		 * generate_useful_gather_paths returned without reaching cost_gather.
+		 */
+		if (ignore &&
+			(root->parse->CTASParallelInsInfo &
+			 CTAS_PARALLEL_INS_TUP_COST_CAN_IGN))
+		{
+			root->parse->CTASParallelInsInfo &=
+										~CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+		}
+	}
+
 	/*
 	 * Reassess which paths are the cheapest, now that we've potentially added
 	 * new Gather (or Gather Merge) and/or Append (or MergeAppend) paths to
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index ed4690305b..0ae2f49e0c 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -39,6 +39,24 @@ typedef struct
 	Oid			object_id;
 } DR_intorel;
 
+/*
+ * Information sent to the planner from CTAS to account for the cost
+ * calculations in cost_gather. We need to do this because, no tuples will be
+ * received by the Gather node if the workers insert the tuples in parallel.
+ */
+typedef enum CTASParallelInsertOpt
+{
+	CTAS_PARALLEL_INS_UNDEF = 0, /* undefined */
+	CTAS_PARALLEL_INS_SELECT = 1 << 0, /* set to this before planning */
+	/*
+	 * Set to this while planning for upper Gather path to ignore parallel
+	 * tuple cost in cost_gather.
+	 */
+	CTAS_PARALLEL_INS_TUP_COST_CAN_IGN = 1 << 1,
+	/* Set to this after the cost is ignored. */
+	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2
+} CTASParallelInsertOpt;
+
 #define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
 #define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
 									 IS_CTAS(((DR_intorel *) dest)->into) && \
@@ -53,5 +71,6 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
 extern void ChooseParallelInsertsInCTAS(IntoClause *into,
-										QueryDesc *queryDesc);
+										QueryDesc *queryDesc,
+										uint8 *tuple_cost_flags);
 #endif							/* CREATEAS_H */
diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h
index ba661d32a6..1a1806dbf1 100644
--- a/src/include/commands/explain.h
+++ b/src/include/commands/explain.h
@@ -91,7 +91,8 @@ extern void ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into,
 						   ExplainState *es, const char *queryString,
 						   ParamListInfo params, QueryEnvironment *queryEnv,
 						   const instr_time *planduration,
-						   const BufferUsage *bufusage);
+						   const BufferUsage *bufusage,
+						   uint8 *ctas_tuple_cost_flags);
 
 extern void ExplainPrintPlan(ExplainState *es, QueryDesc *queryDesc);
 extern void ExplainPrintTriggers(ExplainState *es, QueryDesc *queryDesc);
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 48a79a7657..81b148c383 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -180,6 +180,7 @@ typedef struct Query
 	 */
 	int			stmt_location;	/* start location, or -1 if unknown */
 	int			stmt_len;		/* length in bytes; 0 means "rest of string" */
+	uint8 		CTASParallelInsInfo; /* parallel insert in CTAS info */
 } Query;
 
 
diff --git a/src/include/optimizer/planner.h b/src/include/optimizer/planner.h
index beb7dbbcbe..74b2563828 100644
--- a/src/include/optimizer/planner.h
+++ b/src/include/optimizer/planner.h
@@ -21,6 +21,16 @@
 #include "nodes/pathnodes.h"
 #include "nodes/plannodes.h"
 
+#define HAS_PARENT_PATH_GENERATING_CLAUSE(root) \
+											(root->parse->rowMarks || \
+											 limit_needed(root->parse) || \
+											 root->parse->sortClause ||  \
+											 root->parse->distinctClause ||  \
+											 root->parse->hasWindowFuncs ||  \
+											 root->parse->groupClause ||  \
+											 root->parse->groupingSets ||  \
+											 root->parse->hasAggs ||  \
+											 root->hasHavingQual)
 
 /* Hook for plugins to get control in planner() */
 typedef PlannedStmt *(*planner_hook_type) (Query *parse,
-- 
2.25.1

From f7035babca2529cbb227ac871a678861c9c7197a Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 21 Dec 2020 14:40:49 +0530
Subject: [PATCH v13 3/4] Tests For Parallel Inserts in CTAS

---
 src/test/regress/expected/write_parallel.out | 505 +++++++++++++++++++
 src/test/regress/sql/write_parallel.sql      | 181 +++++++
 2 files changed, 686 insertions(+)

diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 0c4da2591a..84e8f981e1 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -76,4 +76,509 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=10000 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(4 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+                     QUERY PLAN                      
+-----------------------------------------------------
+ LockRows (actual rows=10000 loops=1)
+   ->  Seq Scan on tenk1 (actual rows=10000 loops=1)
+(2 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+    select length(stringu1) from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_mat_view
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_mat_view;
+ count 
+-------
+ 10000
+(1 row)
+
+drop materialized view parallel_mat_view;
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+deallocate parallel_write_prep;
+drop table parallel_write;
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=2000 loops=5)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select nextval('parallel_write_sequence'), four from tenk1;
+                  QUERY PLAN                   
+-----------------------------------------------
+ Seq Scan on tenk1 (actual rows=10000 loops=1)
+(1 row)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;
+                               QUERY PLAN                               
+------------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 4
+   Params Evaluated: $1
+   Workers Launched: 3
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=1 loops=1)
+           ->  Gather (actual rows=1 loops=1)
+                 Workers Planned: 4
+                 Workers Launched: 4
+                 ->  Parallel Seq Scan on tenk2 (actual rows=1 loops=5)
+   ->  Parallel Seq Scan on tenk1 (actual rows=625 loops=4)
+         Filter: (four = 3)
+         Rows Removed by Filter: 1875
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+  2500
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;
+                             QUERY PLAN                              
+---------------------------------------------------------------------
+ Gather (actual rows=2500 loops=1)
+   Workers Planned: 4
+   Workers Launched: 4
+   ->  Parallel Seq Scan on tenk1 (actual rows=500 loops=5)
+         Filter: (four = 3)
+         Rows Removed by Filter: 1500
+   SubPlan 1
+     ->  Function Scan on generate_series (actual rows=1 loops=2500)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+  2500
+(1 row)
+
+drop table parallel_write;
+create table temp1(col1) as select * from generate_series(1,5);
+create table temp2(col2) as select * from temp1;
+create table temp3(col3) as select * from temp1;
+-- parallel inserts must not occur, as there is a limit clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 limit 4;
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ Limit (actual rows=4 loops=1)
+   ->  Gather (actual rows=4 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(5 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an order by clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 order by 1;
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ Gather Merge (actual rows=5 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+   ->  Sort (actual rows=1 loops=4)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         Worker 0:  Sort Method: quicksort  Memory: 25kB
+         Worker 1:  Sort Method: quicksort  Memory: 25kB
+         Worker 2:  Sort Method: quicksort  Memory: 25kB
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(10 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an order by clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 order by 1;
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ Gather Merge (actual rows=5 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+   ->  Sort (actual rows=1 loops=4)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         Worker 0:  Sort Method: quicksort  Memory: 25kB
+         Worker 1:  Sort Method: quicksort  Memory: 25kB
+         Worker 2:  Sort Method: quicksort  Memory: 25kB
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(10 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is a distinct clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select distinct * from temp1;
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ HashAggregate (actual rows=5 loops=1)
+   Group Key: col1
+   Batches: 1  Memory Usage: 40kB
+   ->  Gather (actual rows=5 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(7 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an aggregate and group clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select count(*) from temp1 group by col1;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Finalize HashAggregate (actual rows=5 loops=1)
+   Group Key: col1
+   Batches: 1  Memory Usage: 40kB
+   ->  Gather (actual rows=5 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+         ->  Partial HashAggregate (actual rows=1 loops=4)
+               Group Key: col1
+               Batches: 1  Memory Usage: 40kB
+               Worker 0:  Batches: 1  Memory Usage: 40kB
+               Worker 1:  Batches: 1  Memory Usage: 40kB
+               Worker 2:  Batches: 1  Memory Usage: 40kB
+               ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(13 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an aggregate, group and having
+-- clauses
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select count(col1), (select col3 from
+        (select * from temp3) as tt limit 1) col4 from temp1, temp2
+    where temp1.col1 = temp2.col2 group by col4 having count(col1) > 0;
+                                    QUERY PLAN                                    
+----------------------------------------------------------------------------------
+ Finalize GroupAggregate (actual rows=1 loops=1)
+   Group Key: ($1)
+   Filter: (count(temp1.col1) > 0)
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=1 loops=1)
+           ->  Gather (actual rows=1 loops=1)
+                 Workers Planned: 3
+                 Workers Launched: 3
+                 ->  Parallel Seq Scan on temp3 (actual rows=0 loops=4)
+   ->  Gather (actual rows=1 loops=1)
+         Workers Planned: 3
+         Params Evaluated: $1
+         Workers Launched: 3
+         ->  Partial GroupAggregate (actual rows=0 loops=4)
+               Group Key: $1
+               ->  Parallel Hash Join (actual rows=1 loops=4)
+                     Hash Cond: (temp1.col1 = temp2.col2)
+                     ->  Parallel Seq Scan on temp1 (actual rows=5 loops=1)
+                     ->  Parallel Hash (actual rows=1 loops=4)
+                           Buckets: 4096  Batches: 1  Memory Usage: 64kB
+                           ->  Parallel Seq Scan on temp2 (actual rows=5 loops=1)
+(21 rows)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is a window function
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select avg(col1) OVER (PARTITION BY col1) from temp1;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ WindowAgg (actual rows=5 loops=1)
+   ->  Sort (actual rows=5 loops=1)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         ->  Gather (actual rows=5 loops=1)
+               Workers Planned: 3
+               Workers Launched: 3
+               ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+(8 rows)
+
+drop table parallel_write;
+-- nested loop join is the top node under which Gather node exists, so parallel
+-- inserts must not occur
+set enable_nestloop to on;
+set enable_mergejoin to off;
+set enable_hashjoin to off;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Nested Loop (actual rows=5 loops=1)
+   Join Filter: (temp1.col1 = temp2.col2)
+   Rows Removed by Join Filter: 20
+   ->  Gather (actual rows=5 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+         ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+   ->  Materialize (actual rows=5 loops=5)
+         ->  Gather (actual rows=5 loops=1)
+               Workers Planned: 3
+               Workers Launched: 3
+               ->  Parallel Seq Scan on temp2 (actual rows=1 loops=4)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- the top node is Gather under which merge join happens, so parallel inserts
+-- must occur
+set enable_nestloop to off;
+set enable_mergejoin to on;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+ ->  Create parallel_write
+   ->  Merge Join (actual rows=1 loops=4)
+         Merge Cond: (temp1.col1 = temp2.col2)
+         ->  Sort (actual rows=1 loops=4)
+               Sort Key: temp1.col1
+               Sort Method: quicksort  Memory: 25kB
+               Worker 0:  Sort Method: quicksort  Memory: 25kB
+               Worker 1:  Sort Method: quicksort  Memory: 25kB
+               Worker 2:  Sort Method: quicksort  Memory: 25kB
+               ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+         ->  Sort (actual rows=5 loops=1)
+               Sort Key: temp2.col2
+               Sort Method: quicksort  Memory: 25kB
+               ->  Seq Scan on temp2 (actual rows=5 loops=1)
+(17 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel hash join happens under Gather node, so parallel inserts must occur
+set enable_mergejoin to off;
+set enable_hashjoin to on;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+ ->  Create parallel_write
+   ->  Parallel Hash Join (actual rows=1 loops=4)
+         Hash Cond: (temp1.col1 = temp2.col2)
+         ->  Parallel Seq Scan on temp1 (actual rows=5 loops=1)
+         ->  Parallel Hash (actual rows=1 loops=4)
+               Buckets: 4096  Batches: 1  Memory Usage: 64kB
+               ->  Parallel Seq Scan on temp2 (actual rows=5 loops=1)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+reset enable_nestloop;
+reset enable_mergejoin;
+reset enable_hashjoin;
+drop table temp1;
+drop table temp2;
+drop table temp3;
 rollback;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 78b479cedf..7245cb97c6 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -39,4 +39,185 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
 
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+create temporary table parallel_write as select length(stringu1) from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create unlogged table parallel_write as select length(stringu1) from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into parallel_write from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into temporary parallel_write from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+select length(stringu1) into unlogged parallel_write from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select length(stringu1) from tenk1 for update;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create materialized view parallel_mat_view as
+    select length(stringu1) from tenk1;
+select count(*) from parallel_mat_view;
+drop materialized view parallel_mat_view;
+
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as execute parallel_write_prep;
+select count(*) from parallel_write;
+deallocate parallel_write_prep;
+drop table parallel_write;
+
+-- parallel inserts must occur
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select now(), four from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select nextval('parallel_write_sequence'), four from tenk1;
+select count(*) from parallel_write;
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+create table temp1(col1) as select * from generate_series(1,5);
+create table temp2(col2) as select * from temp1;
+create table temp3(col3) as select * from temp1;
+
+-- parallel inserts must not occur, as there is a limit clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 limit 4;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an order by clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 order by 1;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an order by clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 order by 1;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is a distinct clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select distinct * from temp1;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an aggregate and group clause
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select count(*) from temp1 group by col1;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an aggregate, group and having
+-- clauses
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select count(col1), (select col3 from
+        (select * from temp3) as tt limit 1) col4 from temp1, temp2
+    where temp1.col1 = temp2.col2 group by col4 having count(col1) > 0;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is a window function
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select avg(col1) OVER (PARTITION BY col1) from temp1;
+drop table parallel_write;
+
+-- nested loop join is the top node under which Gather node exists, so parallel
+-- inserts must not occur
+set enable_nestloop to on;
+set enable_mergejoin to off;
+set enable_hashjoin to off;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- the top node is Gather under which merge join happens, so parallel inserts
+-- must occur
+set enable_nestloop to off;
+set enable_mergejoin to on;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel hash join happens under Gather node, so parallel inserts must occur
+set enable_mergejoin to off;
+set enable_hashjoin to on;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+reset enable_nestloop;
+reset enable_mergejoin;
+reset enable_hashjoin;
+
+drop table temp1;
+drop table temp2;
+drop table temp3;
 rollback;
-- 
2.25.1

From de3edce67c45e2104d0789bff3a18bb3ab9562da Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 21 Dec 2020 15:32:33 +0530
Subject: [PATCH v13 4/4] Enable CTAS Parallel Inserts For Append

This patch allows pushing down the CTAS dest receiver even if there
exists Gather node under Top Append node. It also add the code for
influencing the planner to consider parallel tuple cost zero and
asserts for wrong enforcement if later parallel insertion is not
possible. Test cases are also included in this patch.
---
 src/backend/commands/createas.c              | 104 ++-
 src/backend/optimizer/path/allpaths.c        |  39 +
 src/backend/optimizer/plan/planner.c         |  10 +-
 src/include/commands/createas.h              |   4 +-
 src/test/regress/expected/write_parallel.out | 721 +++++++++++++++++++
 src/test/regress/sql/write_parallel.sql      | 221 ++++++
 6 files changed, 1071 insertions(+), 28 deletions(-)

diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 210927d4f4..20d4f805d0 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -655,6 +655,78 @@ intorel_destroy(DestReceiver *self)
 	pfree(self);
 }
 
+/*
+ * PushDownCTASParallelInsertState --- push the dest receiver down to the
+ * Gather nodes.
+ *
+ * In this function we only care Append and Gather nodes.
+ *
+ * Push the dest receiver to Gather node when it is either at the top of the
+ * plan or under top Append node unless it does not have any projections to do.
+ * Required information from the pushed dest receiver is sent to workers so
+ * that they can perform parallel insertions into the target table.
+ *
+ * If the top node is Append, then this function recursively checks the sub
+ * plans for Gather nodes, when found one(and if it does not have projections),
+ * then sets the dest receiver information.
+ *
+ * In any case, this function returns true if at least one Gather node can allow
+ * parallel insertions by the workers. Otherwise returns false.
+ */
+static bool
+PushDownCTASParallelInsertState(DestReceiver *dest, PlanState *ps,
+								bool *gather_exists)
+{
+	bool parallel = false;
+
+	if (ps == NULL)
+		return parallel;
+
+	if (IsA(ps, AppendState))
+	{
+		AppendState *aps = (AppendState *) ps;
+
+		for (int i = 0; i < aps->as_nplans; i++)
+		{
+			parallel |= PushDownCTASParallelInsertState(dest,
+														aps->appendplans[i],
+														gather_exists);
+		}
+	}
+	else if (IsA(ps, GatherState))
+	{
+		/*
+		 * Set to true if there exists at least one Gather node either at the
+		 * top of the plan or as a direct sub node under Append node.
+		 */
+		 *gather_exists |= true;
+
+		if (!ps->ps_ProjInfo)
+		{
+			GatherState *gstate = (GatherState *) ps;
+			parallel = true;
+
+			/*
+			 * For parallelizing inserts in CTAS i.e. making each parallel
+			 * worker insert the tuples, we must send information such as into
+			 * clause (for each worker to build separate dest receiver), object
+			 * id (for each worker to open the created table).
+			 */
+			((DR_intorel *) dest)->is_parallel = true;
+			gstate->dest = dest;
+
+			/*
+			 * Since there are no rows that are transferred from workers to
+			 * Gather node, so we set it to 0 to be visible in estimated row
+			 * count of explain plans.
+			 */
+			ps->plan->plan_rows = 0;
+		}
+	}
+
+	return parallel;
+}
+
 /*
  * ChooseParallelInsertsInCTAS --- determine whether or not parallel
  * insertion is possible, if yes set the parallel insert state i.e. push down
@@ -664,6 +736,7 @@ void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc,
 								 uint8 *tuple_cost_flags)
 {
 	bool allow = false;
+	bool gather_exists = false;
 
 	if (!IS_CTAS(into))
 		return;
@@ -678,30 +751,12 @@ void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc,
 	if (!into->rel || into->rel->relpersistence == RELPERSISTENCE_TEMP)
 		return;
 
-	if (queryDesc && IsA(queryDesc->planstate, GatherState) &&
-		!queryDesc->planstate->ps_ProjInfo)
-	{
-		GatherState *gstate = (GatherState *) queryDesc->planstate;
-		DestReceiver *dest	= queryDesc->dest;
-
-		/*
-		 * For parallelizing inserts in CTAS i.e. making each parallel worker
-		 * insert the tuples, we must send information such as into clause (for
-		 * each worker to build separate dest receiver), object id (for each
-		 * worker to open the created table).
-		 */
-		((DR_intorel *) dest)->is_parallel = true;
-		gstate->dest = dest;
-
-		/*
-		 * Since there are no rows that are transferred from workers to Gather
-		 * node, so we set it to 0 to be visible in estimated row count of
-		 * explain plans.
-		 */
-		queryDesc->planstate->plan->plan_rows = 0;
+	if (!queryDesc)
+		return;
 
-		allow = true;
-	}
+	allow = PushDownCTASParallelInsertState(queryDesc->dest,
+											queryDesc->planstate,
+											&gather_exists);
 
 	/*
 	 * It should not happen that in cost_gather we have ignored the parallel
@@ -712,8 +767,7 @@ void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc,
 	 * which the parallel plan may have been chosen, but we do not allow the
 	 * parallel inserts now.
 	 */
-	if (!allow && tuple_cost_flags && queryDesc &&
-		IsA(queryDesc->planstate, GatherState))
+	if (!allow && tuple_cost_flags && gather_exists)
 	{
 		/*
 		 * If we have correctly ignored parallel tuple cost in planner while
diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c
index 84a69b064a..0e6e2df9cc 100644
--- a/src/backend/optimizer/path/allpaths.c
+++ b/src/backend/optimizer/path/allpaths.c
@@ -23,6 +23,7 @@
 #include "catalog/pg_class.h"
 #include "catalog/pg_operator.h"
 #include "catalog/pg_proc.h"
+#include "commands/createas.h"
 #include "foreign/fdwapi.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1103,6 +1104,44 @@ set_append_rel_size(PlannerInfo *root, RelOptInfo *rel,
 		if (root->glob->parallelModeOK && rel->consider_parallel)
 			set_rel_consider_parallel(root, childrel, childRTE);
 
+		/*
+		 * When subplan is subquery, It's possible to do parallel insert if top
+		 * node of subquery is Gather, so we set the flag to ignore parallel
+		 * tuple cost by the Gather path in cost_gather if the SELECT is for
+		 * CTAS.
+		 */
+		if (childrel->rtekind == RTE_SUBQUERY)
+		{
+			/*
+			 * We set the flag for two cases when there is no parent path will
+			 * be created(such as : limit,sort,distinct...):
+			 * i) query_level is 1
+			 * ii) query_level > 1 then set the flag in the parent_root.
+			 * The case ii) is to check append under append:
+			 * Append
+			 *	->Append
+			 *	->Gather
+			 *	->Other plan
+			 */
+			if (root->parse->CTASParallelInsInfo &
+				CTAS_PARALLEL_INS_SELECT &&
+				(root->query_level == 1 ||
+				 root->parent_root->parse->CTASParallelInsInfo &
+				 CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND) &&
+				!(HAS_PARENT_PATH_GENERATING_CLAUSE(root)))
+			{
+				root->parse->CTASParallelInsInfo |=
+										CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+			}
+		}
+
+		if (root->parse->CTASParallelInsInfo &
+			CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND)
+		{
+			root->parse->CTASParallelInsInfo &=
+										~CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+		}
+
 		/*
 		 * Compute the child's size.
 		 */
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index f1134711b0..7555cde61a 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -7350,8 +7350,14 @@ can_partial_agg(PlannerInfo *root)
 static bool
 ignore_parallel_tuple_cost(PlannerInfo *root)
 {
-	if (root->query_level == 1 &&
-		(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT))
+	if (root->query_level != 1 &&
+		(root->parent_root->parse->CTASParallelInsInfo &
+		 CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND))
+	{
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_SELECT;
+	}
+
+	if (root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT)
 	{
 		/*
 		 * In each of the HAS_PARENT_PATH_GENERATING_CLAUSE cases, a parent
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 0ae2f49e0c..4103ac65f0 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -54,7 +54,9 @@ typedef enum CTASParallelInsertOpt
 	 */
 	CTAS_PARALLEL_INS_TUP_COST_CAN_IGN = 1 << 1,
 	/* Set to this after the cost is ignored. */
-	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2
+	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2,
+	/* Set to this in case tuple cost needs to be ignored for Append cases. */
+	CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND = 1 << 3
 } CTASParallelInsertOpt;
 
 #define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 84e8f981e1..09c9c63c2e 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -578,6 +578,727 @@ drop table parallel_write;
 reset enable_nestloop;
 reset enable_mergejoin;
 reset enable_hashjoin;
+-- test cases for performing parallel inserts when Append node is at the top
+-- and Gather node is in one of its direct sub plans.
+-- case 1: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5 union all
+    select * from temp2 where col2 = 5;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Append (actual rows=0 loops=1)
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 (actual rows=0 loops=4)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: 1
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 (actual rows=0 loops=4)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: 1
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=0 loops=4)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: 1
+(22 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+-- case 2: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp2;
+                               QUERY PLAN                               
+------------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+ ->  Create parallel_write
+   ->  Parallel Append (actual rows=4 loops=4)
+         ->  Parallel Seq Scan on temp1 (actual rows=5 loops=1)
+         ->  Parallel Seq Scan on temp2 (actual rows=5 loops=1)
+         ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=5 loops=1)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select * from temp2;
+                                   QUERY PLAN                                   
+--------------------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 3
+   Params Evaluated: $1, $3
+   Workers Launched: 1
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=1 loops=1)
+           ->  Gather (actual rows=1 loops=1)
+                 Workers Planned: 3
+                 Workers Launched: 3
+                 ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=0 loops=4)
+   InitPlan 2 (returns $3)
+     ->  Limit (actual rows=1 loops=1)
+           ->  Gather (actual rows=1 loops=1)
+                 Workers Planned: 3
+                 Workers Launched: 3
+                 ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=0 loops=4)
+   ->  Parallel Append (actual rows=8 loops=2)
+         ->  Parallel Seq Scan on temp1 (actual rows=5 loops=1)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=5 loops=1)
+         ->  Parallel Seq Scan on temp2 (actual rows=5 loops=1)
+(21 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+-- case 3: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples to
+-- Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Gather
+-- 		->Parallel Seq Scan
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1 union all
+    select * from temp1 where col1 = 5;
+                                  QUERY PLAN                                   
+-------------------------------------------------------------------------------
+ Append (actual rows=5 loops=1)
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=0 loops=4)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: 1
+   ->  Seq Scan on temp1 (actual rows=5 loops=1)
+         SubPlan 1
+           ->  Limit (actual rows=1 loops=5)
+                 ->  Gather (actual rows=1 loops=5)
+                       Workers Planned: 3
+                       Workers Launched: 3
+                       ->  Parallel Seq Scan on temp2 (actual rows=0 loops=20)
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=0 loops=4)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: 1
+(22 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     7
+(1 row)
+
+drop table parallel_write;
+alter table temp2 set (parallel_workers = 0);
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 where col1 = (select 1) union all  select * from temp1 where col1 = (select 1) union all
+ select * from temp2 where col2 = (select 2);
+                               QUERY PLAN                               
+------------------------------------------------------------------------
+ Append (actual rows=1 loops=1)
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Params Evaluated: $0
+         Workers Launched: 3
+       ->  Create parallel_write
+       InitPlan 1 (returns $0)
+           ->  Result (actual rows=1 loops=1)
+         ->  Parallel Seq Scan on temp1 (actual rows=0 loops=4)
+               Filter: (col1 = $0)
+               Rows Removed by Filter: 1
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Params Evaluated: $1
+         Workers Launched: 3
+       ->  Create parallel_write
+       InitPlan 2 (returns $1)
+           ->  Result (actual rows=1 loops=1)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=0 loops=4)
+               Filter: (col1 = $1)
+               Rows Removed by Filter: 1
+   ->  Seq Scan on temp2 (actual rows=1 loops=1)
+         Filter: (col2 = $2)
+         Rows Removed by Filter: 4
+         InitPlan 3 (returns $2)
+           ->  Result (actual rows=1 loops=1)
+(26 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+alter table temp2 reset (parallel_workers);
+drop table parallel_write;
+-- case 4: parallel inserts must not occur as there will be no direct Gather
+-- node under Append node. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1;
+                                      QUERY PLAN                                       
+---------------------------------------------------------------------------------------
+ Append (actual rows=15 loops=1)
+   ->  Seq Scan on temp1 (actual rows=5 loops=1)
+   ->  Seq Scan on temp2 (actual rows=5 loops=1)
+   ->  Seq Scan on temp1 temp1_1 (actual rows=5 loops=1)
+         SubPlan 1
+           ->  Limit (actual rows=1 loops=5)
+                 ->  Gather (actual rows=1 loops=5)
+                       Workers Planned: 3
+                       Workers Launched: 3
+                       ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=0 loops=20)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+-- case 5: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Seq Scan / Join / any other non-Gather node
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+alter table temp2 set (parallel_workers = 0);
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp1;
+                               QUERY PLAN                               
+------------------------------------------------------------------------
+ Gather (actual rows=0 loops=1)
+   Workers Planned: 3
+   Workers Launched: 3
+ ->  Create parallel_write
+   ->  Parallel Append (actual rows=4 loops=4)
+         ->  Seq Scan on temp2 (actual rows=5 loops=1)
+         ->  Parallel Seq Scan on temp1 (actual rows=5 loops=1)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=5 loops=1)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+alter table temp2 reset (parallel_workers);
+-- case 6: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Gather
+--  ->Gather
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp2 where col2 = (select 2);
+                                 QUERY PLAN                                 
+----------------------------------------------------------------------------
+ Append (actual rows=0 loops=1)
+   ->  Append (actual rows=0 loops=1)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=1 loops=1)
+         ->  Gather (actual rows=0 loops=1)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: 3
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Result (actual rows=0 loops=4)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=0 loops=4)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: 1
+         ->  Gather (actual rows=0 loops=1)
+               Workers Planned: 3
+               Params Evaluated: $0, $2
+               Workers Launched: 3
+             ->  Create parallel_write
+             InitPlan 3 (returns $2)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Result (actual rows=0 loops=4)
+                     One-Time Filter: ($2 = $0)
+                     ->  Parallel Seq Scan on temp2 (never executed)
+                           Filter: (col2 = $0)
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Params Evaluated: $3
+         Workers Launched: 3
+       ->  Create parallel_write
+       InitPlan 4 (returns $3)
+           ->  Result (actual rows=1 loops=1)
+         ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=0 loops=4)
+               Filter: (col2 = $3)
+               Rows Removed by Filter: 1
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Params Evaluated: $4
+         Workers Launched: 3
+       ->  Create parallel_write
+       InitPlan 5 (returns $4)
+           ->  Result (actual rows=1 loops=1)
+         ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=0 loops=4)
+               Filter: (col2 = $4)
+               Rows Removed by Filter: 1
+(47 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;
+                                     QUERY PLAN                                     
+------------------------------------------------------------------------------------
+ Append (actual rows=0 loops=1)
+   ->  Append (actual rows=0 loops=1)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=1 loops=1)
+         ->  Gather (actual rows=0 loops=1)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: 3
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Result (actual rows=0 loops=4)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=0 loops=4)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: 1
+         ->  Gather (actual rows=0 loops=1)
+               Workers Planned: 3
+               Params Evaluated: $0, $2
+               Workers Launched: 3
+             ->  Create parallel_write
+             InitPlan 3 (returns $2)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Result (actual rows=0 loops=4)
+                     One-Time Filter: ($2 = $0)
+                     ->  Parallel Seq Scan on temp2 (never executed)
+                           Filter: (col2 = $0)
+   ->  Append (actual rows=0 loops=1)
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=1 loops=1)
+         ->  Gather (actual rows=0 loops=1)
+               Workers Planned: 3
+               Params Evaluated: $3, $4
+               Workers Launched: 3
+             ->  Create parallel_write
+             InitPlan 5 (returns $4)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Result (actual rows=0 loops=4)
+                     One-Time Filter: ($4 = $3)
+                     ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=0 loops=4)
+                           Filter: (col1 = $3)
+                           Rows Removed by Filter: 1
+         ->  Gather (actual rows=0 loops=1)
+               Workers Planned: 3
+               Params Evaluated: $3, $5
+               Workers Launched: 3
+             ->  Create parallel_write
+             InitPlan 6 (returns $5)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Result (actual rows=0 loops=4)
+                     One-Time Filter: ($5 = $3)
+                     ->  Parallel Seq Scan on temp2 temp2_1 (never executed)
+                           Filter: (col2 = $3)
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=0 loops=4)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: 1
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=0 loops=4)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: 1
+(67 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+-- case 7: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Seq Scan / Join / any other non-Gather node
+--  ->Gather
+alter table temp2 set (parallel_workers = 0);
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp1 where col1 = (select 2);
+                                 QUERY PLAN                                 
+----------------------------------------------------------------------------
+ Append (actual rows=1 loops=1)
+   ->  Append (actual rows=0 loops=1)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=1 loops=1)
+         ->  Gather (actual rows=0 loops=1)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: 3
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Result (actual rows=0 loops=4)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=0 loops=4)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: 1
+         ->  Result (actual rows=0 loops=1)
+               One-Time Filter: ($2 = $0)
+               InitPlan 3 (returns $2)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Seq Scan on temp2 (never executed)
+                     Filter: (col2 = $0)
+   ->  Seq Scan on temp2 temp2_1 (actual rows=1 loops=1)
+         Filter: (col2 = $3)
+         Rows Removed by Filter: 4
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=1 loops=1)
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Params Evaluated: $4
+         Workers Launched: 3
+       ->  Create parallel_write
+       InitPlan 5 (returns $4)
+           ->  Result (actual rows=1 loops=1)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=0 loops=4)
+               Filter: (col1 = $4)
+               Rows Removed by Filter: 1
+(37 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;
+                                     QUERY PLAN                                     
+------------------------------------------------------------------------------------
+ Append (actual rows=1 loops=1)
+   ->  Append (actual rows=0 loops=1)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=1 loops=1)
+         ->  Gather (actual rows=0 loops=1)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: 3
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Result (actual rows=0 loops=4)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=0 loops=4)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: 1
+         ->  Result (actual rows=0 loops=1)
+               One-Time Filter: ($2 = $0)
+               InitPlan 3 (returns $2)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Seq Scan on temp2 (never executed)
+                     Filter: (col2 = $0)
+   ->  Append (actual rows=0 loops=1)
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=1 loops=1)
+         ->  Gather (actual rows=0 loops=1)
+               Workers Planned: 3
+               Params Evaluated: $3, $4
+               Workers Launched: 3
+             ->  Create parallel_write
+             InitPlan 5 (returns $4)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Result (actual rows=0 loops=4)
+                     One-Time Filter: ($4 = $3)
+                     ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=0 loops=4)
+                           Filter: (col1 = $3)
+                           Rows Removed by Filter: 1
+         ->  Result (actual rows=0 loops=1)
+               One-Time Filter: ($5 = $3)
+               InitPlan 6 (returns $5)
+                 ->  Result (actual rows=1 loops=1)
+               ->  Seq Scan on temp2 temp2_1 (never executed)
+                     Filter: (col2 = $3)
+   ->  Gather (actual rows=0 loops=1)
+         Workers Planned: 3
+         Workers Launched: 3
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=0 loops=4)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: 1
+   ->  Seq Scan on temp2 temp2_2 (actual rows=1 loops=1)
+         Filter: (col2 = 5)
+         Rows Removed by Filter: 4
+(53 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+alter table temp2 reset (parallel_workers);
+-- case 8: parallel inserts must not occur because there is no Gather or Append
+-- node at the top for union, except/except all, intersect/intersect all
+-- cases.
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 union
+    select * from temp2;
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ HashAggregate (actual rows=5 loops=1)
+   Group Key: temp1.col1
+   Batches: 1  Memory Usage: 217kB
+   ->  Append (actual rows=10 loops=1)
+         ->  Gather (actual rows=5 loops=1)
+               Workers Planned: 3
+               Workers Launched: 3
+               ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+         ->  Gather (actual rows=5 loops=1)
+               Workers Planned: 3
+               Workers Launched: 3
+               ->  Parallel Seq Scan on temp2 (actual rows=1 loops=4)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 except
+    select * from temp2 where col2 < 3;
+                                 QUERY PLAN                                 
+----------------------------------------------------------------------------
+ HashSetOp Except (actual rows=3 loops=1)
+   ->  Append (actual rows=7 loops=1)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=5 loops=1)
+               ->  Gather (actual rows=5 loops=1)
+                     Workers Planned: 3
+                     Workers Launched: 3
+                     ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=2 loops=1)
+               ->  Gather (actual rows=2 loops=1)
+                     Workers Planned: 3
+                     Workers Launched: 3
+                     ->  Parallel Seq Scan on temp2 (actual rows=0 loops=4)
+                           Filter: (col2 < 3)
+                           Rows Removed by Filter: 1
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 except all
+    select * from temp2 where col2 < 3;
+                                 QUERY PLAN                                 
+----------------------------------------------------------------------------
+ HashSetOp Except All (actual rows=3 loops=1)
+   ->  Append (actual rows=7 loops=1)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=5 loops=1)
+               ->  Gather (actual rows=5 loops=1)
+                     Workers Planned: 3
+                     Workers Launched: 3
+                     ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=2 loops=1)
+               ->  Gather (actual rows=2 loops=1)
+                     Workers Planned: 3
+                     Workers Launched: 3
+                     ->  Parallel Seq Scan on temp2 (actual rows=0 loops=4)
+                           Filter: (col2 < 3)
+                           Rows Removed by Filter: 1
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 intersect
+    select * from temp2;
+                                 QUERY PLAN                                 
+----------------------------------------------------------------------------
+ HashSetOp Intersect (actual rows=5 loops=1)
+   ->  Append (actual rows=10 loops=1)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=5 loops=1)
+               ->  Gather (actual rows=5 loops=1)
+                     Workers Planned: 3
+                     Workers Launched: 3
+                     ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=5 loops=1)
+               ->  Gather (actual rows=5 loops=1)
+                     Workers Planned: 3
+                     Workers Launched: 3
+                     ->  Parallel Seq Scan on temp2 (actual rows=1 loops=4)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 intersect all
+    select * from temp2;
+                                 QUERY PLAN                                 
+----------------------------------------------------------------------------
+ HashSetOp Intersect All (actual rows=5 loops=1)
+   ->  Append (actual rows=10 loops=1)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=5 loops=1)
+               ->  Gather (actual rows=5 loops=1)
+                     Workers Planned: 3
+                     Workers Launched: 3
+                     ->  Parallel Seq Scan on temp1 (actual rows=1 loops=4)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=5 loops=1)
+               ->  Gather (actual rows=5 loops=1)
+                     Workers Planned: 3
+                     Workers Launched: 3
+                     ->  Parallel Seq Scan on temp2 (actual rows=1 loops=4)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
 drop table temp1;
 drop table temp2;
 drop table temp3;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 7245cb97c6..937077d79c 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -217,6 +217,227 @@ reset enable_nestloop;
 reset enable_mergejoin;
 reset enable_hashjoin;
 
+-- test cases for performing parallel inserts when Append node is at the top
+-- and Gather node is in one of its direct sub plans.
+
+-- case 1: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5 union all
+    select * from temp2 where col2 = 5;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 2: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select * from temp2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 3: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples to
+-- Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Gather
+-- 		->Parallel Seq Scan
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1 union all
+    select * from temp1 where col1 = 5;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 set (parallel_workers = 0);
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as select * from temp1 where col1 = (select 1) union all  select * from temp1 where col1 = (select 1) union all
+ select * from temp2 where col2 = (select 2);
+select count(*) from parallel_write;
+alter table temp2 reset (parallel_workers);
+drop table parallel_write;
+
+-- case 4: parallel inserts must not occur as there will be no direct Gather
+-- node under Append node. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 5: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Seq Scan / Join / any other non-Gather node
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+
+alter table temp2 set (parallel_workers = 0);
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp1;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 reset (parallel_workers);
+
+-- case 6: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Gather
+--  ->Gather
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp2 where col2 = (select 2);
+select count(*) from parallel_write;
+drop table parallel_write;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 7: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Seq Scan / Join / any other non-Gather node
+--  ->Gather
+
+alter table temp2 set (parallel_workers = 0);
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp1 where col1 = (select 2);
+select count(*) from parallel_write;
+drop table parallel_write;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 reset (parallel_workers);
+
+-- case 8: parallel inserts must not occur because there is no Gather or Append
+-- node at the top for union, except/except all, intersect/intersect all
+-- cases.
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 union
+    select * from temp2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 except
+    select * from temp2 where col2 < 3;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 except all
+    select * from temp2 where col2 < 3;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 intersect
+    select * from temp2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
+explain (costs off, analyze on, timing off, summary off)
+create table parallel_write as
+    select * from temp1 intersect all
+    select * from temp2;
+select count(*) from parallel_write;
+drop table parallel_write;
+
 drop table temp1;
 drop table temp2;
 drop table temp3;
-- 
2.25.1

From e2c7dea3f1f0171b1d42b99f493860ccf1f6ccaa Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 21 Dec 2020 14:35:24 +0530
Subject: [PATCH v14 1/4] Parallel Inserts in CREATE TABLE AS

Allow the leader and each worker insert the tuples in parallel
if the SELECT part of the CTAS is parallelizable.

The design:
The main idea is to push the CTAS dest receiver down to Gather node
and from there the required information will be shared to workers
so that they can perform parallel insertions. Leader will also
participate in insertions. After the planning, check if the upper
plan node is Gather in createas.c and mark a parallelism flag in
the CTAS dest receiver and push it down to Gather node. Each worker
can create its own CTAS dest receiver with the information passed
from the leader. Leader inserts its share of tuples if instructed
to do, and so are workers. Each worker writes atomically its number
of inserted tuples into a shared memory variable, the leader combines
this with its own number of inserted tuples and shares to the client.
---
 src/backend/access/heap/heapam.c       |  11 -
 src/backend/access/transam/xact.c      |  30 ++-
 src/backend/commands/createas.c        | 303 ++++++++++++++++---------
 src/backend/commands/explain.c         |  31 +++
 src/backend/executor/execParallel.c    |  70 +++++-
 src/backend/executor/nodeGather.c      | 113 ++++++++-
 src/backend/executor/nodeGatherMerge.c |   4 +-
 src/include/access/xact.h              |   1 +
 src/include/commands/createas.h        |  25 ++
 src/include/executor/execParallel.h    |   6 +-
 src/include/nodes/execnodes.h          |   3 +
 11 files changed, 457 insertions(+), 140 deletions(-)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index a9583f3103..86347ba273 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2043,17 +2043,6 @@ static HeapTuple
 heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
 					CommandId cid, int options)
 {
-	/*
-	 * To allow parallel inserts, we need to ensure that they are safe to be
-	 * performed in workers. We have the infrastructure to allow parallel
-	 * inserts in general except for the cases where inserts generate a new
-	 * CommandId (eg. inserts into a table having a foreign key column).
-	 */
-	if (IsParallelWorker())
-		ereport(ERROR,
-				(errcode(ERRCODE_INVALID_TRANSACTION_STATE),
-				 errmsg("cannot insert tuples in a parallel worker")));
-
 	tup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
 	tup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
 	tup->t_data->t_infomask |= HEAP_XMAX_INVALID;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 9cd0b7c11b..db6eedd635 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -763,18 +763,34 @@ GetCurrentCommandId(bool used)
 	/* this is global to a transaction, not subtransaction-local */
 	if (used)
 	{
-		/*
-		 * Forbid setting currentCommandIdUsed in a parallel worker, because
-		 * we have no provision for communicating this back to the leader.  We
-		 * could relax this restriction when currentCommandIdUsed was already
-		 * true at the start of the parallel operation.
-		 */
-		Assert(!IsParallelWorker());
+		 /*
+		  * This is a temporary hack for all common parallel insert cases i.e.
+		  * insert into, ctas, copy from. To be changed later. In a parallel
+		  * worker, set currentCommandIdUsed to true only if it was not set to
+		  * true at the start of the parallel operation (by way of
+		  * SetCurrentCommandIdUsedForWorker()). We have to do this because
+		  * GetCurrentCommandId(true) may be called from anywhere, especially
+		  * for parallel inserts, within parallel worker.
+		  */
+		Assert(!(IsParallelWorker() && !currentCommandIdUsed));
 		currentCommandIdUsed = true;
 	}
 	return currentCommandId;
 }
 
+/*
+ *	SetCurrentCommandIdUsedForWorker
+ *
+ * For a parallel worker, record that the currentCommandId has been used. This
+ * must only be called at the start of a parallel operation.
+ */
+void
+SetCurrentCommandIdUsedForWorker(void)
+{
+	Assert(IsParallelWorker() && !currentCommandIdUsed && currentCommandId != InvalidCommandId);
+	currentCommandIdUsed = true;
+}
+
 /*
  *	SetParallelStartTimestamps
  *
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 6bf6c5a310..10f4f2b4d7 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,18 +51,6 @@
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 
-typedef struct
-{
-	DestReceiver pub;			/* publicly-known function pointers */
-	IntoClause *into;			/* target relation specification */
-	/* These fields are filled by intorel_startup: */
-	Relation	rel;			/* relation to write to */
-	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
-	CommandId	output_cid;		/* cmin to insert in output tuples */
-	int			ti_options;		/* table_tuple_insert performance options */
-	BulkInsertState bistate;	/* bulk insert state */
-} DR_intorel;
-
 /* utility functions for CTAS definition creation */
 static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into);
 static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -350,6 +338,14 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		/* call ExecutorStart to prepare the plan for execution */
 		ExecutorStart(queryDesc, GetIntoRelEFlags(into));
 
+		/*
+		 * If SELECT part of the CTAS is parallelizable, then make each
+		 * parallel worker insert the tuples that are resulted in its execution
+		 * into the target table. We need plan state to be initialized by the
+		 * executor to decide whether to allow parallel inserts or not.
+		 */
+		ChooseParallelInsertsInCTAS(into, queryDesc);
+
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
 
@@ -418,6 +414,9 @@ CreateIntoRelDestReceiver(IntoClause *intoClause)
 	self->pub.rDestroy = intorel_destroy;
 	self->pub.mydest = DestIntoRel;
 	self->into = intoClause;
+	self->is_parallel = false;
+	self->is_parallel_worker = false;
+	self->object_id = InvalidOid;
 	/* other private fields will be set during intorel_startup */
 
 	return (DestReceiver *) self;
@@ -430,121 +429,169 @@ static void
 intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 {
 	DR_intorel *myState = (DR_intorel *) self;
-	IntoClause *into = myState->into;
-	bool		is_matview;
-	List	   *attrList;
 	ObjectAddress intoRelationAddr;
 	Relation	intoRelationDesc;
-	ListCell   *lc;
-	int			attnum;
 
-	Assert(into != NULL);		/* else somebody forgot to set it */
-
-	/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
-	is_matview = (into->viewQuery != NULL);
+	if (myState->is_parallel_worker)
+	{
+		/* In the worker */
+		intoRelationDesc = table_open(myState->object_id, AccessExclusiveLock);
+		myState->rel = intoRelationDesc;
+		myState->reladdr = InvalidObjectAddress;
+		myState->ti_options = 0;
+		myState->bistate = GetBulkInsertState();
 
-	/*
-	 * Build column definitions using "pre-cooked" type and collation info. If
-	 * a column name list was specified in CREATE TABLE AS, override the
-	 * column names derived from the query.  (Too few column names are OK, too
-	 * many are not.)
-	 */
-	attrList = NIL;
-	lc = list_head(into->colNames);
-	for (attnum = 0; attnum < typeinfo->natts; attnum++)
+		/*
+		 * Right after the table is created in the leader, the command id is
+		 * incremented (in create_ctas_internal()). The new command id is
+		 * marked as used in intorel_startup(), then the parallel mode is
+		 * entered. The command id and transaction id are serialized into
+		 * parallel DSM, they are then available to all parallel workers. All
+		 * the workers need to mark the command id as used before insertion.
+		 */
+		SetCurrentCommandIdUsedForWorker();
+		myState->output_cid = GetCurrentCommandId(false);
+	}
+	else
 	{
-		Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
-		ColumnDef  *col;
-		char	   *colname;
+		IntoClause *into = myState->into;
+		bool		is_matview;
+		List	   *attrList;
+		ListCell   *lc;
+		int			attnum;
 
-		if (lc)
+		Assert(into != NULL);		/* else somebody forgot to set it */
+
+		/*
+		 * This code supports both CREATE TABLE AS and CREATE MATERIALIZED
+		 * VIEW.
+		 */
+		is_matview = (into->viewQuery != NULL);
+
+		/*
+		 * Build column definitions using "pre-cooked" type and collation info.
+		 * If a column name list was specified in CREATE TABLE AS, override the
+		 * column names derived from the query.  (Too few column names are OK,
+		 * too many are not.)
+		 */
+		attrList = NIL;
+		lc = list_head(into->colNames);
+		for (attnum = 0; attnum < typeinfo->natts; attnum++)
 		{
-			colname = strVal(lfirst(lc));
-			lc = lnext(into->colNames, lc);
+			Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
+			ColumnDef  *col;
+			char	   *colname;
+
+			if (lc)
+			{
+				colname = strVal(lfirst(lc));
+				lc = lnext(into->colNames, lc);
+			}
+			else
+				colname = NameStr(attribute->attname);
+
+			col = makeColumnDef(colname,
+								attribute->atttypid,
+								attribute->atttypmod,
+								attribute->attcollation);
+
+			/*
+			 * It's possible that the column is of a collatable type but the
+			 * collation could not be resolved, so double-check.  (We must
+			 * check this here because DefineRelation would adopt the type's
+			 * default collation rather than complaining.)
+			 */
+			if (!OidIsValid(col->collOid) &&
+				type_is_collatable(col->typeName->typeOid))
+				ereport(ERROR,
+						(errcode(ERRCODE_INDETERMINATE_COLLATION),
+						errmsg("no collation was derived for column \"%s\" with collatable type %s",
+								col->colname,
+								format_type_be(col->typeName->typeOid)),
+						errhint("Use the COLLATE clause to set the collation explicitly.")));
+
+			attrList = lappend(attrList, col);
 		}
-		else
-			colname = NameStr(attribute->attname);
 
-		col = makeColumnDef(colname,
-							attribute->atttypid,
-							attribute->atttypmod,
-							attribute->attcollation);
+		if (lc != NULL)
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					errmsg("too many column names were specified")));
 
 		/*
-		 * It's possible that the column is of a collatable type but the
-		 * collation could not be resolved, so double-check.  (We must check
-		 * this here because DefineRelation would adopt the type's default
-		 * collation rather than complaining.)
+		 * Actually create the target table
 		 */
-		if (!OidIsValid(col->collOid) &&
-			type_is_collatable(col->typeName->typeOid))
-			ereport(ERROR,
-					(errcode(ERRCODE_INDETERMINATE_COLLATION),
-					 errmsg("no collation was derived for column \"%s\" with collatable type %s",
-							col->colname,
-							format_type_be(col->typeName->typeOid)),
-					 errhint("Use the COLLATE clause to set the collation explicitly.")));
+		intoRelationAddr = create_ctas_internal(attrList, into);
 
-		attrList = lappend(attrList, col);
-	}
+		/*
+		 * Finally we can open the target table
+		 */
+		intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
 
-	if (lc != NULL)
-		ereport(ERROR,
-				(errcode(ERRCODE_SYNTAX_ERROR),
-				 errmsg("too many column names were specified")));
+		/*
+		 * Make sure the constructed table does not have RLS enabled.
+		 *
+		 * check_enable_rls() will ereport(ERROR) itself if the user has
+		 * requested something invalid, and otherwise will return RLS_ENABLED
+		 * if RLS should be enabled here.  We don't actually support that
+		 * currently, so throw our own ereport(ERROR) if that happens.
+		 */
+		if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					errmsg("policies not yet implemented for this command")));
 
-	/*
-	 * Actually create the target table
-	 */
-	intoRelationAddr = create_ctas_internal(attrList, into);
+		/*
+		 * Tentatively mark the target as populated, if it's a matview and
+		 * we're going to fill it; otherwise, no change needed.
+		 */
+		if (is_matview && !into->skipData)
+			SetMatViewPopulatedState(intoRelationDesc, true);
 
-	/*
-	 * Finally we can open the target table
-	 */
-	intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
+		/*
+		 * Fill private fields of myState for use by later routines
+		 */
+		myState->rel = intoRelationDesc;
+		myState->reladdr = intoRelationAddr;
+		myState->output_cid = GetCurrentCommandId(true);
+		myState->ti_options = TABLE_INSERT_SKIP_FSM;
 
-	/*
-	 * Make sure the constructed table does not have RLS enabled.
-	 *
-	 * check_enable_rls() will ereport(ERROR) itself if the user has requested
-	 * something invalid, and otherwise will return RLS_ENABLED if RLS should
-	 * be enabled here.  We don't actually support that currently, so throw
-	 * our own ereport(ERROR) if that happens.
-	 */
-	if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
-		ereport(ERROR,
-				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-				 errmsg("policies not yet implemented for this command")));
+		/*
+		 * If WITH NO DATA is specified, there is no need to set up the state
+		 * for bulk inserts as there are no tuples to insert.
+		 */
+		if (!into->skipData)
+			myState->bistate = GetBulkInsertState();
+		else
+			myState->bistate = NULL;
 
-	/*
-	 * Tentatively mark the target as populated, if it's a matview and we're
-	 * going to fill it; otherwise, no change needed.
-	 */
-	if (is_matview && !into->skipData)
-		SetMatViewPopulatedState(intoRelationDesc, true);
+		if (myState->is_parallel)
+		{
+			myState->object_id = intoRelationAddr.objectId;
 
-	/*
-	 * Fill private fields of myState for use by later routines
-	 */
-	myState->rel = intoRelationDesc;
-	myState->reladdr = intoRelationAddr;
-	myState->output_cid = GetCurrentCommandId(true);
-	myState->ti_options = TABLE_INSERT_SKIP_FSM;
+			/*
+			 * We don't need to skip contacting FSM while inserting tuples
+			 * for parallel mode, while extending the relations, workers
+			 * instead of blocking on a page while another worker is inserting,
+			 * can check the FSM for another page that can accommodate the
+			 * tuples. This results in major benefit for parallel inserts.
+			 */
+			myState->ti_options = 0;
 
-	/*
-	 * If WITH NO DATA is specified, there is no need to set up the state for
-	 * bulk inserts as there are no tuples to insert.
-	 */
-	if (!into->skipData)
-		myState->bistate = GetBulkInsertState();
-	else
-		myState->bistate = NULL;
+			/*
+			 * rd_createSubid is marked invalid, otherwise, the table is not
+			 * allowed to be extended by the workers.
+			 */
+			myState->rel->rd_createSubid = InvalidSubTransactionId;
+		}
 
-	/*
-	 * Valid smgr_targblock implies something already wrote to the relation.
-	 * This may be harmless, but this function hasn't planned for it.
-	 */
-	Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+		/*
+		 * Valid smgr_targblock implies something already wrote to the
+		 * relation. This may be harmless, but this function hasn't planned for
+		 * it.
+		 */
+		Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+	}
 }
 
 /*
@@ -606,3 +653,47 @@ intorel_destroy(DestReceiver *self)
 {
 	pfree(self);
 }
+
+/*
+ * ChooseParallelInsertsInCTAS --- determine whether or not parallel
+ * insertion is possible, if yes set the parallel insert state i.e. push down
+ * the dest receiver to the Gather nodes.
+ */
+void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc)
+{
+	if (!IS_CTAS(into))
+		return;
+
+	/*
+	 * Do not allow parallel inserts if the table is temporary. As the
+	 * temporary tables are backend local, workers can not know about them.
+	 * Currently, CTAS supports creation of normal(logged), temporary and
+	 * unlogged tables. It does not support foreign or partition table
+	 * creation. Hence the check for temporary table is enough here.
+	 */
+	if (!into->rel || into->rel->relpersistence == RELPERSISTENCE_TEMP)
+		return;
+
+	if (queryDesc && IsA(queryDesc->planstate, GatherState) &&
+		!queryDesc->planstate->ps_ProjInfo)
+	{
+		GatherState *gstate = (GatherState *) queryDesc->planstate;
+		DestReceiver *dest	= queryDesc->dest;
+
+		/*
+		 * For parallelizing inserts in CTAS i.e. making each parallel worker
+		 * insert the tuples, we must send information such as into clause (for
+		 * each worker to build separate dest receiver), object id (for each
+		 * worker to open the created table).
+		 */
+		((DR_intorel *) dest)->is_parallel = true;
+		gstate->dest = dest;
+
+		/*
+		 * Since there are no rows that are transferred from workers to Gather
+		 * node, so we set it to 0 to be visible in estimated row count of
+		 * explain plans.
+		 */
+		queryDesc->planstate->plan->plan_rows = 0;
+	}
+}
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 43f9b01e83..fbd0bc5a81 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -556,6 +556,14 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	/* call ExecutorStart to prepare the plan for execution */
 	ExecutorStart(queryDesc, eflags);
 
+	/*
+	 * If SELECT part of the CTAS is parallelizable, then make each parallel
+	 * worker insert the tuples that are resulted in its execution into the
+	 * target table. We need plan state to be initialized by the executor to
+	 * decide whether to allow parallel inserts or not.
+	 */
+	ChooseParallelInsertsInCTAS(into, queryDesc);
+
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
 	{
@@ -1775,6 +1783,29 @@ ExplainNode(PlanState *planstate, List *ancestors,
 
 				if (gather->single_copy || es->format != EXPLAIN_FORMAT_TEXT)
 					ExplainPropertyBool("Single Copy", gather->single_copy, es);
+
+				/*
+				 * Show the create table information under Gather node in case
+				 * parallel workers have inserted the rows.
+				 */
+				if (IsA(planstate, GatherState))
+				{
+					GatherState *gstate = (GatherState *) planstate;
+
+					if (IS_PARALLEL_CTAS_DEST(gstate->dest) &&
+						((DR_intorel *) gstate->dest)->into->rel &&
+						((DR_intorel *) gstate->dest)->into->rel->relname)
+					{
+						es->indent--;
+						ExplainIndentText(es);
+						appendStringInfoString(es->str, "->  ");
+						appendStringInfoString(es->str, "Create ");
+						appendStringInfo(es->str, "%s\n",
+							((DR_intorel *) gstate->dest)->into->rel->relname);
+						ExplainIndentText(es);
+						es->indent++;
+					}
+				}
 			}
 			break;
 		case T_GatherMerge:
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index befde52691..9ef33eee54 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -23,6 +23,7 @@
 
 #include "postgres.h"
 
+#include "commands/createas.h"
 #include "executor/execParallel.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
@@ -65,6 +66,7 @@
 #define PARALLEL_KEY_QUERY_TEXT		UINT64CONST(0xE000000000000008)
 #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009)
 #define PARALLEL_KEY_WAL_USAGE			UINT64CONST(0xE00000000000000A)
+#define PARALLEL_KEY_INTO_CLAUSE		UINT64CONST(0xE00000000000000B)
 
 #define PARALLEL_TUPLE_QUEUE_SIZE		65536
 
@@ -77,6 +79,9 @@ typedef struct FixedParallelExecutorState
 	dsa_pointer param_exec;
 	int			eflags;
 	int			jit_flags;
+	Oid			objectid;		/* workers to open relation/table.  */
+	/* Number tuples inserted by all the workers. */
+	pg_atomic_uint64	processed;
 } FixedParallelExecutorState;
 
 /*
@@ -578,7 +583,8 @@ ExecParallelSetupTupleQueues(ParallelContext *pcxt, bool reinitialize)
 ParallelExecutorInfo *
 ExecInitParallelPlan(PlanState *planstate, EState *estate,
 					 Bitmapset *sendParams, int nworkers,
-					 int64 tuples_needed)
+					 int64 tuples_needed, IntoClause *intoclause,
+					 Oid objectid)
 {
 	ParallelExecutorInfo *pei;
 	ParallelContext *pcxt;
@@ -600,6 +606,8 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	Size		dsa_minsize = dsa_minimum_size();
 	char	   *query_string;
 	int			query_len;
+	char 		*intoclausestr = NULL;
+	int			intoclause_len = 0;
 
 	/*
 	 * Force any initplan outputs that we're going to pass to workers to be
@@ -712,6 +720,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_estimate_chunk(&pcxt->estimator, dsa_minsize);
 	shm_toc_estimate_keys(&pcxt->estimator, 1);
 
+	/* Estimate space for into clause for CTAS. */
+	if (IS_CTAS(intoclause) && OidIsValid(objectid))
+	{
+		intoclausestr = nodeToString(intoclause);
+		intoclause_len = strlen(intoclausestr);
+		shm_toc_estimate_chunk(&pcxt->estimator, intoclause_len + 1);
+		shm_toc_estimate_keys(&pcxt->estimator, 1);
+	}
+
 	/* Everyone's had a chance to ask for space, so now create the DSM. */
 	InitializeParallelDSM(pcxt);
 
@@ -729,6 +746,14 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	fpes->param_exec = InvalidDsaPointer;
 	fpes->eflags = estate->es_top_eflags;
 	fpes->jit_flags = estate->es_jit_flags;
+	pg_atomic_init_u64(&fpes->processed, 0);
+	pei->processed = &fpes->processed;
+
+	if (intoclausestr && OidIsValid(objectid))
+		fpes->objectid = objectid;
+	else
+		fpes->objectid = InvalidOid;
+
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, fpes);
 
 	/* Store query string */
@@ -758,8 +783,18 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_WAL_USAGE, walusage_space);
 	pei->wal_usage = walusage_space;
 
-	/* Set up the tuple queues that the workers will write into. */
-	pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	if (intoclausestr)
+	{
+		char *intoclause_space = shm_toc_allocate(pcxt->toc,
+												  intoclause_len + 1);
+		memcpy(intoclause_space, intoclausestr, intoclause_len + 1);
+		shm_toc_insert(pcxt->toc, PARALLEL_KEY_INTO_CLAUSE, intoclause_space);
+	}
+	else
+	{
+		/* Set up the tuple queues that the workers will write into. */
+		pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	}
 
 	/* We don't need the TupleQueueReaders yet, though. */
 	pei->reader = NULL;
@@ -1387,12 +1422,30 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 	void	   *area_space;
 	dsa_area   *area;
 	ParallelWorkerContext pwcxt;
+	char		*intoclausestr = NULL;
+	IntoClause	*intoclause = NULL;
 
 	/* Get fixed-size state. */
 	fpes = shm_toc_lookup(toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
 
-	/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
-	receiver = ExecParallelGetReceiver(seg, toc);
+	intoclausestr = shm_toc_lookup(toc, PARALLEL_KEY_INTO_CLAUSE, true);
+	if (intoclausestr)
+	{
+		/*
+		 * If the worker is for parallel insert in CTAS, then use the proper
+		 * dest receiver.
+		 */
+		intoclause = (IntoClause *) stringToNode(intoclausestr);
+		receiver = CreateIntoRelDestReceiver(intoclause);
+		((DR_intorel *)receiver)->is_parallel_worker = true;
+		((DR_intorel *)receiver)->object_id = fpes->objectid;
+	}
+	else
+	{
+		/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
+		receiver = ExecParallelGetReceiver(seg, toc);
+	}
+
 	instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_INSTRUMENTATION, true);
 	if (instrumentation != NULL)
 		instrument_options = instrumentation->instrument_options;
@@ -1471,6 +1524,13 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 			queryDesc->estate->es_jit->instr;
 	}
 
+	/*
+	 * Write out the number of tuples this worker has inserted. Leader will use
+	 * it to inform the end client.
+	 */
+	if (intoclausestr)
+		pg_atomic_add_fetch_u64(&fpes->processed, queryDesc->estate->es_processed);
+
 	/* Must do this after capturing instrumentation. */
 	ExecutorEnd(queryDesc);
 
diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c
index a01b46af14..e7c588c66a 100644
--- a/src/backend/executor/nodeGather.c
+++ b/src/backend/executor/nodeGather.c
@@ -32,6 +32,7 @@
 
 #include "access/relscan.h"
 #include "access/xact.h"
+#include "commands/createas.h"
 #include "executor/execdebug.h"
 #include "executor/execParallel.h"
 #include "executor/nodeGather.h"
@@ -48,6 +49,7 @@ static TupleTableSlot *ExecGather(PlanState *pstate);
 static TupleTableSlot *gather_getnext(GatherState *gatherstate);
 static MinimalTuple gather_readnext(GatherState *gatherstate);
 static void ExecShutdownGatherWorkers(GatherState *node);
+static void ExecParallelInsertInCTAS(GatherState *node);
 
 
 /* ----------------------------------------------------------------
@@ -131,6 +133,72 @@ ExecInitGather(Gather *node, EState *estate, int eflags)
 	return gatherstate;
 }
 
+/* ----------------------------------------------------------------
+ *		ExecParallelInsertInCTAS(node)
+ *
+ *		Facilitates parallel inserts by parallel workers and/or
+ *		leader for Create Table AS.
+ * ----------------------------------------------------------------
+ */
+static void
+ExecParallelInsertInCTAS(GatherState *node)
+{
+	/* Enable leader to insert in case no parallel workers were launched. */
+	if (node->nworkers_launched == 0)
+		node->need_to_scan_locally = true;
+
+	/*
+	 * By now, for parallel workers (if launched any), would have started their
+	 * work i.e. insertion to target table. In case the leader is chosen to
+	 * participate for parallel inserts in CTAS, then finish its share before
+	 * going to wait for the parallel workers to finish.
+	 */
+	if (node->need_to_scan_locally)
+	{
+		EState	   *estate = node->ps.state;
+		TupleTableSlot *outerTupleSlot;
+
+		for(;;)
+		{
+			/* Install our DSA area while executing the plan. */
+			estate->es_query_dsa =
+					node->pei ? node->pei->area : NULL;
+
+			outerTupleSlot = ExecProcNode(node->ps.lefttree);
+
+			estate->es_query_dsa = NULL;
+
+			if(TupIsNull(outerTupleSlot))
+				break;
+
+			(void) node->dest->receiveSlot(outerTupleSlot, node->dest);
+
+			node->ps.state->es_processed++;
+		}
+
+		node->need_to_scan_locally = false;
+	}
+
+	if (node->nworkers_launched > 0)
+	{
+		/*
+		 * We wait here for the parallel workers to finish their work and
+		 * accumulate the tuples they inserted and also their buffer/WAL usage.
+		 * We do not destroy the parallel context here, it will be done in
+		 * ExecShutdownGather at the end of the plan. Note that the
+		 * ExecShutdownGatherWorkers call from ExecShutdownGather will be a
+		 * no-op.
+		 */
+		ExecShutdownGatherWorkers(node);
+
+		/*
+		 * Add up the total tuples inserted by all workers, to the tuples
+		 * inserted by the leader(if any). This will be shared to client.
+		 */
+		node->ps.state->es_processed += pg_atomic_read_u64(node->pei->processed);
+	}
+}
+
 /* ----------------------------------------------------------------
  *		ExecGather(node)
  *
@@ -157,6 +225,7 @@ ExecGather(PlanState *pstate)
 	{
 		EState	   *estate = node->ps.state;
 		Gather	   *gather = (Gather *) node->ps.plan;
+		bool		isctas = IS_PARALLEL_CTAS_DEST(node->dest);
 
 		/*
 		 * Sometimes we might have to run without parallelism; but if parallel
@@ -165,6 +234,18 @@ ExecGather(PlanState *pstate)
 		if (gather->num_workers > 0 && estate->es_use_parallel_mode)
 		{
 			ParallelContext *pcxt;
+			IntoClause	*intoclause = NULL;
+			Oid			objectid = InvalidOid;
+
+			/*
+			 * Take the necessary information to be passed to workers for
+			 * parallel inserts in CTAS.
+			 */
+			if (isctas)
+			{
+				intoclause = ((DR_intorel *) node->dest)->into;
+				objectid = ((DR_intorel *) node->dest)->object_id;
+			}
 
 			/* Initialize, or re-initialize, shared state needed by workers. */
 			if (!node->pei)
@@ -172,7 +253,10 @@ ExecGather(PlanState *pstate)
 												 estate,
 												 gather->initParam,
 												 gather->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 /* CTAS info */
+												 intoclause,
+												 objectid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
@@ -190,13 +274,16 @@ ExecGather(PlanState *pstate)
 			/* Set up tuple queue readers to read the results. */
 			if (pcxt->nworkers_launched > 0)
 			{
-				ExecParallelCreateReaders(node->pei);
-				/* Make a working array showing the active readers */
-				node->nreaders = pcxt->nworkers_launched;
-				node->reader = (TupleQueueReader **)
-					palloc(node->nreaders * sizeof(TupleQueueReader *));
-				memcpy(node->reader, node->pei->reader,
-					   node->nreaders * sizeof(TupleQueueReader *));
+				if (!isctas)
+				{
+					ExecParallelCreateReaders(node->pei);
+					/* Make a working array showing the active readers */
+					node->nreaders = pcxt->nworkers_launched;
+					node->reader = (TupleQueueReader **)
+						palloc(node->nreaders * sizeof(TupleQueueReader *));
+					memcpy(node->reader, node->pei->reader,
+						node->nreaders * sizeof(TupleQueueReader *));
+				}
 			}
 			else
 			{
@@ -208,9 +295,17 @@ ExecGather(PlanState *pstate)
 		}
 
 		/* Run plan locally if no workers or enabled and not single-copy. */
-		node->need_to_scan_locally = (node->nreaders == 0)
+		node->need_to_scan_locally = (node->nreaders == 0 &&
+			!isctas)
 			|| (!gather->single_copy && parallel_leader_participation);
 		node->initialized = true;
+
+		/* Perform parallel inserts for CTAS. */
+		if (isctas)
+		{
+			ExecParallelInsertInCTAS(node);
+			return NULL;
+		}
 	}
 
 	/*
diff --git a/src/backend/executor/nodeGatherMerge.c b/src/backend/executor/nodeGatherMerge.c
index 47129344f3..ee45272c17 100644
--- a/src/backend/executor/nodeGatherMerge.c
+++ b/src/backend/executor/nodeGatherMerge.c
@@ -216,7 +216,9 @@ ExecGatherMerge(PlanState *pstate)
 												 estate,
 												 gm->initParam,
 												 gm->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 NULL,
+												 InvalidOid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7320de345c..5beae6c617 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -389,6 +389,7 @@ extern FullTransactionId GetCurrentFullTransactionIdIfAny(void);
 extern void MarkCurrentTransactionIdLoggedIfAny(void);
 extern bool SubTransactionIsActive(SubTransactionId subxid);
 extern CommandId GetCurrentCommandId(bool used);
+extern void SetCurrentCommandIdUsedForWorker(void);
 extern void SetParallelStartTimestamps(TimestampTz xact_ts, TimestampTz stmt_ts);
 extern TimestampTz GetCurrentTransactionStartTimestamp(void);
 extern TimestampTz GetCurrentStatementStartTimestamp(void);
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 7629230254..ed4690305b 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -14,12 +14,35 @@
 #ifndef CREATEAS_H
 #define CREATEAS_H
 
+#include "access/heapam.h"
 #include "catalog/objectaddress.h"
+#include "executor/execdesc.h"
 #include "nodes/params.h"
+#include "nodes/plannodes.h"
 #include "parser/parse_node.h"
 #include "tcop/dest.h"
 #include "utils/queryenvironment.h"
 
+typedef struct
+{
+	DestReceiver pub;			/* publicly-known function pointers */
+	IntoClause *into;			/* target relation specification */
+	/* These fields are filled by intorel_startup: */
+	Relation	rel;			/* relation to write to */
+	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
+	CommandId	output_cid;		/* cmin to insert in output tuples */
+	int			ti_options;		/* table_tuple_insert performance options */
+	BulkInsertState bistate;	/* bulk insert state */
+	bool		is_parallel;	/* is parallelism to be considered? */
+	bool		is_parallel_worker; /* true for parallel worker */
+	/* Used for table open by parallel worker. */
+	Oid			object_id;
+} DR_intorel;
+
+#define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
+#define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
+									 IS_CTAS(((DR_intorel *) dest)->into) && \
+									 ((DR_intorel *) dest)->is_parallel)
 
 extern ObjectAddress ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 									   ParamListInfo params, QueryEnvironment *queryEnv,
@@ -29,4 +52,6 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
+extern void ChooseParallelInsertsInCTAS(IntoClause *into,
+										QueryDesc *queryDesc);
 #endif							/* CREATEAS_H */
diff --git a/src/include/executor/execParallel.h b/src/include/executor/execParallel.h
index 5a39a5b29c..9f959f741b 100644
--- a/src/include/executor/execParallel.h
+++ b/src/include/executor/execParallel.h
@@ -35,11 +35,15 @@ typedef struct ParallelExecutorInfo
 	/* These two arrays have pcxt->nworkers_launched entries: */
 	shm_mq_handle **tqueue;		/* tuple queues for worker output */
 	struct TupleQueueReader **reader;	/* tuple reader/writer support */
+	/* Number of tuples inserted by all workers. */
+	volatile pg_atomic_uint64	*processed;
 } ParallelExecutorInfo;
 
 extern ParallelExecutorInfo *ExecInitParallelPlan(PlanState *planstate,
 												  EState *estate, Bitmapset *sendParam, int nworkers,
-												  int64 tuples_needed);
+												  int64 tuples_needed,
+												  IntoClause *intoclause,
+												  Oid objectid);
 extern void ExecParallelCreateReaders(ParallelExecutorInfo *pei);
 extern void ExecParallelFinish(ParallelExecutorInfo *pei);
 extern void ExecParallelCleanup(ParallelExecutorInfo *pei);
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 61ba4c3666..e9c4442c22 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -23,6 +23,7 @@
 #include "nodes/tidbitmap.h"
 #include "partitioning/partdefs.h"
 #include "storage/condition_variable.h"
+#include "tcop/dest.h"
 #include "utils/hsearch.h"
 #include "utils/queryenvironment.h"
 #include "utils/reltrigger.h"
@@ -2326,6 +2327,8 @@ typedef struct GatherState
 	int			nreaders;		/* number of still-active workers */
 	int			nextreader;		/* next one to try to read from */
 	struct TupleQueueReader **reader;	/* array with nreaders active entries */
+	/* Parallel inserts in CTAS related info is specified below. */
+	DestReceiver *dest;
 } GatherState;
 
 /* ----------------
-- 
2.25.1

From 2484cfb3e57c79788448b04bfc1760b59fd20c4d Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 21 Dec 2020 15:19:23 +0530
Subject: [PATCH v14 2/4] Tuple Cost Adjustment for Parallel Inserts in CTAS

Let the planner know that the SELECT is from CTAS in createas.c
so that it can set the number of tuples transferred from the
workers to Gather node to 0. With this change, there are chances
that the planner may choose the parallel plan.
---
 src/backend/commands/createas.c       | 35 +++++++++++++++++-
 src/backend/commands/explain.c        |  7 ++--
 src/backend/commands/prepare.c        |  3 +-
 src/backend/optimizer/path/costsize.c | 22 ++++++++++-
 src/backend/optimizer/plan/planner.c  | 53 +++++++++++++++++++++++++++
 src/include/commands/createas.h       | 21 ++++++++++-
 src/include/commands/explain.h        |  3 +-
 src/include/nodes/parsenodes.h        |  1 +
 src/include/optimizer/planner.h       | 10 +++++
 9 files changed, 146 insertions(+), 9 deletions(-)

diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 10f4f2b4d7..210927d4f4 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -344,7 +344,8 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		 * into the target table. We need plan state to be initialized by the
 		 * executor to decide whether to allow parallel inserts or not.
 		 */
-		ChooseParallelInsertsInCTAS(into, queryDesc);
+		ChooseParallelInsertsInCTAS(into, queryDesc,
+									&query->CTASParallelInsInfo);
 
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
@@ -659,8 +660,11 @@ intorel_destroy(DestReceiver *self)
  * insertion is possible, if yes set the parallel insert state i.e. push down
  * the dest receiver to the Gather nodes.
  */
-void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc)
+void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc,
+								 uint8 *tuple_cost_flags)
 {
+	bool allow = false;
+
 	if (!IS_CTAS(into))
 		return;
 
@@ -695,5 +699,32 @@ void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc)
 		 * explain plans.
 		 */
 		queryDesc->planstate->plan->plan_rows = 0;
+
+		allow = true;
 	}
+
+	/*
+	 * It should not happen that in cost_gather we have ignored the parallel
+	 * tuple cost and now we are not allowing the parallel inserts. And also we
+	 * might need assertion only if the top node is Gather. The main intention
+	 * of assertion is to check if we enforced planner to ignore the parallel
+	 * tuple cost (with the intention of choosing parallel inserts) due to
+	 * which the parallel plan may have been chosen, but we do not allow the
+	 * parallel inserts now.
+	 */
+	if (!allow && tuple_cost_flags && queryDesc &&
+		IsA(queryDesc->planstate, GatherState))
+	{
+		/*
+		 * If we have correctly ignored parallel tuple cost in planner while
+		 * creating Gather path, then this assertion failure should not occur.
+		 * If it occurs, that means the planner may have chosen this parallel
+		 * plan because of our enforcement to ignore the parallel tuple cost.
+		 */
+		Assert(!(*tuple_cost_flags & CTAS_PARALLEL_INS_TUP_COST_IGNORED));
+	}
+
+	if (tuple_cost_flags)
+		*tuple_cost_flags = CTAS_PARALLEL_INS_UNDEF;
+
 }
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index fbd0bc5a81..efdb34d1f0 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -402,7 +402,8 @@ ExplainOneQuery(Query *query, int cursorOptions,
 
 		/* run it (if needed) and produce output */
 		ExplainOnePlan(plan, into, es, queryString, params, queryEnv,
-					   &planduration, (es->buffers ? &bufusage : NULL));
+					   &planduration, (es->buffers ? &bufusage : NULL),
+					   &query->CTASParallelInsInfo);
 	}
 }
 
@@ -496,7 +497,7 @@ void
 ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 			   const char *queryString, ParamListInfo params,
 			   QueryEnvironment *queryEnv, const instr_time *planduration,
-			   const BufferUsage *bufusage)
+			   const BufferUsage *bufusage, uint8 *ctas_tuple_cost_flags)
 {
 	DestReceiver *dest;
 	QueryDesc  *queryDesc;
@@ -562,7 +563,7 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	 * target table. We need plan state to be initialized by the executor to
 	 * decide whether to allow parallel inserts or not.
 	 */
-	ChooseParallelInsertsInCTAS(into, queryDesc);
+	ChooseParallelInsertsInCTAS(into, queryDesc, ctas_tuple_cost_flags);
 
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c
index 89087a7be3..07166479e7 100644
--- a/src/backend/commands/prepare.c
+++ b/src/backend/commands/prepare.c
@@ -672,7 +672,8 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es,
 
 		if (pstmt->commandType != CMD_UTILITY)
 			ExplainOnePlan(pstmt, into, es, query_string, paramLI, queryEnv,
-						   &planduration, (es->buffers ? &bufusage : NULL));
+						   &planduration, (es->buffers ? &bufusage : NULL),
+						   NULL);
 		else
 			ExplainOneUtility(pstmt->utilityStmt, into, es, query_string,
 							  paramLI, queryEnv);
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 22d6935824..800f25903d 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -76,6 +76,7 @@
 #include "access/amapi.h"
 #include "access/htup_details.h"
 #include "access/tsmapi.h"
+#include "commands/createas.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "executor/nodeHash.h"
@@ -378,6 +379,7 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 {
 	Cost		startup_cost = 0;
 	Cost		run_cost = 0;
+	bool		ignore_tuple_cost = false;
 
 	/* Mark the path with the correct row estimate */
 	if (rows)
@@ -393,7 +395,25 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 
 	/* Parallel setup and communication cost. */
 	startup_cost += parallel_setup_cost;
-	run_cost += parallel_tuple_cost * path->path.rows;
+
+	/*
+	 * Do not consider tuple cost in case of we intend to perform parallel
+	 * inserts by workers. We would have set ignore flag in
+	 * apply_scanjoin_target_to_paths before generating Gather path for the
+	 * upper level SELECT part of the CTAS.
+	 */
+	if ((root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT) &&
+		(root->parse->CTASParallelInsInfo &
+		 CTAS_PARALLEL_INS_TUP_COST_CAN_IGN))
+	{
+		ignore_tuple_cost = true;
+		root->parse->CTASParallelInsInfo &=
+									~CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_TUP_COST_IGNORED;
+	}
+
+	if (!ignore_tuple_cost)
+		run_cost += parallel_tuple_cost * path->path.rows;
 
 	path->path.startup_cost = startup_cost;
 	path->path.total_cost = (startup_cost + run_cost);
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index 1a94b58f8b..f1134711b0 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -28,6 +28,7 @@
 #include "catalog/pg_inherits.h"
 #include "catalog/pg_proc.h"
 #include "catalog/pg_type.h"
+#include "commands/createas.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "foreign/fdwapi.h"
@@ -7338,6 +7339,37 @@ can_partial_agg(PlannerInfo *root)
 	return true;
 }
 
+/*
+ * ignore_parallel_tuple_cost
+ *
+ * Gather node will not receive any tuples from the workers in case each worker
+ * inserts them in parallel. So, we set a flag to ignore parallel tuple cost by
+ * the Gather path in cost_gather if the SELECT is for CTAS and we are
+ * generating an upper level Gather path.
+*/
+static bool
+ignore_parallel_tuple_cost(PlannerInfo *root)
+{
+	if (root->query_level == 1 &&
+		(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT))
+	{
+		/*
+		 * In each of the HAS_PARENT_PATH_GENERATING_CLAUSE cases, a parent
+		 * path will be generated for the upper Gather path(in
+		 * grouping_planner), in which case we can not let parallel inserts
+		 * happen. So we do not set ignore tuple cost flag.
+		 */
+		if (HAS_PARENT_PATH_GENERATING_CLAUSE(root))
+			return false;
+
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+
+		return true;
+	}
+
+	return false;
+}
+
 /*
  * apply_scanjoin_target_to_paths
  *
@@ -7557,8 +7589,29 @@ apply_scanjoin_target_to_paths(PlannerInfo *root,
 	 * one of the generated paths may turn out to be the cheapest one.
 	 */
 	if (rel->consider_parallel && !IS_OTHER_REL(rel))
+	{
+		/*
+		 * Set a flag to ignore parallel tuple cost by the Gather path in
+		 * cost_gather if the SELECT is for CTAS and we are generating an upper
+		 * level Gather path.
+		 */
+		bool	ignore = ignore_parallel_tuple_cost(root);
+
 		generate_useful_gather_paths(root, rel, false);
 
+		/*
+		 * Reset the ignore flag, in case we set it but
+		 * generate_useful_gather_paths returned without reaching cost_gather.
+		 */
+		if (ignore &&
+			(root->parse->CTASParallelInsInfo &
+			 CTAS_PARALLEL_INS_TUP_COST_CAN_IGN))
+		{
+			root->parse->CTASParallelInsInfo &=
+										~CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+		}
+	}
+
 	/*
 	 * Reassess which paths are the cheapest, now that we've potentially added
 	 * new Gather (or Gather Merge) and/or Append (or MergeAppend) paths to
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index ed4690305b..0ae2f49e0c 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -39,6 +39,24 @@ typedef struct
 	Oid			object_id;
 } DR_intorel;
 
+/*
+ * Information sent to the planner from CTAS to account for the cost
+ * calculations in cost_gather. We need to do this because, no tuples will be
+ * received by the Gather node if the workers insert the tuples in parallel.
+ */
+typedef enum CTASParallelInsertOpt
+{
+	CTAS_PARALLEL_INS_UNDEF = 0, /* undefined */
+	CTAS_PARALLEL_INS_SELECT = 1 << 0, /* set to this before planning */
+	/*
+	 * Set to this while planning for upper Gather path to ignore parallel
+	 * tuple cost in cost_gather.
+	 */
+	CTAS_PARALLEL_INS_TUP_COST_CAN_IGN = 1 << 1,
+	/* Set to this after the cost is ignored. */
+	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2
+} CTASParallelInsertOpt;
+
 #define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
 #define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
 									 IS_CTAS(((DR_intorel *) dest)->into) && \
@@ -53,5 +71,6 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
 extern void ChooseParallelInsertsInCTAS(IntoClause *into,
-										QueryDesc *queryDesc);
+										QueryDesc *queryDesc,
+										uint8 *tuple_cost_flags);
 #endif							/* CREATEAS_H */
diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h
index ba661d32a6..1a1806dbf1 100644
--- a/src/include/commands/explain.h
+++ b/src/include/commands/explain.h
@@ -91,7 +91,8 @@ extern void ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into,
 						   ExplainState *es, const char *queryString,
 						   ParamListInfo params, QueryEnvironment *queryEnv,
 						   const instr_time *planduration,
-						   const BufferUsage *bufusage);
+						   const BufferUsage *bufusage,
+						   uint8 *ctas_tuple_cost_flags);
 
 extern void ExplainPrintPlan(ExplainState *es, QueryDesc *queryDesc);
 extern void ExplainPrintTriggers(ExplainState *es, QueryDesc *queryDesc);
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 48a79a7657..81b148c383 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -180,6 +180,7 @@ typedef struct Query
 	 */
 	int			stmt_location;	/* start location, or -1 if unknown */
 	int			stmt_len;		/* length in bytes; 0 means "rest of string" */
+	uint8 		CTASParallelInsInfo; /* parallel insert in CTAS info */
 } Query;
 
 
diff --git a/src/include/optimizer/planner.h b/src/include/optimizer/planner.h
index beb7dbbcbe..74b2563828 100644
--- a/src/include/optimizer/planner.h
+++ b/src/include/optimizer/planner.h
@@ -21,6 +21,16 @@
 #include "nodes/pathnodes.h"
 #include "nodes/plannodes.h"
 
+#define HAS_PARENT_PATH_GENERATING_CLAUSE(root) \
+											(root->parse->rowMarks || \
+											 limit_needed(root->parse) || \
+											 root->parse->sortClause ||  \
+											 root->parse->distinctClause ||  \
+											 root->parse->hasWindowFuncs ||  \
+											 root->parse->groupClause ||  \
+											 root->parse->groupingSets ||  \
+											 root->parse->hasAggs ||  \
+											 root->hasHavingQual)
 
 /* Hook for plugins to get control in planner() */
 typedef PlannedStmt *(*planner_hook_type) (Query *parse,
-- 
2.25.1

From d5183d9f831ac5ddc5e3b8417e24e6c29db11011 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Tue, 22 Dec 2020 13:19:22 +0530
Subject: [PATCH v14 3/4] Tests For Parallel Inserts in CTAS

---
 src/test/regress/expected/write_parallel.out | 577 +++++++++++++++++++
 src/test/regress/sql/write_parallel.sql      | 216 +++++++
 2 files changed, 793 insertions(+)

diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 0c4da2591a..357fbbbe8d 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -76,4 +76,581 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+-- Parallel queries won't necessarily get as many workers as the planner
+-- asked for.  This affects not only the "Workers Launched:" field of EXPLAIN
+-- results, but also row counts and loop counts for parallel scans, Gathers,
+-- and everything in between.  This function filters out the values we can't
+-- rely on to be stable.
+-- This removes enough info that you might wonder why bother with EXPLAIN
+-- ANALYZE at all.  The answer is that we need to see whether the parallel
+-- inserts are being done by the workers, the only way is that
+-- Create <<tbl_name>> appears in the explain output.
+create function explain_pictas(text) returns setof text
+language plpgsql as
+$$
+declare
+    ln text;
+begin
+    for ln in
+        execute format('explain (analyze, costs off, summary off, timing off) %s',
+            $1)
+    loop
+        ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N');
+        ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N');
+        ln := regexp_replace(ln, 'Rows Removed by Filter: \d+', 'Rows Removed by Filter: N');
+        return next ln;
+    end loop;
+end;
+$$;
+-- parallel inserts must occur
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'create temporary table parallel_write as select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(4 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+select explain_pictas(
+'create unlogged table parallel_write as select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+select explain_pictas(
+'select length(stringu1) into parallel_write from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'select length(stringu1) into temporary parallel_write from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(4 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+select explain_pictas(
+'select length(stringu1) into unlogged parallel_write from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1 for update;');
+                 explain_pictas                  
+-------------------------------------------------
+ LockRows (actual rows=N loops=N)
+   ->  Seq Scan on tenk1 (actual rows=N loops=N)
+(2 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+select explain_pictas(
+'create materialized view parallel_mat_view as
+    select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_mat_view
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_mat_view;
+ count 
+-------
+ 10000
+(1 row)
+
+drop materialized view parallel_mat_view;
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+select explain_pictas(
+'create table parallel_write as execute parallel_write_prep;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+deallocate parallel_write_prep;
+drop table parallel_write;
+-- parallel inserts must occur
+select explain_pictas(
+'create table parallel_write as select now(), four from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+select explain_pictas(
+E'create table parallel_write as
+    select nextval(\'parallel_write_sequence\'), four from tenk1;');
+              explain_pictas               
+-------------------------------------------
+ Seq Scan on tenk1 (actual rows=N loops=N)
+(1 row)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Params Evaluated: $1
+   Workers Launched: N
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 4
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on tenk2 (actual rows=N loops=N)
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+         Filter: (four = 3)
+         Rows Removed by Filter: N
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+  2500
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;');
+                          explain_pictas                          
+------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+         Filter: (four = 3)
+         Rows Removed by Filter: N
+   SubPlan 1
+     ->  Function Scan on generate_series (actual rows=N loops=N)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+  2500
+(1 row)
+
+drop table parallel_write;
+create table temp1(col1) as select * from generate_series(1,5);
+create table temp2(col2) as select * from temp1;
+create table temp3(col3) as select * from temp1;
+-- parallel inserts must not occur, as there is a limit clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 limit 4;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ Limit (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ Gather Merge (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+   ->  Sort (actual rows=N loops=N)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         Worker 0:  Sort Method: quicksort  Memory: 25kB
+         Worker 1:  Sort Method: quicksort  Memory: 25kB
+         Worker 2:  Sort Method: quicksort  Memory: 25kB
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ Gather Merge (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+   ->  Sort (actual rows=N loops=N)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         Worker 0:  Sort Method: quicksort  Memory: 25kB
+         Worker 1:  Sort Method: quicksort  Memory: 25kB
+         Worker 2:  Sort Method: quicksort  Memory: 25kB
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is a distinct clause
+select explain_pictas(
+'create table parallel_write as select distinct * from temp1;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ HashAggregate (actual rows=N loops=N)
+   Group Key: col1
+   Batches: 1  Memory Usage: 40kB
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(7 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an aggregate and group clause
+select explain_pictas(
+'create table parallel_write as select count(*) from temp1 group by col1;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Finalize HashAggregate (actual rows=N loops=N)
+   Group Key: col1
+   Batches: 1  Memory Usage: 40kB
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Partial HashAggregate (actual rows=N loops=N)
+               Group Key: col1
+               Batches: 1  Memory Usage: 40kB
+               Worker 0:  Batches: 1  Memory Usage: 40kB
+               Worker 1:  Batches: 1  Memory Usage: 40kB
+               Worker 2:  Batches: 1  Memory Usage: 40kB
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(13 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an aggregate, group and having
+-- clauses
+select explain_pictas(
+'create table parallel_write as
+    select count(col1), (select col3 from
+        (select * from temp3) as tt limit 1) col4 from temp1, temp2
+    where temp1.col1 = temp2.col2 group by col4 having count(col1) > 0;');
+                                  explain_pictas                                  
+----------------------------------------------------------------------------------
+ Finalize GroupAggregate (actual rows=N loops=N)
+   Group Key: ($1)
+   Filter: (count(temp1.col1) > 0)
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 3
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on temp3 (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $1
+         Workers Launched: N
+         ->  Partial GroupAggregate (actual rows=N loops=N)
+               Group Key: $1
+               ->  Parallel Hash Join (actual rows=N loops=N)
+                     Hash Cond: (temp1.col1 = temp2.col2)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                     ->  Parallel Hash (actual rows=N loops=N)
+                           Buckets: 4096  Batches: 1  Memory Usage: 64kB
+                           ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(21 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     1
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is a window function
+select explain_pictas(
+'create table parallel_write as
+    select avg(col1) OVER (PARTITION BY col1) from temp1;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ WindowAgg (actual rows=N loops=N)
+   ->  Gather Merge (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Sort (actual rows=N loops=N)
+               Sort Key: col1
+               Sort Method: quicksort  Memory: 25kB
+               Worker 0:  Sort Method: quicksort  Memory: 25kB
+               Worker 1:  Sort Method: quicksort  Memory: 25kB
+               Worker 2:  Sort Method: quicksort  Memory: 25kB
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(11 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- nested loop join is the top node under which Gather node exists, so parallel
+-- inserts must not occur
+set enable_nestloop to on;
+set enable_mergejoin to off;
+set enable_hashjoin to off;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Nested Loop (actual rows=N loops=N)
+   Join Filter: (temp1.col1 = temp2.col2)
+   Rows Removed by Join Filter: 20
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+   ->  Materialize (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Workers Launched: N
+               ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- the top node is Gather under which merge join happens, so parallel inserts
+-- must occur
+set enable_nestloop to off;
+set enable_mergejoin to on;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Merge Join (actual rows=N loops=N)
+         Merge Cond: (temp1.col1 = temp2.col2)
+         ->  Sort (actual rows=N loops=N)
+               Sort Key: temp1.col1
+               Sort Method: quicksort  Memory: 25kB
+               Worker 0:  Sort Method: quicksort  Memory: 25kB
+               Worker 1:  Sort Method: quicksort  Memory: 25kB
+               Worker 2:  Sort Method: quicksort  Memory: 25kB
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Sort (actual rows=N loops=N)
+               Sort Key: temp2.col2
+               Sort Method: quicksort  Memory: 25kB
+               ->  Seq Scan on temp2 (actual rows=N loops=N)
+(17 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel hash join happens under Gather node, so parallel inserts must occur
+set enable_mergejoin to off;
+set enable_hashjoin to on;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Hash Join (actual rows=N loops=N)
+         Hash Cond: (temp1.col1 = temp2.col2)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Hash (actual rows=N loops=N)
+               Buckets: 4096  Batches: 1  Memory Usage: 64kB
+               ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+reset enable_nestloop;
+reset enable_mergejoin;
+reset enable_hashjoin;
+drop table temp1;
+drop table temp2;
+drop table temp3;
+drop function explain_pictas(text);
 rollback;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 78b479cedf..46b958014a 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -39,4 +39,220 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
 
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+
+-- Parallel queries won't necessarily get as many workers as the planner
+-- asked for.  This affects not only the "Workers Launched:" field of EXPLAIN
+-- results, but also row counts and loop counts for parallel scans, Gathers,
+-- and everything in between.  This function filters out the values we can't
+-- rely on to be stable.
+-- This removes enough info that you might wonder why bother with EXPLAIN
+-- ANALYZE at all.  The answer is that we need to see whether the parallel
+-- inserts are being done by the workers, the only way is that
+-- Create <<tbl_name>> appears in the explain output.
+create function explain_pictas(text) returns setof text
+language plpgsql as
+$$
+declare
+    ln text;
+begin
+    for ln in
+        execute format('explain (analyze, costs off, summary off, timing off) %s',
+            $1)
+    loop
+        ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N');
+        ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N');
+        ln := regexp_replace(ln, 'Rows Removed by Filter: \d+', 'Rows Removed by Filter: N');
+        return next ln;
+    end loop;
+end;
+$$;
+
+-- parallel inserts must occur
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'create temporary table parallel_write as select length(stringu1) from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+select explain_pictas(
+'create unlogged table parallel_write as select length(stringu1) from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+select explain_pictas(
+'select length(stringu1) into parallel_write from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'select length(stringu1) into temporary parallel_write from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+select explain_pictas(
+'select length(stringu1) into unlogged parallel_write from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1 for update;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+select explain_pictas(
+'create materialized view parallel_mat_view as
+    select length(stringu1) from tenk1;');
+select count(*) from parallel_mat_view;
+drop materialized view parallel_mat_view;
+
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+select explain_pictas(
+'create table parallel_write as execute parallel_write_prep;');
+select count(*) from parallel_write;
+deallocate parallel_write_prep;
+drop table parallel_write;
+
+-- parallel inserts must occur
+select explain_pictas(
+'create table parallel_write as select now(), four from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+select explain_pictas(
+E'create table parallel_write as
+    select nextval(\'parallel_write_sequence\'), four from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+create table temp1(col1) as select * from generate_series(1,5);
+create table temp2(col2) as select * from temp1;
+create table temp3(col3) as select * from temp1;
+
+-- parallel inserts must not occur, as there is a limit clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 limit 4;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is a distinct clause
+select explain_pictas(
+'create table parallel_write as select distinct * from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an aggregate and group clause
+select explain_pictas(
+'create table parallel_write as select count(*) from temp1 group by col1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an aggregate, group and having
+-- clauses
+select explain_pictas(
+'create table parallel_write as
+    select count(col1), (select col3 from
+        (select * from temp3) as tt limit 1) col4 from temp1, temp2
+    where temp1.col1 = temp2.col2 group by col4 having count(col1) > 0;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is a window function
+select explain_pictas(
+'create table parallel_write as
+    select avg(col1) OVER (PARTITION BY col1) from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- nested loop join is the top node under which Gather node exists, so parallel
+-- inserts must not occur
+set enable_nestloop to on;
+set enable_mergejoin to off;
+set enable_hashjoin to off;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- the top node is Gather under which merge join happens, so parallel inserts
+-- must occur
+set enable_nestloop to off;
+set enable_mergejoin to on;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel hash join happens under Gather node, so parallel inserts must occur
+set enable_mergejoin to off;
+set enable_hashjoin to on;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+reset enable_nestloop;
+reset enable_mergejoin;
+reset enable_hashjoin;
+
+drop table temp1;
+drop table temp2;
+drop table temp3;
+drop function explain_pictas(text);
 rollback;
-- 
2.25.1

From 7f3cc3f322a971cdd23221b45810ceb47e50be84 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Tue, 22 Dec 2020 13:30:22 +0530
Subject: [PATCH v14 4/4] Enable CTAS Parallel Inserts For Append

This patch allows pushing down the CTAS dest receiver even if there
exists Gather node under Top Append node. It also add the code for
influencing the planner to consider parallel tuple cost zero and
asserts for wrong enforcement if later parallel insertion is not
possible. Test cases are also included in this patch.
---
 src/backend/commands/createas.c              | 104 ++-
 src/backend/optimizer/path/allpaths.c        |  39 +
 src/backend/optimizer/plan/planner.c         |  10 +-
 src/include/commands/createas.h              |   4 +-
 src/test/regress/expected/write_parallel.out | 722 +++++++++++++++++++
 src/test/regress/sql/write_parallel.sql      | 222 ++++++
 6 files changed, 1073 insertions(+), 28 deletions(-)

diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 210927d4f4..20d4f805d0 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -655,6 +655,78 @@ intorel_destroy(DestReceiver *self)
 	pfree(self);
 }
 
+/*
+ * PushDownCTASParallelInsertState --- push the dest receiver down to the
+ * Gather nodes.
+ *
+ * In this function we only care Append and Gather nodes.
+ *
+ * Push the dest receiver to Gather node when it is either at the top of the
+ * plan or under top Append node unless it does not have any projections to do.
+ * Required information from the pushed dest receiver is sent to workers so
+ * that they can perform parallel insertions into the target table.
+ *
+ * If the top node is Append, then this function recursively checks the sub
+ * plans for Gather nodes, when found one(and if it does not have projections),
+ * then sets the dest receiver information.
+ *
+ * In any case, this function returns true if at least one Gather node can allow
+ * parallel insertions by the workers. Otherwise returns false.
+ */
+static bool
+PushDownCTASParallelInsertState(DestReceiver *dest, PlanState *ps,
+								bool *gather_exists)
+{
+	bool parallel = false;
+
+	if (ps == NULL)
+		return parallel;
+
+	if (IsA(ps, AppendState))
+	{
+		AppendState *aps = (AppendState *) ps;
+
+		for (int i = 0; i < aps->as_nplans; i++)
+		{
+			parallel |= PushDownCTASParallelInsertState(dest,
+														aps->appendplans[i],
+														gather_exists);
+		}
+	}
+	else if (IsA(ps, GatherState))
+	{
+		/*
+		 * Set to true if there exists at least one Gather node either at the
+		 * top of the plan or as a direct sub node under Append node.
+		 */
+		 *gather_exists |= true;
+
+		if (!ps->ps_ProjInfo)
+		{
+			GatherState *gstate = (GatherState *) ps;
+			parallel = true;
+
+			/*
+			 * For parallelizing inserts in CTAS i.e. making each parallel
+			 * worker insert the tuples, we must send information such as into
+			 * clause (for each worker to build separate dest receiver), object
+			 * id (for each worker to open the created table).
+			 */
+			((DR_intorel *) dest)->is_parallel = true;
+			gstate->dest = dest;
+
+			/*
+			 * Since there are no rows that are transferred from workers to
+			 * Gather node, so we set it to 0 to be visible in estimated row
+			 * count of explain plans.
+			 */
+			ps->plan->plan_rows = 0;
+		}
+	}
+
+	return parallel;
+}
+
 /*
  * ChooseParallelInsertsInCTAS --- determine whether or not parallel
  * insertion is possible, if yes set the parallel insert state i.e. push down
@@ -664,6 +736,7 @@ void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc,
 								 uint8 *tuple_cost_flags)
 {
 	bool allow = false;
+	bool gather_exists = false;
 
 	if (!IS_CTAS(into))
 		return;
@@ -678,30 +751,12 @@ void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc,
 	if (!into->rel || into->rel->relpersistence == RELPERSISTENCE_TEMP)
 		return;
 
-	if (queryDesc && IsA(queryDesc->planstate, GatherState) &&
-		!queryDesc->planstate->ps_ProjInfo)
-	{
-		GatherState *gstate = (GatherState *) queryDesc->planstate;
-		DestReceiver *dest	= queryDesc->dest;
-
-		/*
-		 * For parallelizing inserts in CTAS i.e. making each parallel worker
-		 * insert the tuples, we must send information such as into clause (for
-		 * each worker to build separate dest receiver), object id (for each
-		 * worker to open the created table).
-		 */
-		((DR_intorel *) dest)->is_parallel = true;
-		gstate->dest = dest;
-
-		/*
-		 * Since there are no rows that are transferred from workers to Gather
-		 * node, so we set it to 0 to be visible in estimated row count of
-		 * explain plans.
-		 */
-		queryDesc->planstate->plan->plan_rows = 0;
+	if (!queryDesc)
+		return;
 
-		allow = true;
-	}
+	allow = PushDownCTASParallelInsertState(queryDesc->dest,
+											queryDesc->planstate,
+											&gather_exists);
 
 	/*
 	 * It should not happen that in cost_gather we have ignored the parallel
@@ -712,8 +767,7 @@ void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc,
 	 * which the parallel plan may have been chosen, but we do not allow the
 	 * parallel inserts now.
 	 */
-	if (!allow && tuple_cost_flags && queryDesc &&
-		IsA(queryDesc->planstate, GatherState))
+	if (!allow && tuple_cost_flags && gather_exists)
 	{
 		/*
 		 * If we have correctly ignored parallel tuple cost in planner while
diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c
index 627d08b78a..00819eea3f 100644
--- a/src/backend/optimizer/path/allpaths.c
+++ b/src/backend/optimizer/path/allpaths.c
@@ -23,6 +23,7 @@
 #include "catalog/pg_class.h"
 #include "catalog/pg_operator.h"
 #include "catalog/pg_proc.h"
+#include "commands/createas.h"
 #include "foreign/fdwapi.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1103,6 +1104,44 @@ set_append_rel_size(PlannerInfo *root, RelOptInfo *rel,
 		if (root->glob->parallelModeOK && rel->consider_parallel)
 			set_rel_consider_parallel(root, childrel, childRTE);
 
+		/*
+		 * When subplan is subquery, It's possible to do parallel insert if top
+		 * node of subquery is Gather, so we set the flag to ignore parallel
+		 * tuple cost by the Gather path in cost_gather if the SELECT is for
+		 * CTAS.
+		 */
+		if (childrel->rtekind == RTE_SUBQUERY)
+		{
+			/*
+			 * We set the flag for two cases when there is no parent path will
+			 * be created(such as : limit,sort,distinct...):
+			 * i) query_level is 1
+			 * ii) query_level > 1 then set the flag in the parent_root.
+			 * The case ii) is to check append under append:
+			 * Append
+			 *	->Append
+			 *	->Gather
+			 *	->Other plan
+			 */
+			if (root->parse->CTASParallelInsInfo &
+				CTAS_PARALLEL_INS_SELECT &&
+				(root->query_level == 1 ||
+				 root->parent_root->parse->CTASParallelInsInfo &
+				 CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND) &&
+				!(HAS_PARENT_PATH_GENERATING_CLAUSE(root)))
+			{
+				root->parse->CTASParallelInsInfo |=
+										CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+			}
+		}
+
+		if (root->parse->CTASParallelInsInfo &
+			CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND)
+		{
+			root->parse->CTASParallelInsInfo &=
+										~CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+		}
+
 		/*
 		 * Compute the child's size.
 		 */
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index f1134711b0..7555cde61a 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -7350,8 +7350,14 @@ can_partial_agg(PlannerInfo *root)
 static bool
 ignore_parallel_tuple_cost(PlannerInfo *root)
 {
-	if (root->query_level == 1 &&
-		(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT))
+	if (root->query_level != 1 &&
+		(root->parent_root->parse->CTASParallelInsInfo &
+		 CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND))
+	{
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_SELECT;
+	}
+
+	if (root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT)
 	{
 		/*
 		 * In each of the HAS_PARENT_PATH_GENERATING_CLAUSE cases, a parent
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 0ae2f49e0c..4103ac65f0 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -54,7 +54,9 @@ typedef enum CTASParallelInsertOpt
 	 */
 	CTAS_PARALLEL_INS_TUP_COST_CAN_IGN = 1 << 1,
 	/* Set to this after the cost is ignored. */
-	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2
+	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2,
+	/* Set to this in case tuple cost needs to be ignored for Append cases. */
+	CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND = 1 << 3
 } CTASParallelInsertOpt;
 
 #define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 357fbbbe8d..e0296e88a3 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -649,6 +649,728 @@ drop table parallel_write;
 reset enable_nestloop;
 reset enable_mergejoin;
 reset enable_hashjoin;
+-- test cases for performing parallel inserts when Append node is at the top
+-- and Gather node is in one of its direct sub plans.
+-- case 1: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5 union all
+    select * from temp2 where col2 = 5;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: N
+(22 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+-- case 2: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp2;');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Append (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select * from temp2;');
+                                 explain_pictas                                 
+--------------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Params Evaluated: $1, $3
+   Workers Launched: N
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 3
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+   InitPlan 2 (returns $3)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 3
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+   ->  Parallel Append (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(21 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+-- case 3: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples to
+-- Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Gather
+-- 		->Parallel Seq Scan
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1 union all
+    select * from temp1 where col1 = 5;');
+                                explain_pictas                                
+------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Seq Scan on temp1 (actual rows=N loops=N)
+         SubPlan 1
+           ->  Limit (actual rows=N loops=N)
+                 ->  Gather (actual rows=N loops=N)
+                       Workers Planned: 3
+                       Workers Launched: N
+                       ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+(22 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     7
+(1 row)
+
+drop table parallel_write;
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as select * from temp1 where col1 = (select 1) union all
+    select * from temp1 where col1 = (select 1) union all
+ select * from temp2 where col2 = (select 2);');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $0
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+               Filter: (col1 = $0)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $1
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 2 (returns $1)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+               Filter: (col1 = $1)
+               Rows Removed by Filter: N
+   ->  Seq Scan on temp2 (actual rows=N loops=N)
+         Filter: (col2 = $2)
+         Rows Removed by Filter: N
+         InitPlan 3 (returns $2)
+           ->  Result (actual rows=N loops=N)
+(26 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+alter table temp2 reset (parallel_workers);
+drop table parallel_write;
+-- case 4: parallel inserts must not occur as there will be no direct Gather
+-- node under Append node. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1;');
+                                    explain_pictas                                    
+--------------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Seq Scan on temp1 (actual rows=N loops=N)
+   ->  Seq Scan on temp2 (actual rows=N loops=N)
+   ->  Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+         SubPlan 1
+           ->  Limit (actual rows=N loops=N)
+                 ->  Gather (actual rows=N loops=N)
+                       Workers Planned: 3
+                       Workers Launched: N
+                       ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+-- case 5: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Seq Scan / Join / any other non-Gather node
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp1;');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Append (actual rows=N loops=N)
+         ->  Seq Scan on temp2 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+alter table temp2 reset (parallel_workers);
+-- case 6: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Gather
+--  ->Gather
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp2 where col2 = (select 2);');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $2
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($2 = $0)
+                     ->  Parallel Seq Scan on temp2 (never executed)
+                           Filter: (col2 = $0)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $3
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+               Filter: (col2 = $3)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $4
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 5 (returns $4)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+               Filter: (col2 = $4)
+               Rows Removed by Filter: N
+(47 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+                                   explain_pictas                                   
+------------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $2
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($2 = $0)
+                     ->  Parallel Seq Scan on temp2 (never executed)
+                           Filter: (col2 = $0)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $3, $4
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 5 (returns $4)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($4 = $3)
+                     ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+                           Filter: (col1 = $3)
+                           Rows Removed by Filter: N
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $3, $5
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 6 (returns $5)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($5 = $3)
+                     ->  Parallel Seq Scan on temp2 temp2_1 (never executed)
+                           Filter: (col2 = $3)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: N
+(67 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+-- case 7: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Seq Scan / Join / any other non-Gather node
+--  ->Gather
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp1 where col1 = (select 2);');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Result (actual rows=N loops=N)
+               One-Time Filter: ($2 = $0)
+               InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Seq Scan on temp2 (never executed)
+                     Filter: (col2 = $0)
+   ->  Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+         Filter: (col2 = $3)
+         Rows Removed by Filter: N
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $4
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 5 (returns $4)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+               Filter: (col1 = $4)
+               Rows Removed by Filter: N
+(37 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+                                   explain_pictas                                   
+------------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Result (actual rows=N loops=N)
+               One-Time Filter: ($2 = $0)
+               InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Seq Scan on temp2 (never executed)
+                     Filter: (col2 = $0)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $3, $4
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 5 (returns $4)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($4 = $3)
+                     ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+                           Filter: (col1 = $3)
+                           Rows Removed by Filter: N
+         ->  Result (actual rows=N loops=N)
+               One-Time Filter: ($5 = $3)
+               InitPlan 6 (returns $5)
+                 ->  Result (actual rows=N loops=N)
+               ->  Seq Scan on temp2 temp2_1 (never executed)
+                     Filter: (col2 = $3)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+         Filter: (col2 = 5)
+         Rows Removed by Filter: N
+(53 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+alter table temp2 reset (parallel_workers);
+-- case 8: parallel inserts must not occur because there is no Gather or Append
+-- node at the top for union, except/except all, intersect/intersect all
+-- cases.
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union
+    select * from temp2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ HashAggregate (actual rows=N loops=N)
+   Group Key: temp1.col1
+   Batches: 1  Memory Usage: 217kB
+   ->  Append (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Workers Launched: N
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Workers Launched: N
+               ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except
+    select * from temp2 where col2 < 3;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Except (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+                           Filter: (col2 < 3)
+                           Rows Removed by Filter: N
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except all
+    select * from temp2 where col2 < 3;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Except All (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+                           Filter: (col2 < 3)
+                           Rows Removed by Filter: N
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect
+    select * from temp2;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Intersect (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect all
+    select * from temp2;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Intersect All (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
 drop table temp1;
 drop table temp2;
 drop table temp3;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 46b958014a..4e03d0ab6b 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -251,6 +251,228 @@ reset enable_nestloop;
 reset enable_mergejoin;
 reset enable_hashjoin;
 
+-- test cases for performing parallel inserts when Append node is at the top
+-- and Gather node is in one of its direct sub plans.
+
+-- case 1: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5 union all
+    select * from temp2 where col2 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 2: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 3: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples to
+-- Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Gather
+-- 		->Parallel Seq Scan
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1 union all
+    select * from temp1 where col1 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as select * from temp1 where col1 = (select 1) union all
+    select * from temp1 where col1 = (select 1) union all
+ select * from temp2 where col2 = (select 2);');
+select count(*) from parallel_write;
+alter table temp2 reset (parallel_workers);
+drop table parallel_write;
+
+-- case 4: parallel inserts must not occur as there will be no direct Gather
+-- node under Append node. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 5: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Seq Scan / Join / any other non-Gather node
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+
+alter table temp2 set (parallel_workers = 0);
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 reset (parallel_workers);
+
+-- case 6: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Gather
+--  ->Gather
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp2 where col2 = (select 2);');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 7: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Seq Scan / Join / any other non-Gather node
+--  ->Gather
+
+alter table temp2 set (parallel_workers = 0);
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp1 where col1 = (select 2);');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 reset (parallel_workers);
+
+-- case 8: parallel inserts must not occur because there is no Gather or Append
+-- node at the top for union, except/except all, intersect/intersect all
+-- cases.
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except
+    select * from temp2 where col2 < 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except all
+    select * from temp2 where col2 < 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect all
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
 drop table temp1;
 drop table temp2;
 drop table temp3;
-- 
2.25.1

From a73d0e0e4c300e28d3e5e659eacd6e23141012ec Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Thu, 24 Dec 2020 12:58:26 +0530
Subject: [PATCH v15 1/4] Parallel Inserts in CREATE TABLE AS

Allow the leader and each worker insert the tuples in parallel
if the SELECT part of the CTAS is parallelizable.

The design:
The main idea is to push the CTAS dest receiver down to Gather node
and from there the required information will be shared to workers
so that they can perform parallel insertions. Leader will also
participate in insertions. After the planning, check if the upper
plan node is Gather in createas.c and mark a parallelism flag in
the CTAS dest receiver and push it down to Gather node. Each worker
can create its own CTAS dest receiver with the information passed
from the leader. Leader inserts its share of tuples if instructed
to do, and so are workers. Each worker writes atomically its number
of inserted tuples into a shared memory variable, the leader combines
this with its own number of inserted tuples and shares to the client.
---
 src/backend/access/heap/heapam.c       |  11 -
 src/backend/access/transam/xact.c      |  30 ++-
 src/backend/commands/createas.c        | 303 ++++++++++++++++---------
 src/backend/commands/explain.c         |  31 +++
 src/backend/executor/execParallel.c    |  70 +++++-
 src/backend/executor/nodeGather.c      | 113 ++++++++-
 src/backend/executor/nodeGatherMerge.c |   4 +-
 src/include/access/xact.h              |   1 +
 src/include/commands/createas.h        |  25 ++
 src/include/executor/execParallel.h    |   6 +-
 src/include/nodes/execnodes.h          |   3 +
 11 files changed, 457 insertions(+), 140 deletions(-)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index a9583f3103..86347ba273 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2043,17 +2043,6 @@ static HeapTuple
 heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
 					CommandId cid, int options)
 {
-	/*
-	 * To allow parallel inserts, we need to ensure that they are safe to be
-	 * performed in workers. We have the infrastructure to allow parallel
-	 * inserts in general except for the cases where inserts generate a new
-	 * CommandId (eg. inserts into a table having a foreign key column).
-	 */
-	if (IsParallelWorker())
-		ereport(ERROR,
-				(errcode(ERRCODE_INVALID_TRANSACTION_STATE),
-				 errmsg("cannot insert tuples in a parallel worker")));
-
 	tup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
 	tup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
 	tup->t_data->t_infomask |= HEAP_XMAX_INVALID;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 9cd0b7c11b..db6eedd635 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -763,18 +763,34 @@ GetCurrentCommandId(bool used)
 	/* this is global to a transaction, not subtransaction-local */
 	if (used)
 	{
-		/*
-		 * Forbid setting currentCommandIdUsed in a parallel worker, because
-		 * we have no provision for communicating this back to the leader.  We
-		 * could relax this restriction when currentCommandIdUsed was already
-		 * true at the start of the parallel operation.
-		 */
-		Assert(!IsParallelWorker());
+		 /*
+		  * This is a temporary hack for all common parallel insert cases i.e.
+		  * insert into, ctas, copy from. To be changed later. In a parallel
+		  * worker, set currentCommandIdUsed to true only if it was not set to
+		  * true at the start of the parallel operation (by way of
+		  * SetCurrentCommandIdUsedForWorker()). We have to do this because
+		  * GetCurrentCommandId(true) may be called from anywhere, especially
+		  * for parallel inserts, within parallel worker.
+		  */
+		Assert(!(IsParallelWorker() && !currentCommandIdUsed));
 		currentCommandIdUsed = true;
 	}
 	return currentCommandId;
 }
 
+/*
+ *	SetCurrentCommandIdUsedForWorker
+ *
+ * For a parallel worker, record that the currentCommandId has been used. This
+ * must only be called at the start of a parallel operation.
+ */
+void
+SetCurrentCommandIdUsedForWorker(void)
+{
+	Assert(IsParallelWorker() && !currentCommandIdUsed && currentCommandId != InvalidCommandId);
+	currentCommandIdUsed = true;
+}
+
 /*
  *	SetParallelStartTimestamps
  *
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 6bf6c5a310..10f4f2b4d7 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,18 +51,6 @@
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 
-typedef struct
-{
-	DestReceiver pub;			/* publicly-known function pointers */
-	IntoClause *into;			/* target relation specification */
-	/* These fields are filled by intorel_startup: */
-	Relation	rel;			/* relation to write to */
-	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
-	CommandId	output_cid;		/* cmin to insert in output tuples */
-	int			ti_options;		/* table_tuple_insert performance options */
-	BulkInsertState bistate;	/* bulk insert state */
-} DR_intorel;
-
 /* utility functions for CTAS definition creation */
 static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into);
 static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -350,6 +338,14 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		/* call ExecutorStart to prepare the plan for execution */
 		ExecutorStart(queryDesc, GetIntoRelEFlags(into));
 
+		/*
+		 * If SELECT part of the CTAS is parallelizable, then make each
+		 * parallel worker insert the tuples that are resulted in its execution
+		 * into the target table. We need plan state to be initialized by the
+		 * executor to decide whether to allow parallel inserts or not.
+		 */
+		ChooseParallelInsertsInCTAS(into, queryDesc);
+
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
 
@@ -418,6 +414,9 @@ CreateIntoRelDestReceiver(IntoClause *intoClause)
 	self->pub.rDestroy = intorel_destroy;
 	self->pub.mydest = DestIntoRel;
 	self->into = intoClause;
+	self->is_parallel = false;
+	self->is_parallel_worker = false;
+	self->object_id = InvalidOid;
 	/* other private fields will be set during intorel_startup */
 
 	return (DestReceiver *) self;
@@ -430,121 +429,169 @@ static void
 intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 {
 	DR_intorel *myState = (DR_intorel *) self;
-	IntoClause *into = myState->into;
-	bool		is_matview;
-	List	   *attrList;
 	ObjectAddress intoRelationAddr;
 	Relation	intoRelationDesc;
-	ListCell   *lc;
-	int			attnum;
 
-	Assert(into != NULL);		/* else somebody forgot to set it */
-
-	/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
-	is_matview = (into->viewQuery != NULL);
+	if (myState->is_parallel_worker)
+	{
+		/* In the worker */
+		intoRelationDesc = table_open(myState->object_id, AccessExclusiveLock);
+		myState->rel = intoRelationDesc;
+		myState->reladdr = InvalidObjectAddress;
+		myState->ti_options = 0;
+		myState->bistate = GetBulkInsertState();
 
-	/*
-	 * Build column definitions using "pre-cooked" type and collation info. If
-	 * a column name list was specified in CREATE TABLE AS, override the
-	 * column names derived from the query.  (Too few column names are OK, too
-	 * many are not.)
-	 */
-	attrList = NIL;
-	lc = list_head(into->colNames);
-	for (attnum = 0; attnum < typeinfo->natts; attnum++)
+		/*
+		 * Right after the table is created in the leader, the command id is
+		 * incremented (in create_ctas_internal()). The new command id is
+		 * marked as used in intorel_startup(), then the parallel mode is
+		 * entered. The command id and transaction id are serialized into
+		 * parallel DSM, they are then available to all parallel workers. All
+		 * the workers need to mark the command id as used before insertion.
+		 */
+		SetCurrentCommandIdUsedForWorker();
+		myState->output_cid = GetCurrentCommandId(false);
+	}
+	else
 	{
-		Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
-		ColumnDef  *col;
-		char	   *colname;
+		IntoClause *into = myState->into;
+		bool		is_matview;
+		List	   *attrList;
+		ListCell   *lc;
+		int			attnum;
 
-		if (lc)
+		Assert(into != NULL);		/* else somebody forgot to set it */
+
+		/*
+		 * This code supports both CREATE TABLE AS and CREATE MATERIALIZED
+		 * VIEW.
+		 */
+		is_matview = (into->viewQuery != NULL);
+
+		/*
+		 * Build column definitions using "pre-cooked" type and collation info.
+		 * If a column name list was specified in CREATE TABLE AS, override the
+		 * column names derived from the query.  (Too few column names are OK,
+		 * too many are not.)
+		 */
+		attrList = NIL;
+		lc = list_head(into->colNames);
+		for (attnum = 0; attnum < typeinfo->natts; attnum++)
 		{
-			colname = strVal(lfirst(lc));
-			lc = lnext(into->colNames, lc);
+			Form_pg_attribute attribute = TupleDescAttr(typeinfo, attnum);
+			ColumnDef  *col;
+			char	   *colname;
+
+			if (lc)
+			{
+				colname = strVal(lfirst(lc));
+				lc = lnext(into->colNames, lc);
+			}
+			else
+				colname = NameStr(attribute->attname);
+
+			col = makeColumnDef(colname,
+								attribute->atttypid,
+								attribute->atttypmod,
+								attribute->attcollation);
+
+			/*
+			 * It's possible that the column is of a collatable type but the
+			 * collation could not be resolved, so double-check.  (We must
+			 * check this here because DefineRelation would adopt the type's
+			 * default collation rather than complaining.)
+			 */
+			if (!OidIsValid(col->collOid) &&
+				type_is_collatable(col->typeName->typeOid))
+				ereport(ERROR,
+						(errcode(ERRCODE_INDETERMINATE_COLLATION),
+						errmsg("no collation was derived for column \"%s\" with collatable type %s",
+								col->colname,
+								format_type_be(col->typeName->typeOid)),
+						errhint("Use the COLLATE clause to set the collation explicitly.")));
+
+			attrList = lappend(attrList, col);
 		}
-		else
-			colname = NameStr(attribute->attname);
 
-		col = makeColumnDef(colname,
-							attribute->atttypid,
-							attribute->atttypmod,
-							attribute->attcollation);
+		if (lc != NULL)
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					errmsg("too many column names were specified")));
 
 		/*
-		 * It's possible that the column is of a collatable type but the
-		 * collation could not be resolved, so double-check.  (We must check
-		 * this here because DefineRelation would adopt the type's default
-		 * collation rather than complaining.)
+		 * Actually create the target table
 		 */
-		if (!OidIsValid(col->collOid) &&
-			type_is_collatable(col->typeName->typeOid))
-			ereport(ERROR,
-					(errcode(ERRCODE_INDETERMINATE_COLLATION),
-					 errmsg("no collation was derived for column \"%s\" with collatable type %s",
-							col->colname,
-							format_type_be(col->typeName->typeOid)),
-					 errhint("Use the COLLATE clause to set the collation explicitly.")));
+		intoRelationAddr = create_ctas_internal(attrList, into);
 
-		attrList = lappend(attrList, col);
-	}
+		/*
+		 * Finally we can open the target table
+		 */
+		intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
 
-	if (lc != NULL)
-		ereport(ERROR,
-				(errcode(ERRCODE_SYNTAX_ERROR),
-				 errmsg("too many column names were specified")));
+		/*
+		 * Make sure the constructed table does not have RLS enabled.
+		 *
+		 * check_enable_rls() will ereport(ERROR) itself if the user has
+		 * requested something invalid, and otherwise will return RLS_ENABLED
+		 * if RLS should be enabled here.  We don't actually support that
+		 * currently, so throw our own ereport(ERROR) if that happens.
+		 */
+		if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					errmsg("policies not yet implemented for this command")));
 
-	/*
-	 * Actually create the target table
-	 */
-	intoRelationAddr = create_ctas_internal(attrList, into);
+		/*
+		 * Tentatively mark the target as populated, if it's a matview and
+		 * we're going to fill it; otherwise, no change needed.
+		 */
+		if (is_matview && !into->skipData)
+			SetMatViewPopulatedState(intoRelationDesc, true);
 
-	/*
-	 * Finally we can open the target table
-	 */
-	intoRelationDesc = table_open(intoRelationAddr.objectId, AccessExclusiveLock);
+		/*
+		 * Fill private fields of myState for use by later routines
+		 */
+		myState->rel = intoRelationDesc;
+		myState->reladdr = intoRelationAddr;
+		myState->output_cid = GetCurrentCommandId(true);
+		myState->ti_options = TABLE_INSERT_SKIP_FSM;
 
-	/*
-	 * Make sure the constructed table does not have RLS enabled.
-	 *
-	 * check_enable_rls() will ereport(ERROR) itself if the user has requested
-	 * something invalid, and otherwise will return RLS_ENABLED if RLS should
-	 * be enabled here.  We don't actually support that currently, so throw
-	 * our own ereport(ERROR) if that happens.
-	 */
-	if (check_enable_rls(intoRelationAddr.objectId, InvalidOid, false) == RLS_ENABLED)
-		ereport(ERROR,
-				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-				 errmsg("policies not yet implemented for this command")));
+		/*
+		 * If WITH NO DATA is specified, there is no need to set up the state
+		 * for bulk inserts as there are no tuples to insert.
+		 */
+		if (!into->skipData)
+			myState->bistate = GetBulkInsertState();
+		else
+			myState->bistate = NULL;
 
-	/*
-	 * Tentatively mark the target as populated, if it's a matview and we're
-	 * going to fill it; otherwise, no change needed.
-	 */
-	if (is_matview && !into->skipData)
-		SetMatViewPopulatedState(intoRelationDesc, true);
+		if (myState->is_parallel)
+		{
+			myState->object_id = intoRelationAddr.objectId;
 
-	/*
-	 * Fill private fields of myState for use by later routines
-	 */
-	myState->rel = intoRelationDesc;
-	myState->reladdr = intoRelationAddr;
-	myState->output_cid = GetCurrentCommandId(true);
-	myState->ti_options = TABLE_INSERT_SKIP_FSM;
+			/*
+			 * We don't need to skip contacting FSM while inserting tuples
+			 * for parallel mode, while extending the relations, workers
+			 * instead of blocking on a page while another worker is inserting,
+			 * can check the FSM for another page that can accommodate the
+			 * tuples. This results in major benefit for parallel inserts.
+			 */
+			myState->ti_options = 0;
 
-	/*
-	 * If WITH NO DATA is specified, there is no need to set up the state for
-	 * bulk inserts as there are no tuples to insert.
-	 */
-	if (!into->skipData)
-		myState->bistate = GetBulkInsertState();
-	else
-		myState->bistate = NULL;
+			/*
+			 * rd_createSubid is marked invalid, otherwise, the table is not
+			 * allowed to be extended by the workers.
+			 */
+			myState->rel->rd_createSubid = InvalidSubTransactionId;
+		}
 
-	/*
-	 * Valid smgr_targblock implies something already wrote to the relation.
-	 * This may be harmless, but this function hasn't planned for it.
-	 */
-	Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+		/*
+		 * Valid smgr_targblock implies something already wrote to the
+		 * relation. This may be harmless, but this function hasn't planned for
+		 * it.
+		 */
+		Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber);
+	}
 }
 
 /*
@@ -606,3 +653,47 @@ intorel_destroy(DestReceiver *self)
 {
 	pfree(self);
 }
+
+/*
+ * ChooseParallelInsertsInCTAS --- determine whether or not parallel
+ * insertion is possible, if yes set the parallel insert state i.e. push down
+ * the dest receiver to the Gather nodes.
+ */
+void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc)
+{
+	if (!IS_CTAS(into))
+		return;
+
+	/*
+	 * Do not allow parallel inserts if the table is temporary. As the
+	 * temporary tables are backend local, workers can not know about them.
+	 * Currently, CTAS supports creation of normal(logged), temporary and
+	 * unlogged tables. It does not support foreign or partition table
+	 * creation. Hence the check for temporary table is enough here.
+	 */
+	if (!into->rel || into->rel->relpersistence == RELPERSISTENCE_TEMP)
+		return;
+
+	if (queryDesc && IsA(queryDesc->planstate, GatherState) &&
+		!queryDesc->planstate->ps_ProjInfo)
+	{
+		GatherState *gstate = (GatherState *) queryDesc->planstate;
+		DestReceiver *dest	= queryDesc->dest;
+
+		/*
+		 * For parallelizing inserts in CTAS i.e. making each parallel worker
+		 * insert the tuples, we must send information such as into clause (for
+		 * each worker to build separate dest receiver), object id (for each
+		 * worker to open the created table).
+		 */
+		((DR_intorel *) dest)->is_parallel = true;
+		gstate->dest = dest;
+
+		/*
+		 * Since there are no rows that are transferred from workers to Gather
+		 * node, so we set it to 0 to be visible in estimated row count of
+		 * explain plans.
+		 */
+		queryDesc->planstate->plan->plan_rows = 0;
+	}
+}
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 43f9b01e83..fbd0bc5a81 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -556,6 +556,14 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	/* call ExecutorStart to prepare the plan for execution */
 	ExecutorStart(queryDesc, eflags);
 
+	/*
+	 * If SELECT part of the CTAS is parallelizable, then make each parallel
+	 * worker insert the tuples that are resulted in its execution into the
+	 * target table. We need plan state to be initialized by the executor to
+	 * decide whether to allow parallel inserts or not.
+	 */
+	ChooseParallelInsertsInCTAS(into, queryDesc);
+
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
 	{
@@ -1775,6 +1783,29 @@ ExplainNode(PlanState *planstate, List *ancestors,
 
 				if (gather->single_copy || es->format != EXPLAIN_FORMAT_TEXT)
 					ExplainPropertyBool("Single Copy", gather->single_copy, es);
+
+				/*
+				 * Show the create table information under Gather node in case
+				 * parallel workers have inserted the rows.
+				 */
+				if (IsA(planstate, GatherState))
+				{
+					GatherState *gstate = (GatherState *) planstate;
+
+					if (IS_PARALLEL_CTAS_DEST(gstate->dest) &&
+						((DR_intorel *) gstate->dest)->into->rel &&
+						((DR_intorel *) gstate->dest)->into->rel->relname)
+					{
+						es->indent--;
+						ExplainIndentText(es);
+						appendStringInfoString(es->str, "->  ");
+						appendStringInfoString(es->str, "Create ");
+						appendStringInfo(es->str, "%s\n",
+							((DR_intorel *) gstate->dest)->into->rel->relname);
+						ExplainIndentText(es);
+						es->indent++;
+					}
+				}
 			}
 			break;
 		case T_GatherMerge:
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index befde52691..5aa4bde54d 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -23,6 +23,7 @@
 
 #include "postgres.h"
 
+#include "commands/createas.h"
 #include "executor/execParallel.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
@@ -65,6 +66,7 @@
 #define PARALLEL_KEY_QUERY_TEXT		UINT64CONST(0xE000000000000008)
 #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009)
 #define PARALLEL_KEY_WAL_USAGE			UINT64CONST(0xE00000000000000A)
+#define PARALLEL_KEY_INTO_CLAUSE		UINT64CONST(0xE00000000000000B)
 
 #define PARALLEL_TUPLE_QUEUE_SIZE		65536
 
@@ -77,6 +79,9 @@ typedef struct FixedParallelExecutorState
 	dsa_pointer param_exec;
 	int			eflags;
 	int			jit_flags;
+	Oid			objectid;		/* workers to open relation/table.  */
+	/* Number tuples inserted by all the workers. */
+	pg_atomic_uint64	processed;
 } FixedParallelExecutorState;
 
 /*
@@ -578,7 +583,8 @@ ExecParallelSetupTupleQueues(ParallelContext *pcxt, bool reinitialize)
 ParallelExecutorInfo *
 ExecInitParallelPlan(PlanState *planstate, EState *estate,
 					 Bitmapset *sendParams, int nworkers,
-					 int64 tuples_needed)
+					 int64 tuples_needed, IntoClause *intoclause,
+					 Oid objectid)
 {
 	ParallelExecutorInfo *pei;
 	ParallelContext *pcxt;
@@ -600,6 +606,8 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	Size		dsa_minsize = dsa_minimum_size();
 	char	   *query_string;
 	int			query_len;
+	char 		*intoclausestr = NULL;
+	int			intoclause_len = 0;
 
 	/*
 	 * Force any initplan outputs that we're going to pass to workers to be
@@ -712,6 +720,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_estimate_chunk(&pcxt->estimator, dsa_minsize);
 	shm_toc_estimate_keys(&pcxt->estimator, 1);
 
+	/* Estimate space for into clause for CTAS. */
+	if (IS_CTAS(intoclause) && OidIsValid(objectid))
+	{
+		intoclausestr = nodeToString(intoclause);
+		intoclause_len = strlen(intoclausestr) + 1;
+		shm_toc_estimate_chunk(&pcxt->estimator, intoclause_len);
+		shm_toc_estimate_keys(&pcxt->estimator, 1);
+	}
+
 	/* Everyone's had a chance to ask for space, so now create the DSM. */
 	InitializeParallelDSM(pcxt);
 
@@ -729,6 +746,14 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	fpes->param_exec = InvalidDsaPointer;
 	fpes->eflags = estate->es_top_eflags;
 	fpes->jit_flags = estate->es_jit_flags;
+	pg_atomic_init_u64(&fpes->processed, 0);
+	pei->processed = &fpes->processed;
+
+	if (intoclausestr && OidIsValid(objectid))
+		fpes->objectid = objectid;
+	else
+		fpes->objectid = InvalidOid;
+
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, fpes);
 
 	/* Store query string */
@@ -758,8 +783,18 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_WAL_USAGE, walusage_space);
 	pei->wal_usage = walusage_space;
 
-	/* Set up the tuple queues that the workers will write into. */
-	pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	if (intoclausestr)
+	{
+		char *intoclause_space = shm_toc_allocate(pcxt->toc,
+												  intoclause_len);
+		memcpy(intoclause_space, intoclausestr, intoclause_len);
+		shm_toc_insert(pcxt->toc, PARALLEL_KEY_INTO_CLAUSE, intoclause_space);
+	}
+	else
+	{
+		/* Set up the tuple queues that the workers will write into. */
+		pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	}
 
 	/* We don't need the TupleQueueReaders yet, though. */
 	pei->reader = NULL;
@@ -1387,12 +1422,30 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 	void	   *area_space;
 	dsa_area   *area;
 	ParallelWorkerContext pwcxt;
+	char		*intoclausestr = NULL;
+	IntoClause	*intoclause = NULL;
 
 	/* Get fixed-size state. */
 	fpes = shm_toc_lookup(toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
 
-	/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
-	receiver = ExecParallelGetReceiver(seg, toc);
+	intoclausestr = shm_toc_lookup(toc, PARALLEL_KEY_INTO_CLAUSE, true);
+	if (intoclausestr)
+	{
+		/*
+		 * If the worker is for parallel insert in CTAS, then use the proper
+		 * dest receiver.
+		 */
+		intoclause = (IntoClause *) stringToNode(intoclausestr);
+		receiver = CreateIntoRelDestReceiver(intoclause);
+		((DR_intorel *)receiver)->is_parallel_worker = true;
+		((DR_intorel *)receiver)->object_id = fpes->objectid;
+	}
+	else
+	{
+		/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
+		receiver = ExecParallelGetReceiver(seg, toc);
+	}
+
 	instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_INSTRUMENTATION, true);
 	if (instrumentation != NULL)
 		instrument_options = instrumentation->instrument_options;
@@ -1471,6 +1524,13 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 			queryDesc->estate->es_jit->instr;
 	}
 
+	/*
+	 * Write out the number of tuples this worker has inserted. Leader will use
+	 * it to inform the end client.
+	 */
+	if (intoclausestr)
+		pg_atomic_add_fetch_u64(&fpes->processed, queryDesc->estate->es_processed);
+
 	/* Must do this after capturing instrumentation. */
 	ExecutorEnd(queryDesc);
 
diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c
index a01b46af14..e7c588c66a 100644
--- a/src/backend/executor/nodeGather.c
+++ b/src/backend/executor/nodeGather.c
@@ -32,6 +32,7 @@
 
 #include "access/relscan.h"
 #include "access/xact.h"
+#include "commands/createas.h"
 #include "executor/execdebug.h"
 #include "executor/execParallel.h"
 #include "executor/nodeGather.h"
@@ -48,6 +49,7 @@ static TupleTableSlot *ExecGather(PlanState *pstate);
 static TupleTableSlot *gather_getnext(GatherState *gatherstate);
 static MinimalTuple gather_readnext(GatherState *gatherstate);
 static void ExecShutdownGatherWorkers(GatherState *node);
+static void ExecParallelInsertInCTAS(GatherState *node);
 
 
 /* ----------------------------------------------------------------
@@ -131,6 +133,72 @@ ExecInitGather(Gather *node, EState *estate, int eflags)
 	return gatherstate;
 }
 
+/* ----------------------------------------------------------------
+ *		ExecParallelInsertInCTAS(node)
+ *
+ *		Facilitates parallel inserts by parallel workers and/or
+ *		leader for Create Table AS.
+ * ----------------------------------------------------------------
+ */
+static void
+ExecParallelInsertInCTAS(GatherState *node)
+{
+	/* Enable leader to insert in case no parallel workers were launched. */
+	if (node->nworkers_launched == 0)
+		node->need_to_scan_locally = true;
+
+	/*
+	 * By now, for parallel workers (if launched any), would have started their
+	 * work i.e. insertion to target table. In case the leader is chosen to
+	 * participate for parallel inserts in CTAS, then finish its share before
+	 * going to wait for the parallel workers to finish.
+	 */
+	if (node->need_to_scan_locally)
+	{
+		EState	   *estate = node->ps.state;
+		TupleTableSlot *outerTupleSlot;
+
+		for(;;)
+		{
+			/* Install our DSA area while executing the plan. */
+			estate->es_query_dsa =
+					node->pei ? node->pei->area : NULL;
+
+			outerTupleSlot = ExecProcNode(node->ps.lefttree);
+
+			estate->es_query_dsa = NULL;
+
+			if(TupIsNull(outerTupleSlot))
+				break;
+
+			(void) node->dest->receiveSlot(outerTupleSlot, node->dest);
+
+			node->ps.state->es_processed++;
+		}
+
+		node->need_to_scan_locally = false;
+	}
+
+	if (node->nworkers_launched > 0)
+	{
+		/*
+		 * We wait here for the parallel workers to finish their work and
+		 * accumulate the tuples they inserted and also their buffer/WAL usage.
+		 * We do not destroy the parallel context here, it will be done in
+		 * ExecShutdownGather at the end of the plan. Note that the
+		 * ExecShutdownGatherWorkers call from ExecShutdownGather will be a
+		 * no-op.
+		 */
+		ExecShutdownGatherWorkers(node);
+
+		/*
+		 * Add up the total tuples inserted by all workers, to the tuples
+		 * inserted by the leader(if any). This will be shared to client.
+		 */
+		node->ps.state->es_processed += pg_atomic_read_u64(node->pei->processed);
+	}
+}
+
 /* ----------------------------------------------------------------
  *		ExecGather(node)
  *
@@ -157,6 +225,7 @@ ExecGather(PlanState *pstate)
 	{
 		EState	   *estate = node->ps.state;
 		Gather	   *gather = (Gather *) node->ps.plan;
+		bool		isctas = IS_PARALLEL_CTAS_DEST(node->dest);
 
 		/*
 		 * Sometimes we might have to run without parallelism; but if parallel
@@ -165,6 +234,18 @@ ExecGather(PlanState *pstate)
 		if (gather->num_workers > 0 && estate->es_use_parallel_mode)
 		{
 			ParallelContext *pcxt;
+			IntoClause	*intoclause = NULL;
+			Oid			objectid = InvalidOid;
+
+			/*
+			 * Take the necessary information to be passed to workers for
+			 * parallel inserts in CTAS.
+			 */
+			if (isctas)
+			{
+				intoclause = ((DR_intorel *) node->dest)->into;
+				objectid = ((DR_intorel *) node->dest)->object_id;
+			}
 
 			/* Initialize, or re-initialize, shared state needed by workers. */
 			if (!node->pei)
@@ -172,7 +253,10 @@ ExecGather(PlanState *pstate)
 												 estate,
 												 gather->initParam,
 												 gather->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 /* CTAS info */
+												 intoclause,
+												 objectid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
@@ -190,13 +274,16 @@ ExecGather(PlanState *pstate)
 			/* Set up tuple queue readers to read the results. */
 			if (pcxt->nworkers_launched > 0)
 			{
-				ExecParallelCreateReaders(node->pei);
-				/* Make a working array showing the active readers */
-				node->nreaders = pcxt->nworkers_launched;
-				node->reader = (TupleQueueReader **)
-					palloc(node->nreaders * sizeof(TupleQueueReader *));
-				memcpy(node->reader, node->pei->reader,
-					   node->nreaders * sizeof(TupleQueueReader *));
+				if (!isctas)
+				{
+					ExecParallelCreateReaders(node->pei);
+					/* Make a working array showing the active readers */
+					node->nreaders = pcxt->nworkers_launched;
+					node->reader = (TupleQueueReader **)
+						palloc(node->nreaders * sizeof(TupleQueueReader *));
+					memcpy(node->reader, node->pei->reader,
+						node->nreaders * sizeof(TupleQueueReader *));
+				}
 			}
 			else
 			{
@@ -208,9 +295,17 @@ ExecGather(PlanState *pstate)
 		}
 
 		/* Run plan locally if no workers or enabled and not single-copy. */
-		node->need_to_scan_locally = (node->nreaders == 0)
+		node->need_to_scan_locally = (node->nreaders == 0 &&
+			!isctas)
 			|| (!gather->single_copy && parallel_leader_participation);
 		node->initialized = true;
+
+		/* Perform parallel inserts for CTAS. */
+		if (isctas)
+		{
+			ExecParallelInsertInCTAS(node);
+			return NULL;
+		}
 	}
 
 	/*
diff --git a/src/backend/executor/nodeGatherMerge.c b/src/backend/executor/nodeGatherMerge.c
index 47129344f3..ee45272c17 100644
--- a/src/backend/executor/nodeGatherMerge.c
+++ b/src/backend/executor/nodeGatherMerge.c
@@ -216,7 +216,9 @@ ExecGatherMerge(PlanState *pstate)
 												 estate,
 												 gm->initParam,
 												 gm->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 NULL,
+												 InvalidOid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7320de345c..5beae6c617 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -389,6 +389,7 @@ extern FullTransactionId GetCurrentFullTransactionIdIfAny(void);
 extern void MarkCurrentTransactionIdLoggedIfAny(void);
 extern bool SubTransactionIsActive(SubTransactionId subxid);
 extern CommandId GetCurrentCommandId(bool used);
+extern void SetCurrentCommandIdUsedForWorker(void);
 extern void SetParallelStartTimestamps(TimestampTz xact_ts, TimestampTz stmt_ts);
 extern TimestampTz GetCurrentTransactionStartTimestamp(void);
 extern TimestampTz GetCurrentStatementStartTimestamp(void);
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 7629230254..ed4690305b 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -14,12 +14,35 @@
 #ifndef CREATEAS_H
 #define CREATEAS_H
 
+#include "access/heapam.h"
 #include "catalog/objectaddress.h"
+#include "executor/execdesc.h"
 #include "nodes/params.h"
+#include "nodes/plannodes.h"
 #include "parser/parse_node.h"
 #include "tcop/dest.h"
 #include "utils/queryenvironment.h"
 
+typedef struct
+{
+	DestReceiver pub;			/* publicly-known function pointers */
+	IntoClause *into;			/* target relation specification */
+	/* These fields are filled by intorel_startup: */
+	Relation	rel;			/* relation to write to */
+	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
+	CommandId	output_cid;		/* cmin to insert in output tuples */
+	int			ti_options;		/* table_tuple_insert performance options */
+	BulkInsertState bistate;	/* bulk insert state */
+	bool		is_parallel;	/* is parallelism to be considered? */
+	bool		is_parallel_worker; /* true for parallel worker */
+	/* Used for table open by parallel worker. */
+	Oid			object_id;
+} DR_intorel;
+
+#define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
+#define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
+									 IS_CTAS(((DR_intorel *) dest)->into) && \
+									 ((DR_intorel *) dest)->is_parallel)
 
 extern ObjectAddress ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 									   ParamListInfo params, QueryEnvironment *queryEnv,
@@ -29,4 +52,6 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
+extern void ChooseParallelInsertsInCTAS(IntoClause *into,
+										QueryDesc *queryDesc);
 #endif							/* CREATEAS_H */
diff --git a/src/include/executor/execParallel.h b/src/include/executor/execParallel.h
index 5a39a5b29c..9f959f741b 100644
--- a/src/include/executor/execParallel.h
+++ b/src/include/executor/execParallel.h
@@ -35,11 +35,15 @@ typedef struct ParallelExecutorInfo
 	/* These two arrays have pcxt->nworkers_launched entries: */
 	shm_mq_handle **tqueue;		/* tuple queues for worker output */
 	struct TupleQueueReader **reader;	/* tuple reader/writer support */
+	/* Number of tuples inserted by all workers. */
+	volatile pg_atomic_uint64	*processed;
 } ParallelExecutorInfo;
 
 extern ParallelExecutorInfo *ExecInitParallelPlan(PlanState *planstate,
 												  EState *estate, Bitmapset *sendParam, int nworkers,
-												  int64 tuples_needed);
+												  int64 tuples_needed,
+												  IntoClause *intoclause,
+												  Oid objectid);
 extern void ExecParallelCreateReaders(ParallelExecutorInfo *pei);
 extern void ExecParallelFinish(ParallelExecutorInfo *pei);
 extern void ExecParallelCleanup(ParallelExecutorInfo *pei);
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 61ba4c3666..e9c4442c22 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -23,6 +23,7 @@
 #include "nodes/tidbitmap.h"
 #include "partitioning/partdefs.h"
 #include "storage/condition_variable.h"
+#include "tcop/dest.h"
 #include "utils/hsearch.h"
 #include "utils/queryenvironment.h"
 #include "utils/reltrigger.h"
@@ -2326,6 +2327,8 @@ typedef struct GatherState
 	int			nreaders;		/* number of still-active workers */
 	int			nextreader;		/* next one to try to read from */
 	struct TupleQueueReader **reader;	/* array with nreaders active entries */
+	/* Parallel inserts in CTAS related info is specified below. */
+	DestReceiver *dest;
 } GatherState;
 
 /* ----------------
-- 
2.25.1

From 2484cfb3e57c79788448b04bfc1760b59fd20c4d Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 21 Dec 2020 15:19:23 +0530
Subject: [PATCH v15 2/4] Tuple Cost Adjustment for Parallel Inserts in CTAS

Let the planner know that the SELECT is from CTAS in createas.c
so that it can set the number of tuples transferred from the
workers to Gather node to 0. With this change, there are chances
that the planner may choose the parallel plan.
---
 src/backend/commands/createas.c       | 35 +++++++++++++++++-
 src/backend/commands/explain.c        |  7 ++--
 src/backend/commands/prepare.c        |  3 +-
 src/backend/optimizer/path/costsize.c | 22 ++++++++++-
 src/backend/optimizer/plan/planner.c  | 53 +++++++++++++++++++++++++++
 src/include/commands/createas.h       | 21 ++++++++++-
 src/include/commands/explain.h        |  3 +-
 src/include/nodes/parsenodes.h        |  1 +
 src/include/optimizer/planner.h       | 10 +++++
 9 files changed, 146 insertions(+), 9 deletions(-)

diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 10f4f2b4d7..210927d4f4 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -344,7 +344,8 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		 * into the target table. We need plan state to be initialized by the
 		 * executor to decide whether to allow parallel inserts or not.
 		 */
-		ChooseParallelInsertsInCTAS(into, queryDesc);
+		ChooseParallelInsertsInCTAS(into, queryDesc,
+									&query->CTASParallelInsInfo);
 
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
@@ -659,8 +660,11 @@ intorel_destroy(DestReceiver *self)
  * insertion is possible, if yes set the parallel insert state i.e. push down
  * the dest receiver to the Gather nodes.
  */
-void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc)
+void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc,
+								 uint8 *tuple_cost_flags)
 {
+	bool allow = false;
+
 	if (!IS_CTAS(into))
 		return;
 
@@ -695,5 +699,32 @@ void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc)
 		 * explain plans.
 		 */
 		queryDesc->planstate->plan->plan_rows = 0;
+
+		allow = true;
 	}
+
+	/*
+	 * It should not happen that in cost_gather we have ignored the parallel
+	 * tuple cost and now we are not allowing the parallel inserts. And also we
+	 * might need assertion only if the top node is Gather. The main intention
+	 * of assertion is to check if we enforced planner to ignore the parallel
+	 * tuple cost (with the intention of choosing parallel inserts) due to
+	 * which the parallel plan may have been chosen, but we do not allow the
+	 * parallel inserts now.
+	 */
+	if (!allow && tuple_cost_flags && queryDesc &&
+		IsA(queryDesc->planstate, GatherState))
+	{
+		/*
+		 * If we have correctly ignored parallel tuple cost in planner while
+		 * creating Gather path, then this assertion failure should not occur.
+		 * If it occurs, that means the planner may have chosen this parallel
+		 * plan because of our enforcement to ignore the parallel tuple cost.
+		 */
+		Assert(!(*tuple_cost_flags & CTAS_PARALLEL_INS_TUP_COST_IGNORED));
+	}
+
+	if (tuple_cost_flags)
+		*tuple_cost_flags = CTAS_PARALLEL_INS_UNDEF;
+
 }
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index fbd0bc5a81..efdb34d1f0 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -402,7 +402,8 @@ ExplainOneQuery(Query *query, int cursorOptions,
 
 		/* run it (if needed) and produce output */
 		ExplainOnePlan(plan, into, es, queryString, params, queryEnv,
-					   &planduration, (es->buffers ? &bufusage : NULL));
+					   &planduration, (es->buffers ? &bufusage : NULL),
+					   &query->CTASParallelInsInfo);
 	}
 }
 
@@ -496,7 +497,7 @@ void
 ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 			   const char *queryString, ParamListInfo params,
 			   QueryEnvironment *queryEnv, const instr_time *planduration,
-			   const BufferUsage *bufusage)
+			   const BufferUsage *bufusage, uint8 *ctas_tuple_cost_flags)
 {
 	DestReceiver *dest;
 	QueryDesc  *queryDesc;
@@ -562,7 +563,7 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	 * target table. We need plan state to be initialized by the executor to
 	 * decide whether to allow parallel inserts or not.
 	 */
-	ChooseParallelInsertsInCTAS(into, queryDesc);
+	ChooseParallelInsertsInCTAS(into, queryDesc, ctas_tuple_cost_flags);
 
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c
index 89087a7be3..07166479e7 100644
--- a/src/backend/commands/prepare.c
+++ b/src/backend/commands/prepare.c
@@ -672,7 +672,8 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es,
 
 		if (pstmt->commandType != CMD_UTILITY)
 			ExplainOnePlan(pstmt, into, es, query_string, paramLI, queryEnv,
-						   &planduration, (es->buffers ? &bufusage : NULL));
+						   &planduration, (es->buffers ? &bufusage : NULL),
+						   NULL);
 		else
 			ExplainOneUtility(pstmt->utilityStmt, into, es, query_string,
 							  paramLI, queryEnv);
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 22d6935824..800f25903d 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -76,6 +76,7 @@
 #include "access/amapi.h"
 #include "access/htup_details.h"
 #include "access/tsmapi.h"
+#include "commands/createas.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "executor/nodeHash.h"
@@ -378,6 +379,7 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 {
 	Cost		startup_cost = 0;
 	Cost		run_cost = 0;
+	bool		ignore_tuple_cost = false;
 
 	/* Mark the path with the correct row estimate */
 	if (rows)
@@ -393,7 +395,25 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 
 	/* Parallel setup and communication cost. */
 	startup_cost += parallel_setup_cost;
-	run_cost += parallel_tuple_cost * path->path.rows;
+
+	/*
+	 * Do not consider tuple cost in case of we intend to perform parallel
+	 * inserts by workers. We would have set ignore flag in
+	 * apply_scanjoin_target_to_paths before generating Gather path for the
+	 * upper level SELECT part of the CTAS.
+	 */
+	if ((root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT) &&
+		(root->parse->CTASParallelInsInfo &
+		 CTAS_PARALLEL_INS_TUP_COST_CAN_IGN))
+	{
+		ignore_tuple_cost = true;
+		root->parse->CTASParallelInsInfo &=
+									~CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_TUP_COST_IGNORED;
+	}
+
+	if (!ignore_tuple_cost)
+		run_cost += parallel_tuple_cost * path->path.rows;
 
 	path->path.startup_cost = startup_cost;
 	path->path.total_cost = (startup_cost + run_cost);
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index 1a94b58f8b..f1134711b0 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -28,6 +28,7 @@
 #include "catalog/pg_inherits.h"
 #include "catalog/pg_proc.h"
 #include "catalog/pg_type.h"
+#include "commands/createas.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "foreign/fdwapi.h"
@@ -7338,6 +7339,37 @@ can_partial_agg(PlannerInfo *root)
 	return true;
 }
 
+/*
+ * ignore_parallel_tuple_cost
+ *
+ * Gather node will not receive any tuples from the workers in case each worker
+ * inserts them in parallel. So, we set a flag to ignore parallel tuple cost by
+ * the Gather path in cost_gather if the SELECT is for CTAS and we are
+ * generating an upper level Gather path.
+*/
+static bool
+ignore_parallel_tuple_cost(PlannerInfo *root)
+{
+	if (root->query_level == 1 &&
+		(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT))
+	{
+		/*
+		 * In each of the HAS_PARENT_PATH_GENERATING_CLAUSE cases, a parent
+		 * path will be generated for the upper Gather path(in
+		 * grouping_planner), in which case we can not let parallel inserts
+		 * happen. So we do not set ignore tuple cost flag.
+		 */
+		if (HAS_PARENT_PATH_GENERATING_CLAUSE(root))
+			return false;
+
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+
+		return true;
+	}
+
+	return false;
+}
+
 /*
  * apply_scanjoin_target_to_paths
  *
@@ -7557,8 +7589,29 @@ apply_scanjoin_target_to_paths(PlannerInfo *root,
 	 * one of the generated paths may turn out to be the cheapest one.
 	 */
 	if (rel->consider_parallel && !IS_OTHER_REL(rel))
+	{
+		/*
+		 * Set a flag to ignore parallel tuple cost by the Gather path in
+		 * cost_gather if the SELECT is for CTAS and we are generating an upper
+		 * level Gather path.
+		 */
+		bool	ignore = ignore_parallel_tuple_cost(root);
+
 		generate_useful_gather_paths(root, rel, false);
 
+		/*
+		 * Reset the ignore flag, in case we set it but
+		 * generate_useful_gather_paths returned without reaching cost_gather.
+		 */
+		if (ignore &&
+			(root->parse->CTASParallelInsInfo &
+			 CTAS_PARALLEL_INS_TUP_COST_CAN_IGN))
+		{
+			root->parse->CTASParallelInsInfo &=
+										~CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+		}
+	}
+
 	/*
 	 * Reassess which paths are the cheapest, now that we've potentially added
 	 * new Gather (or Gather Merge) and/or Append (or MergeAppend) paths to
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index ed4690305b..0ae2f49e0c 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -39,6 +39,24 @@ typedef struct
 	Oid			object_id;
 } DR_intorel;
 
+/*
+ * Information sent to the planner from CTAS to account for the cost
+ * calculations in cost_gather. We need to do this because, no tuples will be
+ * received by the Gather node if the workers insert the tuples in parallel.
+ */
+typedef enum CTASParallelInsertOpt
+{
+	CTAS_PARALLEL_INS_UNDEF = 0, /* undefined */
+	CTAS_PARALLEL_INS_SELECT = 1 << 0, /* set to this before planning */
+	/*
+	 * Set to this while planning for upper Gather path to ignore parallel
+	 * tuple cost in cost_gather.
+	 */
+	CTAS_PARALLEL_INS_TUP_COST_CAN_IGN = 1 << 1,
+	/* Set to this after the cost is ignored. */
+	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2
+} CTASParallelInsertOpt;
+
 #define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
 #define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
 									 IS_CTAS(((DR_intorel *) dest)->into) && \
@@ -53,5 +71,6 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
 extern void ChooseParallelInsertsInCTAS(IntoClause *into,
-										QueryDesc *queryDesc);
+										QueryDesc *queryDesc,
+										uint8 *tuple_cost_flags);
 #endif							/* CREATEAS_H */
diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h
index ba661d32a6..1a1806dbf1 100644
--- a/src/include/commands/explain.h
+++ b/src/include/commands/explain.h
@@ -91,7 +91,8 @@ extern void ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into,
 						   ExplainState *es, const char *queryString,
 						   ParamListInfo params, QueryEnvironment *queryEnv,
 						   const instr_time *planduration,
-						   const BufferUsage *bufusage);
+						   const BufferUsage *bufusage,
+						   uint8 *ctas_tuple_cost_flags);
 
 extern void ExplainPrintPlan(ExplainState *es, QueryDesc *queryDesc);
 extern void ExplainPrintTriggers(ExplainState *es, QueryDesc *queryDesc);
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 48a79a7657..81b148c383 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -180,6 +180,7 @@ typedef struct Query
 	 */
 	int			stmt_location;	/* start location, or -1 if unknown */
 	int			stmt_len;		/* length in bytes; 0 means "rest of string" */
+	uint8 		CTASParallelInsInfo; /* parallel insert in CTAS info */
 } Query;
 
 
diff --git a/src/include/optimizer/planner.h b/src/include/optimizer/planner.h
index beb7dbbcbe..74b2563828 100644
--- a/src/include/optimizer/planner.h
+++ b/src/include/optimizer/planner.h
@@ -21,6 +21,16 @@
 #include "nodes/pathnodes.h"
 #include "nodes/plannodes.h"
 
+#define HAS_PARENT_PATH_GENERATING_CLAUSE(root) \
+											(root->parse->rowMarks || \
+											 limit_needed(root->parse) || \
+											 root->parse->sortClause ||  \
+											 root->parse->distinctClause ||  \
+											 root->parse->hasWindowFuncs ||  \
+											 root->parse->groupClause ||  \
+											 root->parse->groupingSets ||  \
+											 root->parse->hasAggs ||  \
+											 root->hasHavingQual)
 
 /* Hook for plugins to get control in planner() */
 typedef PlannedStmt *(*planner_hook_type) (Query *parse,
-- 
2.25.1

From d5183d9f831ac5ddc5e3b8417e24e6c29db11011 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Tue, 22 Dec 2020 13:19:22 +0530
Subject: [PATCH v15 3/4] Tests For Parallel Inserts in CTAS

---
 src/test/regress/expected/write_parallel.out | 577 +++++++++++++++++++
 src/test/regress/sql/write_parallel.sql      | 216 +++++++
 2 files changed, 793 insertions(+)

diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 0c4da2591a..357fbbbe8d 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -76,4 +76,581 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+-- Parallel queries won't necessarily get as many workers as the planner
+-- asked for.  This affects not only the "Workers Launched:" field of EXPLAIN
+-- results, but also row counts and loop counts for parallel scans, Gathers,
+-- and everything in between.  This function filters out the values we can't
+-- rely on to be stable.
+-- This removes enough info that you might wonder why bother with EXPLAIN
+-- ANALYZE at all.  The answer is that we need to see whether the parallel
+-- inserts are being done by the workers, the only way is that
+-- Create <<tbl_name>> appears in the explain output.
+create function explain_pictas(text) returns setof text
+language plpgsql as
+$$
+declare
+    ln text;
+begin
+    for ln in
+        execute format('explain (analyze, costs off, summary off, timing off) %s',
+            $1)
+    loop
+        ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N');
+        ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N');
+        ln := regexp_replace(ln, 'Rows Removed by Filter: \d+', 'Rows Removed by Filter: N');
+        return next ln;
+    end loop;
+end;
+$$;
+-- parallel inserts must occur
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'create temporary table parallel_write as select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(4 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+select explain_pictas(
+'create unlogged table parallel_write as select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+select explain_pictas(
+'select length(stringu1) into parallel_write from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'select length(stringu1) into temporary parallel_write from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(4 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+select explain_pictas(
+'select length(stringu1) into unlogged parallel_write from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1 for update;');
+                 explain_pictas                  
+-------------------------------------------------
+ LockRows (actual rows=N loops=N)
+   ->  Seq Scan on tenk1 (actual rows=N loops=N)
+(2 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+select explain_pictas(
+'create materialized view parallel_mat_view as
+    select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_mat_view
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_mat_view;
+ count 
+-------
+ 10000
+(1 row)
+
+drop materialized view parallel_mat_view;
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+select explain_pictas(
+'create table parallel_write as execute parallel_write_prep;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+deallocate parallel_write_prep;
+drop table parallel_write;
+-- parallel inserts must occur
+select explain_pictas(
+'create table parallel_write as select now(), four from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+select explain_pictas(
+E'create table parallel_write as
+    select nextval(\'parallel_write_sequence\'), four from tenk1;');
+              explain_pictas               
+-------------------------------------------
+ Seq Scan on tenk1 (actual rows=N loops=N)
+(1 row)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Params Evaluated: $1
+   Workers Launched: N
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 4
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on tenk2 (actual rows=N loops=N)
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+         Filter: (four = 3)
+         Rows Removed by Filter: N
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+  2500
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;');
+                          explain_pictas                          
+------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+         Filter: (four = 3)
+         Rows Removed by Filter: N
+   SubPlan 1
+     ->  Function Scan on generate_series (actual rows=N loops=N)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+  2500
+(1 row)
+
+drop table parallel_write;
+create table temp1(col1) as select * from generate_series(1,5);
+create table temp2(col2) as select * from temp1;
+create table temp3(col3) as select * from temp1;
+-- parallel inserts must not occur, as there is a limit clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 limit 4;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ Limit (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ Gather Merge (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+   ->  Sort (actual rows=N loops=N)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         Worker 0:  Sort Method: quicksort  Memory: 25kB
+         Worker 1:  Sort Method: quicksort  Memory: 25kB
+         Worker 2:  Sort Method: quicksort  Memory: 25kB
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ Gather Merge (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+   ->  Sort (actual rows=N loops=N)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         Worker 0:  Sort Method: quicksort  Memory: 25kB
+         Worker 1:  Sort Method: quicksort  Memory: 25kB
+         Worker 2:  Sort Method: quicksort  Memory: 25kB
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is a distinct clause
+select explain_pictas(
+'create table parallel_write as select distinct * from temp1;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ HashAggregate (actual rows=N loops=N)
+   Group Key: col1
+   Batches: 1  Memory Usage: 40kB
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(7 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an aggregate and group clause
+select explain_pictas(
+'create table parallel_write as select count(*) from temp1 group by col1;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Finalize HashAggregate (actual rows=N loops=N)
+   Group Key: col1
+   Batches: 1  Memory Usage: 40kB
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Partial HashAggregate (actual rows=N loops=N)
+               Group Key: col1
+               Batches: 1  Memory Usage: 40kB
+               Worker 0:  Batches: 1  Memory Usage: 40kB
+               Worker 1:  Batches: 1  Memory Usage: 40kB
+               Worker 2:  Batches: 1  Memory Usage: 40kB
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(13 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an aggregate, group and having
+-- clauses
+select explain_pictas(
+'create table parallel_write as
+    select count(col1), (select col3 from
+        (select * from temp3) as tt limit 1) col4 from temp1, temp2
+    where temp1.col1 = temp2.col2 group by col4 having count(col1) > 0;');
+                                  explain_pictas                                  
+----------------------------------------------------------------------------------
+ Finalize GroupAggregate (actual rows=N loops=N)
+   Group Key: ($1)
+   Filter: (count(temp1.col1) > 0)
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 3
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on temp3 (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $1
+         Workers Launched: N
+         ->  Partial GroupAggregate (actual rows=N loops=N)
+               Group Key: $1
+               ->  Parallel Hash Join (actual rows=N loops=N)
+                     Hash Cond: (temp1.col1 = temp2.col2)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                     ->  Parallel Hash (actual rows=N loops=N)
+                           Buckets: 4096  Batches: 1  Memory Usage: 64kB
+                           ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(21 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     1
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is a window function
+select explain_pictas(
+'create table parallel_write as
+    select avg(col1) OVER (PARTITION BY col1) from temp1;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ WindowAgg (actual rows=N loops=N)
+   ->  Gather Merge (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Sort (actual rows=N loops=N)
+               Sort Key: col1
+               Sort Method: quicksort  Memory: 25kB
+               Worker 0:  Sort Method: quicksort  Memory: 25kB
+               Worker 1:  Sort Method: quicksort  Memory: 25kB
+               Worker 2:  Sort Method: quicksort  Memory: 25kB
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(11 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- nested loop join is the top node under which Gather node exists, so parallel
+-- inserts must not occur
+set enable_nestloop to on;
+set enable_mergejoin to off;
+set enable_hashjoin to off;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Nested Loop (actual rows=N loops=N)
+   Join Filter: (temp1.col1 = temp2.col2)
+   Rows Removed by Join Filter: 20
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+   ->  Materialize (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Workers Launched: N
+               ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- the top node is Gather under which merge join happens, so parallel inserts
+-- must occur
+set enable_nestloop to off;
+set enable_mergejoin to on;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Merge Join (actual rows=N loops=N)
+         Merge Cond: (temp1.col1 = temp2.col2)
+         ->  Sort (actual rows=N loops=N)
+               Sort Key: temp1.col1
+               Sort Method: quicksort  Memory: 25kB
+               Worker 0:  Sort Method: quicksort  Memory: 25kB
+               Worker 1:  Sort Method: quicksort  Memory: 25kB
+               Worker 2:  Sort Method: quicksort  Memory: 25kB
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Sort (actual rows=N loops=N)
+               Sort Key: temp2.col2
+               Sort Method: quicksort  Memory: 25kB
+               ->  Seq Scan on temp2 (actual rows=N loops=N)
+(17 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel hash join happens under Gather node, so parallel inserts must occur
+set enable_mergejoin to off;
+set enable_hashjoin to on;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Hash Join (actual rows=N loops=N)
+         Hash Cond: (temp1.col1 = temp2.col2)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Hash (actual rows=N loops=N)
+               Buckets: 4096  Batches: 1  Memory Usage: 64kB
+               ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+reset enable_nestloop;
+reset enable_mergejoin;
+reset enable_hashjoin;
+drop table temp1;
+drop table temp2;
+drop table temp3;
+drop function explain_pictas(text);
 rollback;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 78b479cedf..46b958014a 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -39,4 +39,220 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
 
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+
+-- Parallel queries won't necessarily get as many workers as the planner
+-- asked for.  This affects not only the "Workers Launched:" field of EXPLAIN
+-- results, but also row counts and loop counts for parallel scans, Gathers,
+-- and everything in between.  This function filters out the values we can't
+-- rely on to be stable.
+-- This removes enough info that you might wonder why bother with EXPLAIN
+-- ANALYZE at all.  The answer is that we need to see whether the parallel
+-- inserts are being done by the workers, the only way is that
+-- Create <<tbl_name>> appears in the explain output.
+create function explain_pictas(text) returns setof text
+language plpgsql as
+$$
+declare
+    ln text;
+begin
+    for ln in
+        execute format('explain (analyze, costs off, summary off, timing off) %s',
+            $1)
+    loop
+        ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N');
+        ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N');
+        ln := regexp_replace(ln, 'Rows Removed by Filter: \d+', 'Rows Removed by Filter: N');
+        return next ln;
+    end loop;
+end;
+$$;
+
+-- parallel inserts must occur
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'create temporary table parallel_write as select length(stringu1) from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+select explain_pictas(
+'create unlogged table parallel_write as select length(stringu1) from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+select explain_pictas(
+'select length(stringu1) into parallel_write from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'select length(stringu1) into temporary parallel_write from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+select explain_pictas(
+'select length(stringu1) into unlogged parallel_write from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1 for update;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+select explain_pictas(
+'create materialized view parallel_mat_view as
+    select length(stringu1) from tenk1;');
+select count(*) from parallel_mat_view;
+drop materialized view parallel_mat_view;
+
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+select explain_pictas(
+'create table parallel_write as execute parallel_write_prep;');
+select count(*) from parallel_write;
+deallocate parallel_write_prep;
+drop table parallel_write;
+
+-- parallel inserts must occur
+select explain_pictas(
+'create table parallel_write as select now(), four from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+select explain_pictas(
+E'create table parallel_write as
+    select nextval(\'parallel_write_sequence\'), four from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+create table temp1(col1) as select * from generate_series(1,5);
+create table temp2(col2) as select * from temp1;
+create table temp3(col3) as select * from temp1;
+
+-- parallel inserts must not occur, as there is a limit clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 limit 4;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is a distinct clause
+select explain_pictas(
+'create table parallel_write as select distinct * from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an aggregate and group clause
+select explain_pictas(
+'create table parallel_write as select count(*) from temp1 group by col1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an aggregate, group and having
+-- clauses
+select explain_pictas(
+'create table parallel_write as
+    select count(col1), (select col3 from
+        (select * from temp3) as tt limit 1) col4 from temp1, temp2
+    where temp1.col1 = temp2.col2 group by col4 having count(col1) > 0;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is a window function
+select explain_pictas(
+'create table parallel_write as
+    select avg(col1) OVER (PARTITION BY col1) from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- nested loop join is the top node under which Gather node exists, so parallel
+-- inserts must not occur
+set enable_nestloop to on;
+set enable_mergejoin to off;
+set enable_hashjoin to off;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- the top node is Gather under which merge join happens, so parallel inserts
+-- must occur
+set enable_nestloop to off;
+set enable_mergejoin to on;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel hash join happens under Gather node, so parallel inserts must occur
+set enable_mergejoin to off;
+set enable_hashjoin to on;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+reset enable_nestloop;
+reset enable_mergejoin;
+reset enable_hashjoin;
+
+drop table temp1;
+drop table temp2;
+drop table temp3;
+drop function explain_pictas(text);
 rollback;
-- 
2.25.1

From 7f3cc3f322a971cdd23221b45810ceb47e50be84 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Tue, 22 Dec 2020 13:30:22 +0530
Subject: [PATCH v15 4/4] Enable CTAS Parallel Inserts For Append

This patch allows pushing down the CTAS dest receiver even if there
exists Gather node under Top Append node. It also add the code for
influencing the planner to consider parallel tuple cost zero and
asserts for wrong enforcement if later parallel insertion is not
possible. Test cases are also included in this patch.
---
 src/backend/commands/createas.c              | 104 ++-
 src/backend/optimizer/path/allpaths.c        |  39 +
 src/backend/optimizer/plan/planner.c         |  10 +-
 src/include/commands/createas.h              |   4 +-
 src/test/regress/expected/write_parallel.out | 722 +++++++++++++++++++
 src/test/regress/sql/write_parallel.sql      | 222 ++++++
 6 files changed, 1073 insertions(+), 28 deletions(-)

diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 210927d4f4..20d4f805d0 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -655,6 +655,78 @@ intorel_destroy(DestReceiver *self)
 	pfree(self);
 }
 
+/*
+ * PushDownCTASParallelInsertState --- push the dest receiver down to the
+ * Gather nodes.
+ *
+ * In this function we only care Append and Gather nodes.
+ *
+ * Push the dest receiver to Gather node when it is either at the top of the
+ * plan or under top Append node unless it does not have any projections to do.
+ * Required information from the pushed dest receiver is sent to workers so
+ * that they can perform parallel insertions into the target table.
+ *
+ * If the top node is Append, then this function recursively checks the sub
+ * plans for Gather nodes, when found one(and if it does not have projections),
+ * then sets the dest receiver information.
+ *
+ * In any case, this function returns true if at least one Gather node can allow
+ * parallel insertions by the workers. Otherwise returns false.
+ */
+static bool
+PushDownCTASParallelInsertState(DestReceiver *dest, PlanState *ps,
+								bool *gather_exists)
+{
+	bool parallel = false;
+
+	if (ps == NULL)
+		return parallel;
+
+	if (IsA(ps, AppendState))
+	{
+		AppendState *aps = (AppendState *) ps;
+
+		for (int i = 0; i < aps->as_nplans; i++)
+		{
+			parallel |= PushDownCTASParallelInsertState(dest,
+														aps->appendplans[i],
+														gather_exists);
+		}
+	}
+	else if (IsA(ps, GatherState))
+	{
+		/*
+		 * Set to true if there exists at least one Gather node either at the
+		 * top of the plan or as a direct sub node under Append node.
+		 */
+		 *gather_exists |= true;
+
+		if (!ps->ps_ProjInfo)
+		{
+			GatherState *gstate = (GatherState *) ps;
+			parallel = true;
+
+			/*
+			 * For parallelizing inserts in CTAS i.e. making each parallel
+			 * worker insert the tuples, we must send information such as into
+			 * clause (for each worker to build separate dest receiver), object
+			 * id (for each worker to open the created table).
+			 */
+			((DR_intorel *) dest)->is_parallel = true;
+			gstate->dest = dest;
+
+			/*
+			 * Since there are no rows that are transferred from workers to
+			 * Gather node, so we set it to 0 to be visible in estimated row
+			 * count of explain plans.
+			 */
+			ps->plan->plan_rows = 0;
+		}
+	}
+
+	return parallel;
+}
+
 /*
  * ChooseParallelInsertsInCTAS --- determine whether or not parallel
  * insertion is possible, if yes set the parallel insert state i.e. push down
@@ -664,6 +736,7 @@ void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc,
 								 uint8 *tuple_cost_flags)
 {
 	bool allow = false;
+	bool gather_exists = false;
 
 	if (!IS_CTAS(into))
 		return;
@@ -678,30 +751,12 @@ void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc,
 	if (!into->rel || into->rel->relpersistence == RELPERSISTENCE_TEMP)
 		return;
 
-	if (queryDesc && IsA(queryDesc->planstate, GatherState) &&
-		!queryDesc->planstate->ps_ProjInfo)
-	{
-		GatherState *gstate = (GatherState *) queryDesc->planstate;
-		DestReceiver *dest	= queryDesc->dest;
-
-		/*
-		 * For parallelizing inserts in CTAS i.e. making each parallel worker
-		 * insert the tuples, we must send information such as into clause (for
-		 * each worker to build separate dest receiver), object id (for each
-		 * worker to open the created table).
-		 */
-		((DR_intorel *) dest)->is_parallel = true;
-		gstate->dest = dest;
-
-		/*
-		 * Since there are no rows that are transferred from workers to Gather
-		 * node, so we set it to 0 to be visible in estimated row count of
-		 * explain plans.
-		 */
-		queryDesc->planstate->plan->plan_rows = 0;
+	if (!queryDesc)
+		return;
 
-		allow = true;
-	}
+	allow = PushDownCTASParallelInsertState(queryDesc->dest,
+											queryDesc->planstate,
+											&gather_exists);
 
 	/*
 	 * It should not happen that in cost_gather we have ignored the parallel
@@ -712,8 +767,7 @@ void ChooseParallelInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc,
 	 * which the parallel plan may have been chosen, but we do not allow the
 	 * parallel inserts now.
 	 */
-	if (!allow && tuple_cost_flags && queryDesc &&
-		IsA(queryDesc->planstate, GatherState))
+	if (!allow && tuple_cost_flags && gather_exists)
 	{
 		/*
 		 * If we have correctly ignored parallel tuple cost in planner while
diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c
index 627d08b78a..00819eea3f 100644
--- a/src/backend/optimizer/path/allpaths.c
+++ b/src/backend/optimizer/path/allpaths.c
@@ -23,6 +23,7 @@
 #include "catalog/pg_class.h"
 #include "catalog/pg_operator.h"
 #include "catalog/pg_proc.h"
+#include "commands/createas.h"
 #include "foreign/fdwapi.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1103,6 +1104,44 @@ set_append_rel_size(PlannerInfo *root, RelOptInfo *rel,
 		if (root->glob->parallelModeOK && rel->consider_parallel)
 			set_rel_consider_parallel(root, childrel, childRTE);
 
+		/*
+		 * When subplan is subquery, It's possible to do parallel insert if top
+		 * node of subquery is Gather, so we set the flag to ignore parallel
+		 * tuple cost by the Gather path in cost_gather if the SELECT is for
+		 * CTAS.
+		 */
+		if (childrel->rtekind == RTE_SUBQUERY)
+		{
+			/*
+			 * We set the flag for two cases when there is no parent path will
+			 * be created(such as : limit,sort,distinct...):
+			 * i) query_level is 1
+			 * ii) query_level > 1 then set the flag in the parent_root.
+			 * The case ii) is to check append under append:
+			 * Append
+			 *	->Append
+			 *	->Gather
+			 *	->Other plan
+			 */
+			if (root->parse->CTASParallelInsInfo &
+				CTAS_PARALLEL_INS_SELECT &&
+				(root->query_level == 1 ||
+				 root->parent_root->parse->CTASParallelInsInfo &
+				 CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND) &&
+				!(HAS_PARENT_PATH_GENERATING_CLAUSE(root)))
+			{
+				root->parse->CTASParallelInsInfo |=
+										CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+			}
+		}
+
+		if (root->parse->CTASParallelInsInfo &
+			CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND)
+		{
+			root->parse->CTASParallelInsInfo &=
+										~CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+		}
+
 		/*
 		 * Compute the child's size.
 		 */
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index f1134711b0..7555cde61a 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -7350,8 +7350,14 @@ can_partial_agg(PlannerInfo *root)
 static bool
 ignore_parallel_tuple_cost(PlannerInfo *root)
 {
-	if (root->query_level == 1 &&
-		(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT))
+	if (root->query_level != 1 &&
+		(root->parent_root->parse->CTASParallelInsInfo &
+		 CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND))
+	{
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_SELECT;
+	}
+
+	if (root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT)
 	{
 		/*
 		 * In each of the HAS_PARENT_PATH_GENERATING_CLAUSE cases, a parent
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 0ae2f49e0c..4103ac65f0 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -54,7 +54,9 @@ typedef enum CTASParallelInsertOpt
 	 */
 	CTAS_PARALLEL_INS_TUP_COST_CAN_IGN = 1 << 1,
 	/* Set to this after the cost is ignored. */
-	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2
+	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2,
+	/* Set to this in case tuple cost needs to be ignored for Append cases. */
+	CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND = 1 << 3
 } CTASParallelInsertOpt;
 
 #define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 357fbbbe8d..e0296e88a3 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -649,6 +649,728 @@ drop table parallel_write;
 reset enable_nestloop;
 reset enable_mergejoin;
 reset enable_hashjoin;
+-- test cases for performing parallel inserts when Append node is at the top
+-- and Gather node is in one of its direct sub plans.
+-- case 1: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5 union all
+    select * from temp2 where col2 = 5;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: N
+(22 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+-- case 2: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp2;');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Append (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select * from temp2;');
+                                 explain_pictas                                 
+--------------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Params Evaluated: $1, $3
+   Workers Launched: N
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 3
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+   InitPlan 2 (returns $3)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 3
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+   ->  Parallel Append (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(21 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+-- case 3: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples to
+-- Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Gather
+-- 		->Parallel Seq Scan
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1 union all
+    select * from temp1 where col1 = 5;');
+                                explain_pictas                                
+------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Seq Scan on temp1 (actual rows=N loops=N)
+         SubPlan 1
+           ->  Limit (actual rows=N loops=N)
+                 ->  Gather (actual rows=N loops=N)
+                       Workers Planned: 3
+                       Workers Launched: N
+                       ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+(22 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     7
+(1 row)
+
+drop table parallel_write;
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as select * from temp1 where col1 = (select 1) union all
+    select * from temp1 where col1 = (select 1) union all
+ select * from temp2 where col2 = (select 2);');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $0
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+               Filter: (col1 = $0)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $1
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 2 (returns $1)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+               Filter: (col1 = $1)
+               Rows Removed by Filter: N
+   ->  Seq Scan on temp2 (actual rows=N loops=N)
+         Filter: (col2 = $2)
+         Rows Removed by Filter: N
+         InitPlan 3 (returns $2)
+           ->  Result (actual rows=N loops=N)
+(26 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+alter table temp2 reset (parallel_workers);
+drop table parallel_write;
+-- case 4: parallel inserts must not occur as there will be no direct Gather
+-- node under Append node. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1;');
+                                    explain_pictas                                    
+--------------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Seq Scan on temp1 (actual rows=N loops=N)
+   ->  Seq Scan on temp2 (actual rows=N loops=N)
+   ->  Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+         SubPlan 1
+           ->  Limit (actual rows=N loops=N)
+                 ->  Gather (actual rows=N loops=N)
+                       Workers Planned: 3
+                       Workers Launched: N
+                       ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+-- case 5: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Seq Scan / Join / any other non-Gather node
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp1;');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Append (actual rows=N loops=N)
+         ->  Seq Scan on temp2 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+alter table temp2 reset (parallel_workers);
+-- case 6: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Gather
+--  ->Gather
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp2 where col2 = (select 2);');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $2
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($2 = $0)
+                     ->  Parallel Seq Scan on temp2 (never executed)
+                           Filter: (col2 = $0)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $3
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+               Filter: (col2 = $3)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $4
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 5 (returns $4)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+               Filter: (col2 = $4)
+               Rows Removed by Filter: N
+(47 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+                                   explain_pictas                                   
+------------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $2
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($2 = $0)
+                     ->  Parallel Seq Scan on temp2 (never executed)
+                           Filter: (col2 = $0)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $3, $4
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 5 (returns $4)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($4 = $3)
+                     ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+                           Filter: (col1 = $3)
+                           Rows Removed by Filter: N
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $3, $5
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 6 (returns $5)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($5 = $3)
+                     ->  Parallel Seq Scan on temp2 temp2_1 (never executed)
+                           Filter: (col2 = $3)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: N
+(67 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+-- case 7: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Seq Scan / Join / any other non-Gather node
+--  ->Gather
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp1 where col1 = (select 2);');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Result (actual rows=N loops=N)
+               One-Time Filter: ($2 = $0)
+               InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Seq Scan on temp2 (never executed)
+                     Filter: (col2 = $0)
+   ->  Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+         Filter: (col2 = $3)
+         Rows Removed by Filter: N
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $4
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 5 (returns $4)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+               Filter: (col1 = $4)
+               Rows Removed by Filter: N
+(37 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+                                   explain_pictas                                   
+------------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Result (actual rows=N loops=N)
+               One-Time Filter: ($2 = $0)
+               InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Seq Scan on temp2 (never executed)
+                     Filter: (col2 = $0)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $3, $4
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 5 (returns $4)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($4 = $3)
+                     ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+                           Filter: (col1 = $3)
+                           Rows Removed by Filter: N
+         ->  Result (actual rows=N loops=N)
+               One-Time Filter: ($5 = $3)
+               InitPlan 6 (returns $5)
+                 ->  Result (actual rows=N loops=N)
+               ->  Seq Scan on temp2 temp2_1 (never executed)
+                     Filter: (col2 = $3)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+         Filter: (col2 = 5)
+         Rows Removed by Filter: N
+(53 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+alter table temp2 reset (parallel_workers);
+-- case 8: parallel inserts must not occur because there is no Gather or Append
+-- node at the top for union, except/except all, intersect/intersect all
+-- cases.
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union
+    select * from temp2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ HashAggregate (actual rows=N loops=N)
+   Group Key: temp1.col1
+   Batches: 1  Memory Usage: 217kB
+   ->  Append (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Workers Launched: N
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Workers Launched: N
+               ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except
+    select * from temp2 where col2 < 3;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Except (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+                           Filter: (col2 < 3)
+                           Rows Removed by Filter: N
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except all
+    select * from temp2 where col2 < 3;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Except All (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+                           Filter: (col2 < 3)
+                           Rows Removed by Filter: N
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect
+    select * from temp2;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Intersect (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect all
+    select * from temp2;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Intersect All (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
 drop table temp1;
 drop table temp2;
 drop table temp3;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 46b958014a..4e03d0ab6b 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -251,6 +251,228 @@ reset enable_nestloop;
 reset enable_mergejoin;
 reset enable_hashjoin;
 
+-- test cases for performing parallel inserts when Append node is at the top
+-- and Gather node is in one of its direct sub plans.
+
+-- case 1: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5 union all
+    select * from temp2 where col2 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 2: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 3: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples to
+-- Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Gather
+-- 		->Parallel Seq Scan
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1 union all
+    select * from temp1 where col1 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as select * from temp1 where col1 = (select 1) union all
+    select * from temp1 where col1 = (select 1) union all
+ select * from temp2 where col2 = (select 2);');
+select count(*) from parallel_write;
+alter table temp2 reset (parallel_workers);
+drop table parallel_write;
+
+-- case 4: parallel inserts must not occur as there will be no direct Gather
+-- node under Append node. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 5: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Seq Scan / Join / any other non-Gather node
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+
+alter table temp2 set (parallel_workers = 0);
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 reset (parallel_workers);
+
+-- case 6: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Gather
+--  ->Gather
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp2 where col2 = (select 2);');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 7: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Seq Scan / Join / any other non-Gather node
+--  ->Gather
+
+alter table temp2 set (parallel_workers = 0);
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp1 where col1 = (select 2);');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 reset (parallel_workers);
+
+-- case 8: parallel inserts must not occur because there is no Gather or Append
+-- node at the top for union, except/except all, intersect/intersect all
+-- cases.
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except
+    select * from temp2 where col2 < 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except all
+    select * from temp2 where col2 < 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect all
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
 drop table temp1;
 drop table temp2;
 drop table temp3;
-- 
2.25.1

From 82dceab0abeed669142b90d75c9c120d56e5367e Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Sun, 27 Dec 2020 13:24:57 +0530
Subject: [PATCH v16 1/4] Parallel Inserts in CREATE TABLE AS

Allow the leader and each worker insert the tuples in parallel
if the SELECT part of the CTAS is parallelizable.

The design:
The main idea is to push the CTAS dest receiver down to Gather node
and from there the required information will be shared to workers
so that they can perform parallel insertions. Leader will also
participate in insertions. After the planning, check if the upper
plan node is Gather in createas.c and mark a parallelism flag in
the CTAS dest receiver and push it down to Gather node. Each worker
can create its own CTAS dest receiver with the information passed
from the leader. Leader inserts its share of tuples if instructed
to do, and so are workers. Each worker writes atomically its number
of inserted tuples into a shared memory variable, the leader combines
this with its own number of inserted tuples and shares to the client.
---
 src/backend/access/heap/heapam.c       |  11 ---
 src/backend/access/transam/xact.c      |  28 ++++--
 src/backend/commands/createas.c        | 113 ++++++++++++++++++++++---
 src/backend/commands/explain.c         |  32 +++++++
 src/backend/executor/execParallel.c    |  70 +++++++++++++--
 src/backend/executor/nodeGather.c      | 112 ++++++++++++++++++++++--
 src/backend/executor/nodeGatherMerge.c |   4 +-
 src/include/access/xact.h              |   1 +
 src/include/commands/createas.h        |  25 ++++++
 src/include/executor/execParallel.h    |   6 +-
 src/include/nodes/execnodes.h          |   3 +
 11 files changed, 361 insertions(+), 44 deletions(-)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index a9583f3103..86347ba273 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2043,17 +2043,6 @@ static HeapTuple
 heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
 					CommandId cid, int options)
 {
-	/*
-	 * To allow parallel inserts, we need to ensure that they are safe to be
-	 * performed in workers. We have the infrastructure to allow parallel
-	 * inserts in general except for the cases where inserts generate a new
-	 * CommandId (eg. inserts into a table having a foreign key column).
-	 */
-	if (IsParallelWorker())
-		ereport(ERROR,
-				(errcode(ERRCODE_INVALID_TRANSACTION_STATE),
-				 errmsg("cannot insert tuples in a parallel worker")));
-
 	tup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
 	tup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
 	tup->t_data->t_infomask |= HEAP_XMAX_INVALID;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 9cd0b7c11b..4e5366fe78 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -764,17 +764,35 @@ GetCurrentCommandId(bool used)
 	if (used)
 	{
 		/*
-		 * Forbid setting currentCommandIdUsed in a parallel worker, because
-		 * we have no provision for communicating this back to the leader.  We
-		 * could relax this restriction when currentCommandIdUsed was already
-		 * true at the start of the parallel operation.
+		 * If in a parallel worker, only allow setting currentCommandIdUsed if
+		 * currentCommandIdUsed was already true at the start of the parallel
+		 * operation (by way of SetCurrentCommandIdUsed()), otherwise forbid
+		 * setting currentCommandIdUsed because we have no provision for
+		 * communicating this back to the leader. Once currentCommandIdUsed is
+		 * set, the commandId used by leader and workers can't be changed,
+		 * because CommandCounterIncrement() then prevents any attempted
+		 * increment of the current commandId.
 		 */
-		Assert(!IsParallelWorker());
+		Assert(!(IsParallelWorker() && !currentCommandIdUsed));
 		currentCommandIdUsed = true;
 	}
 	return currentCommandId;
 }
 
+/*
+ * SetCurrentCommandIdUsedForWorker
+ *
+ * For a parallel worker, record that the currentCommandId has been used. This
+ * must only be called at the start of a parallel operation.
+ */
+void
+SetCurrentCommandIdUsedForWorker(void)
+{
+	Assert(IsParallelWorker() && !currentCommandIdUsed && currentCommandId != InvalidCommandId);
+
+	currentCommandIdUsed = true;
+}
+
 /*
  *	SetParallelStartTimestamps
  *
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 6bf6c5a310..62b83425fe 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,18 +51,6 @@
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 
-typedef struct
-{
-	DestReceiver pub;			/* publicly-known function pointers */
-	IntoClause *into;			/* target relation specification */
-	/* These fields are filled by intorel_startup: */
-	Relation	rel;			/* relation to write to */
-	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
-	CommandId	output_cid;		/* cmin to insert in output tuples */
-	int			ti_options;		/* table_tuple_insert performance options */
-	BulkInsertState bistate;	/* bulk insert state */
-} DR_intorel;
-
 /* utility functions for CTAS definition creation */
 static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into);
 static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -350,6 +338,14 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		/* call ExecutorStart to prepare the plan for execution */
 		ExecutorStart(queryDesc, GetIntoRelEFlags(into));
 
+		/*
+		 * If SELECT part of the CTAS is parallelizable, then make each
+		 * parallel worker insert the tuples that are resulted in its execution
+		 * into the target table. We need plan state to be initialized by the
+		 * executor to decide whether to allow parallel inserts or not.
+		 */
+		TryParallelizingInsertsInCTAS(into, queryDesc);
+
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
 
@@ -418,6 +414,9 @@ CreateIntoRelDestReceiver(IntoClause *intoClause)
 	self->pub.rDestroy = intorel_destroy;
 	self->pub.mydest = DestIntoRel;
 	self->into = intoClause;
+	self->is_parallel = false;
+	self->is_parallel_worker = false;
+	self->object_id = InvalidOid;
 	/* other private fields will be set during intorel_startup */
 
 	return (DestReceiver *) self;
@@ -438,6 +437,35 @@ intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 	ListCell   *lc;
 	int			attnum;
 
+	/*
+	 * All the necessary work such as table creation, sanity checks etc. would
+	 * have been done by the leader. So, parallel workers just need to open the
+	 * table, allocate bulk insert state, mark the command id as used, store it
+	 * in the dest recevier and return.
+	 */
+	if (myState->is_parallel_worker)
+	{
+		/* In the worker */
+		intoRelationDesc = table_open(myState->object_id, AccessExclusiveLock);
+		myState->rel = intoRelationDesc;
+		myState->reladdr = InvalidObjectAddress;
+		myState->ti_options = 0;
+		myState->bistate = GetBulkInsertState();
+
+		/*
+		 * Right after the table is created in the leader, the command id is
+		 * incremented (in create_ctas_internal()). The new command id is
+		 * marked as used in intorel_startup(), then the parallel mode is
+		 * entered. The command id and transaction id are serialized into
+		 * parallel DSM, they are then available to all parallel workers. All
+		 * the workers need to mark the command id as used before insertion.
+		 */
+		SetCurrentCommandIdUsedForWorker();
+		myState->output_cid = GetCurrentCommandId(false);
+
+		return;
+	}
+
 	Assert(into != NULL);		/* else somebody forgot to set it */
 
 	/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
@@ -540,6 +568,27 @@ intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 	else
 		myState->bistate = NULL;
 
+	/* If parallel inserts are to be allowed, set few extra information. */
+	if (myState->is_parallel)
+	{
+		myState->object_id = intoRelationAddr.objectId;
+
+		/*
+		 * We don't need to skip contacting FSM while inserting tuples for
+		 * parallel mode, while extending the relations, workers instead of
+		 * blocking on a page while another worker is inserting, can check the
+		 * FSM for another page that can accommodate the tuples. This results
+		 * in major benefit for parallel inserts.
+		 */
+		myState->ti_options = 0;
+
+		/*
+		 * rd_createSubid is marked invalid, otherwise, the table is not
+		 * allowed to be extended by the workers.
+		 */
+		myState->rel->rd_createSubid = InvalidSubTransactionId;
+	}
+
 	/*
 	 * Valid smgr_targblock implies something already wrote to the relation.
 	 * This may be harmless, but this function hasn't planned for it.
@@ -606,3 +655,43 @@ intorel_destroy(DestReceiver *self)
 {
 	pfree(self);
 }
+
+/*
+ * TryParallelizingInsertsInCTAS --- determine whether or not parallel
+ * insertion is possible. If yes set the parallel insert state such as a flag
+ * in the dest receiver and store the dest receiver reference in the Gather
+ * node so that the required information will be sent to workers.
+ */
+void
+TryParallelizingInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc)
+{
+	/*
+	 * Do not allow parallel inserts if the table is temporary. As the
+	 * temporary tables are backend local, workers can not know about them.
+	 * Currently, CTAS supports creation of normal(logged), temporary and
+	 * unlogged tables. It does not support foreign or partition table
+	 * creation. Hence the check for temporary table is enough here.
+	 */
+	if (!into->rel || into->rel->relpersistence == RELPERSISTENCE_TEMP)
+		return;
+
+	if (queryDesc && IsA(queryDesc->planstate, GatherState) &&
+		!queryDesc->planstate->ps_ProjInfo)
+	{
+		GatherState *gstate = (GatherState *) queryDesc->planstate;
+		DestReceiver *dest	= queryDesc->dest;
+
+		/* Okay to parallelize inserts, so mark it. */
+		((DR_intorel *) dest)->is_parallel = true;
+
+		/*
+		 * For parallelizing inserts in CTAS we must send information such as
+		 * into clause (to build separate dest receiver), object id (to open
+		 * the created table) to each workers. Since this information is
+		 * available in the CTAS dest receiver, store a reference to it in the
+		 * Gather state so that it will be used in ExecInitParallelPlan to pick
+		 * the required information.
+		 */
+		gstate->dest = dest;
+	}
+}
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 43f9b01e83..912e2d5b89 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -556,6 +556,15 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	/* call ExecutorStart to prepare the plan for execution */
 	ExecutorStart(queryDesc, eflags);
 
+	/*
+	 * If SELECT part of the CTAS is parallelizable, then make each parallel
+	 * worker insert the tuples that are resulted in its execution into the
+	 * target table. We need plan state to be initialized by the executor to
+	 * decide whether to allow parallel inserts or not.
+	 */
+	if (into)
+		TryParallelizingInsertsInCTAS(into, queryDesc);
+
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
 	{
@@ -1775,6 +1784,29 @@ ExplainNode(PlanState *planstate, List *ancestors,
 
 				if (gather->single_copy || es->format != EXPLAIN_FORMAT_TEXT)
 					ExplainPropertyBool("Single Copy", gather->single_copy, es);
+
+				/*
+				 * Show the create table information under Gather node in case
+				 * parallel workers have inserted the rows.
+				 */
+				if (IsA(planstate, GatherState))
+				{
+					GatherState *gstate = (GatherState *) planstate;
+
+					if (IS_PARALLEL_CTAS_DEST(gstate->dest) &&
+						((DR_intorel *) gstate->dest)->into->rel &&
+						((DR_intorel *) gstate->dest)->into->rel->relname)
+					{
+						es->indent--;
+						ExplainIndentText(es);
+						appendStringInfoString(es->str, "->  ");
+						appendStringInfoString(es->str, "Create ");
+						appendStringInfo(es->str, "%s\n",
+							((DR_intorel *) gstate->dest)->into->rel->relname);
+						ExplainIndentText(es);
+						es->indent++;
+					}
+				}
 			}
 			break;
 		case T_GatherMerge:
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index befde52691..270f8c13d7 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -23,6 +23,7 @@
 
 #include "postgres.h"
 
+#include "commands/createas.h"
 #include "executor/execParallel.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
@@ -65,6 +66,7 @@
 #define PARALLEL_KEY_QUERY_TEXT		UINT64CONST(0xE000000000000008)
 #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009)
 #define PARALLEL_KEY_WAL_USAGE			UINT64CONST(0xE00000000000000A)
+#define PARALLEL_KEY_INTO_CLAUSE		UINT64CONST(0xE00000000000000B)
 
 #define PARALLEL_TUPLE_QUEUE_SIZE		65536
 
@@ -77,6 +79,9 @@ typedef struct FixedParallelExecutorState
 	dsa_pointer param_exec;
 	int			eflags;
 	int			jit_flags;
+	Oid			objectid;		/* workers to open relation/table.  */
+	/* Number of tuples inserted by all the workers. */
+	pg_atomic_uint64	processed;
 } FixedParallelExecutorState;
 
 /*
@@ -578,7 +583,8 @@ ExecParallelSetupTupleQueues(ParallelContext *pcxt, bool reinitialize)
 ParallelExecutorInfo *
 ExecInitParallelPlan(PlanState *planstate, EState *estate,
 					 Bitmapset *sendParams, int nworkers,
-					 int64 tuples_needed)
+					 int64 tuples_needed, IntoClause *intoclause,
+					 Oid objectid)
 {
 	ParallelExecutorInfo *pei;
 	ParallelContext *pcxt;
@@ -600,6 +606,8 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	Size		dsa_minsize = dsa_minimum_size();
 	char	   *query_string;
 	int			query_len;
+	char 		*intoclausestr = NULL;
+	int			intoclause_len = 0;
 
 	/*
 	 * Force any initplan outputs that we're going to pass to workers to be
@@ -712,6 +720,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_estimate_chunk(&pcxt->estimator, dsa_minsize);
 	shm_toc_estimate_keys(&pcxt->estimator, 1);
 
+	/* Estimate space for into clause for CTAS. */
+	if (IS_CTAS(intoclause) && OidIsValid(objectid))
+	{
+		intoclausestr = nodeToString(intoclause);
+		intoclause_len = strlen(intoclausestr) + 1;
+		shm_toc_estimate_chunk(&pcxt->estimator, intoclause_len);
+		shm_toc_estimate_keys(&pcxt->estimator, 1);
+	}
+
 	/* Everyone's had a chance to ask for space, so now create the DSM. */
 	InitializeParallelDSM(pcxt);
 
@@ -729,6 +746,14 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	fpes->param_exec = InvalidDsaPointer;
 	fpes->eflags = estate->es_top_eflags;
 	fpes->jit_flags = estate->es_jit_flags;
+	pg_atomic_init_u64(&fpes->processed, 0);
+	pei->processed = &fpes->processed;
+
+	if (intoclausestr)
+		fpes->objectid = objectid;
+	else
+		fpes->objectid = InvalidOid;
+
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, fpes);
 
 	/* Store query string */
@@ -758,8 +783,18 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_WAL_USAGE, walusage_space);
 	pei->wal_usage = walusage_space;
 
-	/* Set up the tuple queues that the workers will write into. */
-	pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	if (intoclausestr)
+	{
+		char *intoclause_space = shm_toc_allocate(pcxt->toc, intoclause_len);
+
+		memcpy(intoclause_space, intoclausestr, intoclause_len);
+		shm_toc_insert(pcxt->toc, PARALLEL_KEY_INTO_CLAUSE, intoclause_space);
+	}
+	else
+	{
+		/* Set up the tuple queues that the workers will write into. */
+		pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	}
 
 	/* We don't need the TupleQueueReaders yet, though. */
 	pei->reader = NULL;
@@ -1387,12 +1422,30 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 	void	   *area_space;
 	dsa_area   *area;
 	ParallelWorkerContext pwcxt;
+	char		*intoclausestr = NULL;
+	IntoClause	*intoclause = NULL;
 
 	/* Get fixed-size state. */
 	fpes = shm_toc_lookup(toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
 
-	/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
-	receiver = ExecParallelGetReceiver(seg, toc);
+	intoclausestr = shm_toc_lookup(toc, PARALLEL_KEY_INTO_CLAUSE, true);
+	if (intoclausestr)
+	{
+		/*
+		 * If the worker is for parallel insert in CTAS, then use the proper
+		 * dest receiver.
+		 */
+		intoclause = (IntoClause *) stringToNode(intoclausestr);
+		receiver = CreateIntoRelDestReceiver(intoclause);
+		((DR_intorel *)receiver)->is_parallel_worker = true;
+		((DR_intorel *)receiver)->object_id = fpes->objectid;
+	}
+	else
+	{
+		/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
+		receiver = ExecParallelGetReceiver(seg, toc);
+	}
+
 	instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_INSTRUMENTATION, true);
 	if (instrumentation != NULL)
 		instrument_options = instrumentation->instrument_options;
@@ -1471,6 +1524,13 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 			queryDesc->estate->es_jit->instr;
 	}
 
+	/*
+	 * Write out the number of tuples this worker has inserted. Leader will use
+	 * it to inform the end client.
+	 */
+	if (intoclausestr)
+		pg_atomic_add_fetch_u64(&fpes->processed, queryDesc->estate->es_processed);
+
 	/* Must do this after capturing instrumentation. */
 	ExecutorEnd(queryDesc);
 
diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c
index a01b46af14..eeb51a9f43 100644
--- a/src/backend/executor/nodeGather.c
+++ b/src/backend/executor/nodeGather.c
@@ -32,6 +32,7 @@
 
 #include "access/relscan.h"
 #include "access/xact.h"
+#include "commands/createas.h"
 #include "executor/execdebug.h"
 #include "executor/execParallel.h"
 #include "executor/nodeGather.h"
@@ -48,6 +49,7 @@ static TupleTableSlot *ExecGather(PlanState *pstate);
 static TupleTableSlot *gather_getnext(GatherState *gatherstate);
 static MinimalTuple gather_readnext(GatherState *gatherstate);
 static void ExecShutdownGatherWorkers(GatherState *node);
+static void ExecParallelInsertInCTAS(GatherState *node);
 
 
 /* ----------------------------------------------------------------
@@ -131,6 +133,72 @@ ExecInitGather(Gather *node, EState *estate, int eflags)
 	return gatherstate;
 }
 
+/* ----------------------------------------------------------------
+ *		ExecParallelInsertInCTAS(node)
+ *
+ *		Facilitates parallel inserts by parallel workers and/or
+ *		leader for Create Table AS.
+ * ----------------------------------------------------------------
+ */
+static void
+ExecParallelInsertInCTAS(GatherState *node)
+{
+	/* Enable leader to insert in case no parallel workers were launched. */
+	if (node->nworkers_launched == 0)
+		node->need_to_scan_locally = true;
+
+	/*
+	 * By now, for parallel workers (if launched any), would have started their
+	 * work i.e. insertion to target table. In case the leader is chosen to
+	 * participate for parallel inserts in CTAS, then finish its share before
+	 * going to wait for the parallel workers to finish.
+	 */
+	if (node->need_to_scan_locally)
+	{
+		EState	   *estate = node->ps.state;
+		TupleTableSlot *outerTupleSlot;
+
+		for(;;)
+		{
+			/* Install our DSA area while executing the plan. */
+			estate->es_query_dsa =
+					node->pei ? node->pei->area : NULL;
+
+			outerTupleSlot = ExecProcNode(node->ps.lefttree);
+
+			estate->es_query_dsa = NULL;
+
+			if(TupIsNull(outerTupleSlot))
+				break;
+
+			(void) node->dest->receiveSlot(outerTupleSlot, node->dest);
+
+			node->ps.state->es_processed++;
+		}
+
+		node->need_to_scan_locally = false;
+	}
+
+	if (node->nworkers_launched > 0)
+	{
+		/*
+		 * We wait here for the parallel workers to finish their work and
+		 * accumulate the tuples they inserted and also their buffer/WAL usage.
+		 * We do not destroy the parallel context here, it will be done in
+		 * ExecShutdownGather at the end of the plan. Note that the
+		 * ExecShutdownGatherWorkers call from ExecShutdownGather will be a
+		 * no-op.
+		 */
+		ExecShutdownGatherWorkers(node);
+
+		/*
+		 * Add up the total tuples inserted by all workers, to the tuples
+		 * inserted by the leader(if any). This will be shared to client.
+		 */
+		node->ps.state->es_processed += pg_atomic_read_u64(node->pei->processed);
+	}
+}
+
 /* ----------------------------------------------------------------
  *		ExecGather(node)
  *
@@ -157,6 +225,7 @@ ExecGather(PlanState *pstate)
 	{
 		EState	   *estate = node->ps.state;
 		Gather	   *gather = (Gather *) node->ps.plan;
+		bool		isctas = IS_PARALLEL_CTAS_DEST(node->dest);
 
 		/*
 		 * Sometimes we might have to run without parallelism; but if parallel
@@ -165,6 +234,18 @@ ExecGather(PlanState *pstate)
 		if (gather->num_workers > 0 && estate->es_use_parallel_mode)
 		{
 			ParallelContext *pcxt;
+			IntoClause	*intoclause = NULL;
+			Oid			objectid = InvalidOid;
+
+			/*
+			 * Take the necessary information to be passed to workers for
+			 * parallel inserts in CTAS.
+			 */
+			if (isctas)
+			{
+				intoclause = ((DR_intorel *) node->dest)->into;
+				objectid = ((DR_intorel *) node->dest)->object_id;
+			}
 
 			/* Initialize, or re-initialize, shared state needed by workers. */
 			if (!node->pei)
@@ -172,7 +253,10 @@ ExecGather(PlanState *pstate)
 												 estate,
 												 gather->initParam,
 												 gather->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 /* CTAS info */
+												 intoclause,
+												 objectid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
@@ -190,13 +274,16 @@ ExecGather(PlanState *pstate)
 			/* Set up tuple queue readers to read the results. */
 			if (pcxt->nworkers_launched > 0)
 			{
-				ExecParallelCreateReaders(node->pei);
-				/* Make a working array showing the active readers */
-				node->nreaders = pcxt->nworkers_launched;
-				node->reader = (TupleQueueReader **)
-					palloc(node->nreaders * sizeof(TupleQueueReader *));
-				memcpy(node->reader, node->pei->reader,
-					   node->nreaders * sizeof(TupleQueueReader *));
+				if (!isctas)
+				{
+					ExecParallelCreateReaders(node->pei);
+					/* Make a working array showing the active readers */
+					node->nreaders = pcxt->nworkers_launched;
+					node->reader = (TupleQueueReader **)
+						palloc(node->nreaders * sizeof(TupleQueueReader *));
+					memcpy(node->reader, node->pei->reader,
+						node->nreaders * sizeof(TupleQueueReader *));
+				}
 			}
 			else
 			{
@@ -208,9 +295,16 @@ ExecGather(PlanState *pstate)
 		}
 
 		/* Run plan locally if no workers or enabled and not single-copy. */
-		node->need_to_scan_locally = (node->nreaders == 0)
+		node->need_to_scan_locally = (node->nreaders == 0 && !isctas)
 			|| (!gather->single_copy && parallel_leader_participation);
 		node->initialized = true;
+
+		/* Perform parallel inserts for CTAS. */
+		if (isctas)
+		{
+			ExecParallelInsertInCTAS(node);
+			return NULL;
+		}
 	}
 
 	/*
diff --git a/src/backend/executor/nodeGatherMerge.c b/src/backend/executor/nodeGatherMerge.c
index 47129344f3..ee45272c17 100644
--- a/src/backend/executor/nodeGatherMerge.c
+++ b/src/backend/executor/nodeGatherMerge.c
@@ -216,7 +216,9 @@ ExecGatherMerge(PlanState *pstate)
 												 estate,
 												 gm->initParam,
 												 gm->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 NULL,
+												 InvalidOid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7320de345c..5beae6c617 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -389,6 +389,7 @@ extern FullTransactionId GetCurrentFullTransactionIdIfAny(void);
 extern void MarkCurrentTransactionIdLoggedIfAny(void);
 extern bool SubTransactionIsActive(SubTransactionId subxid);
 extern CommandId GetCurrentCommandId(bool used);
+extern void SetCurrentCommandIdUsedForWorker(void);
 extern void SetParallelStartTimestamps(TimestampTz xact_ts, TimestampTz stmt_ts);
 extern TimestampTz GetCurrentTransactionStartTimestamp(void);
 extern TimestampTz GetCurrentStatementStartTimestamp(void);
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 7629230254..fc4a64989f 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -14,12 +14,35 @@
 #ifndef CREATEAS_H
 #define CREATEAS_H
 
+#include "access/heapam.h"
 #include "catalog/objectaddress.h"
+#include "executor/execdesc.h"
 #include "nodes/params.h"
+#include "nodes/plannodes.h"
 #include "parser/parse_node.h"
 #include "tcop/dest.h"
 #include "utils/queryenvironment.h"
 
+typedef struct
+{
+	DestReceiver pub;			/* publicly-known function pointers */
+	IntoClause *into;			/* target relation specification */
+	/* These fields are filled by intorel_startup: */
+	Relation	rel;			/* relation to write to */
+	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
+	CommandId	output_cid;		/* cmin to insert in output tuples */
+	int			ti_options;		/* table_tuple_insert performance options */
+	BulkInsertState bistate;	/* bulk insert state */
+	bool		is_parallel;	/* is parallelism to be considered? */
+	bool		is_parallel_worker; /* true for parallel worker */
+	/* Used by parallel workers for opening the table created in the leader. */
+	Oid			object_id;
+} DR_intorel;
+
+#define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
+#define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
+									 IS_CTAS(((DR_intorel *) dest)->into) && \
+									 ((DR_intorel *) dest)->is_parallel)
 
 extern ObjectAddress ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 									   ParamListInfo params, QueryEnvironment *queryEnv,
@@ -29,4 +52,6 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
+extern void TryParallelizingInsertsInCTAS(IntoClause *into,
+										  QueryDesc *queryDesc);
 #endif							/* CREATEAS_H */
diff --git a/src/include/executor/execParallel.h b/src/include/executor/execParallel.h
index 5a39a5b29c..9f959f741b 100644
--- a/src/include/executor/execParallel.h
+++ b/src/include/executor/execParallel.h
@@ -35,11 +35,15 @@ typedef struct ParallelExecutorInfo
 	/* These two arrays have pcxt->nworkers_launched entries: */
 	shm_mq_handle **tqueue;		/* tuple queues for worker output */
 	struct TupleQueueReader **reader;	/* tuple reader/writer support */
+	/* Number of tuples inserted by all workers. */
+	volatile pg_atomic_uint64	*processed;
 } ParallelExecutorInfo;
 
 extern ParallelExecutorInfo *ExecInitParallelPlan(PlanState *planstate,
 												  EState *estate, Bitmapset *sendParam, int nworkers,
-												  int64 tuples_needed);
+												  int64 tuples_needed,
+												  IntoClause *intoclause,
+												  Oid objectid);
 extern void ExecParallelCreateReaders(ParallelExecutorInfo *pei);
 extern void ExecParallelFinish(ParallelExecutorInfo *pei);
 extern void ExecParallelCleanup(ParallelExecutorInfo *pei);
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 61ba4c3666..a5b281f783 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -23,6 +23,7 @@
 #include "nodes/tidbitmap.h"
 #include "partitioning/partdefs.h"
 #include "storage/condition_variable.h"
+#include "tcop/dest.h"
 #include "utils/hsearch.h"
 #include "utils/queryenvironment.h"
 #include "utils/reltrigger.h"
@@ -2326,6 +2327,8 @@ typedef struct GatherState
 	int			nreaders;		/* number of still-active workers */
 	int			nextreader;		/* next one to try to read from */
 	struct TupleQueueReader **reader;	/* array with nreaders active entries */
+	/* Dest receiver is stored when parallel inserts is allowed in CTAS. */
+	DestReceiver *dest;
 } GatherState;
 
 /* ----------------
-- 
2.25.1

From e580926ce0756b425163d165100f4358e8abbac5 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Sat, 26 Dec 2020 15:19:45 +0530
Subject: [PATCH v16 2/4] Tuple Cost Adjustment for Parallel Inserts in CTAS

Let the planner know that the SELECT is from CTAS in createas.c
so that it can set the number of tuples transferred from the
workers to Gather node to 0. With this change, there are chances
that the planner may choose the parallel plan.
---
 src/backend/commands/createas.c       | 34 ++++++++++++++++-
 src/backend/commands/explain.c        |  7 ++--
 src/backend/commands/prepare.c        |  3 +-
 src/backend/optimizer/path/costsize.c | 22 ++++++++++-
 src/backend/optimizer/plan/planner.c  | 53 +++++++++++++++++++++++++++
 src/include/commands/createas.h       | 21 ++++++++++-
 src/include/commands/explain.h        |  3 +-
 src/include/nodes/parsenodes.h        |  1 +
 src/include/optimizer/planner.h       | 10 +++++
 9 files changed, 145 insertions(+), 9 deletions(-)

diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 27584c6be0..aaed0d8e2b 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -344,7 +344,8 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		 * into the target table. We need plan state to be initialized by the
 		 * executor to decide whether to allow parallel inserts or not.
 		 */
-		TryParallelizingInsertsInCTAS(into, queryDesc);
+		TryParallelizingInsertsInCTAS(into, queryDesc,
+									  &query->CTASParallelInsInfo);
 
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
@@ -663,8 +664,11 @@ intorel_destroy(DestReceiver *self)
  * node so that the required information will be sent to workers.
  */
 void
-TryParallelizingInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc)
+TryParallelizingInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc,
+							  uint8 *tuple_cost_flags)
 {
+	bool allow = false;
+
 	/*
 	 * Do not allow parallel inserts if the table is temporary. As the
 	 * temporary tables are backend local, workers can not know about them.
@@ -693,5 +697,31 @@ TryParallelizingInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc)
 		 * the required information.
 		 */
 		gstate->dest = dest;
+		allow = true;
+	}
+
+	/*
+	 * It should not happen that in cost_gather we have ignored the parallel
+	 * tuple cost and now we are not allowing the parallel inserts. And also we
+	 * might need assertion only if the top node is Gather. The main intention
+	 * of assertion is to check if we enforced planner to ignore the parallel
+	 * tuple cost (with the intention of choosing parallel inserts) due to
+	 * which the parallel plan may have been chosen, but we do not allow the
+	 * parallel inserts now.
+	 */
+	if (!allow && tuple_cost_flags && queryDesc &&
+			IsA(queryDesc->planstate, GatherState))
+	{
+		/*
+		 * If we have correctly ignored parallel tuple cost in planner while
+		 * creating Gather path, then this assertion failure should not occur.
+		 * If it occurs, that means the planner may have chosen this parallel
+		 * plan because of our enforcement to ignore the parallel tuple cost.
+		 */
+		Assert(!(*tuple_cost_flags & CTAS_PARALLEL_INS_TUP_COST_IGNORED));
 	}
+
+	if (tuple_cost_flags)
+		*tuple_cost_flags = CTAS_PARALLEL_INS_UNDEF;
+
 }
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 912e2d5b89..90c614eabc 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -402,7 +402,8 @@ ExplainOneQuery(Query *query, int cursorOptions,
 
 		/* run it (if needed) and produce output */
 		ExplainOnePlan(plan, into, es, queryString, params, queryEnv,
-					   &planduration, (es->buffers ? &bufusage : NULL));
+					   &planduration, (es->buffers ? &bufusage : NULL),
+					   &query->CTASParallelInsInfo);
 	}
 }
 
@@ -496,7 +497,7 @@ void
 ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 			   const char *queryString, ParamListInfo params,
 			   QueryEnvironment *queryEnv, const instr_time *planduration,
-			   const BufferUsage *bufusage)
+			   const BufferUsage *bufusage, uint8 *ctas_tuple_cost_flags)
 {
 	DestReceiver *dest;
 	QueryDesc  *queryDesc;
@@ -563,7 +564,7 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	 * decide whether to allow parallel inserts or not.
 	 */
 	if (into)
-		TryParallelizingInsertsInCTAS(into, queryDesc);
+		TryParallelizingInsertsInCTAS(into, queryDesc, ctas_tuple_cost_flags);
 
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c
index 89087a7be3..07166479e7 100644
--- a/src/backend/commands/prepare.c
+++ b/src/backend/commands/prepare.c
@@ -672,7 +672,8 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es,
 
 		if (pstmt->commandType != CMD_UTILITY)
 			ExplainOnePlan(pstmt, into, es, query_string, paramLI, queryEnv,
-						   &planduration, (es->buffers ? &bufusage : NULL));
+						   &planduration, (es->buffers ? &bufusage : NULL),
+						   NULL);
 		else
 			ExplainOneUtility(pstmt->utilityStmt, into, es, query_string,
 							  paramLI, queryEnv);
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 22d6935824..800f25903d 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -76,6 +76,7 @@
 #include "access/amapi.h"
 #include "access/htup_details.h"
 #include "access/tsmapi.h"
+#include "commands/createas.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "executor/nodeHash.h"
@@ -378,6 +379,7 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 {
 	Cost		startup_cost = 0;
 	Cost		run_cost = 0;
+	bool		ignore_tuple_cost = false;
 
 	/* Mark the path with the correct row estimate */
 	if (rows)
@@ -393,7 +395,25 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 
 	/* Parallel setup and communication cost. */
 	startup_cost += parallel_setup_cost;
-	run_cost += parallel_tuple_cost * path->path.rows;
+
+	/*
+	 * Do not consider tuple cost in case of we intend to perform parallel
+	 * inserts by workers. We would have set ignore flag in
+	 * apply_scanjoin_target_to_paths before generating Gather path for the
+	 * upper level SELECT part of the CTAS.
+	 */
+	if ((root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT) &&
+		(root->parse->CTASParallelInsInfo &
+		 CTAS_PARALLEL_INS_TUP_COST_CAN_IGN))
+	{
+		ignore_tuple_cost = true;
+		root->parse->CTASParallelInsInfo &=
+									~CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_TUP_COST_IGNORED;
+	}
+
+	if (!ignore_tuple_cost)
+		run_cost += parallel_tuple_cost * path->path.rows;
 
 	path->path.startup_cost = startup_cost;
 	path->path.total_cost = (startup_cost + run_cost);
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index 1a94b58f8b..f1134711b0 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -28,6 +28,7 @@
 #include "catalog/pg_inherits.h"
 #include "catalog/pg_proc.h"
 #include "catalog/pg_type.h"
+#include "commands/createas.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "foreign/fdwapi.h"
@@ -7338,6 +7339,37 @@ can_partial_agg(PlannerInfo *root)
 	return true;
 }
 
+/*
+ * ignore_parallel_tuple_cost
+ *
+ * Gather node will not receive any tuples from the workers in case each worker
+ * inserts them in parallel. So, we set a flag to ignore parallel tuple cost by
+ * the Gather path in cost_gather if the SELECT is for CTAS and we are
+ * generating an upper level Gather path.
+*/
+static bool
+ignore_parallel_tuple_cost(PlannerInfo *root)
+{
+	if (root->query_level == 1 &&
+		(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT))
+	{
+		/*
+		 * In each of the HAS_PARENT_PATH_GENERATING_CLAUSE cases, a parent
+		 * path will be generated for the upper Gather path(in
+		 * grouping_planner), in which case we can not let parallel inserts
+		 * happen. So we do not set ignore tuple cost flag.
+		 */
+		if (HAS_PARENT_PATH_GENERATING_CLAUSE(root))
+			return false;
+
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+
+		return true;
+	}
+
+	return false;
+}
+
 /*
  * apply_scanjoin_target_to_paths
  *
@@ -7557,8 +7589,29 @@ apply_scanjoin_target_to_paths(PlannerInfo *root,
 	 * one of the generated paths may turn out to be the cheapest one.
 	 */
 	if (rel->consider_parallel && !IS_OTHER_REL(rel))
+	{
+		/*
+		 * Set a flag to ignore parallel tuple cost by the Gather path in
+		 * cost_gather if the SELECT is for CTAS and we are generating an upper
+		 * level Gather path.
+		 */
+		bool	ignore = ignore_parallel_tuple_cost(root);
+
 		generate_useful_gather_paths(root, rel, false);
 
+		/*
+		 * Reset the ignore flag, in case we set it but
+		 * generate_useful_gather_paths returned without reaching cost_gather.
+		 */
+		if (ignore &&
+			(root->parse->CTASParallelInsInfo &
+			 CTAS_PARALLEL_INS_TUP_COST_CAN_IGN))
+		{
+			root->parse->CTASParallelInsInfo &=
+										~CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+		}
+	}
+
 	/*
 	 * Reassess which paths are the cheapest, now that we've potentially added
 	 * new Gather (or Gather Merge) and/or Append (or MergeAppend) paths to
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index fc4a64989f..919148b10a 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -39,6 +39,24 @@ typedef struct
 	Oid			object_id;
 } DR_intorel;
 
+/*
+ * Information sent to the planner from CTAS to account for the cost
+ * calculations in cost_gather. We need to do this because, no tuples will be
+ * received by the Gather node if the workers insert the tuples in parallel.
+ */
+typedef enum CTASParallelInsertOpt
+{
+	CTAS_PARALLEL_INS_UNDEF = 0, /* undefined */
+	CTAS_PARALLEL_INS_SELECT = 1 << 0, /* set to this before planning */
+	/*
+	 * Set to this while planning for upper Gather path to ignore parallel
+	 * tuple cost in cost_gather.
+	 */
+	CTAS_PARALLEL_INS_TUP_COST_CAN_IGN = 1 << 1,
+	/* Set to this after the cost is ignored. */
+	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2
+} CTASParallelInsertOpt;
+
 #define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
 #define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
 									 IS_CTAS(((DR_intorel *) dest)->into) && \
@@ -53,5 +71,6 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
 extern void TryParallelizingInsertsInCTAS(IntoClause *into,
-										  QueryDesc *queryDesc);
+										  QueryDesc *queryDesc,
+										  uint8 *tuple_cost_flags);
 #endif							/* CREATEAS_H */
diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h
index ba661d32a6..1a1806dbf1 100644
--- a/src/include/commands/explain.h
+++ b/src/include/commands/explain.h
@@ -91,7 +91,8 @@ extern void ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into,
 						   ExplainState *es, const char *queryString,
 						   ParamListInfo params, QueryEnvironment *queryEnv,
 						   const instr_time *planduration,
-						   const BufferUsage *bufusage);
+						   const BufferUsage *bufusage,
+						   uint8 *ctas_tuple_cost_flags);
 
 extern void ExplainPrintPlan(ExplainState *es, QueryDesc *queryDesc);
 extern void ExplainPrintTriggers(ExplainState *es, QueryDesc *queryDesc);
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 48a79a7657..81b148c383 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -180,6 +180,7 @@ typedef struct Query
 	 */
 	int			stmt_location;	/* start location, or -1 if unknown */
 	int			stmt_len;		/* length in bytes; 0 means "rest of string" */
+	uint8 		CTASParallelInsInfo; /* parallel insert in CTAS info */
 } Query;
 
 
diff --git a/src/include/optimizer/planner.h b/src/include/optimizer/planner.h
index beb7dbbcbe..74b2563828 100644
--- a/src/include/optimizer/planner.h
+++ b/src/include/optimizer/planner.h
@@ -21,6 +21,16 @@
 #include "nodes/pathnodes.h"
 #include "nodes/plannodes.h"
 
+#define HAS_PARENT_PATH_GENERATING_CLAUSE(root) \
+											(root->parse->rowMarks || \
+											 limit_needed(root->parse) || \
+											 root->parse->sortClause ||  \
+											 root->parse->distinctClause ||  \
+											 root->parse->hasWindowFuncs ||  \
+											 root->parse->groupClause ||  \
+											 root->parse->groupingSets ||  \
+											 root->parse->hasAggs ||  \
+											 root->hasHavingQual)
 
 /* Hook for plugins to get control in planner() */
 typedef PlannedStmt *(*planner_hook_type) (Query *parse,
-- 
2.25.1

From e0b2344a31ac002cf8cf53d3f9ba2c977a82e39c Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Sat, 26 Dec 2020 15:56:01 +0530
Subject: [PATCH v16 3/4] Tests And Docs For Parallel Inserts in CTAS

---
 doc/src/sgml/ref/create_table_as.sgml        |  31 +-
 src/test/regress/expected/write_parallel.out | 577 +++++++++++++++++++
 src/test/regress/sql/write_parallel.sql      | 216 +++++++
 3 files changed, 819 insertions(+), 5 deletions(-)

diff --git a/doc/src/sgml/ref/create_table_as.sgml b/doc/src/sgml/ref/create_table_as.sgml
index 07558ab56c..35903701ed 100644
--- a/doc/src/sgml/ref/create_table_as.sgml
+++ b/doc/src/sgml/ref/create_table_as.sgml
@@ -37,11 +37,13 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI
 
   <para>
    <command>CREATE TABLE AS</command> creates a table and fills it
-   with data computed by a <command>SELECT</command> command.
-   The table columns have the
-   names and data types associated with the output columns of the
-   <command>SELECT</command> (except that you can override the column
-   names by giving an explicit list of new column names).
+   with data computed by a <command>SELECT</command> command. When the
+   node at the top of the <command>SELECT</command> plan is
+   <literal>Gather</literal> and there are no projections to be performed
+   by it, then the created table can be filled by the workers in parallel.
+   The table columns have the names and data types associated with the output
+   columns of the <command>SELECT</command> (except that you can override the
+   column names by giving an explicit list of new column names).
   </para>
 
   <para>
@@ -297,6 +299,25 @@ PREPARE recentfilms(date) AS
 CREATE TEMP TABLE films_recent ON COMMIT DROP AS
   EXECUTE recentfilms('2002-01-01');
 </programlisting></para>
+
+  <para>
+   Here is an example of a query plan when the data into the created table can
+   be filled by the workers in parallel:
+
+<programlisting>
+EXPLAIN CREATE TABLE bar AS SELECT * FROM foo WHERE i > 5;
+
+                             QUERY PLAN
+-------------------------------------------------------------------&zwsp;--
+Gather  (cost=0.00..23.28 rows=850 width=4)
+   Workers Planned: 2
+ ->  Create bar
+   ->  Parallel Seq Scan on foo  (cost=0.00..23.28 rows=354 width=4)
+         Filter: (i > 5)
+(5 rows)
+</programlisting>
+  </para>
+
  </refsect1>
 
  <refsect1>
diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 0c4da2591a..357fbbbe8d 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -76,4 +76,581 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+-- Parallel queries won't necessarily get as many workers as the planner
+-- asked for.  This affects not only the "Workers Launched:" field of EXPLAIN
+-- results, but also row counts and loop counts for parallel scans, Gathers,
+-- and everything in between.  This function filters out the values we can't
+-- rely on to be stable.
+-- This removes enough info that you might wonder why bother with EXPLAIN
+-- ANALYZE at all.  The answer is that we need to see whether the parallel
+-- inserts are being done by the workers, the only way is that
+-- Create <<tbl_name>> appears in the explain output.
+create function explain_pictas(text) returns setof text
+language plpgsql as
+$$
+declare
+    ln text;
+begin
+    for ln in
+        execute format('explain (analyze, costs off, summary off, timing off) %s',
+            $1)
+    loop
+        ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N');
+        ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N');
+        ln := regexp_replace(ln, 'Rows Removed by Filter: \d+', 'Rows Removed by Filter: N');
+        return next ln;
+    end loop;
+end;
+$$;
+-- parallel inserts must occur
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'create temporary table parallel_write as select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(4 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+select explain_pictas(
+'create unlogged table parallel_write as select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+select explain_pictas(
+'select length(stringu1) into parallel_write from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'select length(stringu1) into temporary parallel_write from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(4 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+select explain_pictas(
+'select length(stringu1) into unlogged parallel_write from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1 for update;');
+                 explain_pictas                  
+-------------------------------------------------
+ LockRows (actual rows=N loops=N)
+   ->  Seq Scan on tenk1 (actual rows=N loops=N)
+(2 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur
+select explain_pictas(
+'create materialized view parallel_mat_view as
+    select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_mat_view
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_mat_view;
+ count 
+-------
+ 10000
+(1 row)
+
+drop materialized view parallel_mat_view;
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+select explain_pictas(
+'create table parallel_write as execute parallel_write_prep;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+deallocate parallel_write_prep;
+drop table parallel_write;
+-- parallel inserts must occur
+select explain_pictas(
+'create table parallel_write as select now(), four from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+select explain_pictas(
+E'create table parallel_write as
+    select nextval(\'parallel_write_sequence\'), four from tenk1;');
+              explain_pictas               
+-------------------------------------------
+ Seq Scan on tenk1 (actual rows=N loops=N)
+(1 row)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Params Evaluated: $1
+   Workers Launched: N
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 4
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on tenk2 (actual rows=N loops=N)
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+         Filter: (four = 3)
+         Rows Removed by Filter: N
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+  2500
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;');
+                          explain_pictas                          
+------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+         Filter: (four = 3)
+         Rows Removed by Filter: N
+   SubPlan 1
+     ->  Function Scan on generate_series (actual rows=N loops=N)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+  2500
+(1 row)
+
+drop table parallel_write;
+create table temp1(col1) as select * from generate_series(1,5);
+create table temp2(col2) as select * from temp1;
+create table temp3(col3) as select * from temp1;
+-- parallel inserts must not occur, as there is a limit clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 limit 4;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ Limit (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ Gather Merge (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+   ->  Sort (actual rows=N loops=N)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         Worker 0:  Sort Method: quicksort  Memory: 25kB
+         Worker 1:  Sort Method: quicksort  Memory: 25kB
+         Worker 2:  Sort Method: quicksort  Memory: 25kB
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ Gather Merge (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+   ->  Sort (actual rows=N loops=N)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         Worker 0:  Sort Method: quicksort  Memory: 25kB
+         Worker 1:  Sort Method: quicksort  Memory: 25kB
+         Worker 2:  Sort Method: quicksort  Memory: 25kB
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is a distinct clause
+select explain_pictas(
+'create table parallel_write as select distinct * from temp1;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ HashAggregate (actual rows=N loops=N)
+   Group Key: col1
+   Batches: 1  Memory Usage: 40kB
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(7 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an aggregate and group clause
+select explain_pictas(
+'create table parallel_write as select count(*) from temp1 group by col1;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Finalize HashAggregate (actual rows=N loops=N)
+   Group Key: col1
+   Batches: 1  Memory Usage: 40kB
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Partial HashAggregate (actual rows=N loops=N)
+               Group Key: col1
+               Batches: 1  Memory Usage: 40kB
+               Worker 0:  Batches: 1  Memory Usage: 40kB
+               Worker 1:  Batches: 1  Memory Usage: 40kB
+               Worker 2:  Batches: 1  Memory Usage: 40kB
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(13 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an aggregate, group and having
+-- clauses
+select explain_pictas(
+'create table parallel_write as
+    select count(col1), (select col3 from
+        (select * from temp3) as tt limit 1) col4 from temp1, temp2
+    where temp1.col1 = temp2.col2 group by col4 having count(col1) > 0;');
+                                  explain_pictas                                  
+----------------------------------------------------------------------------------
+ Finalize GroupAggregate (actual rows=N loops=N)
+   Group Key: ($1)
+   Filter: (count(temp1.col1) > 0)
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 3
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on temp3 (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $1
+         Workers Launched: N
+         ->  Partial GroupAggregate (actual rows=N loops=N)
+               Group Key: $1
+               ->  Parallel Hash Join (actual rows=N loops=N)
+                     Hash Cond: (temp1.col1 = temp2.col2)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                     ->  Parallel Hash (actual rows=N loops=N)
+                           Buckets: 4096  Batches: 1  Memory Usage: 64kB
+                           ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(21 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     1
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is a window function
+select explain_pictas(
+'create table parallel_write as
+    select avg(col1) OVER (PARTITION BY col1) from temp1;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ WindowAgg (actual rows=N loops=N)
+   ->  Gather Merge (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Sort (actual rows=N loops=N)
+               Sort Key: col1
+               Sort Method: quicksort  Memory: 25kB
+               Worker 0:  Sort Method: quicksort  Memory: 25kB
+               Worker 1:  Sort Method: quicksort  Memory: 25kB
+               Worker 2:  Sort Method: quicksort  Memory: 25kB
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(11 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- nested loop join is the top node under which Gather node exists, so parallel
+-- inserts must not occur
+set enable_nestloop to on;
+set enable_mergejoin to off;
+set enable_hashjoin to off;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Nested Loop (actual rows=N loops=N)
+   Join Filter: (temp1.col1 = temp2.col2)
+   Rows Removed by Join Filter: 20
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+   ->  Materialize (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Workers Launched: N
+               ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- the top node is Gather under which merge join happens, so parallel inserts
+-- must occur
+set enable_nestloop to off;
+set enable_mergejoin to on;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Merge Join (actual rows=N loops=N)
+         Merge Cond: (temp1.col1 = temp2.col2)
+         ->  Sort (actual rows=N loops=N)
+               Sort Key: temp1.col1
+               Sort Method: quicksort  Memory: 25kB
+               Worker 0:  Sort Method: quicksort  Memory: 25kB
+               Worker 1:  Sort Method: quicksort  Memory: 25kB
+               Worker 2:  Sort Method: quicksort  Memory: 25kB
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Sort (actual rows=N loops=N)
+               Sort Key: temp2.col2
+               Sort Method: quicksort  Memory: 25kB
+               ->  Seq Scan on temp2 (actual rows=N loops=N)
+(17 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel hash join happens under Gather node, so parallel inserts must occur
+set enable_mergejoin to off;
+set enable_hashjoin to on;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Hash Join (actual rows=N loops=N)
+         Hash Cond: (temp1.col1 = temp2.col2)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Hash (actual rows=N loops=N)
+               Buckets: 4096  Batches: 1  Memory Usage: 64kB
+               ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+reset enable_nestloop;
+reset enable_mergejoin;
+reset enable_hashjoin;
+drop table temp1;
+drop table temp2;
+drop table temp3;
+drop function explain_pictas(text);
 rollback;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 78b479cedf..46b958014a 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -39,4 +39,220 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
 
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+
+-- Parallel queries won't necessarily get as many workers as the planner
+-- asked for.  This affects not only the "Workers Launched:" field of EXPLAIN
+-- results, but also row counts and loop counts for parallel scans, Gathers,
+-- and everything in between.  This function filters out the values we can't
+-- rely on to be stable.
+-- This removes enough info that you might wonder why bother with EXPLAIN
+-- ANALYZE at all.  The answer is that we need to see whether the parallel
+-- inserts are being done by the workers, the only way is that
+-- Create <<tbl_name>> appears in the explain output.
+create function explain_pictas(text) returns setof text
+language plpgsql as
+$$
+declare
+    ln text;
+begin
+    for ln in
+        execute format('explain (analyze, costs off, summary off, timing off) %s',
+            $1)
+    loop
+        ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N');
+        ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N');
+        ln := regexp_replace(ln, 'Rows Removed by Filter: \d+', 'Rows Removed by Filter: N');
+        return next ln;
+    end loop;
+end;
+$$;
+
+-- parallel inserts must occur
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'create temporary table parallel_write as select length(stringu1) from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+select explain_pictas(
+'create unlogged table parallel_write as select length(stringu1) from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+select explain_pictas(
+'select length(stringu1) into parallel_write from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'select length(stringu1) into temporary parallel_write from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+select explain_pictas(
+'select length(stringu1) into unlogged parallel_write from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1 for update;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur
+select explain_pictas(
+'create materialized view parallel_mat_view as
+    select length(stringu1) from tenk1;');
+select count(*) from parallel_mat_view;
+drop materialized view parallel_mat_view;
+
+-- parallel inserts must occur
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+select explain_pictas(
+'create table parallel_write as execute parallel_write_prep;');
+select count(*) from parallel_write;
+deallocate parallel_write_prep;
+drop table parallel_write;
+
+-- parallel inserts must occur
+select explain_pictas(
+'create table parallel_write as select now(), four from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+select explain_pictas(
+E'create table parallel_write as
+    select nextval(\'parallel_write_sequence\'), four from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+create table temp1(col1) as select * from generate_series(1,5);
+create table temp2(col2) as select * from temp1;
+create table temp3(col3) as select * from temp1;
+
+-- parallel inserts must not occur, as there is a limit clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 limit 4;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is a distinct clause
+select explain_pictas(
+'create table parallel_write as select distinct * from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an aggregate and group clause
+select explain_pictas(
+'create table parallel_write as select count(*) from temp1 group by col1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an aggregate, group and having
+-- clauses
+select explain_pictas(
+'create table parallel_write as
+    select count(col1), (select col3 from
+        (select * from temp3) as tt limit 1) col4 from temp1, temp2
+    where temp1.col1 = temp2.col2 group by col4 having count(col1) > 0;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is a window function
+select explain_pictas(
+'create table parallel_write as
+    select avg(col1) OVER (PARTITION BY col1) from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- nested loop join is the top node under which Gather node exists, so parallel
+-- inserts must not occur
+set enable_nestloop to on;
+set enable_mergejoin to off;
+set enable_hashjoin to off;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- the top node is Gather under which merge join happens, so parallel inserts
+-- must occur
+set enable_nestloop to off;
+set enable_mergejoin to on;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel hash join happens under Gather node, so parallel inserts must occur
+set enable_mergejoin to off;
+set enable_hashjoin to on;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+reset enable_nestloop;
+reset enable_mergejoin;
+reset enable_hashjoin;
+
+drop table temp1;
+drop table temp2;
+drop table temp3;
+drop function explain_pictas(text);
 rollback;
-- 
2.25.1

From 1f9cad8986141a9a5efb81e448ed088d3bf4efc7 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Sat, 26 Dec 2020 15:30:15 +0530
Subject: [PATCH v16 4/4] Enable CTAS Parallel Inserts For Append

This patch allows pushing down the CTAS dest receiver even if there
exists Gather node under Top Append node. It also add the code for
influencing the planner to consider parallel tuple cost zero and
asserts for wrong enforcement if later parallel insertion is not
possible. Test cases are also included in this patch.
---
 src/backend/commands/createas.c              |  99 ++-
 src/backend/optimizer/path/allpaths.c        |  39 +
 src/backend/optimizer/plan/planner.c         |  10 +-
 src/include/commands/createas.h              |   4 +-
 src/test/regress/expected/write_parallel.out | 722 +++++++++++++++++++
 src/test/regress/sql/write_parallel.sql      | 222 ++++++
 6 files changed, 1071 insertions(+), 25 deletions(-)

diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index aaed0d8e2b..25da4e6035 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -657,6 +657,76 @@ intorel_destroy(DestReceiver *self)
 	pfree(self);
 }
 
+/*
+ * PushDownCTASParallelInsertState --- push the dest receiver down to the
+ * Gather nodes.
+ *
+ * In this function we only care Append and Gather nodes.
+ *
+ * Push the dest receiver to Gather node when it is either at the top of the
+ * plan or under top Append node unless it does not have any projections to do.
+ * Required information from the pushed dest receiver is sent to workers so
+ * that they can perform parallel insertions into the target table.
+ *
+ * If the top node is Append, then this function recursively checks the sub
+ * plans for Gather nodes, when found one(and if it does not have projections),
+ * then sets the dest receiver information.
+ *
+ * In any case, this function returns true if at least one Gather node can allow
+ * parallel insertions by the workers. Otherwise returns false.
+ */
+static bool
+PushDownCTASParallelInsertState(DestReceiver *dest, PlanState *ps,
+								bool *gather_exists)
+{
+	bool parallel = false;
+
+	if (ps == NULL)
+		return parallel;
+
+	if (IsA(ps, AppendState))
+	{
+		AppendState *aps = (AppendState *) ps;
+
+		for (int i = 0; i < aps->as_nplans; i++)
+		{
+			parallel |= PushDownCTASParallelInsertState(dest,
+														aps->appendplans[i],
+														gather_exists);
+		}
+	}
+	else if (IsA(ps, GatherState))
+	{
+		/*
+		 * Set to true if there exists at least one Gather node either at the
+		 * top of the plan or as a direct sub node under Append node.
+		 */
+		 *gather_exists |= true;
+
+		if (!ps->ps_ProjInfo)
+		{
+			GatherState *gstate = (GatherState *) ps;
+
+			parallel = true;
+
+			/* Okay to parallelize inserts, so mark it. */
+			((DR_intorel *) dest)->is_parallel = true;
+
+			/*
+			 * For parallelizing inserts in CTAS we must send information such
+			 * as into clause (to build separate dest receiver), object id (to
+			 * open the created table) to each workers. Since this information
+			 * is available in the CTAS dest receiver, store a reference to it
+			 * in the Gather state so that it will be used in
+			 * ExecInitParallelPlan to pick the required information.
+			 */
+			gstate->dest = dest;
+		}
+	}
+
+	return parallel;
+}
+
 /*
  * TryParallelizingInsertsInCTAS --- determine whether or not parallel
  * insertion is possible. If yes set the parallel insert state such as a flag
@@ -668,6 +738,7 @@ TryParallelizingInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc,
 							  uint8 *tuple_cost_flags)
 {
 	bool allow = false;
+	bool gather_exists = false;
 
 	/*
 	 * Do not allow parallel inserts if the table is temporary. As the
@@ -679,26 +750,12 @@ TryParallelizingInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc,
 	if (!into->rel || into->rel->relpersistence == RELPERSISTENCE_TEMP)
 		return;
 
-	if (queryDesc && IsA(queryDesc->planstate, GatherState) &&
-		!queryDesc->planstate->ps_ProjInfo)
-	{
-		GatherState *gstate = (GatherState *) queryDesc->planstate;
-		DestReceiver *dest	= queryDesc->dest;
+	if (!queryDesc)
+		return;
 
-		/* Okay to parallelize inserts, so mark it. */
-		((DR_intorel *) dest)->is_parallel = true;
-
-		/*
-		 * For parallelizing inserts in CTAS we must send information such as
-		 * into clause (to build separate dest receiver), object id (to open
-		 * the created table) to each workers. Since this information is
-		 * available in the CTAS dest receiver, store a reference to it in the
-		 * Gather state so that it will be used in ExecInitParallelPlan to pick
-		 * the required information.
-		 */
-		gstate->dest = dest;
-		allow = true;
-	}
+	allow = PushDownCTASParallelInsertState(queryDesc->dest,
+											queryDesc->planstate,
+											&gather_exists);
 
 	/*
 	 * It should not happen that in cost_gather we have ignored the parallel
@@ -709,8 +766,7 @@ TryParallelizingInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc,
 	 * which the parallel plan may have been chosen, but we do not allow the
 	 * parallel inserts now.
 	 */
-	if (!allow && tuple_cost_flags && queryDesc &&
-			IsA(queryDesc->planstate, GatherState))
+	if (!allow && tuple_cost_flags && gather_exists)
 	{
 		/*
 		 * If we have correctly ignored parallel tuple cost in planner while
@@ -723,5 +779,4 @@ TryParallelizingInsertsInCTAS(IntoClause *into, QueryDesc *queryDesc,
 
 	if (tuple_cost_flags)
 		*tuple_cost_flags = CTAS_PARALLEL_INS_UNDEF;
-
 }
diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c
index 627d08b78a..00819eea3f 100644
--- a/src/backend/optimizer/path/allpaths.c
+++ b/src/backend/optimizer/path/allpaths.c
@@ -23,6 +23,7 @@
 #include "catalog/pg_class.h"
 #include "catalog/pg_operator.h"
 #include "catalog/pg_proc.h"
+#include "commands/createas.h"
 #include "foreign/fdwapi.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1103,6 +1104,44 @@ set_append_rel_size(PlannerInfo *root, RelOptInfo *rel,
 		if (root->glob->parallelModeOK && rel->consider_parallel)
 			set_rel_consider_parallel(root, childrel, childRTE);
 
+		/*
+		 * When subplan is subquery, It's possible to do parallel insert if top
+		 * node of subquery is Gather, so we set the flag to ignore parallel
+		 * tuple cost by the Gather path in cost_gather if the SELECT is for
+		 * CTAS.
+		 */
+		if (childrel->rtekind == RTE_SUBQUERY)
+		{
+			/*
+			 * We set the flag for two cases when there is no parent path will
+			 * be created(such as : limit,sort,distinct...):
+			 * i) query_level is 1
+			 * ii) query_level > 1 then set the flag in the parent_root.
+			 * The case ii) is to check append under append:
+			 * Append
+			 *	->Append
+			 *	->Gather
+			 *	->Other plan
+			 */
+			if (root->parse->CTASParallelInsInfo &
+				CTAS_PARALLEL_INS_SELECT &&
+				(root->query_level == 1 ||
+				 root->parent_root->parse->CTASParallelInsInfo &
+				 CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND) &&
+				!(HAS_PARENT_PATH_GENERATING_CLAUSE(root)))
+			{
+				root->parse->CTASParallelInsInfo |=
+										CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+			}
+		}
+
+		if (root->parse->CTASParallelInsInfo &
+			CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND)
+		{
+			root->parse->CTASParallelInsInfo &=
+										~CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+		}
+
 		/*
 		 * Compute the child's size.
 		 */
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index f1134711b0..7555cde61a 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -7350,8 +7350,14 @@ can_partial_agg(PlannerInfo *root)
 static bool
 ignore_parallel_tuple_cost(PlannerInfo *root)
 {
-	if (root->query_level == 1 &&
-		(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT))
+	if (root->query_level != 1 &&
+		(root->parent_root->parse->CTASParallelInsInfo &
+		 CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND))
+	{
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_SELECT;
+	}
+
+	if (root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT)
 	{
 		/*
 		 * In each of the HAS_PARENT_PATH_GENERATING_CLAUSE cases, a parent
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 919148b10a..8dbe537693 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -54,7 +54,9 @@ typedef enum CTASParallelInsertOpt
 	 */
 	CTAS_PARALLEL_INS_TUP_COST_CAN_IGN = 1 << 1,
 	/* Set to this after the cost is ignored. */
-	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2
+	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2,
+	/* Set to this in case tuple cost needs to be ignored for Append cases. */
+	CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND = 1 << 3
 } CTASParallelInsertOpt;
 
 #define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 357fbbbe8d..e0296e88a3 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -649,6 +649,728 @@ drop table parallel_write;
 reset enable_nestloop;
 reset enable_mergejoin;
 reset enable_hashjoin;
+-- test cases for performing parallel inserts when Append node is at the top
+-- and Gather node is in one of its direct sub plans.
+-- case 1: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5 union all
+    select * from temp2 where col2 = 5;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: N
+(22 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+-- case 2: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp2;');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Append (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select * from temp2;');
+                                 explain_pictas                                 
+--------------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Params Evaluated: $1, $3
+   Workers Launched: N
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 3
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+   InitPlan 2 (returns $3)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 3
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+   ->  Parallel Append (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(21 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+-- case 3: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples to
+-- Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Gather
+-- 		->Parallel Seq Scan
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1 union all
+    select * from temp1 where col1 = 5;');
+                                explain_pictas                                
+------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Seq Scan on temp1 (actual rows=N loops=N)
+         SubPlan 1
+           ->  Limit (actual rows=N loops=N)
+                 ->  Gather (actual rows=N loops=N)
+                       Workers Planned: 3
+                       Workers Launched: N
+                       ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+(22 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     7
+(1 row)
+
+drop table parallel_write;
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as select * from temp1 where col1 = (select 1) union all
+    select * from temp1 where col1 = (select 1) union all
+ select * from temp2 where col2 = (select 2);');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $0
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+               Filter: (col1 = $0)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $1
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 2 (returns $1)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+               Filter: (col1 = $1)
+               Rows Removed by Filter: N
+   ->  Seq Scan on temp2 (actual rows=N loops=N)
+         Filter: (col2 = $2)
+         Rows Removed by Filter: N
+         InitPlan 3 (returns $2)
+           ->  Result (actual rows=N loops=N)
+(26 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+alter table temp2 reset (parallel_workers);
+drop table parallel_write;
+-- case 4: parallel inserts must not occur as there will be no direct Gather
+-- node under Append node. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1;');
+                                    explain_pictas                                    
+--------------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Seq Scan on temp1 (actual rows=N loops=N)
+   ->  Seq Scan on temp2 (actual rows=N loops=N)
+   ->  Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+         SubPlan 1
+           ->  Limit (actual rows=N loops=N)
+                 ->  Gather (actual rows=N loops=N)
+                       Workers Planned: 3
+                       Workers Launched: N
+                       ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+-- case 5: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Seq Scan / Join / any other non-Gather node
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp1;');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Append (actual rows=N loops=N)
+         ->  Seq Scan on temp2 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+alter table temp2 reset (parallel_workers);
+-- case 6: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Gather
+--  ->Gather
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp2 where col2 = (select 2);');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $2
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($2 = $0)
+                     ->  Parallel Seq Scan on temp2 (never executed)
+                           Filter: (col2 = $0)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $3
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+               Filter: (col2 = $3)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $4
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 5 (returns $4)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+               Filter: (col2 = $4)
+               Rows Removed by Filter: N
+(47 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+                                   explain_pictas                                   
+------------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $2
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($2 = $0)
+                     ->  Parallel Seq Scan on temp2 (never executed)
+                           Filter: (col2 = $0)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $3, $4
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 5 (returns $4)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($4 = $3)
+                     ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+                           Filter: (col1 = $3)
+                           Rows Removed by Filter: N
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $3, $5
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 6 (returns $5)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($5 = $3)
+                     ->  Parallel Seq Scan on temp2 temp2_1 (never executed)
+                           Filter: (col2 = $3)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: N
+(67 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+-- case 7: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Seq Scan / Join / any other non-Gather node
+--  ->Gather
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp1 where col1 = (select 2);');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Result (actual rows=N loops=N)
+               One-Time Filter: ($2 = $0)
+               InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Seq Scan on temp2 (never executed)
+                     Filter: (col2 = $0)
+   ->  Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+         Filter: (col2 = $3)
+         Rows Removed by Filter: N
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $4
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 5 (returns $4)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+               Filter: (col1 = $4)
+               Rows Removed by Filter: N
+(37 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+                                   explain_pictas                                   
+------------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Result (actual rows=N loops=N)
+               One-Time Filter: ($2 = $0)
+               InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Seq Scan on temp2 (never executed)
+                     Filter: (col2 = $0)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $3, $4
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 5 (returns $4)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($4 = $3)
+                     ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+                           Filter: (col1 = $3)
+                           Rows Removed by Filter: N
+         ->  Result (actual rows=N loops=N)
+               One-Time Filter: ($5 = $3)
+               InitPlan 6 (returns $5)
+                 ->  Result (actual rows=N loops=N)
+               ->  Seq Scan on temp2 temp2_1 (never executed)
+                     Filter: (col2 = $3)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+         Filter: (col2 = 5)
+         Rows Removed by Filter: N
+(53 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+alter table temp2 reset (parallel_workers);
+-- case 8: parallel inserts must not occur because there is no Gather or Append
+-- node at the top for union, except/except all, intersect/intersect all
+-- cases.
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union
+    select * from temp2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ HashAggregate (actual rows=N loops=N)
+   Group Key: temp1.col1
+   Batches: 1  Memory Usage: 217kB
+   ->  Append (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Workers Launched: N
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Workers Launched: N
+               ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except
+    select * from temp2 where col2 < 3;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Except (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+                           Filter: (col2 < 3)
+                           Rows Removed by Filter: N
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except all
+    select * from temp2 where col2 < 3;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Except All (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+                           Filter: (col2 < 3)
+                           Rows Removed by Filter: N
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect
+    select * from temp2;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Intersect (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect all
+    select * from temp2;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Intersect All (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
 drop table temp1;
 drop table temp2;
 drop table temp3;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 46b958014a..4e03d0ab6b 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -251,6 +251,228 @@ reset enable_nestloop;
 reset enable_mergejoin;
 reset enable_hashjoin;
 
+-- test cases for performing parallel inserts when Append node is at the top
+-- and Gather node is in one of its direct sub plans.
+
+-- case 1: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5 union all
+    select * from temp2 where col2 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 2: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 3: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples to
+-- Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Gather
+-- 		->Parallel Seq Scan
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1 union all
+    select * from temp1 where col1 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as select * from temp1 where col1 = (select 1) union all
+    select * from temp1 where col1 = (select 1) union all
+ select * from temp2 where col2 = (select 2);');
+select count(*) from parallel_write;
+alter table temp2 reset (parallel_workers);
+drop table parallel_write;
+
+-- case 4: parallel inserts must not occur as there will be no direct Gather
+-- node under Append node. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 5: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Seq Scan / Join / any other non-Gather node
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+
+alter table temp2 set (parallel_workers = 0);
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 reset (parallel_workers);
+
+-- case 6: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Gather
+--  ->Gather
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp2 where col2 = (select 2);');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 7: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Seq Scan / Join / any other non-Gather node
+--  ->Gather
+
+alter table temp2 set (parallel_workers = 0);
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp1 where col1 = (select 2);');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 reset (parallel_workers);
+
+-- case 8: parallel inserts must not occur because there is no Gather or Append
+-- node at the top for union, except/except all, intersect/intersect all
+-- cases.
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except
+    select * from temp2 where col2 < 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except all
+    select * from temp2 where col2 < 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect all
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
 drop table temp1;
 drop table temp2;
 drop table temp3;
-- 
2.25.1

From d36c81fb6e13a8ec3c36fd6cd3bb096d783715e7 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 28 Dec 2020 14:30:45 +0530
Subject: [PATCH v17 1/4] Parallel Inserts in CREATE TABLE AS

Allow the leader and each worker insert the tuples in parallel
if the SELECT part of the CTAS is parallelizable.

The design:
The main idea is to push the CTAS dest receiver down to Gather node
and from there the required information will be shared to workers
so that they can perform parallel insertions. Leader will also
participate in insertions. After the planning, check if the upper
plan node is Gather in createas.c and mark a parallelism flag in
the CTAS dest receiver and push it down to Gather node. Each worker
can create its own CTAS dest receiver with the information passed
from the leader. Leader inserts its share of tuples if instructed
to do, and so are workers. Each worker writes atomically its number
of inserted tuples into a shared memory variable, the leader combines
this with its own number of inserted tuples and shares to the client.
---
 src/backend/access/heap/heapam.c       |  11 --
 src/backend/access/transam/xact.c      |  28 ++++-
 src/backend/commands/createas.c        | 137 ++++++++++++++++++++++---
 src/backend/commands/explain.c         |  31 ++++++
 src/backend/executor/execParallel.c    |  70 ++++++++++++-
 src/backend/executor/nodeGather.c      | 111 ++++++++++++++++++--
 src/backend/executor/nodeGatherMerge.c |   4 +-
 src/include/access/xact.h              |   1 +
 src/include/commands/createas.h        |  26 +++++
 src/include/executor/execParallel.h    |   6 +-
 src/include/nodes/execnodes.h          |   3 +
 11 files changed, 384 insertions(+), 44 deletions(-)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index a9583f3103..86347ba273 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2043,17 +2043,6 @@ static HeapTuple
 heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
 					CommandId cid, int options)
 {
-	/*
-	 * To allow parallel inserts, we need to ensure that they are safe to be
-	 * performed in workers. We have the infrastructure to allow parallel
-	 * inserts in general except for the cases where inserts generate a new
-	 * CommandId (eg. inserts into a table having a foreign key column).
-	 */
-	if (IsParallelWorker())
-		ereport(ERROR,
-				(errcode(ERRCODE_INVALID_TRANSACTION_STATE),
-				 errmsg("cannot insert tuples in a parallel worker")));
-
 	tup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
 	tup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
 	tup->t_data->t_infomask |= HEAP_XMAX_INVALID;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 9cd0b7c11b..4e5366fe78 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -764,17 +764,35 @@ GetCurrentCommandId(bool used)
 	if (used)
 	{
 		/*
-		 * Forbid setting currentCommandIdUsed in a parallel worker, because
-		 * we have no provision for communicating this back to the leader.  We
-		 * could relax this restriction when currentCommandIdUsed was already
-		 * true at the start of the parallel operation.
+		 * If in a parallel worker, only allow setting currentCommandIdUsed if
+		 * currentCommandIdUsed was already true at the start of the parallel
+		 * operation (by way of SetCurrentCommandIdUsed()), otherwise forbid
+		 * setting currentCommandIdUsed because we have no provision for
+		 * communicating this back to the leader. Once currentCommandIdUsed is
+		 * set, the commandId used by leader and workers can't be changed,
+		 * because CommandCounterIncrement() then prevents any attempted
+		 * increment of the current commandId.
 		 */
-		Assert(!IsParallelWorker());
+		Assert(!(IsParallelWorker() && !currentCommandIdUsed));
 		currentCommandIdUsed = true;
 	}
 	return currentCommandId;
 }
 
+/*
+ * SetCurrentCommandIdUsedForWorker
+ *
+ * For a parallel worker, record that the currentCommandId has been used. This
+ * must only be called at the start of a parallel operation.
+ */
+void
+SetCurrentCommandIdUsedForWorker(void)
+{
+	Assert(IsParallelWorker() && !currentCommandIdUsed && currentCommandId != InvalidCommandId);
+
+	currentCommandIdUsed = true;
+}
+
 /*
  *	SetParallelStartTimestamps
  *
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 6bf6c5a310..2beccf60df 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,18 +51,6 @@
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 
-typedef struct
-{
-	DestReceiver pub;			/* publicly-known function pointers */
-	IntoClause *into;			/* target relation specification */
-	/* These fields are filled by intorel_startup: */
-	Relation	rel;			/* relation to write to */
-	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
-	CommandId	output_cid;		/* cmin to insert in output tuples */
-	int			ti_options;		/* table_tuple_insert performance options */
-	BulkInsertState bistate;	/* bulk insert state */
-} DR_intorel;
-
 /* utility functions for CTAS definition creation */
 static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into);
 static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -350,6 +338,14 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		/* call ExecutorStart to prepare the plan for execution */
 		ExecutorStart(queryDesc, GetIntoRelEFlags(into));
 
+		/*
+		 * If the SELECT part of the CTAS is parallelizable, then set the
+		 * parallel insert state. We need plan state to be initialized by the
+		 * executor to decide whether to allow parallel inserts or not.
+		 */
+		if (IsParallelInsertionAllowedInCTAS(into))
+			SetCTASParallelInsertState(queryDesc);
+
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
 
@@ -418,6 +414,9 @@ CreateIntoRelDestReceiver(IntoClause *intoClause)
 	self->pub.rDestroy = intorel_destroy;
 	self->pub.mydest = DestIntoRel;
 	self->into = intoClause;
+	self->is_parallel = false;
+	self->is_parallel_worker = false;
+	self->object_id = InvalidOid;
 	/* other private fields will be set during intorel_startup */
 
 	return (DestReceiver *) self;
@@ -438,6 +437,35 @@ intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 	ListCell   *lc;
 	int			attnum;
 
+	/*
+	 * All the necessary work such as table creation, sanity checks etc. would
+	 * have been done by the leader. So, parallel workers just need to open the
+	 * table, allocate bulk insert state, mark the command id as used, store it
+	 * in the dest receiver and return.
+	 */
+	if (myState->is_parallel_worker)
+	{
+		/* In the worker */
+		intoRelationDesc = table_open(myState->object_id, AccessExclusiveLock);
+		myState->rel = intoRelationDesc;
+		myState->reladdr = InvalidObjectAddress;
+		myState->ti_options = 0;
+		myState->bistate = GetBulkInsertState();
+
+		/*
+		 * Right after the table is created in the leader, the command id is
+		 * incremented (in create_ctas_internal()). The new command id is
+		 * marked as used in intorel_startup(), then the parallel mode is
+		 * entered. The command id and transaction id are serialized into
+		 * parallel DSM, they are then available to all parallel workers. All
+		 * the workers need to mark the command id as used before insertion.
+		 */
+		SetCurrentCommandIdUsedForWorker();
+		myState->output_cid = GetCurrentCommandId(false);
+
+		return;
+	}
+
 	Assert(into != NULL);		/* else somebody forgot to set it */
 
 	/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
@@ -540,6 +568,27 @@ intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 	else
 		myState->bistate = NULL;
 
+	/* If parallel inserts are to be allowed, set few extra information. */
+	if (myState->is_parallel)
+	{
+		myState->object_id = intoRelationAddr.objectId;
+
+		/*
+		 * We don't need to skip contacting FSM while inserting tuples for
+		 * parallel mode, while extending the relations, workers instead of
+		 * blocking on a page while another worker is inserting, can check the
+		 * FSM for another page that can accommodate the tuples. This results
+		 * in major benefit for parallel inserts.
+		 */
+		myState->ti_options = 0;
+
+		/*
+		 * rd_createSubid is marked invalid, otherwise, the table is not
+		 * allowed to be extended by the workers.
+		 */
+		myState->rel->rd_createSubid = InvalidSubTransactionId;
+	}
+
 	/*
 	 * Valid smgr_targblock implies something already wrote to the relation.
 	 * This may be harmless, but this function hasn't planned for it.
@@ -606,3 +655,67 @@ intorel_destroy(DestReceiver *self)
 {
 	pfree(self);
 }
+
+/*
+ * IsParallelInsertionAllowedInCTAS --- check if parallel insertion is allowed
+ *
+ * Do not allow parallel inserts if the table is temporary. As the temporary
+ * tables are backend local, workers can not know about them. Currently, CTAS
+ * supports creation of normal(logged), temporary and unlogged tables. It does
+ * not support foreign or partition table creation. Hence the check for
+ * temporary table is enough here.
+ *
+ * Return false either if the into clause is NULL or if the table is
+ * temporary, otherwise true.
+ */
+bool
+IsParallelInsertionAllowedInCTAS(IntoClause *into)
+{
+	/* Below check may hit in case this function is called from explain.c. */
+	if (!IS_CTAS(into))
+		return false;
+
+	if (into->rel && into->rel->relpersistence == RELPERSISTENCE_TEMP)
+		return false;
+
+	return true;
+}
+
+/*
+ * SetCTASParallelInsertState --- set the parallel insert state
+ *
+ * See if the upper node is Gather and it doesn't have any projections, then
+ * set the parallel insert state such as a flag in the dest receiver and also
+ * store the dest receiver reference in the Gather node so that the required
+ * information will be sent to workers.
+ */
+void
+SetCTASParallelInsertState(QueryDesc *queryDesc)
+{
+	GatherState *gstate;
+	DestReceiver *dest;
+
+	Assert(queryDesc);
+
+	gstate = (GatherState *) queryDesc->planstate;
+	dest	= queryDesc->dest;
+
+	/*
+	 * Parallel insertions are not possible either if the upper node is not
+	 * Gather or it's a Gather but it have some projections to perform.
+	 */
+	if (!IsA(gstate, GatherState) || gstate->ps.ps_ProjInfo)
+		return;
+
+	/* Okay to parallelize inserts, so mark it. */
+	((DR_intorel *) dest)->is_parallel = true;
+
+	/*
+	 * For parallelizing inserts in CTAS we must send information such as into
+	 * clause (to build separate dest receiver), object id (to open the created
+	 * table) to each workers. Since this information is available in the CTAS
+	 * dest receiver, store a reference to it in the Gather state so that it
+	 * will be used in ExecInitParallelPlan to pick the required information.
+	 */
+	gstate->dest = dest;
+}
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 43f9b01e83..9a412c3e6b 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -556,6 +556,14 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	/* call ExecutorStart to prepare the plan for execution */
 	ExecutorStart(queryDesc, eflags);
 
+	/*
+	 * If the SELECT part of the CTAS is parallelizable, then set the parallel
+	 * insert state. We need plan state to be initialized by the executor to
+	 * decide whether to allow parallel inserts or not.
+	 */
+	if (IsParallelInsertionAllowedInCTAS(into))
+		SetCTASParallelInsertState(queryDesc);
+
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
 	{
@@ -1775,6 +1783,29 @@ ExplainNode(PlanState *planstate, List *ancestors,
 
 				if (gather->single_copy || es->format != EXPLAIN_FORMAT_TEXT)
 					ExplainPropertyBool("Single Copy", gather->single_copy, es);
+
+				/*
+				 * Show the create table information under Gather node in case
+				 * parallel workers have inserted the rows.
+				 */
+				if (IsA(planstate, GatherState))
+				{
+					GatherState *gstate = (GatherState *) planstate;
+
+					if (IS_PARALLEL_CTAS_DEST(gstate->dest) &&
+						((DR_intorel *) gstate->dest)->into->rel &&
+						((DR_intorel *) gstate->dest)->into->rel->relname)
+					{
+						es->indent--;
+						ExplainIndentText(es);
+						appendStringInfoString(es->str, "->  ");
+						appendStringInfoString(es->str, "Create ");
+						appendStringInfo(es->str, "%s\n",
+							((DR_intorel *) gstate->dest)->into->rel->relname);
+						ExplainIndentText(es);
+						es->indent++;
+					}
+				}
 			}
 			break;
 		case T_GatherMerge:
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index befde52691..270f8c13d7 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -23,6 +23,7 @@
 
 #include "postgres.h"
 
+#include "commands/createas.h"
 #include "executor/execParallel.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
@@ -65,6 +66,7 @@
 #define PARALLEL_KEY_QUERY_TEXT		UINT64CONST(0xE000000000000008)
 #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009)
 #define PARALLEL_KEY_WAL_USAGE			UINT64CONST(0xE00000000000000A)
+#define PARALLEL_KEY_INTO_CLAUSE		UINT64CONST(0xE00000000000000B)
 
 #define PARALLEL_TUPLE_QUEUE_SIZE		65536
 
@@ -77,6 +79,9 @@ typedef struct FixedParallelExecutorState
 	dsa_pointer param_exec;
 	int			eflags;
 	int			jit_flags;
+	Oid			objectid;		/* workers to open relation/table.  */
+	/* Number of tuples inserted by all the workers. */
+	pg_atomic_uint64	processed;
 } FixedParallelExecutorState;
 
 /*
@@ -578,7 +583,8 @@ ExecParallelSetupTupleQueues(ParallelContext *pcxt, bool reinitialize)
 ParallelExecutorInfo *
 ExecInitParallelPlan(PlanState *planstate, EState *estate,
 					 Bitmapset *sendParams, int nworkers,
-					 int64 tuples_needed)
+					 int64 tuples_needed, IntoClause *intoclause,
+					 Oid objectid)
 {
 	ParallelExecutorInfo *pei;
 	ParallelContext *pcxt;
@@ -600,6 +606,8 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	Size		dsa_minsize = dsa_minimum_size();
 	char	   *query_string;
 	int			query_len;
+	char 		*intoclausestr = NULL;
+	int			intoclause_len = 0;
 
 	/*
 	 * Force any initplan outputs that we're going to pass to workers to be
@@ -712,6 +720,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_estimate_chunk(&pcxt->estimator, dsa_minsize);
 	shm_toc_estimate_keys(&pcxt->estimator, 1);
 
+	/* Estimate space for into clause for CTAS. */
+	if (IS_CTAS(intoclause) && OidIsValid(objectid))
+	{
+		intoclausestr = nodeToString(intoclause);
+		intoclause_len = strlen(intoclausestr) + 1;
+		shm_toc_estimate_chunk(&pcxt->estimator, intoclause_len);
+		shm_toc_estimate_keys(&pcxt->estimator, 1);
+	}
+
 	/* Everyone's had a chance to ask for space, so now create the DSM. */
 	InitializeParallelDSM(pcxt);
 
@@ -729,6 +746,14 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	fpes->param_exec = InvalidDsaPointer;
 	fpes->eflags = estate->es_top_eflags;
 	fpes->jit_flags = estate->es_jit_flags;
+	pg_atomic_init_u64(&fpes->processed, 0);
+	pei->processed = &fpes->processed;
+
+	if (intoclausestr)
+		fpes->objectid = objectid;
+	else
+		fpes->objectid = InvalidOid;
+
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, fpes);
 
 	/* Store query string */
@@ -758,8 +783,18 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_WAL_USAGE, walusage_space);
 	pei->wal_usage = walusage_space;
 
-	/* Set up the tuple queues that the workers will write into. */
-	pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	if (intoclausestr)
+	{
+		char *intoclause_space = shm_toc_allocate(pcxt->toc, intoclause_len);
+
+		memcpy(intoclause_space, intoclausestr, intoclause_len);
+		shm_toc_insert(pcxt->toc, PARALLEL_KEY_INTO_CLAUSE, intoclause_space);
+	}
+	else
+	{
+		/* Set up the tuple queues that the workers will write into. */
+		pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	}
 
 	/* We don't need the TupleQueueReaders yet, though. */
 	pei->reader = NULL;
@@ -1387,12 +1422,30 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 	void	   *area_space;
 	dsa_area   *area;
 	ParallelWorkerContext pwcxt;
+	char		*intoclausestr = NULL;
+	IntoClause	*intoclause = NULL;
 
 	/* Get fixed-size state. */
 	fpes = shm_toc_lookup(toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
 
-	/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
-	receiver = ExecParallelGetReceiver(seg, toc);
+	intoclausestr = shm_toc_lookup(toc, PARALLEL_KEY_INTO_CLAUSE, true);
+	if (intoclausestr)
+	{
+		/*
+		 * If the worker is for parallel insert in CTAS, then use the proper
+		 * dest receiver.
+		 */
+		intoclause = (IntoClause *) stringToNode(intoclausestr);
+		receiver = CreateIntoRelDestReceiver(intoclause);
+		((DR_intorel *)receiver)->is_parallel_worker = true;
+		((DR_intorel *)receiver)->object_id = fpes->objectid;
+	}
+	else
+	{
+		/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
+		receiver = ExecParallelGetReceiver(seg, toc);
+	}
+
 	instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_INSTRUMENTATION, true);
 	if (instrumentation != NULL)
 		instrument_options = instrumentation->instrument_options;
@@ -1471,6 +1524,13 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 			queryDesc->estate->es_jit->instr;
 	}
 
+	/*
+	 * Write out the number of tuples this worker has inserted. Leader will use
+	 * it to inform the end client.
+	 */
+	if (intoclausestr)
+		pg_atomic_add_fetch_u64(&fpes->processed, queryDesc->estate->es_processed);
+
 	/* Must do this after capturing instrumentation. */
 	ExecutorEnd(queryDesc);
 
diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c
index a01b46af14..96d745229d 100644
--- a/src/backend/executor/nodeGather.c
+++ b/src/backend/executor/nodeGather.c
@@ -32,6 +32,7 @@
 
 #include "access/relscan.h"
 #include "access/xact.h"
+#include "commands/createas.h"
 #include "executor/execdebug.h"
 #include "executor/execParallel.h"
 #include "executor/nodeGather.h"
@@ -48,6 +49,7 @@ static TupleTableSlot *ExecGather(PlanState *pstate);
 static TupleTableSlot *gather_getnext(GatherState *gatherstate);
 static MinimalTuple gather_readnext(GatherState *gatherstate);
 static void ExecShutdownGatherWorkers(GatherState *node);
+static void ExecParallelInsertInCTAS(GatherState *node);
 
 
 /* ----------------------------------------------------------------
@@ -131,6 +133,71 @@ ExecInitGather(Gather *node, EState *estate, int eflags)
 	return gatherstate;
 }
 
+/* ----------------------------------------------------------------
+ *		ExecParallelInsertInCTAS(node)
+ *
+ *		Facilitates parallel inserts by parallel workers and/or
+ *		leader for Create Table AS.
+ * ----------------------------------------------------------------
+ */
+static void
+ExecParallelInsertInCTAS(GatherState *node)
+{
+	/*
+	 * By now, parallel workers if launched any, would have started their work
+	 * i.e. insertion to target table. In case the leader is also chosen to
+	 * participate, then finish its share before going to wait for the parallel
+	 * workers to finish.
+	 *
+	 * In case if no workers were launched, allow the leader to insert entire
+	 * tuples.
+	 */
+	if (node->need_to_scan_locally || node->nworkers_launched == 0)
+	{
+		EState	   *estate = node->ps.state;
+		TupleTableSlot *outerTupleSlot;
+
+		for(;;)
+		{
+			/* Install our DSA area while executing the plan. */
+			estate->es_query_dsa =
+					node->pei ? node->pei->area : NULL;
+
+			outerTupleSlot = ExecProcNode(node->ps.lefttree);
+
+			estate->es_query_dsa = NULL;
+
+			if(TupIsNull(outerTupleSlot))
+				break;
+
+			(void) node->dest->receiveSlot(outerTupleSlot, node->dest);
+
+			node->ps.state->es_processed++;
+		}
+
+		node->need_to_scan_locally = false;
+	}
+
+	if (node->nworkers_launched > 0)
+	{
+		/*
+		 * We wait here for the parallel workers to finish their work and
+		 * accumulate the tuples they inserted and also their buffer/WAL usage.
+		 * We do not destroy the parallel context here, it will be done in
+		 * ExecShutdownGather at the end of the plan. Note that the
+		 * ExecShutdownGatherWorkers call from ExecShutdownGather will be a
+		 * no-op.
+		 */
+		ExecShutdownGatherWorkers(node);
+
+		/*
+		 * Add up the total tuples inserted by all workers, to the tuples
+		 * inserted by the leader(if any). This will be shared to client.
+		 */
+		node->ps.state->es_processed += pg_atomic_read_u64(node->pei->processed);
+	}
+}
+
 /* ----------------------------------------------------------------
  *		ExecGather(node)
  *
@@ -157,6 +224,7 @@ ExecGather(PlanState *pstate)
 	{
 		EState	   *estate = node->ps.state;
 		Gather	   *gather = (Gather *) node->ps.plan;
+		bool		isctas = IS_PARALLEL_CTAS_DEST(node->dest);
 
 		/*
 		 * Sometimes we might have to run without parallelism; but if parallel
@@ -165,6 +233,18 @@ ExecGather(PlanState *pstate)
 		if (gather->num_workers > 0 && estate->es_use_parallel_mode)
 		{
 			ParallelContext *pcxt;
+			IntoClause	*intoclause = NULL;
+			Oid			objectid = InvalidOid;
+
+			/*
+			 * Take the necessary information to be passed to workers for
+			 * parallel inserts in CTAS.
+			 */
+			if (isctas)
+			{
+				intoclause = ((DR_intorel *) node->dest)->into;
+				objectid = ((DR_intorel *) node->dest)->object_id;
+			}
 
 			/* Initialize, or re-initialize, shared state needed by workers. */
 			if (!node->pei)
@@ -172,7 +252,10 @@ ExecGather(PlanState *pstate)
 												 estate,
 												 gather->initParam,
 												 gather->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 /* CTAS info */
+												 intoclause,
+												 objectid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
@@ -190,13 +273,16 @@ ExecGather(PlanState *pstate)
 			/* Set up tuple queue readers to read the results. */
 			if (pcxt->nworkers_launched > 0)
 			{
-				ExecParallelCreateReaders(node->pei);
-				/* Make a working array showing the active readers */
-				node->nreaders = pcxt->nworkers_launched;
-				node->reader = (TupleQueueReader **)
-					palloc(node->nreaders * sizeof(TupleQueueReader *));
-				memcpy(node->reader, node->pei->reader,
-					   node->nreaders * sizeof(TupleQueueReader *));
+				if (!isctas)
+				{
+					ExecParallelCreateReaders(node->pei);
+					/* Make a working array showing the active readers */
+					node->nreaders = pcxt->nworkers_launched;
+					node->reader = (TupleQueueReader **)
+						palloc(node->nreaders * sizeof(TupleQueueReader *));
+					memcpy(node->reader, node->pei->reader,
+						node->nreaders * sizeof(TupleQueueReader *));
+				}
 			}
 			else
 			{
@@ -208,9 +294,16 @@ ExecGather(PlanState *pstate)
 		}
 
 		/* Run plan locally if no workers or enabled and not single-copy. */
-		node->need_to_scan_locally = (node->nreaders == 0)
+		node->need_to_scan_locally = (node->nreaders == 0 && !isctas)
 			|| (!gather->single_copy && parallel_leader_participation);
 		node->initialized = true;
+
+		/* Perform parallel inserts for CTAS. */
+		if (isctas)
+		{
+			ExecParallelInsertInCTAS(node);
+			return NULL;
+		}
 	}
 
 	/*
diff --git a/src/backend/executor/nodeGatherMerge.c b/src/backend/executor/nodeGatherMerge.c
index 47129344f3..ee45272c17 100644
--- a/src/backend/executor/nodeGatherMerge.c
+++ b/src/backend/executor/nodeGatherMerge.c
@@ -216,7 +216,9 @@ ExecGatherMerge(PlanState *pstate)
 												 estate,
 												 gm->initParam,
 												 gm->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 NULL,
+												 InvalidOid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7320de345c..5beae6c617 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -389,6 +389,7 @@ extern FullTransactionId GetCurrentFullTransactionIdIfAny(void);
 extern void MarkCurrentTransactionIdLoggedIfAny(void);
 extern bool SubTransactionIsActive(SubTransactionId subxid);
 extern CommandId GetCurrentCommandId(bool used);
+extern void SetCurrentCommandIdUsedForWorker(void);
 extern void SetParallelStartTimestamps(TimestampTz xact_ts, TimestampTz stmt_ts);
 extern TimestampTz GetCurrentTransactionStartTimestamp(void);
 extern TimestampTz GetCurrentStatementStartTimestamp(void);
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 7629230254..ab74c69b9f 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -14,12 +14,35 @@
 #ifndef CREATEAS_H
 #define CREATEAS_H
 
+#include "access/heapam.h"
 #include "catalog/objectaddress.h"
+#include "executor/execdesc.h"
 #include "nodes/params.h"
+#include "nodes/plannodes.h"
 #include "parser/parse_node.h"
 #include "tcop/dest.h"
 #include "utils/queryenvironment.h"
 
+typedef struct
+{
+	DestReceiver pub;			/* publicly-known function pointers */
+	IntoClause *into;			/* target relation specification */
+	/* These fields are filled by intorel_startup: */
+	Relation	rel;			/* relation to write to */
+	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
+	CommandId	output_cid;		/* cmin to insert in output tuples */
+	int			ti_options;		/* table_tuple_insert performance options */
+	BulkInsertState bistate;	/* bulk insert state */
+	bool		is_parallel;	/* is parallelism to be considered? */
+	bool		is_parallel_worker; /* true for parallel worker */
+	/* Used by parallel workers for opening the table created in the leader. */
+	Oid			object_id;
+} DR_intorel;
+
+#define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
+#define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
+									 IS_CTAS(((DR_intorel *) dest)->into) && \
+									 ((DR_intorel *) dest)->is_parallel)
 
 extern ObjectAddress ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 									   ParamListInfo params, QueryEnvironment *queryEnv,
@@ -29,4 +52,7 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
+extern bool IsParallelInsertionAllowedInCTAS(IntoClause *into);
+
+extern void SetCTASParallelInsertState(QueryDesc *queryDesc);
 #endif							/* CREATEAS_H */
diff --git a/src/include/executor/execParallel.h b/src/include/executor/execParallel.h
index 5a39a5b29c..9f959f741b 100644
--- a/src/include/executor/execParallel.h
+++ b/src/include/executor/execParallel.h
@@ -35,11 +35,15 @@ typedef struct ParallelExecutorInfo
 	/* These two arrays have pcxt->nworkers_launched entries: */
 	shm_mq_handle **tqueue;		/* tuple queues for worker output */
 	struct TupleQueueReader **reader;	/* tuple reader/writer support */
+	/* Number of tuples inserted by all workers. */
+	volatile pg_atomic_uint64	*processed;
 } ParallelExecutorInfo;
 
 extern ParallelExecutorInfo *ExecInitParallelPlan(PlanState *planstate,
 												  EState *estate, Bitmapset *sendParam, int nworkers,
-												  int64 tuples_needed);
+												  int64 tuples_needed,
+												  IntoClause *intoclause,
+												  Oid objectid);
 extern void ExecParallelCreateReaders(ParallelExecutorInfo *pei);
 extern void ExecParallelFinish(ParallelExecutorInfo *pei);
 extern void ExecParallelCleanup(ParallelExecutorInfo *pei);
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 61ba4c3666..a5b281f783 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -23,6 +23,7 @@
 #include "nodes/tidbitmap.h"
 #include "partitioning/partdefs.h"
 #include "storage/condition_variable.h"
+#include "tcop/dest.h"
 #include "utils/hsearch.h"
 #include "utils/queryenvironment.h"
 #include "utils/reltrigger.h"
@@ -2326,6 +2327,8 @@ typedef struct GatherState
 	int			nreaders;		/* number of still-active workers */
 	int			nextreader;		/* next one to try to read from */
 	struct TupleQueueReader **reader;	/* array with nreaders active entries */
+	/* Dest receiver is stored when parallel inserts is allowed in CTAS. */
+	DestReceiver *dest;
 } GatherState;
 
 /* ----------------
-- 
2.25.1

From 9ce3e49f0f11d7144566704f5153d1e579a3814c Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 28 Dec 2020 15:06:01 +0530
Subject: [PATCH v17 2/4] Tuple Cost Adjustment for Parallel Inserts in CTAS

Let the planner know that the SELECT is from CTAS in createas.c
so that it can set the number of tuples transferred from the
workers to Gather node to 0. With this change, there are chances
that the planner may choose the parallel plan.
---
 src/backend/commands/createas.c       | 61 +++++++++++++++++++++------
 src/backend/commands/explain.c        | 16 +++++--
 src/backend/commands/prepare.c        |  3 +-
 src/backend/optimizer/path/costsize.c | 22 +++++++++-
 src/backend/optimizer/plan/planner.c  | 53 +++++++++++++++++++++++
 src/include/commands/createas.h       | 21 ++++++++-
 src/include/commands/explain.h        |  3 +-
 src/include/nodes/parsenodes.h        |  1 +
 src/include/optimizer/planner.h       | 10 +++++
 9 files changed, 169 insertions(+), 21 deletions(-)

diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 2beccf60df..5046e02aac 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -316,6 +316,15 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		query = linitial_node(Query, rewritten);
 		Assert(query->commandType == CMD_SELECT);
 
+		/*
+		 * Turn on a flag to indicate the planner so that it can ignore the
+		 * parallel tuple cost while generating Gather path.
+		 */
+		if (IsParallelInsertionAllowedInCTAS(into))
+			query->CTASParallelInsInfo |= CTAS_PARALLEL_INS_SELECT;
+		else
+			query->CTASParallelInsInfo = CTAS_PARALLEL_INS_UNDEF;
+
 		/* plan the query */
 		plan = pg_plan_query(query, pstate->p_sourcetext,
 							 CURSOR_OPT_PARALLEL_OK, params);
@@ -344,7 +353,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		 * executor to decide whether to allow parallel inserts or not.
 		 */
 		if (IsParallelInsertionAllowedInCTAS(into))
-			SetCTASParallelInsertState(queryDesc);
+			SetCTASParallelInsertState(queryDesc, &query->CTASParallelInsInfo);
 
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
@@ -690,7 +699,7 @@ IsParallelInsertionAllowedInCTAS(IntoClause *into)
  * information will be sent to workers.
  */
 void
-SetCTASParallelInsertState(QueryDesc *queryDesc)
+SetCTASParallelInsertState(QueryDesc *queryDesc, uint8 *tuple_cost_flags)
 {
 	GatherState *gstate;
 	DestReceiver *dest;
@@ -701,21 +710,45 @@ SetCTASParallelInsertState(QueryDesc *queryDesc)
 	dest	= queryDesc->dest;
 
 	/*
-	 * Parallel insertions are not possible either if the upper node is not
-	 * Gather or it's a Gather but it have some projections to perform.
+	 * Parallel insertions are not possible if the upper node is not Gather.
 	 */
-	if (!IsA(gstate, GatherState) || gstate->ps.ps_ProjInfo)
+	if (!IsA(gstate, GatherState))
 		return;
 
-	/* Okay to parallelize inserts, so mark it. */
-	((DR_intorel *) dest)->is_parallel = true;
-
 	/*
-	 * For parallelizing inserts in CTAS we must send information such as into
-	 * clause (to build separate dest receiver), object id (to open the created
-	 * table) to each workers. Since this information is available in the CTAS
-	 * dest receiver, store a reference to it in the Gather state so that it
-	 * will be used in ExecInitParallelPlan to pick the required information.
+	 * If the upper Gather node has some projections to perform, then we can
+	 * not allow parallel insertions. But before returning, ensure that we have
+	 * not done wrong parallel tuple cost enforcement in the planner.
+	 *
+	 * The main reason for this assertion is to check if we enforced planner to
+	 * ignore the parallel tuple cost (with the intention of choosing parallel
+	 * inserts) due to which the parallel plan may have been chosen, but we do
+	 * not allow the parallel inserts now.
+	 *
+	 * If we have correctly ignored parallel tuple cost in planner while
+	 * creating Gather path, then this assertion failure should not occur. In
+	 * case it occurs, that means the planner may have chosen this parallel
+	 * plan because of our wrong enforcement. So let's catch that here.
 	 */
-	gstate->dest = dest;
+	if (tuple_cost_flags && gstate->ps.ps_ProjInfo)
+		Assert(!(*tuple_cost_flags & CTAS_PARALLEL_INS_TUP_COST_IGNORED));
+
+	if (!gstate->ps.ps_ProjInfo)
+	{
+		/* Okay to parallelize inserts, so mark it. */
+		((DR_intorel *) dest)->is_parallel = true;
+
+		/*
+		 * For parallelizing inserts in CTAS we must send information such as
+		 * into clause (to build separate dest receiver), object id (to open
+		 * the created table) to each workers. Since this information is
+		 * available in the CTAS dest receiver, store a reference to it in the
+		 * Gather state so that it will be used in ExecInitParallelPlan to pick
+		 * the required information.
+		 */
+		gstate->dest = dest;
+	}
+
+	if (tuple_cost_flags)
+		*tuple_cost_flags = CTAS_PARALLEL_INS_UNDEF;
 }
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 9a412c3e6b..50a1fc2f36 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -387,6 +387,15 @@ ExplainOneQuery(Query *query, int cursorOptions,
 			bufusage_start = pgBufferUsage;
 		INSTR_TIME_SET_CURRENT(planstart);
 
+		/*
+		 * Turn on a flag to indicate the planner so that it can ignore the
+		 * parallel tuple cost while generating Gather path.
+		 */
+		if (IsParallelInsertionAllowedInCTAS(into))
+			query->CTASParallelInsInfo |= CTAS_PARALLEL_INS_SELECT;
+		else
+			query->CTASParallelInsInfo = CTAS_PARALLEL_INS_UNDEF;
+
 		/* plan the query */
 		plan = pg_plan_query(query, queryString, cursorOptions, params);
 
@@ -402,7 +411,8 @@ ExplainOneQuery(Query *query, int cursorOptions,
 
 		/* run it (if needed) and produce output */
 		ExplainOnePlan(plan, into, es, queryString, params, queryEnv,
-					   &planduration, (es->buffers ? &bufusage : NULL));
+					   &planduration, (es->buffers ? &bufusage : NULL),
+					   &query->CTASParallelInsInfo);
 	}
 }
 
@@ -496,7 +506,7 @@ void
 ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 			   const char *queryString, ParamListInfo params,
 			   QueryEnvironment *queryEnv, const instr_time *planduration,
-			   const BufferUsage *bufusage)
+			   const BufferUsage *bufusage, uint8 *ctas_tuple_cost_flags)
 {
 	DestReceiver *dest;
 	QueryDesc  *queryDesc;
@@ -562,7 +572,7 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	 * decide whether to allow parallel inserts or not.
 	 */
 	if (IsParallelInsertionAllowedInCTAS(into))
-		SetCTASParallelInsertState(queryDesc);
+		SetCTASParallelInsertState(queryDesc, ctas_tuple_cost_flags);
 
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c
index 89087a7be3..07166479e7 100644
--- a/src/backend/commands/prepare.c
+++ b/src/backend/commands/prepare.c
@@ -672,7 +672,8 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es,
 
 		if (pstmt->commandType != CMD_UTILITY)
 			ExplainOnePlan(pstmt, into, es, query_string, paramLI, queryEnv,
-						   &planduration, (es->buffers ? &bufusage : NULL));
+						   &planduration, (es->buffers ? &bufusage : NULL),
+						   NULL);
 		else
 			ExplainOneUtility(pstmt->utilityStmt, into, es, query_string,
 							  paramLI, queryEnv);
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 22d6935824..e706219102 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -76,6 +76,7 @@
 #include "access/amapi.h"
 #include "access/htup_details.h"
 #include "access/tsmapi.h"
+#include "commands/createas.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "executor/nodeHash.h"
@@ -378,6 +379,7 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 {
 	Cost		startup_cost = 0;
 	Cost		run_cost = 0;
+	bool		ignore_tuple_cost = false;
 
 	/* Mark the path with the correct row estimate */
 	if (rows)
@@ -393,7 +395,25 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 
 	/* Parallel setup and communication cost. */
 	startup_cost += parallel_setup_cost;
-	run_cost += parallel_tuple_cost * path->path.rows;
+
+	/*
+	 * Do not consider tuple cost in case of we intend to perform parallel
+	 * inserts by workers. We would have turned on the ignore flag in
+	 * apply_scanjoin_target_to_paths before generating Gather path for the
+	 * upper level SELECT part of the CTAS.
+	 */
+	if ((root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT) &&
+		(root->parse->CTASParallelInsInfo &
+		 CTAS_PARALLEL_INS_TUP_COST_CAN_IGN))
+	{
+		ignore_tuple_cost = true;
+		root->parse->CTASParallelInsInfo &=
+									~CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_TUP_COST_IGNORED;
+	}
+
+	if (!ignore_tuple_cost)
+		run_cost += parallel_tuple_cost * path->path.rows;
 
 	path->path.startup_cost = startup_cost;
 	path->path.total_cost = (startup_cost + run_cost);
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index 1a94b58f8b..20a18628a7 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -28,6 +28,7 @@
 #include "catalog/pg_inherits.h"
 #include "catalog/pg_proc.h"
 #include "catalog/pg_type.h"
+#include "commands/createas.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "foreign/fdwapi.h"
@@ -7338,6 +7339,37 @@ can_partial_agg(PlannerInfo *root)
 	return true;
 }
 
+/*
+ * ignore_parallel_tuple_cost
+ *
+ * Gather node will not receive any tuples from the workers in case each worker
+ * inserts them in parallel. So, we turn on a flag to ignore parallel tuple
+ * cost by the Gather path in cost_gather if the SELECT is for CTAS and we are
+ * generating an upper level Gather path.
+*/
+static bool
+ignore_parallel_tuple_cost(PlannerInfo *root)
+{
+	if (root->query_level == 1 &&
+		(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT))
+	{
+		/*
+		 * In each of the HAS_PARENT_PATH_GENERATING_CLAUSE cases, a parent
+		 * path will be generated for the upper Gather path(in
+		 * grouping_planner), in which case we can not let parallel inserts
+		 * happen. So we do not turn on ignore tuple cost flag.
+		 */
+		if (HAS_PARENT_PATH_GENERATING_CLAUSE(root))
+			return false;
+
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+
+		return true;
+	}
+
+	return false;
+}
+
 /*
  * apply_scanjoin_target_to_paths
  *
@@ -7557,8 +7589,29 @@ apply_scanjoin_target_to_paths(PlannerInfo *root,
 	 * one of the generated paths may turn out to be the cheapest one.
 	 */
 	if (rel->consider_parallel && !IS_OTHER_REL(rel))
+	{
+		/*
+		 * Turn on a flag to ignore parallel tuple cost by the Gather path in
+		 * cost_gather if the SELECT is for CTAS and we are generating an upper
+		 * level Gather path.
+		 */
+		bool	ignore = ignore_parallel_tuple_cost(root);
+
 		generate_useful_gather_paths(root, rel, false);
 
+		/*
+		 * Reset the ignore flag, in case we turned it on but
+		 * generate_useful_gather_paths returned without reaching cost_gather.
+		 * If we reached cost_gather, we would have been reset it there.
+		 */
+		if (ignore && (root->parse->CTASParallelInsInfo &
+			CTAS_PARALLEL_INS_TUP_COST_CAN_IGN))
+		{
+			root->parse->CTASParallelInsInfo &=
+										~CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+		}
+	}
+
 	/*
 	 * Reassess which paths are the cheapest, now that we've potentially added
 	 * new Gather (or Gather Merge) and/or Append (or MergeAppend) paths to
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index ab74c69b9f..140226ad5a 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -39,6 +39,24 @@ typedef struct
 	Oid			object_id;
 } DR_intorel;
 
+/*
+ * Information sent to the planner from CTAS to account for the cost
+ * calculations in cost_gather. We need to do this because, no tuples will be
+ * received by the Gather node if the workers insert the tuples in parallel.
+ */
+typedef enum CTASParallelInsertOpt
+{
+	CTAS_PARALLEL_INS_UNDEF = 0, /* undefined */
+	CTAS_PARALLEL_INS_SELECT = 1 << 0, /* turn on this before planning */
+	/*
+	 * Turn on this while planning for upper Gather path to ignore parallel
+	 * tuple cost in cost_gather.
+	 */
+	CTAS_PARALLEL_INS_TUP_COST_CAN_IGN = 1 << 1,
+	/* Turn on this after the cost is ignored. */
+	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2
+} CTASParallelInsertOpt;
+
 #define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
 #define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
 									 IS_CTAS(((DR_intorel *) dest)->into) && \
@@ -54,5 +72,6 @@ extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
 extern bool IsParallelInsertionAllowedInCTAS(IntoClause *into);
 
-extern void SetCTASParallelInsertState(QueryDesc *queryDesc);
+extern void SetCTASParallelInsertState(QueryDesc *queryDesc,
+									   uint8 *tuple_cost_flags);
 #endif							/* CREATEAS_H */
diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h
index ba661d32a6..1a1806dbf1 100644
--- a/src/include/commands/explain.h
+++ b/src/include/commands/explain.h
@@ -91,7 +91,8 @@ extern void ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into,
 						   ExplainState *es, const char *queryString,
 						   ParamListInfo params, QueryEnvironment *queryEnv,
 						   const instr_time *planduration,
-						   const BufferUsage *bufusage);
+						   const BufferUsage *bufusage,
+						   uint8 *ctas_tuple_cost_flags);
 
 extern void ExplainPrintPlan(ExplainState *es, QueryDesc *queryDesc);
 extern void ExplainPrintTriggers(ExplainState *es, QueryDesc *queryDesc);
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 48a79a7657..81b148c383 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -180,6 +180,7 @@ typedef struct Query
 	 */
 	int			stmt_location;	/* start location, or -1 if unknown */
 	int			stmt_len;		/* length in bytes; 0 means "rest of string" */
+	uint8 		CTASParallelInsInfo; /* parallel insert in CTAS info */
 } Query;
 
 
diff --git a/src/include/optimizer/planner.h b/src/include/optimizer/planner.h
index beb7dbbcbe..74b2563828 100644
--- a/src/include/optimizer/planner.h
+++ b/src/include/optimizer/planner.h
@@ -21,6 +21,16 @@
 #include "nodes/pathnodes.h"
 #include "nodes/plannodes.h"
 
+#define HAS_PARENT_PATH_GENERATING_CLAUSE(root) \
+											(root->parse->rowMarks || \
+											 limit_needed(root->parse) || \
+											 root->parse->sortClause ||  \
+											 root->parse->distinctClause ||  \
+											 root->parse->hasWindowFuncs ||  \
+											 root->parse->groupClause ||  \
+											 root->parse->groupingSets ||  \
+											 root->parse->hasAggs ||  \
+											 root->hasHavingQual)
 
 /* Hook for plugins to get control in planner() */
 typedef PlannedStmt *(*planner_hook_type) (Query *parse,
-- 
2.25.1

From 61641393e4c1d90a47f2a070d6e9e020e6f014e4 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 28 Dec 2020 16:49:43 +0530
Subject: [PATCH v17 3/4] Tests And Docs For Parallel Inserts in CTAS

---
 doc/src/sgml/ref/create_table_as.sgml        |  31 +-
 src/test/regress/expected/write_parallel.out | 559 +++++++++++++++++++
 src/test/regress/sql/write_parallel.sql      | 211 +++++++
 3 files changed, 796 insertions(+), 5 deletions(-)

diff --git a/doc/src/sgml/ref/create_table_as.sgml b/doc/src/sgml/ref/create_table_as.sgml
index 07558ab56c..35903701ed 100644
--- a/doc/src/sgml/ref/create_table_as.sgml
+++ b/doc/src/sgml/ref/create_table_as.sgml
@@ -37,11 +37,13 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI
 
   <para>
    <command>CREATE TABLE AS</command> creates a table and fills it
-   with data computed by a <command>SELECT</command> command.
-   The table columns have the
-   names and data types associated with the output columns of the
-   <command>SELECT</command> (except that you can override the column
-   names by giving an explicit list of new column names).
+   with data computed by a <command>SELECT</command> command. When the
+   node at the top of the <command>SELECT</command> plan is
+   <literal>Gather</literal> and there are no projections to be performed
+   by it, then the created table can be filled by the workers in parallel.
+   The table columns have the names and data types associated with the output
+   columns of the <command>SELECT</command> (except that you can override the
+   column names by giving an explicit list of new column names).
   </para>
 
   <para>
@@ -297,6 +299,25 @@ PREPARE recentfilms(date) AS
 CREATE TEMP TABLE films_recent ON COMMIT DROP AS
   EXECUTE recentfilms('2002-01-01');
 </programlisting></para>
+
+  <para>
+   Here is an example of a query plan when the data into the created table can
+   be filled by the workers in parallel:
+
+<programlisting>
+EXPLAIN CREATE TABLE bar AS SELECT * FROM foo WHERE i > 5;
+
+                             QUERY PLAN
+-------------------------------------------------------------------&zwsp;--
+Gather  (cost=0.00..23.28 rows=850 width=4)
+   Workers Planned: 2
+ ->  Create bar
+   ->  Parallel Seq Scan on foo  (cost=0.00..23.28 rows=354 width=4)
+         Filter: (i > 5)
+(5 rows)
+</programlisting>
+  </para>
+
  </refsect1>
 
  <refsect1>
diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 0c4da2591a..38a18c5a9b 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -76,4 +76,563 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+-- Parallel queries won't necessarily get as many workers as the planner
+-- asked for.  This affects not only the "Workers Launched:" field of EXPLAIN
+-- results, but also row counts and loop counts for parallel scans, Gathers,
+-- and everything in between.  This function filters out the values we can't
+-- rely on to be stable.
+-- This removes enough info that you might wonder why bother with EXPLAIN
+-- ANALYZE at all.  The answer is that we need to see whether the parallel
+-- inserts are being done by the workers, the only way is that
+-- Create <<tbl_name>> appears in the explain output.
+create function explain_pictas(text) returns setof text
+language plpgsql as
+$$
+declare
+    ln text;
+begin
+    for ln in
+        execute format('explain (analyze, costs off, summary off, timing off) %s',
+            $1)
+    loop
+        ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N');
+        ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N');
+        ln := regexp_replace(ln, 'Rows Removed by Filter: \d+', 'Rows Removed by Filter: N');
+        return next ln;
+    end loop;
+end;
+$$;
+-- parallel inserts must occur as the CTAS creates a normal table
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'create temporary table parallel_write as select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(4 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur as the CTAS creates an unlogged table
+select explain_pictas(
+'create unlogged table parallel_write as select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur as the select into creates a normal table
+select explain_pictas(
+'select length(stringu1) into parallel_write from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'select length(stringu1) into temporary parallel_write from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(4 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur as the select into creates an unlogged table
+select explain_pictas(
+'select length(stringu1) into unlogged parallel_write from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1 for update;');
+                 explain_pictas                  
+-------------------------------------------------
+ LockRows (actual rows=N loops=N)
+   ->  Seq Scan on tenk1 (actual rows=N loops=N)
+(2 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur as the materialized view is being created here
+select explain_pictas(
+'create materialized view parallel_mat_view as
+    select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_mat_view
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_mat_view;
+ count 
+-------
+ 10000
+(1 row)
+
+drop materialized view parallel_mat_view;
+-- parallel inserts must occur as the CTAS creates the table using prepared
+-- statement for which parallelism would have been picked
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+select explain_pictas(
+'create table parallel_write as execute parallel_write_prep;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+deallocate parallel_write_prep;
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+select explain_pictas(
+E'create table parallel_write as
+    select nextval(\'parallel_write_sequence\'), four from tenk1;');
+              explain_pictas               
+-------------------------------------------
+ Seq Scan on tenk1 (actual rows=N loops=N)
+(1 row)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Params Evaluated: $1
+   Workers Launched: N
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 4
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on tenk2 (actual rows=N loops=N)
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+         Filter: (four = 3)
+         Rows Removed by Filter: N
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+  2500
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;');
+                          explain_pictas                          
+------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+         Filter: (four = 3)
+         Rows Removed by Filter: N
+   SubPlan 1
+     ->  Function Scan on generate_series (actual rows=N loops=N)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+  2500
+(1 row)
+
+drop table parallel_write;
+create table temp1(col1) as select * from generate_series(1,5);
+create table temp2(col2) as select * from temp1;
+create table temp3(col3) as select * from temp1;
+-- parallel inserts must not occur, as there is a limit clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 limit 4;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ Limit (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ Gather Merge (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+   ->  Sort (actual rows=N loops=N)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         Worker 0:  Sort Method: quicksort  Memory: 25kB
+         Worker 1:  Sort Method: quicksort  Memory: 25kB
+         Worker 2:  Sort Method: quicksort  Memory: 25kB
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ Gather Merge (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+   ->  Sort (actual rows=N loops=N)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         Worker 0:  Sort Method: quicksort  Memory: 25kB
+         Worker 1:  Sort Method: quicksort  Memory: 25kB
+         Worker 2:  Sort Method: quicksort  Memory: 25kB
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is a distinct clause
+select explain_pictas(
+'create table parallel_write as select distinct * from temp1;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ HashAggregate (actual rows=N loops=N)
+   Group Key: col1
+   Batches: 1  Memory Usage: 40kB
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(7 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an aggregate and group clause
+select explain_pictas(
+'create table parallel_write as select count(*) from temp1 group by col1;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Finalize HashAggregate (actual rows=N loops=N)
+   Group Key: col1
+   Batches: 1  Memory Usage: 40kB
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Partial HashAggregate (actual rows=N loops=N)
+               Group Key: col1
+               Batches: 1  Memory Usage: 40kB
+               Worker 0:  Batches: 1  Memory Usage: 40kB
+               Worker 1:  Batches: 1  Memory Usage: 40kB
+               Worker 2:  Batches: 1  Memory Usage: 40kB
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(13 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an aggregate, group and having
+-- clauses
+select explain_pictas(
+'create table parallel_write as
+    select count(col1), (select col3 from
+        (select * from temp3) as tt limit 1) col4 from temp1, temp2
+    where temp1.col1 = temp2.col2 group by col4 having count(col1) > 0;');
+                                  explain_pictas                                  
+----------------------------------------------------------------------------------
+ Finalize GroupAggregate (actual rows=N loops=N)
+   Group Key: ($1)
+   Filter: (count(temp1.col1) > 0)
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 3
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on temp3 (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $1
+         Workers Launched: N
+         ->  Partial GroupAggregate (actual rows=N loops=N)
+               Group Key: $1
+               ->  Parallel Hash Join (actual rows=N loops=N)
+                     Hash Cond: (temp1.col1 = temp2.col2)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                     ->  Parallel Hash (actual rows=N loops=N)
+                           Buckets: 4096  Batches: 1  Memory Usage: 64kB
+                           ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(21 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     1
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is a window function
+select explain_pictas(
+'create table parallel_write as
+    select avg(col1) OVER (PARTITION BY col1) from temp1;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ WindowAgg (actual rows=N loops=N)
+   ->  Gather Merge (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Sort (actual rows=N loops=N)
+               Sort Key: col1
+               Sort Method: quicksort  Memory: 25kB
+               Worker 0:  Sort Method: quicksort  Memory: 25kB
+               Worker 1:  Sort Method: quicksort  Memory: 25kB
+               Worker 2:  Sort Method: quicksort  Memory: 25kB
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(11 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- nested loop join is the top node under which Gather node exists, so parallel
+-- inserts must not occur
+set enable_nestloop to on;
+set enable_mergejoin to off;
+set enable_hashjoin to off;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Nested Loop (actual rows=N loops=N)
+   Join Filter: (temp1.col1 = temp2.col2)
+   Rows Removed by Join Filter: 20
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+   ->  Materialize (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Workers Launched: N
+               ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- the top node is Gather under which merge join happens, so parallel inserts
+-- must occur
+set enable_nestloop to off;
+set enable_mergejoin to on;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Merge Join (actual rows=N loops=N)
+         Merge Cond: (temp1.col1 = temp2.col2)
+         ->  Sort (actual rows=N loops=N)
+               Sort Key: temp1.col1
+               Sort Method: quicksort  Memory: 25kB
+               Worker 0:  Sort Method: quicksort  Memory: 25kB
+               Worker 1:  Sort Method: quicksort  Memory: 25kB
+               Worker 2:  Sort Method: quicksort  Memory: 25kB
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Sort (actual rows=N loops=N)
+               Sort Key: temp2.col2
+               Sort Method: quicksort  Memory: 25kB
+               ->  Seq Scan on temp2 (actual rows=N loops=N)
+(17 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel hash join happens under Gather node, so parallel inserts must occur
+set enable_mergejoin to off;
+set enable_hashjoin to on;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Hash Join (actual rows=N loops=N)
+         Hash Cond: (temp1.col1 = temp2.col2)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Hash (actual rows=N loops=N)
+               Buckets: 4096  Batches: 1  Memory Usage: 64kB
+               ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+reset enable_nestloop;
+reset enable_mergejoin;
+reset enable_hashjoin;
+drop table temp1;
+drop table temp2;
+drop table temp3;
+drop function explain_pictas(text);
 rollback;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 78b479cedf..40aadafc2a 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -39,4 +39,215 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
 
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+
+-- Parallel queries won't necessarily get as many workers as the planner
+-- asked for.  This affects not only the "Workers Launched:" field of EXPLAIN
+-- results, but also row counts and loop counts for parallel scans, Gathers,
+-- and everything in between.  This function filters out the values we can't
+-- rely on to be stable.
+-- This removes enough info that you might wonder why bother with EXPLAIN
+-- ANALYZE at all.  The answer is that we need to see whether the parallel
+-- inserts are being done by the workers, the only way is that
+-- Create <<tbl_name>> appears in the explain output.
+create function explain_pictas(text) returns setof text
+language plpgsql as
+$$
+declare
+    ln text;
+begin
+    for ln in
+        execute format('explain (analyze, costs off, summary off, timing off) %s',
+            $1)
+    loop
+        ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N');
+        ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N');
+        ln := regexp_replace(ln, 'Rows Removed by Filter: \d+', 'Rows Removed by Filter: N');
+        return next ln;
+    end loop;
+end;
+$$;
+
+-- parallel inserts must occur as the CTAS creates a normal table
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'create temporary table parallel_write as select length(stringu1) from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur as the CTAS creates an unlogged table
+select explain_pictas(
+'create unlogged table parallel_write as select length(stringu1) from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur as the select into creates a normal table
+select explain_pictas(
+'select length(stringu1) into parallel_write from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'select length(stringu1) into temporary parallel_write from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur as the select into creates an unlogged table
+select explain_pictas(
+'select length(stringu1) into unlogged parallel_write from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1 for update;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur as the materialized view is being created here
+select explain_pictas(
+'create materialized view parallel_mat_view as
+    select length(stringu1) from tenk1;');
+select count(*) from parallel_mat_view;
+drop materialized view parallel_mat_view;
+
+-- parallel inserts must occur as the CTAS creates the table using prepared
+-- statement for which parallelism would have been picked
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+select explain_pictas(
+'create table parallel_write as execute parallel_write_prep;');
+select count(*) from parallel_write;
+deallocate parallel_write_prep;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+select explain_pictas(
+E'create table parallel_write as
+    select nextval(\'parallel_write_sequence\'), four from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+create table temp1(col1) as select * from generate_series(1,5);
+create table temp2(col2) as select * from temp1;
+create table temp3(col3) as select * from temp1;
+
+-- parallel inserts must not occur, as there is a limit clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 limit 4;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is a distinct clause
+select explain_pictas(
+'create table parallel_write as select distinct * from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an aggregate and group clause
+select explain_pictas(
+'create table parallel_write as select count(*) from temp1 group by col1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an aggregate, group and having
+-- clauses
+select explain_pictas(
+'create table parallel_write as
+    select count(col1), (select col3 from
+        (select * from temp3) as tt limit 1) col4 from temp1, temp2
+    where temp1.col1 = temp2.col2 group by col4 having count(col1) > 0;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is a window function
+select explain_pictas(
+'create table parallel_write as
+    select avg(col1) OVER (PARTITION BY col1) from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- nested loop join is the top node under which Gather node exists, so parallel
+-- inserts must not occur
+set enable_nestloop to on;
+set enable_mergejoin to off;
+set enable_hashjoin to off;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- the top node is Gather under which merge join happens, so parallel inserts
+-- must occur
+set enable_nestloop to off;
+set enable_mergejoin to on;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel hash join happens under Gather node, so parallel inserts must occur
+set enable_mergejoin to off;
+set enable_hashjoin to on;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+reset enable_nestloop;
+reset enable_mergejoin;
+reset enable_hashjoin;
+
+drop table temp1;
+drop table temp2;
+drop table temp3;
+drop function explain_pictas(text);
 rollback;
-- 
2.25.1

From 4f71a8b1e0cf50d488af8c925151aed335bb2e8c Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 28 Dec 2020 17:00:38 +0530
Subject: [PATCH v17 4/4] Enable CTAS Parallel Inserts For Append

This patch allows pushing down the CTAS dest receiver even if there
exists Gather node under Top Append node. It also add the code for
influencing the planner to consider parallel tuple cost zero and
asserts for wrong enforcement if later parallel insertion is not
possible. Test cases are also included in this patch.
---
 src/backend/commands/createas.c              | 118 ++-
 src/backend/optimizer/path/allpaths.c        |  38 +
 src/backend/optimizer/plan/planner.c         |  10 +-
 src/include/commands/createas.h              |   4 +-
 src/test/regress/expected/write_parallel.out | 722 +++++++++++++++++++
 src/test/regress/sql/write_parallel.sql      | 222 ++++++
 6 files changed, 1076 insertions(+), 38 deletions(-)

diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 5046e02aac..148e7c7ea2 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -690,35 +690,99 @@ IsParallelInsertionAllowedInCTAS(IntoClause *into)
 	return true;
 }
 
+/*
+ * PushDownCTASParallelInsertState --- push the dest receiver down to the
+ * Gather nodes.
+ *
+ * In this function we only care about Append and Gather nodes.
+ *
+ * Push the dest receiver to Gather node when it is either at the top of the
+ * plan or under top Append node unless it does not have any projections to do.
+ * Required information from the pushed dest receiver is sent to workers so
+ * that they can perform parallel insertions into the target table.
+ *
+ * If the top node is Append, then this function recursively checks the sub
+ * plans for Gather nodes, when found one(and if it does not have projections),
+ * then sets the dest receiver information.
+ *
+ * In any case, this function returns true if at least one Gather node can allow
+ * parallel insertions by the workers. Otherwise returns false.
+ */
+static bool
+PushDownCTASParallelInsertState(DestReceiver *dest, PlanState *ps,
+								bool *gather_exists)
+{
+	bool parallel = false;
+
+	if (ps == NULL)
+		return parallel;
+
+	if (IsA(ps, AppendState))
+	{
+		AppendState *aps = (AppendState *) ps;
+
+		for (int i = 0; i < aps->as_nplans; i++)
+		{
+			parallel |= PushDownCTASParallelInsertState(dest,
+														aps->appendplans[i],
+														gather_exists);
+		}
+	}
+	else if (IsA(ps, GatherState))
+	{
+		/*
+		 * Set to true if there exists at least one Gather node either at the
+		 * top of the plan or as a direct sub node under Append node.
+		 */
+		 *gather_exists |= true;
+
+		if (!ps->ps_ProjInfo)
+		{
+			GatherState *gstate = (GatherState *) ps;
+
+			parallel = true;
+
+			/* Okay to parallelize inserts, so mark it. */
+			((DR_intorel *) dest)->is_parallel = true;
+
+			/*
+			 * For parallelizing inserts in CTAS we must send information such
+			 * as into clause (to build separate dest receiver), object id (to
+			 * open the created table) to each workers. Since this information
+			 * is available in the CTAS dest receiver, store a reference to it
+			 * in the Gather state so that it will be used in
+			 * ExecInitParallelPlan to pick the required information.
+			 */
+			gstate->dest = dest;
+		}
+	}
+
+	return parallel;
+}
+
 /*
  * SetCTASParallelInsertState --- set the parallel insert state
  *
- * See if the upper node is Gather and it doesn't have any projections, then
- * set the parallel insert state such as a flag in the dest receiver and also
- * store the dest receiver reference in the Gather node so that the required
- * information will be sent to workers.
+ * See if the upper node is either Gather or Append but it does have Gather
+ * nodes under it, then set the parallel insert state in respective Gather
+ * nodes if they do not have any projections. The parallel insert state
+ * includes a flag in the dest receiver and also a dest receiver reference in
+ * the Gather node so that the required information will be sent to workers.
  */
 void
 SetCTASParallelInsertState(QueryDesc *queryDesc, uint8 *tuple_cost_flags)
 {
-	GatherState *gstate;
-	DestReceiver *dest;
+	bool allow = false;
+	bool gather_exists = false;
 
 	Assert(queryDesc);
 
-	gstate = (GatherState *) queryDesc->planstate;
-	dest	= queryDesc->dest;
-
+	allow = PushDownCTASParallelInsertState(queryDesc->dest,
+											queryDesc->planstate,
+											&gather_exists);
 	/*
-	 * Parallel insertions are not possible if the upper node is not Gather.
-	 */
-	if (!IsA(gstate, GatherState))
-		return;
-
-	/*
-	 * If the upper Gather node has some projections to perform, then we can
-	 * not allow parallel insertions. But before returning, ensure that we have
-	 * not done wrong parallel tuple cost enforcement in the planner.
+	 * Ensure that we have not done wrong parallel tuple cost enforcement in
+	 * the planner.
 	 *
 	 * The main reason for this assertion is to check if we enforced planner to
 	 * ignore the parallel tuple cost (with the intention of choosing parallel
@@ -730,25 +794,9 @@ SetCTASParallelInsertState(QueryDesc *queryDesc, uint8 *tuple_cost_flags)
 	 * case it occurs, that means the planner may have chosen this parallel
 	 * plan because of our wrong enforcement. So let's catch that here.
 	 */
-	if (tuple_cost_flags && gstate->ps.ps_ProjInfo)
+	if (!allow && tuple_cost_flags && gather_exists)
 		Assert(!(*tuple_cost_flags & CTAS_PARALLEL_INS_TUP_COST_IGNORED));
 
-	if (!gstate->ps.ps_ProjInfo)
-	{
-		/* Okay to parallelize inserts, so mark it. */
-		((DR_intorel *) dest)->is_parallel = true;
-
-		/*
-		 * For parallelizing inserts in CTAS we must send information such as
-		 * into clause (to build separate dest receiver), object id (to open
-		 * the created table) to each workers. Since this information is
-		 * available in the CTAS dest receiver, store a reference to it in the
-		 * Gather state so that it will be used in ExecInitParallelPlan to pick
-		 * the required information.
-		 */
-		gstate->dest = dest;
-	}
-
 	if (tuple_cost_flags)
 		*tuple_cost_flags = CTAS_PARALLEL_INS_UNDEF;
 }
diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c
index 627d08b78a..b0835c32bd 100644
--- a/src/backend/optimizer/path/allpaths.c
+++ b/src/backend/optimizer/path/allpaths.c
@@ -23,6 +23,7 @@
 #include "catalog/pg_class.h"
 #include "catalog/pg_operator.h"
 #include "catalog/pg_proc.h"
+#include "commands/createas.h"
 #include "foreign/fdwapi.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1103,11 +1104,48 @@ set_append_rel_size(PlannerInfo *root, RelOptInfo *rel,
 		if (root->glob->parallelModeOK && rel->consider_parallel)
 			set_rel_consider_parallel(root, childrel, childRTE);
 
+		/*
+		 * When subplan is subquery, It's possible to do parallel insert if top
+		 * node of subquery is Gather, so we turn on a flag to ignore parallel
+		 * tuple cost in cost_gather if the SELECT is for CTAS.
+		 */
+		if (childrel->rtekind == RTE_SUBQUERY)
+		{
+			/*
+			 * When there is no parent path generating clause(such as limit,
+			 * sort, distinct...), we can turn on the flag for two cases:
+			 * i) query_level is 1
+			 * ii) query_level > 1 then turn on the flag in the parent_root.
+			 * The case ii) is to check append under append:
+			 * Append
+			 *	->Append
+			 *	->Gather
+			 *	->Other plan
+			 */
+			if (root->parse->CTASParallelInsInfo &
+				CTAS_PARALLEL_INS_SELECT &&
+				(root->query_level == 1 ||
+				 root->parent_root->parse->CTASParallelInsInfo &
+				 CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND) &&
+				!(HAS_PARENT_PATH_GENERATING_CLAUSE(root)))
+			{
+				root->parse->CTASParallelInsInfo |=
+										CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+			}
+		}
+
 		/*
 		 * Compute the child's size.
 		 */
 		set_rel_size(root, childrel, childRTindex, childRTE);
 
+		if (root->parse->CTASParallelInsInfo &
+			CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND)
+		{
+			root->parse->CTASParallelInsInfo &=
+										~CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+		}
+
 		/*
 		 * It is possible that constraint exclusion detected a contradiction
 		 * within a child subquery, even though we didn't prove one above. If
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index 20a18628a7..5e607f598a 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -7350,8 +7350,14 @@ can_partial_agg(PlannerInfo *root)
 static bool
 ignore_parallel_tuple_cost(PlannerInfo *root)
 {
-	if (root->query_level == 1 &&
-		(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT))
+	if (root->query_level != 1 &&
+		(root->parent_root->parse->CTASParallelInsInfo &
+		 CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND))
+	{
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_SELECT;
+	}
+
+	if (root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT)
 	{
 		/*
 		 * In each of the HAS_PARENT_PATH_GENERATING_CLAUSE cases, a parent
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 140226ad5a..bf52b05ea0 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -54,7 +54,9 @@ typedef enum CTASParallelInsertOpt
 	 */
 	CTAS_PARALLEL_INS_TUP_COST_CAN_IGN = 1 << 1,
 	/* Turn on this after the cost is ignored. */
-	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2
+	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2,
+	/* Turn on this in case tuple cost needs to be ignored for Append cases. */
+	CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND = 1 << 3
 } CTASParallelInsertOpt;
 
 #define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 38a18c5a9b..356a2d0002 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -631,6 +631,728 @@ drop table parallel_write;
 reset enable_nestloop;
 reset enable_mergejoin;
 reset enable_hashjoin;
+-- test cases for performing parallel inserts when Append node is at the top
+-- and Gather node is in one of its direct sub plans.
+-- case 1: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5 union all
+    select * from temp2 where col2 = 5;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: N
+(22 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+-- case 2: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp2;');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Append (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select * from temp2;');
+                                 explain_pictas                                 
+--------------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Params Evaluated: $1, $3
+   Workers Launched: N
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 3
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+   InitPlan 2 (returns $3)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 3
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+   ->  Parallel Append (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(21 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+-- case 3: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples to
+-- Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Gather
+-- 		->Parallel Seq Scan
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1 union all
+    select * from temp1 where col1 = 5;');
+                                explain_pictas                                
+------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Seq Scan on temp1 (actual rows=N loops=N)
+         SubPlan 1
+           ->  Limit (actual rows=N loops=N)
+                 ->  Gather (actual rows=N loops=N)
+                       Workers Planned: 3
+                       Workers Launched: N
+                       ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+(22 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     7
+(1 row)
+
+drop table parallel_write;
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as select * from temp1 where col1 = (select 1) union all
+    select * from temp1 where col1 = (select 1) union all
+ select * from temp2 where col2 = (select 2);');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $0
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+               Filter: (col1 = $0)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $1
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 2 (returns $1)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+               Filter: (col1 = $1)
+               Rows Removed by Filter: N
+   ->  Seq Scan on temp2 (actual rows=N loops=N)
+         Filter: (col2 = $2)
+         Rows Removed by Filter: N
+         InitPlan 3 (returns $2)
+           ->  Result (actual rows=N loops=N)
+(26 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+alter table temp2 reset (parallel_workers);
+drop table parallel_write;
+-- case 4: parallel inserts must not occur as there will be no direct Gather
+-- node under Append node. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1;');
+                                    explain_pictas                                    
+--------------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Seq Scan on temp1 (actual rows=N loops=N)
+   ->  Seq Scan on temp2 (actual rows=N loops=N)
+   ->  Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+         SubPlan 1
+           ->  Limit (actual rows=N loops=N)
+                 ->  Gather (actual rows=N loops=N)
+                       Workers Planned: 3
+                       Workers Launched: N
+                       ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+-- case 5: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Seq Scan / Join / any other non-Gather node
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp1;');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Append (actual rows=N loops=N)
+         ->  Seq Scan on temp2 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+alter table temp2 reset (parallel_workers);
+-- case 6: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Gather
+--  ->Gather
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp2 where col2 = (select 2);');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $2
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($2 = $0)
+                     ->  Parallel Seq Scan on temp2 (never executed)
+                           Filter: (col2 = $0)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $3
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+               Filter: (col2 = $3)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $4
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 5 (returns $4)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+               Filter: (col2 = $4)
+               Rows Removed by Filter: N
+(47 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+                                   explain_pictas                                   
+------------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $2
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($2 = $0)
+                     ->  Parallel Seq Scan on temp2 (never executed)
+                           Filter: (col2 = $0)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $3, $4
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 5 (returns $4)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($4 = $3)
+                     ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+                           Filter: (col1 = $3)
+                           Rows Removed by Filter: N
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $3, $5
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 6 (returns $5)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($5 = $3)
+                     ->  Parallel Seq Scan on temp2 temp2_1 (never executed)
+                           Filter: (col2 = $3)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: N
+(67 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+-- case 7: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Seq Scan / Join / any other non-Gather node
+--  ->Gather
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp1 where col1 = (select 2);');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Result (actual rows=N loops=N)
+               One-Time Filter: ($2 = $0)
+               InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Seq Scan on temp2 (never executed)
+                     Filter: (col2 = $0)
+   ->  Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+         Filter: (col2 = $3)
+         Rows Removed by Filter: N
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $4
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 5 (returns $4)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+               Filter: (col1 = $4)
+               Rows Removed by Filter: N
+(37 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+                                   explain_pictas                                   
+------------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Result (actual rows=N loops=N)
+               One-Time Filter: ($2 = $0)
+               InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Seq Scan on temp2 (never executed)
+                     Filter: (col2 = $0)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $3, $4
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 5 (returns $4)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($4 = $3)
+                     ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+                           Filter: (col1 = $3)
+                           Rows Removed by Filter: N
+         ->  Result (actual rows=N loops=N)
+               One-Time Filter: ($5 = $3)
+               InitPlan 6 (returns $5)
+                 ->  Result (actual rows=N loops=N)
+               ->  Seq Scan on temp2 temp2_1 (never executed)
+                     Filter: (col2 = $3)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+         Filter: (col2 = 5)
+         Rows Removed by Filter: N
+(53 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+alter table temp2 reset (parallel_workers);
+-- case 8: parallel inserts must not occur because there is no Gather or Append
+-- node at the top for union, except/except all, intersect/intersect all
+-- cases.
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union
+    select * from temp2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ HashAggregate (actual rows=N loops=N)
+   Group Key: temp1.col1
+   Batches: 1  Memory Usage: 217kB
+   ->  Append (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Workers Launched: N
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Workers Launched: N
+               ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except
+    select * from temp2 where col2 < 3;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Except (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+                           Filter: (col2 < 3)
+                           Rows Removed by Filter: N
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except all
+    select * from temp2 where col2 < 3;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Except All (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+                           Filter: (col2 < 3)
+                           Rows Removed by Filter: N
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect
+    select * from temp2;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Intersect (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect all
+    select * from temp2;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Intersect All (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
 drop table temp1;
 drop table temp2;
 drop table temp3;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 40aadafc2a..32e6ad8636 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -246,6 +246,228 @@ reset enable_nestloop;
 reset enable_mergejoin;
 reset enable_hashjoin;
 
+-- test cases for performing parallel inserts when Append node is at the top
+-- and Gather node is in one of its direct sub plans.
+
+-- case 1: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5 union all
+    select * from temp2 where col2 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 2: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 3: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples to
+-- Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Gather
+-- 		->Parallel Seq Scan
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1 union all
+    select * from temp1 where col1 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as select * from temp1 where col1 = (select 1) union all
+    select * from temp1 where col1 = (select 1) union all
+ select * from temp2 where col2 = (select 2);');
+select count(*) from parallel_write;
+alter table temp2 reset (parallel_workers);
+drop table parallel_write;
+
+-- case 4: parallel inserts must not occur as there will be no direct Gather
+-- node under Append node. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 5: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Seq Scan / Join / any other non-Gather node
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+
+alter table temp2 set (parallel_workers = 0);
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 reset (parallel_workers);
+
+-- case 6: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Gather
+--  ->Gather
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp2 where col2 = (select 2);');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 7: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Seq Scan / Join / any other non-Gather node
+--  ->Gather
+
+alter table temp2 set (parallel_workers = 0);
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp1 where col1 = (select 2);');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 reset (parallel_workers);
+
+-- case 8: parallel inserts must not occur because there is no Gather or Append
+-- node at the top for union, except/except all, intersect/intersect all
+-- cases.
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except
+    select * from temp2 where col2 < 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except all
+    select * from temp2 where col2 < 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect all
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
 drop table temp1;
 drop table temp2;
 drop table temp3;
-- 
2.25.1

From d36c81fb6e13a8ec3c36fd6cd3bb096d783715e7 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 28 Dec 2020 14:30:45 +0530
Subject: [PATCH v18 1/4] Parallel Inserts in CREATE TABLE AS

Allow the leader and each worker insert the tuples in parallel
if the SELECT part of the CTAS is parallelizable.

The design:
The main idea is to push the CTAS dest receiver down to Gather node
and from there the required information will be shared to workers
so that they can perform parallel insertions. Leader will also
participate in insertions. After the planning, check if the upper
plan node is Gather in createas.c and mark a parallelism flag in
the CTAS dest receiver and push it down to Gather node. Each worker
can create its own CTAS dest receiver with the information passed
from the leader. Leader inserts its share of tuples if instructed
to do, and so are workers. Each worker writes atomically its number
of inserted tuples into a shared memory variable, the leader combines
this with its own number of inserted tuples and shares to the client.
---
 src/backend/access/heap/heapam.c       |  11 --
 src/backend/access/transam/xact.c      |  28 ++++-
 src/backend/commands/createas.c        | 137 ++++++++++++++++++++++---
 src/backend/commands/explain.c         |  31 ++++++
 src/backend/executor/execParallel.c    |  70 ++++++++++++-
 src/backend/executor/nodeGather.c      | 111 ++++++++++++++++++--
 src/backend/executor/nodeGatherMerge.c |   4 +-
 src/include/access/xact.h              |   1 +
 src/include/commands/createas.h        |  26 +++++
 src/include/executor/execParallel.h    |   6 +-
 src/include/nodes/execnodes.h          |   3 +
 11 files changed, 384 insertions(+), 44 deletions(-)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index a9583f3103..86347ba273 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2043,17 +2043,6 @@ static HeapTuple
 heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
 					CommandId cid, int options)
 {
-	/*
-	 * To allow parallel inserts, we need to ensure that they are safe to be
-	 * performed in workers. We have the infrastructure to allow parallel
-	 * inserts in general except for the cases where inserts generate a new
-	 * CommandId (eg. inserts into a table having a foreign key column).
-	 */
-	if (IsParallelWorker())
-		ereport(ERROR,
-				(errcode(ERRCODE_INVALID_TRANSACTION_STATE),
-				 errmsg("cannot insert tuples in a parallel worker")));
-
 	tup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
 	tup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
 	tup->t_data->t_infomask |= HEAP_XMAX_INVALID;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 9cd0b7c11b..4e5366fe78 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -764,17 +764,35 @@ GetCurrentCommandId(bool used)
 	if (used)
 	{
 		/*
-		 * Forbid setting currentCommandIdUsed in a parallel worker, because
-		 * we have no provision for communicating this back to the leader.  We
-		 * could relax this restriction when currentCommandIdUsed was already
-		 * true at the start of the parallel operation.
+		 * If in a parallel worker, only allow setting currentCommandIdUsed if
+		 * currentCommandIdUsed was already true at the start of the parallel
+		 * operation (by way of SetCurrentCommandIdUsed()), otherwise forbid
+		 * setting currentCommandIdUsed because we have no provision for
+		 * communicating this back to the leader. Once currentCommandIdUsed is
+		 * set, the commandId used by leader and workers can't be changed,
+		 * because CommandCounterIncrement() then prevents any attempted
+		 * increment of the current commandId.
 		 */
-		Assert(!IsParallelWorker());
+		Assert(!(IsParallelWorker() && !currentCommandIdUsed));
 		currentCommandIdUsed = true;
 	}
 	return currentCommandId;
 }
 
+/*
+ * SetCurrentCommandIdUsedForWorker
+ *
+ * For a parallel worker, record that the currentCommandId has been used. This
+ * must only be called at the start of a parallel operation.
+ */
+void
+SetCurrentCommandIdUsedForWorker(void)
+{
+	Assert(IsParallelWorker() && !currentCommandIdUsed && currentCommandId != InvalidCommandId);
+
+	currentCommandIdUsed = true;
+}
+
 /*
  *	SetParallelStartTimestamps
  *
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 6bf6c5a310..2beccf60df 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -51,18 +51,6 @@
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 
-typedef struct
-{
-	DestReceiver pub;			/* publicly-known function pointers */
-	IntoClause *into;			/* target relation specification */
-	/* These fields are filled by intorel_startup: */
-	Relation	rel;			/* relation to write to */
-	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
-	CommandId	output_cid;		/* cmin to insert in output tuples */
-	int			ti_options;		/* table_tuple_insert performance options */
-	BulkInsertState bistate;	/* bulk insert state */
-} DR_intorel;
-
 /* utility functions for CTAS definition creation */
 static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into);
 static ObjectAddress create_ctas_nodata(List *tlist, IntoClause *into);
@@ -350,6 +338,14 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		/* call ExecutorStart to prepare the plan for execution */
 		ExecutorStart(queryDesc, GetIntoRelEFlags(into));
 
+		/*
+		 * If the SELECT part of the CTAS is parallelizable, then set the
+		 * parallel insert state. We need plan state to be initialized by the
+		 * executor to decide whether to allow parallel inserts or not.
+		 */
+		if (IsParallelInsertionAllowedInCTAS(into))
+			SetCTASParallelInsertState(queryDesc);
+
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
 
@@ -418,6 +414,9 @@ CreateIntoRelDestReceiver(IntoClause *intoClause)
 	self->pub.rDestroy = intorel_destroy;
 	self->pub.mydest = DestIntoRel;
 	self->into = intoClause;
+	self->is_parallel = false;
+	self->is_parallel_worker = false;
+	self->object_id = InvalidOid;
 	/* other private fields will be set during intorel_startup */
 
 	return (DestReceiver *) self;
@@ -438,6 +437,35 @@ intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 	ListCell   *lc;
 	int			attnum;
 
+	/*
+	 * All the necessary work such as table creation, sanity checks etc. would
+	 * have been done by the leader. So, parallel workers just need to open the
+	 * table, allocate bulk insert state, mark the command id as used, store it
+	 * in the dest receiver and return.
+	 */
+	if (myState->is_parallel_worker)
+	{
+		/* In the worker */
+		intoRelationDesc = table_open(myState->object_id, AccessExclusiveLock);
+		myState->rel = intoRelationDesc;
+		myState->reladdr = InvalidObjectAddress;
+		myState->ti_options = 0;
+		myState->bistate = GetBulkInsertState();
+
+		/*
+		 * Right after the table is created in the leader, the command id is
+		 * incremented (in create_ctas_internal()). The new command id is
+		 * marked as used in intorel_startup(), then the parallel mode is
+		 * entered. The command id and transaction id are serialized into
+		 * parallel DSM, they are then available to all parallel workers. All
+		 * the workers need to mark the command id as used before insertion.
+		 */
+		SetCurrentCommandIdUsedForWorker();
+		myState->output_cid = GetCurrentCommandId(false);
+
+		return;
+	}
+
 	Assert(into != NULL);		/* else somebody forgot to set it */
 
 	/* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */
@@ -540,6 +568,27 @@ intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
 	else
 		myState->bistate = NULL;
 
+	/* If parallel inserts are to be allowed, set few extra information. */
+	if (myState->is_parallel)
+	{
+		myState->object_id = intoRelationAddr.objectId;
+
+		/*
+		 * We don't need to skip contacting FSM while inserting tuples for
+		 * parallel mode, while extending the relations, workers instead of
+		 * blocking on a page while another worker is inserting, can check the
+		 * FSM for another page that can accommodate the tuples. This results
+		 * in major benefit for parallel inserts.
+		 */
+		myState->ti_options = 0;
+
+		/*
+		 * rd_createSubid is marked invalid, otherwise, the table is not
+		 * allowed to be extended by the workers.
+		 */
+		myState->rel->rd_createSubid = InvalidSubTransactionId;
+	}
+
 	/*
 	 * Valid smgr_targblock implies something already wrote to the relation.
 	 * This may be harmless, but this function hasn't planned for it.
@@ -606,3 +655,67 @@ intorel_destroy(DestReceiver *self)
 {
 	pfree(self);
 }
+
+/*
+ * IsParallelInsertionAllowedInCTAS --- check if parallel insertion is allowed
+ *
+ * Do not allow parallel inserts if the table is temporary. As the temporary
+ * tables are backend local, workers can not know about them. Currently, CTAS
+ * supports creation of normal(logged), temporary and unlogged tables. It does
+ * not support foreign or partition table creation. Hence the check for
+ * temporary table is enough here.
+ *
+ * Return false either if the into clause is NULL or if the table is
+ * temporary, otherwise true.
+ */
+bool
+IsParallelInsertionAllowedInCTAS(IntoClause *into)
+{
+	/* Below check may hit in case this function is called from explain.c. */
+	if (!IS_CTAS(into))
+		return false;
+
+	if (into->rel && into->rel->relpersistence == RELPERSISTENCE_TEMP)
+		return false;
+
+	return true;
+}
+
+/*
+ * SetCTASParallelInsertState --- set the parallel insert state
+ *
+ * See if the upper node is Gather and it doesn't have any projections, then
+ * set the parallel insert state such as a flag in the dest receiver and also
+ * store the dest receiver reference in the Gather node so that the required
+ * information will be sent to workers.
+ */
+void
+SetCTASParallelInsertState(QueryDesc *queryDesc)
+{
+	GatherState *gstate;
+	DestReceiver *dest;
+
+	Assert(queryDesc);
+
+	gstate = (GatherState *) queryDesc->planstate;
+	dest	= queryDesc->dest;
+
+	/*
+	 * Parallel insertions are not possible either if the upper node is not
+	 * Gather or it's a Gather but it have some projections to perform.
+	 */
+	if (!IsA(gstate, GatherState) || gstate->ps.ps_ProjInfo)
+		return;
+
+	/* Okay to parallelize inserts, so mark it. */
+	((DR_intorel *) dest)->is_parallel = true;
+
+	/*
+	 * For parallelizing inserts in CTAS we must send information such as into
+	 * clause (to build separate dest receiver), object id (to open the created
+	 * table) to each workers. Since this information is available in the CTAS
+	 * dest receiver, store a reference to it in the Gather state so that it
+	 * will be used in ExecInitParallelPlan to pick the required information.
+	 */
+	gstate->dest = dest;
+}
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 43f9b01e83..9a412c3e6b 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -556,6 +556,14 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	/* call ExecutorStart to prepare the plan for execution */
 	ExecutorStart(queryDesc, eflags);
 
+	/*
+	 * If the SELECT part of the CTAS is parallelizable, then set the parallel
+	 * insert state. We need plan state to be initialized by the executor to
+	 * decide whether to allow parallel inserts or not.
+	 */
+	if (IsParallelInsertionAllowedInCTAS(into))
+		SetCTASParallelInsertState(queryDesc);
+
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
 	{
@@ -1775,6 +1783,29 @@ ExplainNode(PlanState *planstate, List *ancestors,
 
 				if (gather->single_copy || es->format != EXPLAIN_FORMAT_TEXT)
 					ExplainPropertyBool("Single Copy", gather->single_copy, es);
+
+				/*
+				 * Show the create table information under Gather node in case
+				 * parallel workers have inserted the rows.
+				 */
+				if (IsA(planstate, GatherState))
+				{
+					GatherState *gstate = (GatherState *) planstate;
+
+					if (IS_PARALLEL_CTAS_DEST(gstate->dest) &&
+						((DR_intorel *) gstate->dest)->into->rel &&
+						((DR_intorel *) gstate->dest)->into->rel->relname)
+					{
+						es->indent--;
+						ExplainIndentText(es);
+						appendStringInfoString(es->str, "->  ");
+						appendStringInfoString(es->str, "Create ");
+						appendStringInfo(es->str, "%s\n",
+							((DR_intorel *) gstate->dest)->into->rel->relname);
+						ExplainIndentText(es);
+						es->indent++;
+					}
+				}
 			}
 			break;
 		case T_GatherMerge:
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index befde52691..270f8c13d7 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -23,6 +23,7 @@
 
 #include "postgres.h"
 
+#include "commands/createas.h"
 #include "executor/execParallel.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
@@ -65,6 +66,7 @@
 #define PARALLEL_KEY_QUERY_TEXT		UINT64CONST(0xE000000000000008)
 #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009)
 #define PARALLEL_KEY_WAL_USAGE			UINT64CONST(0xE00000000000000A)
+#define PARALLEL_KEY_INTO_CLAUSE		UINT64CONST(0xE00000000000000B)
 
 #define PARALLEL_TUPLE_QUEUE_SIZE		65536
 
@@ -77,6 +79,9 @@ typedef struct FixedParallelExecutorState
 	dsa_pointer param_exec;
 	int			eflags;
 	int			jit_flags;
+	Oid			objectid;		/* workers to open relation/table.  */
+	/* Number of tuples inserted by all the workers. */
+	pg_atomic_uint64	processed;
 } FixedParallelExecutorState;
 
 /*
@@ -578,7 +583,8 @@ ExecParallelSetupTupleQueues(ParallelContext *pcxt, bool reinitialize)
 ParallelExecutorInfo *
 ExecInitParallelPlan(PlanState *planstate, EState *estate,
 					 Bitmapset *sendParams, int nworkers,
-					 int64 tuples_needed)
+					 int64 tuples_needed, IntoClause *intoclause,
+					 Oid objectid)
 {
 	ParallelExecutorInfo *pei;
 	ParallelContext *pcxt;
@@ -600,6 +606,8 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	Size		dsa_minsize = dsa_minimum_size();
 	char	   *query_string;
 	int			query_len;
+	char 		*intoclausestr = NULL;
+	int			intoclause_len = 0;
 
 	/*
 	 * Force any initplan outputs that we're going to pass to workers to be
@@ -712,6 +720,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_estimate_chunk(&pcxt->estimator, dsa_minsize);
 	shm_toc_estimate_keys(&pcxt->estimator, 1);
 
+	/* Estimate space for into clause for CTAS. */
+	if (IS_CTAS(intoclause) && OidIsValid(objectid))
+	{
+		intoclausestr = nodeToString(intoclause);
+		intoclause_len = strlen(intoclausestr) + 1;
+		shm_toc_estimate_chunk(&pcxt->estimator, intoclause_len);
+		shm_toc_estimate_keys(&pcxt->estimator, 1);
+	}
+
 	/* Everyone's had a chance to ask for space, so now create the DSM. */
 	InitializeParallelDSM(pcxt);
 
@@ -729,6 +746,14 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	fpes->param_exec = InvalidDsaPointer;
 	fpes->eflags = estate->es_top_eflags;
 	fpes->jit_flags = estate->es_jit_flags;
+	pg_atomic_init_u64(&fpes->processed, 0);
+	pei->processed = &fpes->processed;
+
+	if (intoclausestr)
+		fpes->objectid = objectid;
+	else
+		fpes->objectid = InvalidOid;
+
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, fpes);
 
 	/* Store query string */
@@ -758,8 +783,18 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate,
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_WAL_USAGE, walusage_space);
 	pei->wal_usage = walusage_space;
 
-	/* Set up the tuple queues that the workers will write into. */
-	pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	if (intoclausestr)
+	{
+		char *intoclause_space = shm_toc_allocate(pcxt->toc, intoclause_len);
+
+		memcpy(intoclause_space, intoclausestr, intoclause_len);
+		shm_toc_insert(pcxt->toc, PARALLEL_KEY_INTO_CLAUSE, intoclause_space);
+	}
+	else
+	{
+		/* Set up the tuple queues that the workers will write into. */
+		pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
+	}
 
 	/* We don't need the TupleQueueReaders yet, though. */
 	pei->reader = NULL;
@@ -1387,12 +1422,30 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 	void	   *area_space;
 	dsa_area   *area;
 	ParallelWorkerContext pwcxt;
+	char		*intoclausestr = NULL;
+	IntoClause	*intoclause = NULL;
 
 	/* Get fixed-size state. */
 	fpes = shm_toc_lookup(toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
 
-	/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
-	receiver = ExecParallelGetReceiver(seg, toc);
+	intoclausestr = shm_toc_lookup(toc, PARALLEL_KEY_INTO_CLAUSE, true);
+	if (intoclausestr)
+	{
+		/*
+		 * If the worker is for parallel insert in CTAS, then use the proper
+		 * dest receiver.
+		 */
+		intoclause = (IntoClause *) stringToNode(intoclausestr);
+		receiver = CreateIntoRelDestReceiver(intoclause);
+		((DR_intorel *)receiver)->is_parallel_worker = true;
+		((DR_intorel *)receiver)->object_id = fpes->objectid;
+	}
+	else
+	{
+		/* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
+		receiver = ExecParallelGetReceiver(seg, toc);
+	}
+
 	instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_INSTRUMENTATION, true);
 	if (instrumentation != NULL)
 		instrument_options = instrumentation->instrument_options;
@@ -1471,6 +1524,13 @@ ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
 			queryDesc->estate->es_jit->instr;
 	}
 
+	/*
+	 * Write out the number of tuples this worker has inserted. Leader will use
+	 * it to inform the end client.
+	 */
+	if (intoclausestr)
+		pg_atomic_add_fetch_u64(&fpes->processed, queryDesc->estate->es_processed);
+
 	/* Must do this after capturing instrumentation. */
 	ExecutorEnd(queryDesc);
 
diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c
index a01b46af14..96d745229d 100644
--- a/src/backend/executor/nodeGather.c
+++ b/src/backend/executor/nodeGather.c
@@ -32,6 +32,7 @@
 
 #include "access/relscan.h"
 #include "access/xact.h"
+#include "commands/createas.h"
 #include "executor/execdebug.h"
 #include "executor/execParallel.h"
 #include "executor/nodeGather.h"
@@ -48,6 +49,7 @@ static TupleTableSlot *ExecGather(PlanState *pstate);
 static TupleTableSlot *gather_getnext(GatherState *gatherstate);
 static MinimalTuple gather_readnext(GatherState *gatherstate);
 static void ExecShutdownGatherWorkers(GatherState *node);
+static void ExecParallelInsertInCTAS(GatherState *node);
 
 
 /* ----------------------------------------------------------------
@@ -131,6 +133,71 @@ ExecInitGather(Gather *node, EState *estate, int eflags)
 	return gatherstate;
 }
 
+/* ----------------------------------------------------------------
+ *		ExecParallelInsertInCTAS(node)
+ *
+ *		Facilitates parallel inserts by parallel workers and/or
+ *		leader for Create Table AS.
+ * ----------------------------------------------------------------
+ */
+static void
+ExecParallelInsertInCTAS(GatherState *node)
+{
+	/*
+	 * By now, parallel workers if launched any, would have started their work
+	 * i.e. insertion to target table. In case the leader is also chosen to
+	 * participate, then finish its share before going to wait for the parallel
+	 * workers to finish.
+	 *
+	 * In case if no workers were launched, allow the leader to insert entire
+	 * tuples.
+	 */
+	if (node->need_to_scan_locally || node->nworkers_launched == 0)
+	{
+		EState	   *estate = node->ps.state;
+		TupleTableSlot *outerTupleSlot;
+
+		for(;;)
+		{
+			/* Install our DSA area while executing the plan. */
+			estate->es_query_dsa =
+					node->pei ? node->pei->area : NULL;
+
+			outerTupleSlot = ExecProcNode(node->ps.lefttree);
+
+			estate->es_query_dsa = NULL;
+
+			if(TupIsNull(outerTupleSlot))
+				break;
+
+			(void) node->dest->receiveSlot(outerTupleSlot, node->dest);
+
+			node->ps.state->es_processed++;
+		}
+
+		node->need_to_scan_locally = false;
+	}
+
+	if (node->nworkers_launched > 0)
+	{
+		/*
+		 * We wait here for the parallel workers to finish their work and
+		 * accumulate the tuples they inserted and also their buffer/WAL usage.
+		 * We do not destroy the parallel context here, it will be done in
+		 * ExecShutdownGather at the end of the plan. Note that the
+		 * ExecShutdownGatherWorkers call from ExecShutdownGather will be a
+		 * no-op.
+		 */
+		ExecShutdownGatherWorkers(node);
+
+		/*
+		 * Add up the total tuples inserted by all workers, to the tuples
+		 * inserted by the leader(if any). This will be shared to client.
+		 */
+		node->ps.state->es_processed += pg_atomic_read_u64(node->pei->processed);
+	}
+}
+
 /* ----------------------------------------------------------------
  *		ExecGather(node)
  *
@@ -157,6 +224,7 @@ ExecGather(PlanState *pstate)
 	{
 		EState	   *estate = node->ps.state;
 		Gather	   *gather = (Gather *) node->ps.plan;
+		bool		isctas = IS_PARALLEL_CTAS_DEST(node->dest);
 
 		/*
 		 * Sometimes we might have to run without parallelism; but if parallel
@@ -165,6 +233,18 @@ ExecGather(PlanState *pstate)
 		if (gather->num_workers > 0 && estate->es_use_parallel_mode)
 		{
 			ParallelContext *pcxt;
+			IntoClause	*intoclause = NULL;
+			Oid			objectid = InvalidOid;
+
+			/*
+			 * Take the necessary information to be passed to workers for
+			 * parallel inserts in CTAS.
+			 */
+			if (isctas)
+			{
+				intoclause = ((DR_intorel *) node->dest)->into;
+				objectid = ((DR_intorel *) node->dest)->object_id;
+			}
 
 			/* Initialize, or re-initialize, shared state needed by workers. */
 			if (!node->pei)
@@ -172,7 +252,10 @@ ExecGather(PlanState *pstate)
 												 estate,
 												 gather->initParam,
 												 gather->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 /* CTAS info */
+												 intoclause,
+												 objectid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
@@ -190,13 +273,16 @@ ExecGather(PlanState *pstate)
 			/* Set up tuple queue readers to read the results. */
 			if (pcxt->nworkers_launched > 0)
 			{
-				ExecParallelCreateReaders(node->pei);
-				/* Make a working array showing the active readers */
-				node->nreaders = pcxt->nworkers_launched;
-				node->reader = (TupleQueueReader **)
-					palloc(node->nreaders * sizeof(TupleQueueReader *));
-				memcpy(node->reader, node->pei->reader,
-					   node->nreaders * sizeof(TupleQueueReader *));
+				if (!isctas)
+				{
+					ExecParallelCreateReaders(node->pei);
+					/* Make a working array showing the active readers */
+					node->nreaders = pcxt->nworkers_launched;
+					node->reader = (TupleQueueReader **)
+						palloc(node->nreaders * sizeof(TupleQueueReader *));
+					memcpy(node->reader, node->pei->reader,
+						node->nreaders * sizeof(TupleQueueReader *));
+				}
 			}
 			else
 			{
@@ -208,9 +294,16 @@ ExecGather(PlanState *pstate)
 		}
 
 		/* Run plan locally if no workers or enabled and not single-copy. */
-		node->need_to_scan_locally = (node->nreaders == 0)
+		node->need_to_scan_locally = (node->nreaders == 0 && !isctas)
 			|| (!gather->single_copy && parallel_leader_participation);
 		node->initialized = true;
+
+		/* Perform parallel inserts for CTAS. */
+		if (isctas)
+		{
+			ExecParallelInsertInCTAS(node);
+			return NULL;
+		}
 	}
 
 	/*
diff --git a/src/backend/executor/nodeGatherMerge.c b/src/backend/executor/nodeGatherMerge.c
index 47129344f3..ee45272c17 100644
--- a/src/backend/executor/nodeGatherMerge.c
+++ b/src/backend/executor/nodeGatherMerge.c
@@ -216,7 +216,9 @@ ExecGatherMerge(PlanState *pstate)
 												 estate,
 												 gm->initParam,
 												 gm->num_workers,
-												 node->tuples_needed);
+												 node->tuples_needed,
+												 NULL,
+												 InvalidOid);
 			else
 				ExecParallelReinitialize(node->ps.lefttree,
 										 node->pei,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7320de345c..5beae6c617 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -389,6 +389,7 @@ extern FullTransactionId GetCurrentFullTransactionIdIfAny(void);
 extern void MarkCurrentTransactionIdLoggedIfAny(void);
 extern bool SubTransactionIsActive(SubTransactionId subxid);
 extern CommandId GetCurrentCommandId(bool used);
+extern void SetCurrentCommandIdUsedForWorker(void);
 extern void SetParallelStartTimestamps(TimestampTz xact_ts, TimestampTz stmt_ts);
 extern TimestampTz GetCurrentTransactionStartTimestamp(void);
 extern TimestampTz GetCurrentStatementStartTimestamp(void);
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 7629230254..ab74c69b9f 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -14,12 +14,35 @@
 #ifndef CREATEAS_H
 #define CREATEAS_H
 
+#include "access/heapam.h"
 #include "catalog/objectaddress.h"
+#include "executor/execdesc.h"
 #include "nodes/params.h"
+#include "nodes/plannodes.h"
 #include "parser/parse_node.h"
 #include "tcop/dest.h"
 #include "utils/queryenvironment.h"
 
+typedef struct
+{
+	DestReceiver pub;			/* publicly-known function pointers */
+	IntoClause *into;			/* target relation specification */
+	/* These fields are filled by intorel_startup: */
+	Relation	rel;			/* relation to write to */
+	ObjectAddress reladdr;		/* address of rel, for ExecCreateTableAs */
+	CommandId	output_cid;		/* cmin to insert in output tuples */
+	int			ti_options;		/* table_tuple_insert performance options */
+	BulkInsertState bistate;	/* bulk insert state */
+	bool		is_parallel;	/* is parallelism to be considered? */
+	bool		is_parallel_worker; /* true for parallel worker */
+	/* Used by parallel workers for opening the table created in the leader. */
+	Oid			object_id;
+} DR_intorel;
+
+#define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
+#define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
+									 IS_CTAS(((DR_intorel *) dest)->into) && \
+									 ((DR_intorel *) dest)->is_parallel)
 
 extern ObjectAddress ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 									   ParamListInfo params, QueryEnvironment *queryEnv,
@@ -29,4 +52,7 @@ extern int	GetIntoRelEFlags(IntoClause *intoClause);
 
 extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
+extern bool IsParallelInsertionAllowedInCTAS(IntoClause *into);
+
+extern void SetCTASParallelInsertState(QueryDesc *queryDesc);
 #endif							/* CREATEAS_H */
diff --git a/src/include/executor/execParallel.h b/src/include/executor/execParallel.h
index 5a39a5b29c..9f959f741b 100644
--- a/src/include/executor/execParallel.h
+++ b/src/include/executor/execParallel.h
@@ -35,11 +35,15 @@ typedef struct ParallelExecutorInfo
 	/* These two arrays have pcxt->nworkers_launched entries: */
 	shm_mq_handle **tqueue;		/* tuple queues for worker output */
 	struct TupleQueueReader **reader;	/* tuple reader/writer support */
+	/* Number of tuples inserted by all workers. */
+	volatile pg_atomic_uint64	*processed;
 } ParallelExecutorInfo;
 
 extern ParallelExecutorInfo *ExecInitParallelPlan(PlanState *planstate,
 												  EState *estate, Bitmapset *sendParam, int nworkers,
-												  int64 tuples_needed);
+												  int64 tuples_needed,
+												  IntoClause *intoclause,
+												  Oid objectid);
 extern void ExecParallelCreateReaders(ParallelExecutorInfo *pei);
 extern void ExecParallelFinish(ParallelExecutorInfo *pei);
 extern void ExecParallelCleanup(ParallelExecutorInfo *pei);
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 61ba4c3666..a5b281f783 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -23,6 +23,7 @@
 #include "nodes/tidbitmap.h"
 #include "partitioning/partdefs.h"
 #include "storage/condition_variable.h"
+#include "tcop/dest.h"
 #include "utils/hsearch.h"
 #include "utils/queryenvironment.h"
 #include "utils/reltrigger.h"
@@ -2326,6 +2327,8 @@ typedef struct GatherState
 	int			nreaders;		/* number of still-active workers */
 	int			nextreader;		/* next one to try to read from */
 	struct TupleQueueReader **reader;	/* array with nreaders active entries */
+	/* Dest receiver is stored when parallel inserts is allowed in CTAS. */
+	DestReceiver *dest;
 } GatherState;
 
 /* ----------------
-- 
2.25.1

From 9ce3e49f0f11d7144566704f5153d1e579a3814c Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 28 Dec 2020 15:06:01 +0530
Subject: [PATCH v18 2/4] Tuple Cost Adjustment for Parallel Inserts in CTAS

Let the planner know that the SELECT is from CTAS in createas.c
so that it can set the number of tuples transferred from the
workers to Gather node to 0. With this change, there are chances
that the planner may choose the parallel plan.
---
 src/backend/commands/createas.c       | 61 +++++++++++++++++++++------
 src/backend/commands/explain.c        | 16 +++++--
 src/backend/commands/prepare.c        |  3 +-
 src/backend/optimizer/path/costsize.c | 22 +++++++++-
 src/backend/optimizer/plan/planner.c  | 53 +++++++++++++++++++++++
 src/include/commands/createas.h       | 21 ++++++++-
 src/include/commands/explain.h        |  3 +-
 src/include/nodes/parsenodes.h        |  1 +
 src/include/optimizer/planner.h       | 10 +++++
 9 files changed, 169 insertions(+), 21 deletions(-)

diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 2beccf60df..5046e02aac 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -316,6 +316,15 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		query = linitial_node(Query, rewritten);
 		Assert(query->commandType == CMD_SELECT);
 
+		/*
+		 * Turn on a flag to indicate the planner so that it can ignore the
+		 * parallel tuple cost while generating Gather path.
+		 */
+		if (IsParallelInsertionAllowedInCTAS(into))
+			query->CTASParallelInsInfo |= CTAS_PARALLEL_INS_SELECT;
+		else
+			query->CTASParallelInsInfo = CTAS_PARALLEL_INS_UNDEF;
+
 		/* plan the query */
 		plan = pg_plan_query(query, pstate->p_sourcetext,
 							 CURSOR_OPT_PARALLEL_OK, params);
@@ -344,7 +353,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
 		 * executor to decide whether to allow parallel inserts or not.
 		 */
 		if (IsParallelInsertionAllowedInCTAS(into))
-			SetCTASParallelInsertState(queryDesc);
+			SetCTASParallelInsertState(queryDesc, &query->CTASParallelInsInfo);
 
 		/* run the plan to completion */
 		ExecutorRun(queryDesc, ForwardScanDirection, 0L, true);
@@ -690,7 +699,7 @@ IsParallelInsertionAllowedInCTAS(IntoClause *into)
  * information will be sent to workers.
  */
 void
-SetCTASParallelInsertState(QueryDesc *queryDesc)
+SetCTASParallelInsertState(QueryDesc *queryDesc, uint8 *tuple_cost_flags)
 {
 	GatherState *gstate;
 	DestReceiver *dest;
@@ -701,21 +710,45 @@ SetCTASParallelInsertState(QueryDesc *queryDesc)
 	dest	= queryDesc->dest;
 
 	/*
-	 * Parallel insertions are not possible either if the upper node is not
-	 * Gather or it's a Gather but it have some projections to perform.
+	 * Parallel insertions are not possible if the upper node is not Gather.
 	 */
-	if (!IsA(gstate, GatherState) || gstate->ps.ps_ProjInfo)
+	if (!IsA(gstate, GatherState))
 		return;
 
-	/* Okay to parallelize inserts, so mark it. */
-	((DR_intorel *) dest)->is_parallel = true;
-
 	/*
-	 * For parallelizing inserts in CTAS we must send information such as into
-	 * clause (to build separate dest receiver), object id (to open the created
-	 * table) to each workers. Since this information is available in the CTAS
-	 * dest receiver, store a reference to it in the Gather state so that it
-	 * will be used in ExecInitParallelPlan to pick the required information.
+	 * If the upper Gather node has some projections to perform, then we can
+	 * not allow parallel insertions. But before returning, ensure that we have
+	 * not done wrong parallel tuple cost enforcement in the planner.
+	 *
+	 * The main reason for this assertion is to check if we enforced planner to
+	 * ignore the parallel tuple cost (with the intention of choosing parallel
+	 * inserts) due to which the parallel plan may have been chosen, but we do
+	 * not allow the parallel inserts now.
+	 *
+	 * If we have correctly ignored parallel tuple cost in planner while
+	 * creating Gather path, then this assertion failure should not occur. In
+	 * case it occurs, that means the planner may have chosen this parallel
+	 * plan because of our wrong enforcement. So let's catch that here.
 	 */
-	gstate->dest = dest;
+	if (tuple_cost_flags && gstate->ps.ps_ProjInfo)
+		Assert(!(*tuple_cost_flags & CTAS_PARALLEL_INS_TUP_COST_IGNORED));
+
+	if (!gstate->ps.ps_ProjInfo)
+	{
+		/* Okay to parallelize inserts, so mark it. */
+		((DR_intorel *) dest)->is_parallel = true;
+
+		/*
+		 * For parallelizing inserts in CTAS we must send information such as
+		 * into clause (to build separate dest receiver), object id (to open
+		 * the created table) to each workers. Since this information is
+		 * available in the CTAS dest receiver, store a reference to it in the
+		 * Gather state so that it will be used in ExecInitParallelPlan to pick
+		 * the required information.
+		 */
+		gstate->dest = dest;
+	}
+
+	if (tuple_cost_flags)
+		*tuple_cost_flags = CTAS_PARALLEL_INS_UNDEF;
 }
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 9a412c3e6b..50a1fc2f36 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -387,6 +387,15 @@ ExplainOneQuery(Query *query, int cursorOptions,
 			bufusage_start = pgBufferUsage;
 		INSTR_TIME_SET_CURRENT(planstart);
 
+		/*
+		 * Turn on a flag to indicate the planner so that it can ignore the
+		 * parallel tuple cost while generating Gather path.
+		 */
+		if (IsParallelInsertionAllowedInCTAS(into))
+			query->CTASParallelInsInfo |= CTAS_PARALLEL_INS_SELECT;
+		else
+			query->CTASParallelInsInfo = CTAS_PARALLEL_INS_UNDEF;
+
 		/* plan the query */
 		plan = pg_plan_query(query, queryString, cursorOptions, params);
 
@@ -402,7 +411,8 @@ ExplainOneQuery(Query *query, int cursorOptions,
 
 		/* run it (if needed) and produce output */
 		ExplainOnePlan(plan, into, es, queryString, params, queryEnv,
-					   &planduration, (es->buffers ? &bufusage : NULL));
+					   &planduration, (es->buffers ? &bufusage : NULL),
+					   &query->CTASParallelInsInfo);
 	}
 }
 
@@ -496,7 +506,7 @@ void
 ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 			   const char *queryString, ParamListInfo params,
 			   QueryEnvironment *queryEnv, const instr_time *planduration,
-			   const BufferUsage *bufusage)
+			   const BufferUsage *bufusage, uint8 *ctas_tuple_cost_flags)
 {
 	DestReceiver *dest;
 	QueryDesc  *queryDesc;
@@ -562,7 +572,7 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
 	 * decide whether to allow parallel inserts or not.
 	 */
 	if (IsParallelInsertionAllowedInCTAS(into))
-		SetCTASParallelInsertState(queryDesc);
+		SetCTASParallelInsertState(queryDesc, ctas_tuple_cost_flags);
 
 	/* Execute the plan for statistics if asked for */
 	if (es->analyze)
diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c
index 89087a7be3..07166479e7 100644
--- a/src/backend/commands/prepare.c
+++ b/src/backend/commands/prepare.c
@@ -672,7 +672,8 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es,
 
 		if (pstmt->commandType != CMD_UTILITY)
 			ExplainOnePlan(pstmt, into, es, query_string, paramLI, queryEnv,
-						   &planduration, (es->buffers ? &bufusage : NULL));
+						   &planduration, (es->buffers ? &bufusage : NULL),
+						   NULL);
 		else
 			ExplainOneUtility(pstmt->utilityStmt, into, es, query_string,
 							  paramLI, queryEnv);
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 22d6935824..e706219102 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -76,6 +76,7 @@
 #include "access/amapi.h"
 #include "access/htup_details.h"
 #include "access/tsmapi.h"
+#include "commands/createas.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "executor/nodeHash.h"
@@ -378,6 +379,7 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 {
 	Cost		startup_cost = 0;
 	Cost		run_cost = 0;
+	bool		ignore_tuple_cost = false;
 
 	/* Mark the path with the correct row estimate */
 	if (rows)
@@ -393,7 +395,25 @@ cost_gather(GatherPath *path, PlannerInfo *root,
 
 	/* Parallel setup and communication cost. */
 	startup_cost += parallel_setup_cost;
-	run_cost += parallel_tuple_cost * path->path.rows;
+
+	/*
+	 * Do not consider tuple cost in case of we intend to perform parallel
+	 * inserts by workers. We would have turned on the ignore flag in
+	 * apply_scanjoin_target_to_paths before generating Gather path for the
+	 * upper level SELECT part of the CTAS.
+	 */
+	if ((root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT) &&
+		(root->parse->CTASParallelInsInfo &
+		 CTAS_PARALLEL_INS_TUP_COST_CAN_IGN))
+	{
+		ignore_tuple_cost = true;
+		root->parse->CTASParallelInsInfo &=
+									~CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_TUP_COST_IGNORED;
+	}
+
+	if (!ignore_tuple_cost)
+		run_cost += parallel_tuple_cost * path->path.rows;
 
 	path->path.startup_cost = startup_cost;
 	path->path.total_cost = (startup_cost + run_cost);
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index 1a94b58f8b..20a18628a7 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -28,6 +28,7 @@
 #include "catalog/pg_inherits.h"
 #include "catalog/pg_proc.h"
 #include "catalog/pg_type.h"
+#include "commands/createas.h"
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "foreign/fdwapi.h"
@@ -7338,6 +7339,37 @@ can_partial_agg(PlannerInfo *root)
 	return true;
 }
 
+/*
+ * ignore_parallel_tuple_cost
+ *
+ * Gather node will not receive any tuples from the workers in case each worker
+ * inserts them in parallel. So, we turn on a flag to ignore parallel tuple
+ * cost by the Gather path in cost_gather if the SELECT is for CTAS and we are
+ * generating an upper level Gather path.
+*/
+static bool
+ignore_parallel_tuple_cost(PlannerInfo *root)
+{
+	if (root->query_level == 1 &&
+		(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT))
+	{
+		/*
+		 * In each of the HAS_PARENT_PATH_GENERATING_CLAUSE cases, a parent
+		 * path will be generated for the upper Gather path(in
+		 * grouping_planner), in which case we can not let parallel inserts
+		 * happen. So we do not turn on ignore tuple cost flag.
+		 */
+		if (HAS_PARENT_PATH_GENERATING_CLAUSE(root))
+			return false;
+
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+
+		return true;
+	}
+
+	return false;
+}
+
 /*
  * apply_scanjoin_target_to_paths
  *
@@ -7557,8 +7589,29 @@ apply_scanjoin_target_to_paths(PlannerInfo *root,
 	 * one of the generated paths may turn out to be the cheapest one.
 	 */
 	if (rel->consider_parallel && !IS_OTHER_REL(rel))
+	{
+		/*
+		 * Turn on a flag to ignore parallel tuple cost by the Gather path in
+		 * cost_gather if the SELECT is for CTAS and we are generating an upper
+		 * level Gather path.
+		 */
+		bool	ignore = ignore_parallel_tuple_cost(root);
+
 		generate_useful_gather_paths(root, rel, false);
 
+		/*
+		 * Reset the ignore flag, in case we turned it on but
+		 * generate_useful_gather_paths returned without reaching cost_gather.
+		 * If we reached cost_gather, we would have been reset it there.
+		 */
+		if (ignore && (root->parse->CTASParallelInsInfo &
+			CTAS_PARALLEL_INS_TUP_COST_CAN_IGN))
+		{
+			root->parse->CTASParallelInsInfo &=
+										~CTAS_PARALLEL_INS_TUP_COST_CAN_IGN;
+		}
+	}
+
 	/*
 	 * Reassess which paths are the cheapest, now that we've potentially added
 	 * new Gather (or Gather Merge) and/or Append (or MergeAppend) paths to
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index ab74c69b9f..140226ad5a 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -39,6 +39,24 @@ typedef struct
 	Oid			object_id;
 } DR_intorel;
 
+/*
+ * Information sent to the planner from CTAS to account for the cost
+ * calculations in cost_gather. We need to do this because, no tuples will be
+ * received by the Gather node if the workers insert the tuples in parallel.
+ */
+typedef enum CTASParallelInsertOpt
+{
+	CTAS_PARALLEL_INS_UNDEF = 0, /* undefined */
+	CTAS_PARALLEL_INS_SELECT = 1 << 0, /* turn on this before planning */
+	/*
+	 * Turn on this while planning for upper Gather path to ignore parallel
+	 * tuple cost in cost_gather.
+	 */
+	CTAS_PARALLEL_INS_TUP_COST_CAN_IGN = 1 << 1,
+	/* Turn on this after the cost is ignored. */
+	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2
+} CTASParallelInsertOpt;
+
 #define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
 #define IS_PARALLEL_CTAS_DEST(dest) (dest && dest->mydest == DestIntoRel && \
 									 IS_CTAS(((DR_intorel *) dest)->into) && \
@@ -54,5 +72,6 @@ extern DestReceiver *CreateIntoRelDestReceiver(IntoClause *intoClause);
 
 extern bool IsParallelInsertionAllowedInCTAS(IntoClause *into);
 
-extern void SetCTASParallelInsertState(QueryDesc *queryDesc);
+extern void SetCTASParallelInsertState(QueryDesc *queryDesc,
+									   uint8 *tuple_cost_flags);
 #endif							/* CREATEAS_H */
diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h
index ba661d32a6..1a1806dbf1 100644
--- a/src/include/commands/explain.h
+++ b/src/include/commands/explain.h
@@ -91,7 +91,8 @@ extern void ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into,
 						   ExplainState *es, const char *queryString,
 						   ParamListInfo params, QueryEnvironment *queryEnv,
 						   const instr_time *planduration,
-						   const BufferUsage *bufusage);
+						   const BufferUsage *bufusage,
+						   uint8 *ctas_tuple_cost_flags);
 
 extern void ExplainPrintPlan(ExplainState *es, QueryDesc *queryDesc);
 extern void ExplainPrintTriggers(ExplainState *es, QueryDesc *queryDesc);
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 48a79a7657..81b148c383 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -180,6 +180,7 @@ typedef struct Query
 	 */
 	int			stmt_location;	/* start location, or -1 if unknown */
 	int			stmt_len;		/* length in bytes; 0 means "rest of string" */
+	uint8 		CTASParallelInsInfo; /* parallel insert in CTAS info */
 } Query;
 
 
diff --git a/src/include/optimizer/planner.h b/src/include/optimizer/planner.h
index beb7dbbcbe..74b2563828 100644
--- a/src/include/optimizer/planner.h
+++ b/src/include/optimizer/planner.h
@@ -21,6 +21,16 @@
 #include "nodes/pathnodes.h"
 #include "nodes/plannodes.h"
 
+#define HAS_PARENT_PATH_GENERATING_CLAUSE(root) \
+											(root->parse->rowMarks || \
+											 limit_needed(root->parse) || \
+											 root->parse->sortClause ||  \
+											 root->parse->distinctClause ||  \
+											 root->parse->hasWindowFuncs ||  \
+											 root->parse->groupClause ||  \
+											 root->parse->groupingSets ||  \
+											 root->parse->hasAggs ||  \
+											 root->hasHavingQual)
 
 /* Hook for plugins to get control in planner() */
 typedef PlannedStmt *(*planner_hook_type) (Query *parse,
-- 
2.25.1

From 61641393e4c1d90a47f2a070d6e9e020e6f014e4 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 28 Dec 2020 16:49:43 +0530
Subject: [PATCH v18 3/4] Tests And Docs For Parallel Inserts in CTAS

---
 doc/src/sgml/ref/create_table_as.sgml        |  31 +-
 src/test/regress/expected/write_parallel.out | 559 +++++++++++++++++++
 src/test/regress/sql/write_parallel.sql      | 211 +++++++
 3 files changed, 796 insertions(+), 5 deletions(-)

diff --git a/doc/src/sgml/ref/create_table_as.sgml b/doc/src/sgml/ref/create_table_as.sgml
index 07558ab56c..35903701ed 100644
--- a/doc/src/sgml/ref/create_table_as.sgml
+++ b/doc/src/sgml/ref/create_table_as.sgml
@@ -37,11 +37,13 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI
 
   <para>
    <command>CREATE TABLE AS</command> creates a table and fills it
-   with data computed by a <command>SELECT</command> command.
-   The table columns have the
-   names and data types associated with the output columns of the
-   <command>SELECT</command> (except that you can override the column
-   names by giving an explicit list of new column names).
+   with data computed by a <command>SELECT</command> command. When the
+   node at the top of the <command>SELECT</command> plan is
+   <literal>Gather</literal> and there are no projections to be performed
+   by it, then the created table can be filled by the workers in parallel.
+   The table columns have the names and data types associated with the output
+   columns of the <command>SELECT</command> (except that you can override the
+   column names by giving an explicit list of new column names).
   </para>
 
   <para>
@@ -297,6 +299,25 @@ PREPARE recentfilms(date) AS
 CREATE TEMP TABLE films_recent ON COMMIT DROP AS
   EXECUTE recentfilms('2002-01-01');
 </programlisting></para>
+
+  <para>
+   Here is an example of a query plan when the data into the created table can
+   be filled by the workers in parallel:
+
+<programlisting>
+EXPLAIN CREATE TABLE bar AS SELECT * FROM foo WHERE i > 5;
+
+                             QUERY PLAN
+-------------------------------------------------------------------&zwsp;--
+Gather  (cost=0.00..23.28 rows=850 width=4)
+   Workers Planned: 2
+ ->  Create bar
+   ->  Parallel Seq Scan on foo  (cost=0.00..23.28 rows=354 width=4)
+         Filter: (i > 5)
+(5 rows)
+</programlisting>
+  </para>
+
  </refsect1>
 
  <refsect1>
diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 0c4da2591a..38a18c5a9b 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -76,4 +76,563 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+-- Parallel queries won't necessarily get as many workers as the planner
+-- asked for.  This affects not only the "Workers Launched:" field of EXPLAIN
+-- results, but also row counts and loop counts for parallel scans, Gathers,
+-- and everything in between.  This function filters out the values we can't
+-- rely on to be stable.
+-- This removes enough info that you might wonder why bother with EXPLAIN
+-- ANALYZE at all.  The answer is that we need to see whether the parallel
+-- inserts are being done by the workers, the only way is that
+-- Create <<tbl_name>> appears in the explain output.
+create function explain_pictas(text) returns setof text
+language plpgsql as
+$$
+declare
+    ln text;
+begin
+    for ln in
+        execute format('explain (analyze, costs off, summary off, timing off) %s',
+            $1)
+    loop
+        ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N');
+        ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N');
+        ln := regexp_replace(ln, 'Rows Removed by Filter: \d+', 'Rows Removed by Filter: N');
+        return next ln;
+    end loop;
+end;
+$$;
+-- parallel inserts must occur as the CTAS creates a normal table
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'create temporary table parallel_write as select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(4 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur as the CTAS creates an unlogged table
+select explain_pictas(
+'create unlogged table parallel_write as select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur as the select into creates a normal table
+select explain_pictas(
+'select length(stringu1) into parallel_write from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'select length(stringu1) into temporary parallel_write from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(4 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur as the select into creates an unlogged table
+select explain_pictas(
+'select length(stringu1) into unlogged parallel_write from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1 for update;');
+                 explain_pictas                  
+-------------------------------------------------
+ LockRows (actual rows=N loops=N)
+   ->  Seq Scan on tenk1 (actual rows=N loops=N)
+(2 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must occur as the materialized view is being created here
+select explain_pictas(
+'create materialized view parallel_mat_view as
+    select length(stringu1) from tenk1;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_mat_view
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_mat_view;
+ count 
+-------
+ 10000
+(1 row)
+
+drop materialized view parallel_mat_view;
+-- parallel inserts must occur as the CTAS creates the table using prepared
+-- statement for which parallelism would have been picked
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+select explain_pictas(
+'create table parallel_write as execute parallel_write_prep;');
+                      explain_pictas                      
+----------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+deallocate parallel_write_prep;
+drop table parallel_write;
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+select explain_pictas(
+E'create table parallel_write as
+    select nextval(\'parallel_write_sequence\'), four from tenk1;');
+              explain_pictas               
+-------------------------------------------
+ Seq Scan on tenk1 (actual rows=N loops=N)
+(1 row)
+
+select count(*) from parallel_write;
+ count 
+-------
+ 10000
+(1 row)
+
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Params Evaluated: $1
+   Workers Launched: N
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 4
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on tenk2 (actual rows=N loops=N)
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+         Filter: (four = 3)
+         Rows Removed by Filter: N
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+  2500
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;');
+                          explain_pictas                          
+------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 4
+   Workers Launched: N
+   ->  Parallel Seq Scan on tenk1 (actual rows=N loops=N)
+         Filter: (four = 3)
+         Rows Removed by Filter: N
+   SubPlan 1
+     ->  Function Scan on generate_series (actual rows=N loops=N)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+  2500
+(1 row)
+
+drop table parallel_write;
+create table temp1(col1) as select * from generate_series(1,5);
+create table temp2(col2) as select * from temp1;
+create table temp3(col3) as select * from temp1;
+-- parallel inserts must not occur, as there is a limit clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 limit 4;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ Limit (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(5 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ Gather Merge (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+   ->  Sort (actual rows=N loops=N)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         Worker 0:  Sort Method: quicksort  Memory: 25kB
+         Worker 1:  Sort Method: quicksort  Memory: 25kB
+         Worker 2:  Sort Method: quicksort  Memory: 25kB
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ Gather Merge (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+   ->  Sort (actual rows=N loops=N)
+         Sort Key: col1
+         Sort Method: quicksort  Memory: 25kB
+         Worker 0:  Sort Method: quicksort  Memory: 25kB
+         Worker 1:  Sort Method: quicksort  Memory: 25kB
+         Worker 2:  Sort Method: quicksort  Memory: 25kB
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is a distinct clause
+select explain_pictas(
+'create table parallel_write as select distinct * from temp1;');
+                         explain_pictas                         
+----------------------------------------------------------------
+ HashAggregate (actual rows=N loops=N)
+   Group Key: col1
+   Batches: 1  Memory Usage: 40kB
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(7 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an aggregate and group clause
+select explain_pictas(
+'create table parallel_write as select count(*) from temp1 group by col1;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Finalize HashAggregate (actual rows=N loops=N)
+   Group Key: col1
+   Batches: 1  Memory Usage: 40kB
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Partial HashAggregate (actual rows=N loops=N)
+               Group Key: col1
+               Batches: 1  Memory Usage: 40kB
+               Worker 0:  Batches: 1  Memory Usage: 40kB
+               Worker 1:  Batches: 1  Memory Usage: 40kB
+               Worker 2:  Batches: 1  Memory Usage: 40kB
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(13 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is an aggregate, group and having
+-- clauses
+select explain_pictas(
+'create table parallel_write as
+    select count(col1), (select col3 from
+        (select * from temp3) as tt limit 1) col4 from temp1, temp2
+    where temp1.col1 = temp2.col2 group by col4 having count(col1) > 0;');
+                                  explain_pictas                                  
+----------------------------------------------------------------------------------
+ Finalize GroupAggregate (actual rows=N loops=N)
+   Group Key: ($1)
+   Filter: (count(temp1.col1) > 0)
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 3
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on temp3 (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $1
+         Workers Launched: N
+         ->  Partial GroupAggregate (actual rows=N loops=N)
+               Group Key: $1
+               ->  Parallel Hash Join (actual rows=N loops=N)
+                     Hash Cond: (temp1.col1 = temp2.col2)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                     ->  Parallel Hash (actual rows=N loops=N)
+                           Buckets: 4096  Batches: 1  Memory Usage: 64kB
+                           ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(21 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     1
+(1 row)
+
+drop table parallel_write;
+-- parallel inserts must not occur, as there is a window function
+select explain_pictas(
+'create table parallel_write as
+    select avg(col1) OVER (PARTITION BY col1) from temp1;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ WindowAgg (actual rows=N loops=N)
+   ->  Gather Merge (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Sort (actual rows=N loops=N)
+               Sort Key: col1
+               Sort Method: quicksort  Memory: 25kB
+               Worker 0:  Sort Method: quicksort  Memory: 25kB
+               Worker 1:  Sort Method: quicksort  Memory: 25kB
+               Worker 2:  Sort Method: quicksort  Memory: 25kB
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+(11 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- nested loop join is the top node under which Gather node exists, so parallel
+-- inserts must not occur
+set enable_nestloop to on;
+set enable_mergejoin to off;
+set enable_hashjoin to off;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Nested Loop (actual rows=N loops=N)
+   Join Filter: (temp1.col1 = temp2.col2)
+   Rows Removed by Join Filter: 20
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+   ->  Materialize (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Workers Launched: N
+               ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- the top node is Gather under which merge join happens, so parallel inserts
+-- must occur
+set enable_nestloop to off;
+set enable_mergejoin to on;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Merge Join (actual rows=N loops=N)
+         Merge Cond: (temp1.col1 = temp2.col2)
+         ->  Sort (actual rows=N loops=N)
+               Sort Key: temp1.col1
+               Sort Method: quicksort  Memory: 25kB
+               Worker 0:  Sort Method: quicksort  Memory: 25kB
+               Worker 1:  Sort Method: quicksort  Memory: 25kB
+               Worker 2:  Sort Method: quicksort  Memory: 25kB
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Sort (actual rows=N loops=N)
+               Sort Key: temp2.col2
+               Sort Method: quicksort  Memory: 25kB
+               ->  Seq Scan on temp2 (actual rows=N loops=N)
+(17 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+-- parallel hash join happens under Gather node, so parallel inserts must occur
+set enable_mergejoin to off;
+set enable_hashjoin to on;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Hash Join (actual rows=N loops=N)
+         Hash Cond: (temp1.col1 = temp2.col2)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Hash (actual rows=N loops=N)
+               Buckets: 4096  Batches: 1  Memory Usage: 64kB
+               ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+reset enable_nestloop;
+reset enable_mergejoin;
+reset enable_hashjoin;
+drop table temp1;
+drop table temp2;
+drop table temp3;
+drop function explain_pictas(text);
 rollback;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 78b479cedf..40aadafc2a 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -39,4 +39,215 @@ explain (costs off) create table parallel_write as execute prep_stmt;
 create table parallel_write as execute prep_stmt;
 drop table parallel_write;
 
+--
+-- Test parallel inserts in create table as/select into/create materialized
+-- view.
+--
+
+-- Parallel queries won't necessarily get as many workers as the planner
+-- asked for.  This affects not only the "Workers Launched:" field of EXPLAIN
+-- results, but also row counts and loop counts for parallel scans, Gathers,
+-- and everything in between.  This function filters out the values we can't
+-- rely on to be stable.
+-- This removes enough info that you might wonder why bother with EXPLAIN
+-- ANALYZE at all.  The answer is that we need to see whether the parallel
+-- inserts are being done by the workers, the only way is that
+-- Create <<tbl_name>> appears in the explain output.
+create function explain_pictas(text) returns setof text
+language plpgsql as
+$$
+declare
+    ln text;
+begin
+    for ln in
+        execute format('explain (analyze, costs off, summary off, timing off) %s',
+            $1)
+    loop
+        ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N');
+        ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N');
+        ln := regexp_replace(ln, 'Rows Removed by Filter: \d+', 'Rows Removed by Filter: N');
+        return next ln;
+    end loop;
+end;
+$$;
+
+-- parallel inserts must occur as the CTAS creates a normal table
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'create temporary table parallel_write as select length(stringu1) from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur as the CTAS creates an unlogged table
+select explain_pictas(
+'create unlogged table parallel_write as select length(stringu1) from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur as the select into creates a normal table
+select explain_pictas(
+'select length(stringu1) into parallel_write from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the table is temporary
+select explain_pictas(
+'select length(stringu1) into temporary parallel_write from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur as the select into creates an unlogged table
+select explain_pictas(
+'select length(stringu1) into unlogged parallel_write from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of for update clause
+select explain_pictas(
+'create table parallel_write as select length(stringu1) from tenk1 for update;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must occur as the materialized view is being created here
+select explain_pictas(
+'create materialized view parallel_mat_view as
+    select length(stringu1) from tenk1;');
+select count(*) from parallel_mat_view;
+drop materialized view parallel_mat_view;
+
+-- parallel inserts must occur as the CTAS creates the table using prepared
+-- statement for which parallelism would have been picked
+prepare parallel_write_prep as select length(stringu1) from tenk1;
+select explain_pictas(
+'create table parallel_write as execute parallel_write_prep;');
+select count(*) from parallel_write;
+deallocate parallel_write_prep;
+drop table parallel_write;
+
+-- parallel inserts must not occur as the parallelism will not be picked
+-- for select part because of the parallel unsafe function
+create sequence parallel_write_sequence;
+select explain_pictas(
+E'create table parallel_write as
+    select nextval(\'parallel_write_sequence\'), four from tenk1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+drop sequence parallel_write_sequence;
+
+-- parallel inserts must occur, as there is init plan that gets executed by
+-- each parallel worker
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select two from (select * from tenk2) as tt limit 1) col2
+    from tenk1  where tenk1.four = 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is sub plan that gets executed by
+-- the Gather node in leader
+select explain_pictas(
+'create table parallel_write as select two col1,
+    (select tenk1.two from generate_series(1,1)) col2
+    from tenk1  where tenk1.four = 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+create table temp1(col1) as select * from generate_series(1,5);
+create table temp2(col2) as select * from temp1;
+create table temp3(col3) as select * from temp1;
+
+-- parallel inserts must not occur, as there is a limit clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 limit 4;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an order by clause
+select explain_pictas(
+'create table parallel_write as select * from temp1 order by 1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is a distinct clause
+select explain_pictas(
+'create table parallel_write as select distinct * from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an aggregate and group clause
+select explain_pictas(
+'create table parallel_write as select count(*) from temp1 group by col1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is an aggregate, group and having
+-- clauses
+select explain_pictas(
+'create table parallel_write as
+    select count(col1), (select col3 from
+        (select * from temp3) as tt limit 1) col4 from temp1, temp2
+    where temp1.col1 = temp2.col2 group by col4 having count(col1) > 0;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel inserts must not occur, as there is a window function
+select explain_pictas(
+'create table parallel_write as
+    select avg(col1) OVER (PARTITION BY col1) from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- nested loop join is the top node under which Gather node exists, so parallel
+-- inserts must not occur
+set enable_nestloop to on;
+set enable_mergejoin to off;
+set enable_hashjoin to off;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- the top node is Gather under which merge join happens, so parallel inserts
+-- must occur
+set enable_nestloop to off;
+set enable_mergejoin to on;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- parallel hash join happens under Gather node, so parallel inserts must occur
+set enable_mergejoin to off;
+set enable_hashjoin to on;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1, temp2  where temp1.col1 = temp2.col2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+reset enable_nestloop;
+reset enable_mergejoin;
+reset enable_hashjoin;
+
+drop table temp1;
+drop table temp2;
+drop table temp3;
+drop function explain_pictas(text);
 rollback;
-- 
2.25.1

From 4f71a8b1e0cf50d488af8c925151aed335bb2e8c Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddy@enterprisedb.com>
Date: Mon, 28 Dec 2020 17:00:38 +0530
Subject: [PATCH v18 4/4] Enable CTAS Parallel Inserts For Append

This patch allows pushing down the CTAS dest receiver even if there
exists Gather node under Top Append node. It also add the code for
influencing the planner to consider parallel tuple cost zero and
asserts for wrong enforcement if later parallel insertion is not
possible. Test cases are also included in this patch.
---
 src/backend/commands/createas.c              | 118 ++-
 src/backend/optimizer/path/allpaths.c        |  38 +
 src/backend/optimizer/plan/planner.c         |  10 +-
 src/include/commands/createas.h              |   4 +-
 src/test/regress/expected/write_parallel.out | 722 +++++++++++++++++++
 src/test/regress/sql/write_parallel.sql      | 222 ++++++
 6 files changed, 1076 insertions(+), 38 deletions(-)

diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 5046e02aac..148e7c7ea2 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -690,35 +690,99 @@ IsParallelInsertionAllowedInCTAS(IntoClause *into)
 	return true;
 }
 
+/*
+ * PushDownCTASParallelInsertState --- push the dest receiver down to the
+ * Gather nodes.
+ *
+ * In this function we only care about Append and Gather nodes.
+ *
+ * Push the dest receiver to Gather node when it is either at the top of the
+ * plan or under top Append node and if it does not have any projections to do.
+ * Required information from the pushed dest receiver is sent to workers so
+ * that they can perform parallel insertions into the target table.
+ *
+ * If the top node is Append, then this function recursively checks the sub
+ * plans for Gather nodes, when found one(and if it does not have projections),
+ * then sets the dest receiver information.
+ *
+ * In any case, this function returns true if at least one Gather node can allow
+ * parallel insertions by the workers. Otherwise returns false.
+ */
+static bool
+PushDownCTASParallelInsertState(DestReceiver *dest, PlanState *ps,
+								bool *gather_exists)
+{
+	bool parallel = false;
+
+	if (ps == NULL)
+		return parallel;
+
+	if (IsA(ps, AppendState))
+	{
+		AppendState *aps = (AppendState *) ps;
+
+		for (int i = 0; i < aps->as_nplans; i++)
+		{
+			parallel |= PushDownCTASParallelInsertState(dest,
+														aps->appendplans[i],
+														gather_exists);
+		}
+	}
+	else if (IsA(ps, GatherState))
+	{
+		/*
+		 * Set to true if there exists at least one Gather node either at the
+		 * top of the plan or as a direct sub node under Append node.
+		 */
+		 *gather_exists |= true;
+
+		if (!ps->ps_ProjInfo)
+		{
+			GatherState *gstate = (GatherState *) ps;
+
+			parallel = true;
+
+			/* Okay to parallelize inserts, so mark it. */
+			((DR_intorel *) dest)->is_parallel = true;
+
+			/*
+			 * For parallelizing inserts in CTAS we must send information such
+			 * as into clause (to build separate dest receiver), object id (to
+			 * open the created table) to each workers. Since this information
+			 * is available in the CTAS dest receiver, store a reference to it
+			 * in the Gather state so that it will be used in
+			 * ExecInitParallelPlan to pick the required information.
+			 */
+			gstate->dest = dest;
+		}
+	}
+
+	return parallel;
+}
+
 /*
  * SetCTASParallelInsertState --- set the parallel insert state
  *
- * See if the upper node is Gather and it doesn't have any projections, then
- * set the parallel insert state such as a flag in the dest receiver and also
- * store the dest receiver reference in the Gather node so that the required
- * information will be sent to workers.
+ * See if the upper node is either Gather or Append but it does have Gather
+ * nodes under it, then set the parallel insert state in respective Gather
+ * nodes if they do not have any projections. The parallel insert state
+ * includes a flag in the dest receiver and also a dest receiver reference in
+ * the Gather node so that the required information will be sent to workers.
  */
 void
 SetCTASParallelInsertState(QueryDesc *queryDesc, uint8 *tuple_cost_flags)
 {
-	GatherState *gstate;
-	DestReceiver *dest;
+	bool allow = false;
+	bool gather_exists = false;
 
 	Assert(queryDesc);
 
-	gstate = (GatherState *) queryDesc->planstate;
-	dest	= queryDesc->dest;
-
+	allow = PushDownCTASParallelInsertState(queryDesc->dest,
+											queryDesc->planstate,
+											&gather_exists);
 	/*
-	 * Parallel insertions are not possible if the upper node is not Gather.
-	 */
-	if (!IsA(gstate, GatherState))
-		return;
-
-	/*
-	 * If the upper Gather node has some projections to perform, then we can
-	 * not allow parallel insertions. But before returning, ensure that we have
-	 * not done wrong parallel tuple cost enforcement in the planner.
+	 * Ensure that we have not done wrong parallel tuple cost enforcement in
+	 * the planner.
 	 *
 	 * The main reason for this assertion is to check if we enforced planner to
 	 * ignore the parallel tuple cost (with the intention of choosing parallel
@@ -730,25 +794,9 @@ SetCTASParallelInsertState(QueryDesc *queryDesc, uint8 *tuple_cost_flags)
 	 * case it occurs, that means the planner may have chosen this parallel
 	 * plan because of our wrong enforcement. So let's catch that here.
 	 */
-	if (tuple_cost_flags && gstate->ps.ps_ProjInfo)
+	if (!allow && tuple_cost_flags && gather_exists)
 		Assert(!(*tuple_cost_flags & CTAS_PARALLEL_INS_TUP_COST_IGNORED));
 
-	if (!gstate->ps.ps_ProjInfo)
-	{
-		/* Okay to parallelize inserts, so mark it. */
-		((DR_intorel *) dest)->is_parallel = true;
-
-		/*
-		 * For parallelizing inserts in CTAS we must send information such as
-		 * into clause (to build separate dest receiver), object id (to open
-		 * the created table) to each workers. Since this information is
-		 * available in the CTAS dest receiver, store a reference to it in the
-		 * Gather state so that it will be used in ExecInitParallelPlan to pick
-		 * the required information.
-		 */
-		gstate->dest = dest;
-	}
-
 	if (tuple_cost_flags)
 		*tuple_cost_flags = CTAS_PARALLEL_INS_UNDEF;
 }
diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c
index 627d08b78a..b0835c32bd 100644
--- a/src/backend/optimizer/path/allpaths.c
+++ b/src/backend/optimizer/path/allpaths.c
@@ -23,6 +23,7 @@
 #include "catalog/pg_class.h"
 #include "catalog/pg_operator.h"
 #include "catalog/pg_proc.h"
+#include "commands/createas.h"
 #include "foreign/fdwapi.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1103,11 +1104,48 @@ set_append_rel_size(PlannerInfo *root, RelOptInfo *rel,
 		if (root->glob->parallelModeOK && rel->consider_parallel)
 			set_rel_consider_parallel(root, childrel, childRTE);
 
+		/*
+		 * When subplan is subquery, It's possible to do parallel insert if top
+		 * node of subquery is Gather, so we turn on a flag to ignore parallel
+		 * tuple cost in cost_gather if the SELECT is for CTAS.
+		 */
+		if (childrel->rtekind == RTE_SUBQUERY)
+		{
+			/*
+			 * When there is no parent path generating clause(such as limit,
+			 * sort, distinct...), we can turn on the flag for two cases:
+			 * i) query_level is 1
+			 * ii) query_level > 1 then turn on the flag in the parent_root.
+			 * The case ii) is to check append under append:
+			 * Append
+			 *	->Append
+			 *	->Gather
+			 *	->Other plan
+			 */
+			if (root->parse->CTASParallelInsInfo &
+				CTAS_PARALLEL_INS_SELECT &&
+				(root->query_level == 1 ||
+				 root->parent_root->parse->CTASParallelInsInfo &
+				 CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND) &&
+				!(HAS_PARENT_PATH_GENERATING_CLAUSE(root)))
+			{
+				root->parse->CTASParallelInsInfo |=
+										CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+			}
+		}
+
 		/*
 		 * Compute the child's size.
 		 */
 		set_rel_size(root, childrel, childRTindex, childRTE);
 
+		if (root->parse->CTASParallelInsInfo &
+			CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND)
+		{
+			root->parse->CTASParallelInsInfo &=
+										~CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND;
+		}
+
 		/*
 		 * It is possible that constraint exclusion detected a contradiction
 		 * within a child subquery, even though we didn't prove one above. If
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index 20a18628a7..5e607f598a 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -7350,8 +7350,14 @@ can_partial_agg(PlannerInfo *root)
 static bool
 ignore_parallel_tuple_cost(PlannerInfo *root)
 {
-	if (root->query_level == 1 &&
-		(root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT))
+	if (root->query_level != 1 &&
+		(root->parent_root->parse->CTASParallelInsInfo &
+		 CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND))
+	{
+		root->parse->CTASParallelInsInfo |= CTAS_PARALLEL_INS_SELECT;
+	}
+
+	if (root->parse->CTASParallelInsInfo & CTAS_PARALLEL_INS_SELECT)
 	{
 		/*
 		 * In each of the HAS_PARENT_PATH_GENERATING_CLAUSE cases, a parent
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 140226ad5a..bf52b05ea0 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -54,7 +54,9 @@ typedef enum CTASParallelInsertOpt
 	 */
 	CTAS_PARALLEL_INS_TUP_COST_CAN_IGN = 1 << 1,
 	/* Turn on this after the cost is ignored. */
-	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2
+	CTAS_PARALLEL_INS_TUP_COST_IGNORED = 1 << 2,
+	/* Turn on this in case tuple cost needs to be ignored for Append cases. */
+	CTAS_PARALLEL_INS_IGN_TUP_COST_APPEND = 1 << 3
 } CTASParallelInsertOpt;
 
 #define IS_CTAS(intoclause) (intoclause && IsA(intoclause, IntoClause))
diff --git a/src/test/regress/expected/write_parallel.out b/src/test/regress/expected/write_parallel.out
index 38a18c5a9b..356a2d0002 100644
--- a/src/test/regress/expected/write_parallel.out
+++ b/src/test/regress/expected/write_parallel.out
@@ -631,6 +631,728 @@ drop table parallel_write;
 reset enable_nestloop;
 reset enable_mergejoin;
 reset enable_hashjoin;
+-- test cases for performing parallel inserts when Append node is at the top
+-- and Gather node is in one of its direct sub plans.
+-- case 1: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5 union all
+    select * from temp2 where col2 = 5;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: N
+(22 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+-- case 2: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp2;');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Append (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select * from temp2;');
+                                 explain_pictas                                 
+--------------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Params Evaluated: $1, $3
+   Workers Launched: N
+ ->  Create parallel_write
+   InitPlan 1 (returns $1)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 3
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+   InitPlan 2 (returns $3)
+     ->  Limit (actual rows=N loops=N)
+           ->  Gather (actual rows=N loops=N)
+                 Workers Planned: 3
+                 Workers Launched: N
+                 ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+   ->  Parallel Append (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(21 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+-- case 3: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples to
+-- Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Gather
+-- 		->Parallel Seq Scan
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1 union all
+    select * from temp1 where col1 = 5;');
+                                explain_pictas                                
+------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Seq Scan on temp1 (actual rows=N loops=N)
+         SubPlan 1
+           ->  Limit (actual rows=N loops=N)
+                 ->  Gather (actual rows=N loops=N)
+                       Workers Planned: 3
+                       Workers Launched: N
+                       ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+(22 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     7
+(1 row)
+
+drop table parallel_write;
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as select * from temp1 where col1 = (select 1) union all
+    select * from temp1 where col1 = (select 1) union all
+ select * from temp2 where col2 = (select 2);');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $0
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+               Filter: (col1 = $0)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $1
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 2 (returns $1)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+               Filter: (col1 = $1)
+               Rows Removed by Filter: N
+   ->  Seq Scan on temp2 (actual rows=N loops=N)
+         Filter: (col2 = $2)
+         Rows Removed by Filter: N
+         InitPlan 3 (returns $2)
+           ->  Result (actual rows=N loops=N)
+(26 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+alter table temp2 reset (parallel_workers);
+drop table parallel_write;
+-- case 4: parallel inserts must not occur as there will be no direct Gather
+-- node under Append node. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1;');
+                                    explain_pictas                                    
+--------------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Seq Scan on temp1 (actual rows=N loops=N)
+   ->  Seq Scan on temp2 (actual rows=N loops=N)
+   ->  Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+         SubPlan 1
+           ->  Limit (actual rows=N loops=N)
+                 ->  Gather (actual rows=N loops=N)
+                       Workers Planned: 3
+                       Workers Launched: N
+                       ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+(10 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+-- case 5: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Seq Scan / Join / any other non-Gather node
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp1;');
+                             explain_pictas                             
+------------------------------------------------------------------------
+ Gather (actual rows=N loops=N)
+   Workers Planned: 3
+   Workers Launched: N
+ ->  Create parallel_write
+   ->  Parallel Append (actual rows=N loops=N)
+         ->  Seq Scan on temp2 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+(8 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+    15
+(1 row)
+
+drop table parallel_write;
+alter table temp2 reset (parallel_workers);
+-- case 6: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Gather
+--  ->Gather
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp2 where col2 = (select 2);');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $2
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($2 = $0)
+                     ->  Parallel Seq Scan on temp2 (never executed)
+                           Filter: (col2 = $0)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $3
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+               Filter: (col2 = $3)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $4
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 5 (returns $4)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+               Filter: (col2 = $4)
+               Rows Removed by Filter: N
+(47 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+                                   explain_pictas                                   
+------------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $2
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($2 = $0)
+                     ->  Parallel Seq Scan on temp2 (never executed)
+                           Filter: (col2 = $0)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $3, $4
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 5 (returns $4)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($4 = $3)
+                     ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+                           Filter: (col1 = $3)
+                           Rows Removed by Filter: N
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $3, $5
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 6 (returns $5)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($5 = $3)
+                     ->  Parallel Seq Scan on temp2 temp2_1 (never executed)
+                           Filter: (col2 = $3)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+               Filter: (col2 = 5)
+               Rows Removed by Filter: N
+(67 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+-- case 7: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Seq Scan / Join / any other non-Gather node
+--  ->Gather
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp1 where col1 = (select 2);');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Result (actual rows=N loops=N)
+               One-Time Filter: ($2 = $0)
+               InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Seq Scan on temp2 (never executed)
+                     Filter: (col2 = $0)
+   ->  Seq Scan on temp2 temp2_1 (actual rows=N loops=N)
+         Filter: (col2 = $3)
+         Rows Removed by Filter: N
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Params Evaluated: $4
+         Workers Launched: N
+       ->  Create parallel_write
+       InitPlan 5 (returns $4)
+           ->  Result (actual rows=N loops=N)
+         ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+               Filter: (col1 = $4)
+               Rows Removed by Filter: N
+(37 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+                                   explain_pictas                                   
+------------------------------------------------------------------------------------
+ Append (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 1 (returns $0)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $0, $1
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 2 (returns $1)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($1 = $0)
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+                           Filter: (col1 = $0)
+                           Rows Removed by Filter: N
+         ->  Result (actual rows=N loops=N)
+               One-Time Filter: ($2 = $0)
+               InitPlan 3 (returns $2)
+                 ->  Result (actual rows=N loops=N)
+               ->  Seq Scan on temp2 (never executed)
+                     Filter: (col2 = $0)
+   ->  Append (actual rows=N loops=N)
+         InitPlan 4 (returns $3)
+           ->  Result (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Params Evaluated: $3, $4
+               Workers Launched: N
+             ->  Create parallel_write
+             InitPlan 5 (returns $4)
+                 ->  Result (actual rows=N loops=N)
+               ->  Result (actual rows=N loops=N)
+                     One-Time Filter: ($4 = $3)
+                     ->  Parallel Seq Scan on temp1 temp1_1 (actual rows=N loops=N)
+                           Filter: (col1 = $3)
+                           Rows Removed by Filter: N
+         ->  Result (actual rows=N loops=N)
+               One-Time Filter: ($5 = $3)
+               InitPlan 6 (returns $5)
+                 ->  Result (actual rows=N loops=N)
+               ->  Seq Scan on temp2 temp2_1 (never executed)
+                     Filter: (col2 = $3)
+   ->  Gather (actual rows=N loops=N)
+         Workers Planned: 3
+         Workers Launched: N
+       ->  Create parallel_write
+       ->  Parallel Seq Scan on temp1 temp1_2 (actual rows=N loops=N)
+               Filter: (col1 = 5)
+               Rows Removed by Filter: N
+   ->  Seq Scan on temp2 temp2_2 (actual rows=N loops=N)
+         Filter: (col2 = 5)
+         Rows Removed by Filter: N
+(53 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     4
+(1 row)
+
+drop table parallel_write;
+alter table temp2 reset (parallel_workers);
+-- case 8: parallel inserts must not occur because there is no Gather or Append
+-- node at the top for union, except/except all, intersect/intersect all
+-- cases.
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union
+    select * from temp2;');
+                            explain_pictas                            
+----------------------------------------------------------------------
+ HashAggregate (actual rows=N loops=N)
+   Group Key: temp1.col1
+   Batches: 1  Memory Usage: 217kB
+   ->  Append (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Workers Launched: N
+               ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Gather (actual rows=N loops=N)
+               Workers Planned: 3
+               Workers Launched: N
+               ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except
+    select * from temp2 where col2 < 3;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Except (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+                           Filter: (col2 < 3)
+                           Rows Removed by Filter: N
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except all
+    select * from temp2 where col2 < 3;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Except All (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+                           Filter: (col2 < 3)
+                           Rows Removed by Filter: N
+(14 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     3
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect
+    select * from temp2;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Intersect (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect all
+    select * from temp2;');
+                               explain_pictas                               
+----------------------------------------------------------------------------
+ HashSetOp Intersect All (actual rows=N loops=N)
+   ->  Append (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 1" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp1 (actual rows=N loops=N)
+         ->  Subquery Scan on "*SELECT* 2" (actual rows=N loops=N)
+               ->  Gather (actual rows=N loops=N)
+                     Workers Planned: 3
+                     Workers Launched: N
+                     ->  Parallel Seq Scan on temp2 (actual rows=N loops=N)
+(12 rows)
+
+select count(*) from parallel_write;
+ count 
+-------
+     5
+(1 row)
+
+drop table parallel_write;
 drop table temp1;
 drop table temp2;
 drop table temp3;
diff --git a/src/test/regress/sql/write_parallel.sql b/src/test/regress/sql/write_parallel.sql
index 40aadafc2a..32e6ad8636 100644
--- a/src/test/regress/sql/write_parallel.sql
+++ b/src/test/regress/sql/write_parallel.sql
@@ -246,6 +246,228 @@ reset enable_nestloop;
 reset enable_mergejoin;
 reset enable_hashjoin;
 
+-- test cases for performing parallel inserts when Append node is at the top
+-- and Gather node is in one of its direct sub plans.
+
+-- case 1: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Gather
+-- 		->Parallel Seq Scan
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5 union all
+    select * from temp2 where col2 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 2: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select (select col2 from temp2 limit 1) col2 from temp1 union all
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 3: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples to
+-- Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Gather
+-- 		->Parallel Seq Scan
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Gather
+-- 		->Parallel Seq Scan
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 where col1 = 5 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1 union all
+    select * from temp1 where col1 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 set (parallel_workers = 0);
+select explain_pictas(
+'create table parallel_write as select * from temp1 where col1 = (select 1) union all
+    select * from temp1 where col1 = (select 1) union all
+ select * from temp2 where col2 = (select 2);');
+select count(*) from parallel_write;
+alter table temp2 reset (parallel_workers);
+drop table parallel_write;
+
+-- case 4: parallel inserts must not occur as there will be no direct Gather
+-- node under Append node. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+-- 	->Seq Scan / Join / any other non-Gather node
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select (select temp1.col1 from temp2 limit 1) col2 from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 5: parallel inserts must occur at the top Gather node as we can push
+-- the CTAS dest receiver to it.
+-- Gather
+-- 	->Parallel Append
+-- 		->Seq Scan / Join / any other non-Gather node
+-- 		->Parallel Seq Scan
+-- 		->Parallel Seq Scan
+
+alter table temp2 set (parallel_workers = 0);
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union all
+    select * from temp2 union all
+    select * from temp1;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 reset (parallel_workers);
+
+-- case 6: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Gather
+--  ->Gather
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp2 where col2 = (select 2);');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+-- case 7: parallel inserts must occur at each Gather node as we can push the
+-- CTAS dest receiver. Non-Gather nodes will do inserts by sending tuples
+-- to Append and from there to CTAS dest receiver.
+-- Append
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Append
+--     ->Gather
+-- 		->Parallel Seq Scan
+--  ->Seq Scan / Join / any other non-Gather node
+--  ->Gather
+
+alter table temp2 set (parallel_workers = 0);
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp2 where col2 = (select 2) union all
+    select * from temp1 where col1 = (select 2);');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from (select * from temp1 where col1 = (select 1) union all
+            select * from temp2 where col2 = (select 2)) as tt
+        where col1 = (select 1) union all
+    select * from temp1 where col1 = 5 union all
+    select * from temp2 where col2 = 5;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+alter table temp2 reset (parallel_workers);
+
+-- case 8: parallel inserts must not occur because there is no Gather or Append
+-- node at the top for union, except/except all, intersect/intersect all
+-- cases.
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 union
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except
+    select * from temp2 where col2 < 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 except all
+    select * from temp2 where col2 < 3;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
+select explain_pictas(
+'create table parallel_write as
+    select * from temp1 intersect all
+    select * from temp2;');
+select count(*) from parallel_write;
+drop table parallel_write;
+
 drop table temp1;
 drop table temp2;
 drop table temp3;
-- 
2.25.1