Wired if-statement in gen_partprune_steps_internal

Started by Andy Fanover 5 years ago23 messages
#1Andy Fan
zhihui.fan1213@gmail.com

Hi:

I found the following code in gen_partprune_steps_internal, which
looks the if-statement to be always true since list_length(results) > 1;
I added an Assert(step_ids != NIL) and all the test cases passed.
if the if-statement is always true, shall we remove it to avoid confusion?

gen_partprune_steps_internal(GeneratePruningStepsContext *context,

if (list_length(result) > 1)
{
List *step_ids = NIL;

foreach(lc, result)
{
PartitionPruneStep *step = lfirst(lc);

step_ids = lappend_int(step_ids, step->step_id);
}
Assert(step_ids != NIL);
if (step_ids != NIL) // This should always be true.
{
PartitionPruneStep *step;

step = gen_prune_step_combine(context, step_ids,

PARTPRUNE_COMBINE_INTERSECT);
result = lappend(result, step);
}
}

--
Best Regards
Andy Fan

#2Amit Langote
amitlangote09@gmail.com
In reply to: Andy Fan (#1)
1 attachment(s)
Re: Wired if-statement in gen_partprune_steps_internal

Hi,

On Thu, Oct 8, 2020 at 6:56 PM Andy Fan <zhihui.fan1213@gmail.com> wrote:

Hi:

I found the following code in gen_partprune_steps_internal, which
looks the if-statement to be always true since list_length(results) > 1;
I added an Assert(step_ids != NIL) and all the test cases passed.
if the if-statement is always true, shall we remove it to avoid confusion?

gen_partprune_steps_internal(GeneratePruningStepsContext *context,

if (list_length(result) > 1)
{
List *step_ids = NIL;

foreach(lc, result)
{
PartitionPruneStep *step = lfirst(lc);

step_ids = lappend_int(step_ids, step->step_id);
}
Assert(step_ids != NIL);
if (step_ids != NIL) // This should always be true.
{
PartitionPruneStep *step;

step = gen_prune_step_combine(context, step_ids,
PARTPRUNE_COMBINE_INTERSECT);
result = lappend(result, step);
}
}

That seems fine to me.

Looking at this piece of code, I remembered that exactly the same
piece of logic is also present in gen_prune_steps_from_opexps(), which
looks like this:

/* Lastly, add a combine step to mutually AND these op steps, if needed */
if (list_length(opsteps) > 1)
{
List *opstep_ids = NIL;

foreach(lc, opsteps)
{
PartitionPruneStep *step = lfirst(lc);

opstep_ids = lappend_int(opstep_ids, step->step_id);
}

if (opstep_ids != NIL)
return gen_prune_step_combine(context, opstep_ids,
PARTPRUNE_COMBINE_INTERSECT);
return NULL;
}
else if (opsteps != NIL)
return linitial(opsteps);

I think we should remove this duplicative logic and return the
generated steps in a list from this function, which the code in
gen_partprune_steps_internal() then "combines" using an INTERSECT
step. See attached a patch to show what I mean.

--
Amit Langote
EDB: http://www.enterprisedb.com

Attachments:

partprune-code-tweaks.patchapplication/octet-stream; name=partprune-code-tweaks.patchDownload
diff --git a/src/backend/partitioning/partprune.c b/src/backend/partitioning/partprune.c
index 6268623..e64f494 100644
--- a/src/backend/partitioning/partprune.c
+++ b/src/backend/partitioning/partprune.c
@@ -154,8 +154,8 @@ static PartitionPruneStep *gen_prune_step_op(GeneratePruningStepsContext *contex
 static PartitionPruneStep *gen_prune_step_combine(GeneratePruningStepsContext *context,
 												  List *source_stepids,
 												  PartitionPruneCombineOp combineOp);
-static PartitionPruneStep *gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
-													   List **keyclauses, Bitmapset *nullkeys);
+static List *gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
+							List **keyclauses, Bitmapset *nullkeys);
 static PartClauseMatchStatus match_clause_to_partition_key(GeneratePruningStepsContext *context,
 														   Expr *clause, Expr *partkey, int partkeyidx,
 														   bool *clause_is_not_null,
@@ -1147,12 +1147,11 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 	}
 	else if (generate_opsteps)
 	{
-		PartitionPruneStep *step;
+		List   *opsteps;
 
 		/* Strategy 2 */
-		step = gen_prune_steps_from_opexps(context, keyclauses, nullkeys);
-		if (step != NULL)
-			result = lappend(result, step);
+		opsteps = gen_prune_steps_from_opexps(context, keyclauses, nullkeys);
+		result = list_concat(result, opsteps);
 	}
 	else if (bms_num_members(notnullkeys) == part_scheme->partnatts)
 	{
@@ -1165,12 +1164,13 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 	}
 
 	/*
-	 * Finally, results from all entries appearing in result should be
-	 * combined using an INTERSECT combine step, if more than one.
+	 * Finally, if there are multiple steps, add an INTERSECT step to combine
+	 * the partition sets resulting from them.
 	 */
 	if (list_length(result) > 1)
 	{
 		List	   *step_ids = NIL;
+		PartitionPruneStep *step;
 
 		foreach(lc, result)
 		{
@@ -1179,14 +1179,9 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 			step_ids = lappend_int(step_ids, step->step_id);
 		}
 
-		if (step_ids != NIL)
-		{
-			PartitionPruneStep *step;
-
-			step = gen_prune_step_combine(context, step_ids,
-										  PARTPRUNE_COMBINE_INTERSECT);
-			result = lappend(result, step);
-		}
+		step = gen_prune_step_combine(context, step_ids,
+									  PARTPRUNE_COMBINE_INTERSECT);
+		result = lappend(result, step);
 	}
 
 	return result;
@@ -1257,8 +1252,11 @@ gen_prune_step_combine(GeneratePruningStepsContext *context,
  * cases, (depending on the type of partitioning being used) if we didn't
  * find clauses for a given key, we discard clauses that may have been
  * found for any subsequent keys; see specific notes below.
+ *
+ * Partition sets obtained by executing each of the returned steps must be
+ * combined by the caller using an additional INTERSECT step.
  */
-static PartitionPruneStep *
+static List *
 gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
 							List **keyclauses, Bitmapset *nullkeys)
 {
@@ -1291,7 +1289,7 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
 		 */
 		if (part_scheme->strategy == PARTITION_STRATEGY_HASH &&
 			clauselist == NIL && !bms_is_member(i, nullkeys))
-			return NULL;
+			return NIL;
 
 		foreach(lc, clauselist)
 		{
@@ -1622,27 +1620,7 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
 			break;
 	}
 
-	/* Lastly, add a combine step to mutually AND these op steps, if needed */
-	if (list_length(opsteps) > 1)
-	{
-		List	   *opstep_ids = NIL;
-
-		foreach(lc, opsteps)
-		{
-			PartitionPruneStep *step = lfirst(lc);
-
-			opstep_ids = lappend_int(opstep_ids, step->step_id);
-		}
-
-		if (opstep_ids != NIL)
-			return gen_prune_step_combine(context, opstep_ids,
-										  PARTPRUNE_COMBINE_INTERSECT);
-		return NULL;
-	}
-	else if (opsteps != NIL)
-		return linitial(opsteps);
-
-	return NULL;
+	return opsteps;
 }
 
 /*
#3Andy Fan
zhihui.fan1213@gmail.com
In reply to: Amit Langote (#2)
Re: Wired if-statement in gen_partprune_steps_internal

On Mon, Oct 12, 2020 at 4:37 PM Amit Langote <amitlangote09@gmail.com>
wrote:

Hi,

On Thu, Oct 8, 2020 at 6:56 PM Andy Fan <zhihui.fan1213@gmail.com> wrote:

Hi:

I found the following code in gen_partprune_steps_internal, which
looks the if-statement to be always true since list_length(results) > 1;
I added an Assert(step_ids != NIL) and all the test cases passed.
if the if-statement is always true, shall we remove it to avoid

confusion?

gen_partprune_steps_internal(GeneratePruningStepsContext *context,

if (list_length(result) > 1)
{
List *step_ids = NIL;

foreach(lc, result)
{
PartitionPruneStep *step = lfirst(lc);

step_ids = lappend_int(step_ids, step->step_id);
}
Assert(step_ids != NIL);
if (step_ids != NIL) // This should always be true.
{
PartitionPruneStep *step;

step = gen_prune_step_combine(context, step_ids,

PARTPRUNE_COMBINE_INTERSECT);

result = lappend(result, step);
}
}

That seems fine to me.

Looking at this piece of code, I remembered that exactly the same
piece of logic is also present in gen_prune_steps_from_opexps(), which
looks like this:

/* Lastly, add a combine step to mutually AND these op steps, if
needed */
if (list_length(opsteps) > 1)
{
List *opstep_ids = NIL;

foreach(lc, opsteps)
{
PartitionPruneStep *step = lfirst(lc);

opstep_ids = lappend_int(opstep_ids, step->step_id);
}

if (opstep_ids != NIL)
return gen_prune_step_combine(context, opstep_ids,
PARTPRUNE_COMBINE_INTERSECT);
return NULL;
}
else if (opsteps != NIL)
return linitial(opsteps);

I think we should remove this duplicative logic and return the
generated steps in a list from this function, which the code in
gen_partprune_steps_internal() then "combines" using an INTERSECT
step. See attached a patch to show what I mean.

This changes LGTM, and "make check" PASSED, thanks for the patch!

--
Best Regards
Andy Fan

#4Kyotaro Horiguchi
horikyota.ntt@gmail.com
In reply to: Andy Fan (#3)
Re: Wired if-statement in gen_partprune_steps_internal

At Wed, 14 Oct 2020 11:26:33 +0800, Andy Fan <zhihui.fan1213@gmail.com> wrote in

On Mon, Oct 12, 2020 at 4:37 PM Amit Langote <amitlangote09@gmail.com>
wrote:

I think we should remove this duplicative logic and return the
generated steps in a list from this function, which the code in
gen_partprune_steps_internal() then "combines" using an INTERSECT
step. See attached a patch to show what I mean.

This changes LGTM, and "make check" PASSED, thanks for the patch!

FWIW, both looks fine to me.

By the way, I guess that some of the caller sites of
gen_prune_step_combine(PARTPRUNE_COMBINE_INTERSECT) is useless if we
do that later?

(Diff1 below)

Mmm. I was wrong. *All the other caller site* than that at the end of
gen_partprune_steps_internal is useless?

(Note: The Diff1 alone leads to assertion failure at partprune.c:945@master.
See below.)

By the way, I'm confused to see the following portion in
gen_partprune_steps_internal.

/*
* Finally, results from all entries appearing in result should be
* combined using an INTERSECT combine step, if more than one.
*/
if (list_length(result) > 1)

...

step = gen_prune_step_combine(context, step_ids,
PARTPRUNE_COMBINE_INTERSECT);
result = lappend(result, step);

The result contains both the source terms and the combined term. If I
understand it correctly, we should replace the source terms with
combined one. (With this change the assertion above doesn't fire and
passes all regression tests.)

=====
@@ -1180,13 +1163,9 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
}

 		if (step_ids != NIL)
-		{
-			PartitionPruneStep *step;
-
-			step = gen_prune_step_combine(context, step_ids,
-										  PARTPRUNE_COMBINE_INTERSECT);
-			result = lappend(result, step);
-		}
+			result =
+				list_make1(gen_prune_step_combine(context, step_ids,
+												  PARTPRUNE_COMBINE_INTERSECT));
 	}

return result;
=====

regards.

Diff1
======
@@ -983,9 +983,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 			else if (is_andclause(clause))
 			{
 				List	   *args = ((BoolExpr *) clause)->args;
-				List	   *argsteps,
-						   *arg_stepids = NIL;
-				ListCell   *lc1;
+				List	   *argsteps;

/*
* args may itself contain clauses of arbitrary type, so just
@@ -998,21 +996,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
if (context->contradictory)
return NIL;

-				foreach(lc1, argsteps)
-				{
-					PartitionPruneStep *step = lfirst(lc1);
-
-					arg_stepids = lappend_int(arg_stepids, step->step_id);
-				}
-
-				if (arg_stepids != NIL)
-				{
-					PartitionPruneStep *step;
-
-					step = gen_prune_step_combine(context, arg_stepids,
-												  PARTPRUNE_COMBINE_INTERSECT);
-					result = lappend(result, step);
-				}
+				result = list_concat(result, argsteps);
 				continue;
 			}
==== 

--
Kyotaro Horiguchi
NTT Open Source Software Center

#5Andy Fan
zhihui.fan1213@gmail.com
In reply to: Andy Fan (#3)
Re: Wired if-statement in gen_partprune_steps_internal

On Wed, Oct 14, 2020 at 11:26 AM Andy Fan <zhihui.fan1213@gmail.com> wrote:

On Mon, Oct 12, 2020 at 4:37 PM Amit Langote <amitlangote09@gmail.com>
wrote:

Hi,

On Thu, Oct 8, 2020 at 6:56 PM Andy Fan <zhihui.fan1213@gmail.com> wrote:

Hi:

I found the following code in gen_partprune_steps_internal, which
looks the if-statement to be always true since list_length(results) > 1;
I added an Assert(step_ids != NIL) and all the test cases passed.
if the if-statement is always true, shall we remove it to avoid

confusion?

gen_partprune_steps_internal(GeneratePruningStepsContext *context,

if (list_length(result) > 1)
{
List *step_ids = NIL;

foreach(lc, result)
{
PartitionPruneStep *step = lfirst(lc);

step_ids = lappend_int(step_ids, step->step_id);
}
Assert(step_ids != NIL);
if (step_ids != NIL) // This should always be true.
{
PartitionPruneStep *step;

step = gen_prune_step_combine(context, step_ids,

PARTPRUNE_COMBINE_INTERSECT);

result = lappend(result, step);
}
}

That seems fine to me.

Looking at this piece of code, I remembered that exactly the same
piece of logic is also present in gen_prune_steps_from_opexps(), which
looks like this:

/* Lastly, add a combine step to mutually AND these op steps, if
needed */
if (list_length(opsteps) > 1)
{
List *opstep_ids = NIL;

foreach(lc, opsteps)
{
PartitionPruneStep *step = lfirst(lc);

opstep_ids = lappend_int(opstep_ids, step->step_id);
}

if (opstep_ids != NIL)
return gen_prune_step_combine(context, opstep_ids,
PARTPRUNE_COMBINE_INTERSECT);
return NULL;
}
else if (opsteps != NIL)
return linitial(opsteps);

I think we should remove this duplicative logic and return the
generated steps in a list from this function, which the code in
gen_partprune_steps_internal() then "combines" using an INTERSECT
step. See attached a patch to show what I mean.

This changes LGTM, and "make check" PASSED, thanks for the patch!

I created https://commitfest.postgresql.org/30/2771/ so that this patch
will not
be lost. Thanks!

--
Best Regards
Andy Fan

#6Amit Langote
amitlangote09@gmail.com
In reply to: Andy Fan (#5)
Re: Wired if-statement in gen_partprune_steps_internal

Hi Andy,

On Tue, Oct 20, 2020 at 4:05 PM Andy Fan <zhihui.fan1213@gmail.com> wrote:

On Wed, Oct 14, 2020 at 11:26 AM Andy Fan <zhihui.fan1213@gmail.com> wrote:

On Mon, Oct 12, 2020 at 4:37 PM Amit Langote <amitlangote09@gmail.com> wrote:

I think we should remove this duplicative logic and return the
generated steps in a list from this function, which the code in
gen_partprune_steps_internal() then "combines" using an INTERSECT
step. See attached a patch to show what I mean.

This changes LGTM, and "make check" PASSED, thanks for the patch!

I created https://commitfest.postgresql.org/30/2771/ so that this patch will not
be lost. Thanks!

Thanks for doing that.

I had updated the patch last week to address Horiguchi-san's comments
but didn't manage to post a polished-enough version. I will try again
this week.

--
Amit Langote
EDB: http://www.enterprisedb.com

#7Ryan Lambert
ryan@rustprooflabs.com
In reply to: Amit Langote (#6)
Re: Wired if-statement in gen_partprune_steps_internal

The following review has been posted through the commitfest application:
make installcheck-world: tested, passed
Implements feature: not tested
Spec compliant: not tested
Documentation: not tested

The original patch still applies and passes make installcheck-world. An updated patch was mentioned but has not been attached. Updating status to Waiting on Author.

Cheers,

-- Ryan Lambert

The new status of this patch is: Waiting on Author

#8Amit Langote
amitlangote09@gmail.com
In reply to: Amit Langote (#6)
1 attachment(s)
Re: Wired if-statement in gen_partprune_steps_internal

On Tue, Oct 20, 2020 at 9:46 PM Amit Langote <amitlangote09@gmail.com> wrote:

On Tue, Oct 20, 2020 at 4:05 PM Andy Fan <zhihui.fan1213@gmail.com> wrote:

On Wed, Oct 14, 2020 at 11:26 AM Andy Fan <zhihui.fan1213@gmail.com> wrote:

On Mon, Oct 12, 2020 at 4:37 PM Amit Langote <amitlangote09@gmail.com> wrote:

I think we should remove this duplicative logic and return the
generated steps in a list from this function, which the code in
gen_partprune_steps_internal() then "combines" using an INTERSECT
step. See attached a patch to show what I mean.

This changes LGTM, and "make check" PASSED, thanks for the patch!

I created https://commitfest.postgresql.org/30/2771/ so that this patch will not
be lost. Thanks!

Thanks for doing that.

I had updated the patch last week to address Horiguchi-san's comments
but didn't manage to post a polished-enough version. I will try again
this week.

Sorry, this seems to have totally slipped my mind.

Attached is the patch I had promised.

Also, I have updated the title of the CF entry to "Some cosmetic
improvements of partition pruning code", which I think is more
appropriate.

--
Amit Langote
EDB: http://www.enterprisedb.com

Attachments:

v2-0001-Cosmetic-improvements-to-partition-pruning-step-g.patchapplication/octet-stream; name=v2-0001-Cosmetic-improvements-to-partition-pruning-step-g.patchDownload
From ea51647b47455c5447e3fbc61ace962288b56f6f Mon Sep 17 00:00:00 2001
From: amitlan <amitlangote09@gmail.com>
Date: Thu, 4 Mar 2021 14:30:57 +0900
Subject: [PATCH v2] Cosmetic improvements to partition pruning step generation
 code

Make gen_partprune_steps_from_opexprs() return the "op" steps
(PartitionPruneOpStep) it generates in the list instead of a single
"combine" step that combines their result.  The new behavior better
matches its name.

Instead make it the job of its higher level caller
gen_partprune_steps_internal() to add the "combine" step if needed.
To match, also change its return type to be PartitionPruneStep *
rather than List *.  This also makes the job of its recursive
callers a bit simpler, because they either expect a single step
to be returned or combine the multiple steps into a single step
anyway.

While at it, also fix some comments.
---
 src/backend/partitioning/partprune.c | 191 +++++++++++----------------
 1 file changed, 79 insertions(+), 112 deletions(-)

diff --git a/src/backend/partitioning/partprune.c b/src/backend/partitioning/partprune.c
index d08739127b..018f069b53 100644
--- a/src/backend/partitioning/partprune.c
+++ b/src/backend/partitioning/partprune.c
@@ -148,7 +148,7 @@ static List *make_partitionedrel_pruneinfo(PlannerInfo *root,
 static void gen_partprune_steps(RelOptInfo *rel, List *clauses,
 								PartClauseTarget target,
 								GeneratePruningStepsContext *context);
-static List *gen_partprune_steps_internal(GeneratePruningStepsContext *context,
+static PartitionPruneStep *gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 										  List *clauses);
 static PartitionPruneStep *gen_prune_step_op(GeneratePruningStepsContext *context,
 											 StrategyNumber opstrategy, bool op_is_ne,
@@ -156,12 +156,13 @@ static PartitionPruneStep *gen_prune_step_op(GeneratePruningStepsContext *contex
 static PartitionPruneStep *gen_prune_step_combine(GeneratePruningStepsContext *context,
 												  List *source_stepids,
 												  PartitionPruneCombineOp combineOp);
-static PartitionPruneStep *gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
-													   List **keyclauses, Bitmapset *nullkeys);
+static List *gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
+							List **keyclauses, Bitmapset *nullkeys);
 static PartClauseMatchStatus match_clause_to_partition_key(GeneratePruningStepsContext *context,
 														   Expr *clause, Expr *partkey, int partkeyidx,
 														   bool *clause_is_not_null,
-														   PartClauseInfo **pc, List **clause_steps);
+														   PartClauseInfo **pc,
+														   PartitionPruneStep **clause_step);
 static List *get_steps_using_prefix(GeneratePruningStepsContext *context,
 									StrategyNumber step_opstrategy,
 									bool step_op_is_ne,
@@ -926,28 +927,28 @@ get_matching_partitions(PartitionPruneContext *context, List *pruning_steps)
  * gen_partprune_steps_internal
  *		Processes 'clauses' to generate partition pruning steps.
  *
- * From OpExpr clauses that are mutually AND'd, we find combinations of those
- * that match to the partition key columns and for every such combination,
- * we emit a PartitionPruneStepOp containing a vector of expressions whose
- * values are used as a look up key to search partitions by comparing the
- * values with partition bounds.  Relevant details of the operator and a
- * vector of (possibly cross-type) comparison functions is also included with
- * each step.
+ * From OpExpr clauses that are mutually ANDed, we find combinations of those
+ * that match the partition key columns and make one or more "op" steps
+ * (PartitionPruneStepOp) from those clause combinations; details of how the
+ * combinations can be found in gen_prune_steps_from_opexps(). Each step thus
+ * created is added to context->steps
  *
- * For BoolExpr clauses, we recursively generate steps for each argument, and
- * return a PartitionPruneStepCombine of their results.
+ * For BoolExpr clauses, each argument is processed recursively and to
+ * mix the results of the steps resulting from each, a "combine" step
+ * (PartitionPruneStepCombine) is added to context->steps.
  *
- * The return value is a list of the steps generated, which are also added to
- * the context's steps list.  Each step is assigned a step identifier, unique
- * even across recursive calls.
+ * The return value is the final step whose evaluation will give the partition
+ * set satisfying the provided clauses.  Actually, even if multiple steps may
+ * actually get added into context->steps, this still returns a single
+ * "combine" step which intersects partition sets of those steps.
  *
  * If we find clauses that are mutually contradictory, or contradictory with
  * the partitioning constraint, or a pseudoconstant clause that contains
- * false, we set context->contradictory to true and return NIL (that is, no
+ * false, we set context->contradictory to true and return NULL (that is, no
  * pruning steps).  Caller should consider all partitions as pruned in that
  * case.
  */
-static List *
+static PartitionPruneStep *
 gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 							 List *clauses)
 {
@@ -956,7 +957,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 	Bitmapset  *nullkeys = NULL,
 			   *notnullkeys = NULL;
 	bool		generate_opsteps = false;
-	List	   *result = NIL;
+	List	   *steps = NIL;
 	ListCell   *lc;
 
 	/*
@@ -975,7 +976,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 		predicate_refuted_by(context->rel->partition_qual, clauses, false))
 	{
 		context->contradictory = true;
-		return NIL;
+		return NULL;
 	}
 
 	memset(keyclauses, 0, sizeof(keyclauses));
@@ -994,7 +995,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 			 !DatumGetBool(((Const *) clause)->constvalue)))
 		{
 			context->contradictory = true;
-			return NIL;
+			return NULL;
 		}
 
 		/* Get the BoolExpr's out of the way. */
@@ -1028,10 +1029,10 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 				{
 					Expr	   *arg = lfirst(lc1);
 					bool		arg_contradictory;
-					List	   *argsteps;
+					PartitionPruneStep *step;
 
-					argsteps = gen_partprune_steps_internal(context,
-															list_make1(arg));
+					step = gen_partprune_steps_internal(context,
+														list_make1(arg));
 					arg_contradictory = context->contradictory;
 					/* Keep context->contradictory clear till we're done */
 					context->contradictory = false;
@@ -1044,14 +1045,8 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 					else
 						all_args_contradictory = false;
 
-					if (argsteps != NIL)
-					{
-						PartitionPruneStep *step;
-
-						Assert(list_length(argsteps) == 1);
-						step = (PartitionPruneStep *) linitial(argsteps);
+					if (step != NULL)
 						arg_stepids = lappend_int(arg_stepids, step->step_id);
-					}
 					else
 					{
 						PartitionPruneStep *orstep;
@@ -1073,7 +1068,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 				if (all_args_contradictory)
 				{
 					context->contradictory = true;
-					return NIL;
+					return NULL;
 				}
 
 				if (arg_stepids != NIL)
@@ -1082,43 +1077,28 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 
 					step = gen_prune_step_combine(context, arg_stepids,
 												  PARTPRUNE_COMBINE_UNION);
-					result = lappend(result, step);
+					steps = lappend(steps, step);
 				}
 				continue;
 			}
 			else if (is_andclause(clause))
 			{
 				List	   *args = ((BoolExpr *) clause)->args;
-				List	   *argsteps,
-						   *arg_stepids = NIL;
-				ListCell   *lc1;
+				PartitionPruneStep *step;
 
 				/*
 				 * args may itself contain clauses of arbitrary type, so just
 				 * recurse and later combine the component partitions sets
 				 * using a combine step.
 				 */
-				argsteps = gen_partprune_steps_internal(context, args);
+				step = gen_partprune_steps_internal(context, args);
 
 				/* If any AND arm is contradictory, we can stop immediately */
 				if (context->contradictory)
-					return NIL;
+					return NULL;
 
-				foreach(lc1, argsteps)
-				{
-					PartitionPruneStep *step = lfirst(lc1);
-
-					arg_stepids = lappend_int(arg_stepids, step->step_id);
-				}
+				steps = lappend(steps, step);
 
-				if (arg_stepids != NIL)
-				{
-					PartitionPruneStep *step;
-
-					step = gen_prune_step_combine(context, arg_stepids,
-												  PARTPRUNE_COMBINE_INTERSECT);
-					result = lappend(result, step);
-				}
 				continue;
 			}
 
@@ -1138,12 +1118,12 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 			Expr	   *partkey = linitial(context->rel->partexprs[i]);
 			bool		clause_is_not_null = false;
 			PartClauseInfo *pc = NULL;
-			List	   *clause_steps = NIL;
+			PartitionPruneStep   *clause_step = NULL;
 
 			switch (match_clause_to_partition_key(context,
 												  clause, partkey, i,
 												  &clause_is_not_null,
-												  &pc, &clause_steps))
+												  &pc, &clause_step))
 			{
 				case PARTCLAUSE_MATCH_CLAUSE:
 					Assert(pc != NULL);
@@ -1155,7 +1135,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 					if (bms_is_member(i, nullkeys))
 					{
 						context->contradictory = true;
-						return NIL;
+						return NULL;
 					}
 					generate_opsteps = true;
 					keyclauses[i] = lappend(keyclauses[i], pc);
@@ -1172,7 +1152,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 							keyclauses[i] != NIL)
 						{
 							context->contradictory = true;
-							return NIL;
+							return NULL;
 						}
 						nullkeys = bms_add_member(nullkeys, i);
 					}
@@ -1182,21 +1162,21 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 						if (bms_is_member(i, nullkeys))
 						{
 							context->contradictory = true;
-							return NIL;
+							return NULL;
 						}
 						notnullkeys = bms_add_member(notnullkeys, i);
 					}
 					break;
 
 				case PARTCLAUSE_MATCH_STEPS:
-					Assert(clause_steps != NIL);
-					result = list_concat(result, clause_steps);
+					Assert(clause_step != NULL);
+					steps = lappend(steps, clause_step);
 					break;
 
 				case PARTCLAUSE_MATCH_CONTRADICT:
 					/* We've nothing more to do if a contradiction was found. */
 					context->contradictory = true;
-					return NIL;
+					return NULL;
 
 				case PARTCLAUSE_NOMATCH:
 
@@ -1249,16 +1229,15 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 		/* Strategy 1 */
 		step = gen_prune_step_op(context, InvalidStrategy,
 								 false, NIL, NIL, nullkeys);
-		result = lappend(result, step);
+		steps = lappend(steps, step);
 	}
 	else if (generate_opsteps)
 	{
-		PartitionPruneStep *step;
+		List   *opsteps;
 
 		/* Strategy 2 */
-		step = gen_prune_steps_from_opexps(context, keyclauses, nullkeys);
-		if (step != NULL)
-			result = lappend(result, step);
+		opsteps = gen_prune_steps_from_opexps(context, keyclauses, nullkeys);
+		steps = list_concat(steps, opsteps);
 	}
 	else if (bms_num_members(notnullkeys) == part_scheme->partnatts)
 	{
@@ -1267,35 +1246,30 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 		/* Strategy 3 */
 		step = gen_prune_step_op(context, InvalidStrategy,
 								 false, NIL, NIL, NULL);
-		result = lappend(result, step);
+		steps = lappend(steps, step);
 	}
 
 	/*
-	 * Finally, results from all entries appearing in result should be
-	 * combined using an INTERSECT combine step, if more than one.
+	 * Finally, if there are multiple steps, add an INTERSECT step to combine
+	 * the partition sets resulting from them and return it.
 	 */
-	if (list_length(result) > 1)
+	if (list_length(steps) > 1)
 	{
 		List	   *step_ids = NIL;
 
-		foreach(lc, result)
+		foreach(lc, steps)
 		{
 			PartitionPruneStep *step = lfirst(lc);
 
 			step_ids = lappend_int(step_ids, step->step_id);
 		}
 
-		if (step_ids != NIL)
-		{
-			PartitionPruneStep *step;
-
-			step = gen_prune_step_combine(context, step_ids,
-										  PARTPRUNE_COMBINE_INTERSECT);
-			result = lappend(result, step);
-		}
+		return gen_prune_step_combine(context, step_ids,
+									  PARTPRUNE_COMBINE_INTERSECT);
 	}
 
-	return result;
+	/* A single step or no pruning possible with the provided clauses. */
+	return steps ? linitial(steps) : NULL;
 }
 
 /*
@@ -1356,15 +1330,28 @@ gen_prune_step_combine(GeneratePruningStepsContext *context,
 
 /*
  * gen_prune_steps_from_opexps
- *		Generate pruning steps based on clauses for partition keys
+ *		Generate pruning "op" steps (PartitionPruneStepOp) based on OpExpr
+ *		clauses that have been matched to partition keys
  *
- * 'keyclauses' contains one list of clauses per partition key.  We check here
- * if we have found clauses for a valid subset of the partition key. In some
- * cases, (depending on the type of partitioning being used) if we didn't
- * find clauses for a given key, we discard clauses that may have been
+ * 'keyclauses' contains a list of clauses for each partition key.  We check
+ * here if we have found clauses for a valid subset of the partition keys. In
+ * some cases, depending on the type of partitioning being used, if we don't
+ * see clauses for a given key, we discard clauses that may have been
  * found for any subsequent keys; see specific notes below.
+ *
+ * Each "op" step generated here will contain a vector of expressions whose
+ * values will constitute a tuple to be used as the look up key to search
+ * partitions by comparing it with partition bound tuples in the
+ * PartitionBoundInfo of the parent table, along with the effective operator
+ * strategy to use in those comparisons (=, >, <, etc.), and a vector of
+ * possibly cross-type comparison functions.
+ *
+ * Note that if multiple steps are returned, the partition set satisfying all
+ * the clauses is the one obtained by intersecting the partition sets of those
+ * individual steps, which the caller should do by adding a relevant "combine"
+ * step.
  */
-static PartitionPruneStep *
+static List *
 gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
 							List **keyclauses, Bitmapset *nullkeys)
 {
@@ -1397,7 +1384,7 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
 		 */
 		if (part_scheme->strategy == PARTITION_STRATEGY_HASH &&
 			clauselist == NIL && !bms_is_member(i, nullkeys))
-			return NULL;
+			return NIL;
 
 		foreach(lc, clauselist)
 		{
@@ -1728,27 +1715,7 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
 			break;
 	}
 
-	/* Lastly, add a combine step to mutually AND these op steps, if needed */
-	if (list_length(opsteps) > 1)
-	{
-		List	   *opstep_ids = NIL;
-
-		foreach(lc, opsteps)
-		{
-			PartitionPruneStep *step = lfirst(lc);
-
-			opstep_ids = lappend_int(opstep_ids, step->step_id);
-		}
-
-		if (opstep_ids != NIL)
-			return gen_prune_step_combine(context, opstep_ids,
-										  PARTPRUNE_COMBINE_INTERSECT);
-		return NULL;
-	}
-	else if (opsteps != NIL)
-		return linitial(opsteps);
-
-	return NULL;
+	return opsteps;
 }
 
 /*
@@ -1782,8 +1749,8 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
  *   true otherwise.
  *
  * * PARTCLAUSE_MATCH_STEPS if there is a match.
- *   Output arguments: *clause_steps is set to a list of PartitionPruneStep
- *   generated for the clause.
+ *   Output arguments: *clause_step is set to the PartitionPruneStep generated
+ *   for the clause.
  *
  * * PARTCLAUSE_MATCH_CONTRADICT if the clause is self-contradictory, ie
  *   it provably returns FALSE or NULL.
@@ -1798,7 +1765,7 @@ static PartClauseMatchStatus
 match_clause_to_partition_key(GeneratePruningStepsContext *context,
 							  Expr *clause, Expr *partkey, int partkeyidx,
 							  bool *clause_is_not_null, PartClauseInfo **pc,
-							  List **clause_steps)
+							  PartitionPruneStep **clause_step)
 {
 	PartClauseMatchStatus boolmatchstatus;
 	PartitionScheme part_scheme = context->rel->part_scheme;
@@ -2309,10 +2276,10 @@ match_clause_to_partition_key(GeneratePruningStepsContext *context,
 			elem_clauses = list_make1(makeBoolExpr(OR_EXPR, elem_clauses, -1));
 
 		/* Finally, generate steps */
-		*clause_steps = gen_partprune_steps_internal(context, elem_clauses);
+		*clause_step = gen_partprune_steps_internal(context, elem_clauses);
 		if (context->contradictory)
 			return PARTCLAUSE_MATCH_CONTRADICT;
-		else if (*clause_steps == NIL)
+		else if (*clause_step == NULL)
 			return PARTCLAUSE_UNSUPPORTED;	/* step generation failed */
 		return PARTCLAUSE_MATCH_STEPS;
 	}
-- 
2.24.1

#9Ryan Lambert
ryan@rustprooflabs.com
In reply to: Amit Langote (#8)
Re: Wired if-statement in gen_partprune_steps_internal

On Wed, Mar 3, 2021 at 11:03 PM Amit Langote <amitlangote09@gmail.com>
wrote:

Sorry, this seems to have totally slipped my mind.

Attached is the patch I had promised.

Also, I have updated the title of the CF entry to "Some cosmetic
improvements of partition pruning code", which I think is more
appropriate.

--
Amit Langote
EDB: http://www.enterprisedb.com

Thank you. The updated patch passes installcheck-world. I ran a handful
of test queries with a small number of partitions and observed the same
plans before and after the patch. I cannot speak to the quality of the
code, though am happy to test any additional use cases that should be
verified.

Ryan Lambert

#10Amit Langote
amitlangote09@gmail.com
In reply to: Ryan Lambert (#9)
Re: Wired if-statement in gen_partprune_steps_internal

On Fri, Mar 5, 2021 at 7:50 AM Ryan Lambert <ryan@rustprooflabs.com> wrote:

On Wed, Mar 3, 2021 at 11:03 PM Amit Langote <amitlangote09@gmail.com> wrote:

Sorry, this seems to have totally slipped my mind.

Attached is the patch I had promised.

Also, I have updated the title of the CF entry to "Some cosmetic
improvements of partition pruning code", which I think is more
appropriate.

Thank you. The updated patch passes installcheck-world. I ran a handful of test queries with a small number of partitions and observed the same plans before and after the patch. I cannot speak to the quality of the code, though am happy to test any additional use cases that should be verified.

Thanks Ryan.

There's no need to test it extensively, because no functionality is
changed with this patch.

--
Amit Langote
EDB: http://www.enterprisedb.com

#11Ryan Lambert
ryan@rustprooflabs.com
In reply to: Amit Langote (#10)
Re: Wired if-statement in gen_partprune_steps_internal

Should the status of this patch be updated to ready for comitter to get in
line for Pg 14 deadline?

*Ryan Lambert*

On Sun, Mar 7, 2021 at 11:38 PM Amit Langote <amitlangote09@gmail.com>
wrote:

Show quoted text

On Fri, Mar 5, 2021 at 7:50 AM Ryan Lambert <ryan@rustprooflabs.com>
wrote:

On Wed, Mar 3, 2021 at 11:03 PM Amit Langote <amitlangote09@gmail.com>

wrote:

Sorry, this seems to have totally slipped my mind.

Attached is the patch I had promised.

Also, I have updated the title of the CF entry to "Some cosmetic
improvements of partition pruning code", which I think is more
appropriate.

Thank you. The updated patch passes installcheck-world. I ran a

handful of test queries with a small number of partitions and observed the
same plans before and after the patch. I cannot speak to the quality of the
code, though am happy to test any additional use cases that should be
verified.

Thanks Ryan.

There's no need to test it extensively, because no functionality is
changed with this patch.

--
Amit Langote
EDB: http://www.enterprisedb.com

#12Amit Langote
amitlangote09@gmail.com
In reply to: Ryan Lambert (#11)
Re: Wired if-statement in gen_partprune_steps_internal

Hi Ryan,

On Tue, Mar 23, 2021 at 2:24 AM Ryan Lambert <ryan@rustprooflabs.com> wrote:

Should the status of this patch be updated to ready for comitter to get in line for Pg 14 deadline?

Yes, I've done that. Thanks for the reminder.

--
Amit Langote
EDB: http://www.enterprisedb.com

#13David Rowley
dgrowleyml@gmail.com
In reply to: Amit Langote (#8)
Re: Wired if-statement in gen_partprune_steps_internal

On Thu, 4 Mar 2021 at 19:03, Amit Langote <amitlangote09@gmail.com> wrote:

On Tue, Oct 20, 2020 at 9:46 PM Amit Langote <amitlangote09@gmail.com> wrote:

I had updated the patch last week to address Horiguchi-san's comments
but didn't manage to post a polished-enough version. I will try again
this week.

Sorry, this seems to have totally slipped my mind.

Attached is the patch I had promised.

I've been looking at this patch today and spent quite a bit of time
staring at the following fragment:

  case PARTCLAUSE_MATCH_STEPS:
- Assert(clause_steps != NIL);
- result = list_concat(result, clause_steps);
+ Assert(clause_step != NULL);
+ steps = lappend(steps, clause_step);
  break;

So here, we used to use list_concat to add the steps that
match_clause_to_partition_key() output, but now we lappend() the
single step that match_clause_to_partition_key set in its output arg.

This appears to be ok as we only return PARTCLAUSE_MATCH_STEPS from
match_clause_to_partition_key() when we process a ScalarArrayOpExpr.
There we just transform the IN(<list of consts>) into a Boolean OR
clause with a set of OpExprs which are equivalent to the
ScalarArrayOpExpr. e.g. "a IN (1,2)" becomes "a = 1 OR a = 2". The
code path which processes the list of OR clauses in
gen_partprune_steps_internal() will always just output a single
PARTPRUNE_COMBINE_UNION combine step. So it does not appear that there
are any behavioural changes there. The list_concat would always have
been just adding a single item to the list before anyway.

However, it does change the meaning of what PARTCLAUSE_MATCH_STEPS
does. If we ever needed to expand what PARTCLAUSE_MATCH_STEPS does,
then we'll have less flexibility with the newly updated code. For
example if we needed to return multiple steps and only combine them at
the top level then we now can't. I feel there's a good possibility
that we'll never need to do that, but I'm not certain of that.

I'm keen to hear your opinion on this.

David

#14Amit Langote
amitlangote09@gmail.com
In reply to: David Rowley (#13)
Re: Wired if-statement in gen_partprune_steps_internal

On Wed, Apr 7, 2021 at 4:43 PM David Rowley <dgrowleyml@gmail.com> wrote:

On Thu, 4 Mar 2021 at 19:03, Amit Langote <amitlangote09@gmail.com> wrote:

On Tue, Oct 20, 2020 at 9:46 PM Amit Langote <amitlangote09@gmail.com> wrote:

I had updated the patch last week to address Horiguchi-san's comments
but didn't manage to post a polished-enough version. I will try again
this week.

Sorry, this seems to have totally slipped my mind.

Attached is the patch I had promised.

I've been looking at this patch today and spent quite a bit of time
staring at the following fragment:

Thanks a lot for looking at this.

case PARTCLAUSE_MATCH_STEPS:
- Assert(clause_steps != NIL);
- result = list_concat(result, clause_steps);
+ Assert(clause_step != NULL);
+ steps = lappend(steps, clause_step);
break;

So here, we used to use list_concat to add the steps that
match_clause_to_partition_key() output, but now we lappend() the
single step that match_clause_to_partition_key set in its output arg.

This appears to be ok as we only return PARTCLAUSE_MATCH_STEPS from
match_clause_to_partition_key() when we process a ScalarArrayOpExpr.
There we just transform the IN(<list of consts>) into a Boolean OR
clause with a set of OpExprs which are equivalent to the
ScalarArrayOpExpr. e.g. "a IN (1,2)" becomes "a = 1 OR a = 2". The
code path which processes the list of OR clauses in
gen_partprune_steps_internal() will always just output a single
PARTPRUNE_COMBINE_UNION combine step. So it does not appear that there
are any behavioural changes there. The list_concat would always have
been just adding a single item to the list before anyway.

Right, that was my observation as well.

However, it does change the meaning of what PARTCLAUSE_MATCH_STEPS
does. If we ever needed to expand what PARTCLAUSE_MATCH_STEPS does,
then we'll have less flexibility with the newly updated code. For
example if we needed to return multiple steps and only combine them at
the top level then we now can't. I feel there's a good possibility
that we'll never need to do that, but I'm not certain of that.

I'm keen to hear your opinion on this.

That's a good point. So maybe gen_partprune_steps_internal() should
continue to return a list of steps, the last of which would be an
intersect step to combine the results of the earlier multiple steps.
We should still fix the originally reported issue that
gen_prune_steps_from_opexps() seems to needlessly add an intersect
step.

--
Amit Langote
EDB: http://www.enterprisedb.com

#15David Rowley
dgrowleyml@gmail.com
In reply to: Amit Langote (#14)
Re: Wired if-statement in gen_partprune_steps_internal

On Wed, 7 Apr 2021 at 21:04, Amit Langote <amitlangote09@gmail.com> wrote:

On Wed, Apr 7, 2021 at 4:43 PM David Rowley <dgrowleyml@gmail.com> wrote:

However, it does change the meaning of what PARTCLAUSE_MATCH_STEPS
does. If we ever needed to expand what PARTCLAUSE_MATCH_STEPS does,
then we'll have less flexibility with the newly updated code. For
example if we needed to return multiple steps and only combine them at
the top level then we now can't. I feel there's a good possibility
that we'll never need to do that, but I'm not certain of that.

I'm keen to hear your opinion on this.

That's a good point. So maybe gen_partprune_steps_internal() should
continue to return a list of steps, the last of which would be an
intersect step to combine the results of the earlier multiple steps.
We should still fix the originally reported issue that
gen_prune_steps_from_opexps() seems to needlessly add an intersect
step.

I was hoping you'd just say that we'll likely not need to do that and
if we ever did we could adapt the code at that time. :)

Thinking more about it, these steps we're talking about are generated
from a recursive call to gen_partprune_steps_internal(). I'm finding
it very hard to imagine that we'd want to combine steps generated in
some recursive call with steps from outside that same call. Right now
we recuse into AND BoolExprs OR BoolExprs. I'm struggling to think of
why we'd want to combine a set of steps we generated processing some
of those with steps from outside that BoolExpr. If we did, we might
want to consider teaching canonicalize_qual() to fix it beforehand.

e.g.

postgres=# explain select * from ab where (a = 1 and b = 1) or (a = 1
and b = 2);
QUERY PLAN
---------------------------------------------------
Seq Scan on ab (cost=0.00..49.55 rows=1 width=8)
Filter: ((a = 1) AND ((b = 1) OR (b = 2)))
(2 rows)

If canonicalize_qual() had been unable to rewrite that WHERE clause
then I could see that we might want to combine steps from other
recursive quals. I'm thinking right now that I'm glad
canonicalize_qual() does that hard work for us. (I think partprune.c
could handle the original WHERE clause as-is in this example
anyway...)

David

#16David Rowley
dgrowleyml@gmail.com
In reply to: David Rowley (#15)
1 attachment(s)
Re: Wired if-statement in gen_partprune_steps_internal

On Wed, 7 Apr 2021 at 21:53, David Rowley <dgrowleyml@gmail.com> wrote:

If canonicalize_qual() had been unable to rewrite that WHERE clause
then I could see that we might want to combine steps from other
recursive quals. I'm thinking right now that I'm glad
canonicalize_qual() does that hard work for us. (I think partprune.c
could handle the original WHERE clause as-is in this example
anyway...)

I made a pass over the v2 patch and since it's been a long time since
I'd looked at partprune.c I ended doing further rewriting of the
comments you'd changed.

There's only one small code change as I didn't like the following:

- return result;
+ /* A single step or no pruning possible with the provided clauses. */
+ return steps ? linitial(steps) : NULL;

I ended up breaking that out into an if condition.

All the other changes are around the comments.

Can you look over this and let me know if you're happy with the changes?

David

Attachments:

v3-0001-Cosmetic-improvements-to-partition-pruning-step-g.patchapplication/octet-stream; name=v3-0001-Cosmetic-improvements-to-partition-pruning-step-g.patchDownload
From b18d1bd8d0cbd834c0eb9dc20a0883800508fe3f Mon Sep 17 00:00:00 2001
From: "dgrowley@gmail.com" <dgrowley@gmail.com>
Date: Wed, 7 Apr 2021 23:19:08 +1200
Subject: [PATCH v3] Cosmetic improvements to partition pruning step generation
 code

Make gen_partprune_steps_from_opexprs() return the "op" steps
(PartitionPruneOpStep) it generates in the list instead of a single
"combine" step that combines their result.  The new behavior better
matches its name.

Instead make it the job of its higher level caller
gen_partprune_steps_internal() to add the "combine" step if needed.
To match, also change its return type to be PartitionPruneStep *
rather than List *.  This also makes the job of its recursive
callers a bit simpler, because they either expect a single step
to be returned or combine the multiple steps into a single step
anyway.

While at it, also fix some comments.
---
 src/backend/partitioning/partbounds.c |   2 +-
 src/backend/partitioning/partprune.c  | 219 ++++++++++++--------------
 src/include/nodes/plannodes.h         |   2 +-
 3 files changed, 102 insertions(+), 121 deletions(-)

diff --git a/src/backend/partitioning/partbounds.c b/src/backend/partitioning/partbounds.c
index 2b2b1dc1ad..bfa6e27e9b 100644
--- a/src/backend/partitioning/partbounds.c
+++ b/src/backend/partitioning/partbounds.c
@@ -4067,7 +4067,7 @@ get_qual_for_list(Relation parent, PartitionBoundSpec *spec)
 	if (!list_has_null)
 	{
 		/*
-		 * Gin up a "col IS NOT NULL" test that will be AND'd with the main
+		 * Gin up a "col IS NOT NULL" test that will be ANDed with the main
 		 * expression.  This might seem redundant, but the partition routing
 		 * machinery needs it.
 		 */
diff --git a/src/backend/partitioning/partprune.c b/src/backend/partitioning/partprune.c
index d08739127b..5ab00cdcf1 100644
--- a/src/backend/partitioning/partprune.c
+++ b/src/backend/partitioning/partprune.c
@@ -148,7 +148,7 @@ static List *make_partitionedrel_pruneinfo(PlannerInfo *root,
 static void gen_partprune_steps(RelOptInfo *rel, List *clauses,
 								PartClauseTarget target,
 								GeneratePruningStepsContext *context);
-static List *gen_partprune_steps_internal(GeneratePruningStepsContext *context,
+static PartitionPruneStep *gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 										  List *clauses);
 static PartitionPruneStep *gen_prune_step_op(GeneratePruningStepsContext *context,
 											 StrategyNumber opstrategy, bool op_is_ne,
@@ -156,12 +156,13 @@ static PartitionPruneStep *gen_prune_step_op(GeneratePruningStepsContext *contex
 static PartitionPruneStep *gen_prune_step_combine(GeneratePruningStepsContext *context,
 												  List *source_stepids,
 												  PartitionPruneCombineOp combineOp);
-static PartitionPruneStep *gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
-													   List **keyclauses, Bitmapset *nullkeys);
+static List *gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
+							List **keyclauses, Bitmapset *nullkeys);
 static PartClauseMatchStatus match_clause_to_partition_key(GeneratePruningStepsContext *context,
 														   Expr *clause, Expr *partkey, int partkeyidx,
 														   bool *clause_is_not_null,
-														   PartClauseInfo **pc, List **clause_steps);
+														   PartClauseInfo **pc,
+														   PartitionPruneStep **clause_step);
 static List *get_steps_using_prefix(GeneratePruningStepsContext *context,
 									StrategyNumber step_opstrategy,
 									bool step_op_is_ne,
@@ -924,30 +925,39 @@ get_matching_partitions(PartitionPruneContext *context, List *pruning_steps)
 
 /*
  * gen_partprune_steps_internal
- *		Processes 'clauses' to generate partition pruning steps.
- *
- * From OpExpr clauses that are mutually AND'd, we find combinations of those
- * that match to the partition key columns and for every such combination,
- * we emit a PartitionPruneStepOp containing a vector of expressions whose
- * values are used as a look up key to search partitions by comparing the
- * values with partition bounds.  Relevant details of the operator and a
- * vector of (possibly cross-type) comparison functions is also included with
- * each step.
- *
- * For BoolExpr clauses, we recursively generate steps for each argument, and
- * return a PartitionPruneStepCombine of their results.
- *
- * The return value is a list of the steps generated, which are also added to
- * the context's steps list.  Each step is assigned a step identifier, unique
- * even across recursive calls.
+ *		Processes 'clauses' to generate a set of partition pruning steps.  We
+ *		return a single PartitionPruneStep, which may be a single step or
+ *		several steps combined into either a PARTPRUNE_COMBINE_INTERSECT or
+ *		PARTPRUNE_COMBINE_UNION step.  We return NULL when no steps were
+ *		generated.
+ *
+ * Combine steps instruct the partition pruning code how it should produce a
+ * single set of partitions from multiple input pruning steps.  A
+ * PARTPRUNE_COMBINE_INTERSECT type combine step will merge its input steps
+ * to produce a result which only contains the partitions which are present in
+ * all of the input steps.  A PARTPRUNE_COMBINE_UNION combine step will
+ * produce a result that has all of the partitions from each of the input
+ * steps.
+ *
+ * From OpExpr clauses that are mutually ANDed, we find combinations of those
+ * that match the partition key columns and make one or more "op" steps
+ * (PartitionPruneStepOp) from those clause combinations.
+ *
+ * For BoolExpr clauses, each argument is processed recursively. Steps
+ * generated from processing an OR BoolExpr will be combined using
+ * PARTPRUNE_COMBINE_UNION.  AND BoolExprs get combined using
+ * PARTPRUNE_COMBINE_INTERSECT.
+ *
+ * The return value in the top-level call is the final step whose evaluation
+ * will give the partition set satisfying the provided clauses.
  *
  * If we find clauses that are mutually contradictory, or contradictory with
  * the partitioning constraint, or a pseudoconstant clause that contains
- * false, we set context->contradictory to true and return NIL (that is, no
- * pruning steps).  Caller should consider all partitions as pruned in that
+ * false, we set context->contradictory to true and return NULL (that is, no
+ * pruning steps). The caller should consider all partitions as pruned in that
  * case.
  */
-static List *
+static PartitionPruneStep *
 gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 							 List *clauses)
 {
@@ -956,7 +966,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 	Bitmapset  *nullkeys = NULL,
 			   *notnullkeys = NULL;
 	bool		generate_opsteps = false;
-	List	   *result = NIL;
+	List	   *steps = NIL;
 	ListCell   *lc;
 
 	/*
@@ -975,7 +985,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 		predicate_refuted_by(context->rel->partition_qual, clauses, false))
 	{
 		context->contradictory = true;
-		return NIL;
+		return NULL;
 	}
 
 	memset(keyclauses, 0, sizeof(keyclauses));
@@ -994,7 +1004,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 			 !DatumGetBool(((Const *) clause)->constvalue)))
 		{
 			context->contradictory = true;
-			return NIL;
+			return NULL;
 		}
 
 		/* Get the BoolExpr's out of the way. */
@@ -1028,10 +1038,10 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 				{
 					Expr	   *arg = lfirst(lc1);
 					bool		arg_contradictory;
-					List	   *argsteps;
+					PartitionPruneStep *step;
 
-					argsteps = gen_partprune_steps_internal(context,
-															list_make1(arg));
+					step = gen_partprune_steps_internal(context,
+														list_make1(arg));
 					arg_contradictory = context->contradictory;
 					/* Keep context->contradictory clear till we're done */
 					context->contradictory = false;
@@ -1044,14 +1054,8 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 					else
 						all_args_contradictory = false;
 
-					if (argsteps != NIL)
-					{
-						PartitionPruneStep *step;
-
-						Assert(list_length(argsteps) == 1);
-						step = (PartitionPruneStep *) linitial(argsteps);
+					if (step != NULL)
 						arg_stepids = lappend_int(arg_stepids, step->step_id);
-					}
 					else
 					{
 						PartitionPruneStep *orstep;
@@ -1073,7 +1077,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 				if (all_args_contradictory)
 				{
 					context->contradictory = true;
-					return NIL;
+					return NULL;
 				}
 
 				if (arg_stepids != NIL)
@@ -1082,43 +1086,28 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 
 					step = gen_prune_step_combine(context, arg_stepids,
 												  PARTPRUNE_COMBINE_UNION);
-					result = lappend(result, step);
+					steps = lappend(steps, step);
 				}
 				continue;
 			}
 			else if (is_andclause(clause))
 			{
 				List	   *args = ((BoolExpr *) clause)->args;
-				List	   *argsteps,
-						   *arg_stepids = NIL;
-				ListCell   *lc1;
+				PartitionPruneStep *step;
 
 				/*
 				 * args may itself contain clauses of arbitrary type, so just
 				 * recurse and later combine the component partitions sets
 				 * using a combine step.
 				 */
-				argsteps = gen_partprune_steps_internal(context, args);
+				step = gen_partprune_steps_internal(context, args);
 
 				/* If any AND arm is contradictory, we can stop immediately */
 				if (context->contradictory)
-					return NIL;
+					return NULL;
 
-				foreach(lc1, argsteps)
-				{
-					PartitionPruneStep *step = lfirst(lc1);
+				steps = lappend(steps, step);
 
-					arg_stepids = lappend_int(arg_stepids, step->step_id);
-				}
-
-				if (arg_stepids != NIL)
-				{
-					PartitionPruneStep *step;
-
-					step = gen_prune_step_combine(context, arg_stepids,
-												  PARTPRUNE_COMBINE_INTERSECT);
-					result = lappend(result, step);
-				}
 				continue;
 			}
 
@@ -1138,12 +1127,12 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 			Expr	   *partkey = linitial(context->rel->partexprs[i]);
 			bool		clause_is_not_null = false;
 			PartClauseInfo *pc = NULL;
-			List	   *clause_steps = NIL;
+			PartitionPruneStep   *clause_step = NULL;
 
 			switch (match_clause_to_partition_key(context,
 												  clause, partkey, i,
 												  &clause_is_not_null,
-												  &pc, &clause_steps))
+												  &pc, &clause_step))
 			{
 				case PARTCLAUSE_MATCH_CLAUSE:
 					Assert(pc != NULL);
@@ -1155,7 +1144,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 					if (bms_is_member(i, nullkeys))
 					{
 						context->contradictory = true;
-						return NIL;
+						return NULL;
 					}
 					generate_opsteps = true;
 					keyclauses[i] = lappend(keyclauses[i], pc);
@@ -1172,7 +1161,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 							keyclauses[i] != NIL)
 						{
 							context->contradictory = true;
-							return NIL;
+							return NULL;
 						}
 						nullkeys = bms_add_member(nullkeys, i);
 					}
@@ -1182,21 +1171,21 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 						if (bms_is_member(i, nullkeys))
 						{
 							context->contradictory = true;
-							return NIL;
+							return NULL;
 						}
 						notnullkeys = bms_add_member(notnullkeys, i);
 					}
 					break;
 
 				case PARTCLAUSE_MATCH_STEPS:
-					Assert(clause_steps != NIL);
-					result = list_concat(result, clause_steps);
+					Assert(clause_step != NULL);
+					steps = lappend(steps, clause_step);
 					break;
 
 				case PARTCLAUSE_MATCH_CONTRADICT:
 					/* We've nothing more to do if a contradiction was found. */
 					context->contradictory = true;
-					return NIL;
+					return NULL;
 
 				case PARTCLAUSE_NOMATCH:
 
@@ -1249,16 +1238,15 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 		/* Strategy 1 */
 		step = gen_prune_step_op(context, InvalidStrategy,
 								 false, NIL, NIL, nullkeys);
-		result = lappend(result, step);
+		steps = lappend(steps, step);
 	}
 	else if (generate_opsteps)
 	{
-		PartitionPruneStep *step;
+		List   *opsteps;
 
 		/* Strategy 2 */
-		step = gen_prune_steps_from_opexps(context, keyclauses, nullkeys);
-		if (step != NULL)
-			result = lappend(result, step);
+		opsteps = gen_prune_steps_from_opexps(context, keyclauses, nullkeys);
+		steps = list_concat(steps, opsteps);
 	}
 	else if (bms_num_members(notnullkeys) == part_scheme->partnatts)
 	{
@@ -1267,35 +1255,37 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 		/* Strategy 3 */
 		step = gen_prune_step_op(context, InvalidStrategy,
 								 false, NIL, NIL, NULL);
-		result = lappend(result, step);
+		steps = lappend(steps, step);
 	}
 
+	/* No pruning possible without any steps */
+	if (steps == NIL)
+		return NULL;
+
 	/*
-	 * Finally, results from all entries appearing in result should be
-	 * combined using an INTERSECT combine step, if more than one.
+	 * Finally, if there are multiple steps, combine these using an INTERSECT
+	 * step and return that.
 	 */
-	if (list_length(result) > 1)
+	if (list_length(steps) > 1)
 	{
 		List	   *step_ids = NIL;
 
-		foreach(lc, result)
+		foreach(lc, steps)
 		{
 			PartitionPruneStep *step = lfirst(lc);
 
 			step_ids = lappend_int(step_ids, step->step_id);
 		}
 
-		if (step_ids != NIL)
-		{
-			PartitionPruneStep *step;
-
-			step = gen_prune_step_combine(context, step_ids,
-										  PARTPRUNE_COMBINE_INTERSECT);
-			result = lappend(result, step);
-		}
+		return gen_prune_step_combine(context, step_ids,
+									  PARTPRUNE_COMBINE_INTERSECT);
 	}
 
-	return result;
+	/*
+	 * steps contains just a single step.  There's no sense in adding a
+	 * combine on a single step, so just return it as-is.
+	 */
+	return linitial(steps);
 }
 
 /*
@@ -1356,15 +1346,26 @@ gen_prune_step_combine(GeneratePruningStepsContext *context,
 
 /*
  * gen_prune_steps_from_opexps
- *		Generate pruning steps based on clauses for partition keys
- *
- * 'keyclauses' contains one list of clauses per partition key.  We check here
- * if we have found clauses for a valid subset of the partition key. In some
- * cases, (depending on the type of partitioning being used) if we didn't
- * find clauses for a given key, we discard clauses that may have been
- * found for any subsequent keys; see specific notes below.
+ *		Generate and return a list of PartitionPruneStepOp that are based on
+ *		OpExpr and BooleanTest clauses that have been matched to the partition
+ *		key.
+ *
+ * 'keyclauses' is an array of List pointers, indexed by the partition key's
+ * index.  Each List element in the array can contain clauses that match to
+ * the corresponding partition key column.  Partition key columns without any
+ * matched clauses will have an empty List.
+ *
+ * Some partitioning strategies allow pruning to still occur when we only have
+ * clauses for a prefix of the partition key columns, for example, RANGE
+ * partitioning.  Other strategies, such as HASH partitioning, require clauses
+ * for all partition key columns.
+ *
+ * When we return multiple pruning steps here, it's up to the caller to add a
+ * relevant "combine" step to combine the returned steps.  This is not done
+ * here as callers may wish to include additional pruning steps before
+ * combining them all.
  */
-static PartitionPruneStep *
+static List *
 gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
 							List **keyclauses, Bitmapset *nullkeys)
 {
@@ -1397,7 +1398,7 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
 		 */
 		if (part_scheme->strategy == PARTITION_STRATEGY_HASH &&
 			clauselist == NIL && !bms_is_member(i, nullkeys))
-			return NULL;
+			return NIL;
 
 		foreach(lc, clauselist)
 		{
@@ -1728,27 +1729,7 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
 			break;
 	}
 
-	/* Lastly, add a combine step to mutually AND these op steps, if needed */
-	if (list_length(opsteps) > 1)
-	{
-		List	   *opstep_ids = NIL;
-
-		foreach(lc, opsteps)
-		{
-			PartitionPruneStep *step = lfirst(lc);
-
-			opstep_ids = lappend_int(opstep_ids, step->step_id);
-		}
-
-		if (opstep_ids != NIL)
-			return gen_prune_step_combine(context, opstep_ids,
-										  PARTPRUNE_COMBINE_INTERSECT);
-		return NULL;
-	}
-	else if (opsteps != NIL)
-		return linitial(opsteps);
-
-	return NULL;
+	return opsteps;
 }
 
 /*
@@ -1782,8 +1763,8 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
  *   true otherwise.
  *
  * * PARTCLAUSE_MATCH_STEPS if there is a match.
- *   Output arguments: *clause_steps is set to a list of PartitionPruneStep
- *   generated for the clause.
+ *   Output arguments: *clause_step is set to the PartitionPruneStep generated
+ *   for the clause.
  *
  * * PARTCLAUSE_MATCH_CONTRADICT if the clause is self-contradictory, ie
  *   it provably returns FALSE or NULL.
@@ -1798,7 +1779,7 @@ static PartClauseMatchStatus
 match_clause_to_partition_key(GeneratePruningStepsContext *context,
 							  Expr *clause, Expr *partkey, int partkeyidx,
 							  bool *clause_is_not_null, PartClauseInfo **pc,
-							  List **clause_steps)
+							  PartitionPruneStep **clause_step)
 {
 	PartClauseMatchStatus boolmatchstatus;
 	PartitionScheme part_scheme = context->rel->part_scheme;
@@ -2309,10 +2290,10 @@ match_clause_to_partition_key(GeneratePruningStepsContext *context,
 			elem_clauses = list_make1(makeBoolExpr(OR_EXPR, elem_clauses, -1));
 
 		/* Finally, generate steps */
-		*clause_steps = gen_partprune_steps_internal(context, elem_clauses);
+		*clause_step = gen_partprune_steps_internal(context, elem_clauses);
 		if (context->contradictory)
 			return PARTCLAUSE_MATCH_CONTRADICT;
-		else if (*clause_steps == NIL)
+		else if (*clause_step == NULL)
 			return PARTCLAUSE_UNSUPPORTED;	/* step generation failed */
 		return PARTCLAUSE_MATCH_STEPS;
 	}
diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h
index 1678bd66fe..d671328dfd 100644
--- a/src/include/nodes/plannodes.h
+++ b/src/include/nodes/plannodes.h
@@ -1215,7 +1215,7 @@ typedef struct PartitionPruneStep
 } PartitionPruneStep;
 
 /*
- * PartitionPruneStepOp - Information to prune using a set of mutually AND'd
+ * PartitionPruneStepOp - Information to prune using a set of mutually ANDed
  *							OpExpr clauses
  *
  * This contains information extracted from up to partnatts OpExpr clauses,
-- 
2.27.0

#17Amit Langote
amitlangote09@gmail.com
In reply to: David Rowley (#16)
1 attachment(s)
Re: Wired if-statement in gen_partprune_steps_internal

On Wed, Apr 7, 2021 at 8:44 PM David Rowley <dgrowleyml@gmail.com> wrote:

On Wed, 7 Apr 2021 at 21:53, David Rowley <dgrowleyml@gmail.com> wrote:

If canonicalize_qual() had been unable to rewrite that WHERE clause
then I could see that we might want to combine steps from other
recursive quals. I'm thinking right now that I'm glad
canonicalize_qual() does that hard work for us. (I think partprune.c
could handle the original WHERE clause as-is in this example
anyway...)

I made a pass over the v2 patch and since it's been a long time since
I'd looked at partprune.c I ended doing further rewriting of the
comments you'd changed.

There's only one small code change as I didn't like the following:

- return result;
+ /* A single step or no pruning possible with the provided clauses. */
+ return steps ? linitial(steps) : NULL;

I ended up breaking that out into an if condition.

All the other changes are around the comments.

Can you look over this and let me know if you're happy with the changes?

Thanks David. Actually, I was busy updating the patch to revert to
gen_partprune_steps_internal() returning a list and was almost done
with it when I saw your message.

I read through v3 and can say that it certainly looks better than v2.
If you are happy with gen_partprune_steps_internal() no longer
returning a list, I would not object if you wanted to go ahead and
commit the v3.

I've attached the patch I had ended up with and was about to post as
v3, just in case you wanted to glance.

--
Amit Langote
EDB: http://www.enterprisedb.com

Attachments:

v3_amit.diffapplication/octet-stream; name=v3_amit.diffDownload
diff --git a/src/backend/partitioning/partprune.c b/src/backend/partitioning/partprune.c
index d08739127b..9215f566b7 100644
--- a/src/backend/partitioning/partprune.c
+++ b/src/backend/partitioning/partprune.c
@@ -156,8 +156,8 @@ static PartitionPruneStep *gen_prune_step_op(GeneratePruningStepsContext *contex
 static PartitionPruneStep *gen_prune_step_combine(GeneratePruningStepsContext *context,
 												  List *source_stepids,
 												  PartitionPruneCombineOp combineOp);
-static PartitionPruneStep *gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
-													   List **keyclauses, Bitmapset *nullkeys);
+static List *gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
+							List **keyclauses, Bitmapset *nullkeys);
 static PartClauseMatchStatus match_clause_to_partition_key(GeneratePruningStepsContext *context,
 														   Expr *clause, Expr *partkey, int partkeyidx,
 														   bool *clause_is_not_null,
@@ -926,24 +926,23 @@ get_matching_partitions(PartitionPruneContext *context, List *pruning_steps)
  * gen_partprune_steps_internal
  *		Processes 'clauses' to generate partition pruning steps.
  *
- * From OpExpr clauses that are mutually AND'd, we find combinations of those
- * that match to the partition key columns and for every such combination,
- * we emit a PartitionPruneStepOp containing a vector of expressions whose
- * values are used as a look up key to search partitions by comparing the
- * values with partition bounds.  Relevant details of the operator and a
- * vector of (possibly cross-type) comparison functions is also included with
- * each step.
+ * From OpExpr clauses that are mutually ANDed, we find combinations of those
+ * that match the partition key columns and make one or more "op" steps
+ * (PartitionPruneStepOp) from those clause combinations; details of how the
+ * combinations can be found in gen_prune_steps_from_opexps(). Each step thus
+ * created is added to context->steps
  *
- * For BoolExpr clauses, we recursively generate steps for each argument, and
- * return a PartitionPruneStepCombine of their results.
+ * For BoolExpr clauses, each argument is processed recursively and to
+ * mix the results of the steps resulting from each, a "combine" step
+ * (PartitionPruneStepCombine) is added to context->steps.
  *
- * The return value is a list of the steps generated, which are also added to
- * the context's steps list.  Each step is assigned a step identifier, unique
- * even across recursive calls.
+ * The return value is the list of steps to evaluate to get the partition set
+ * satisfying the provided clauses.  Note that all of those steps would also
+ * have been added to context->steps.
  *
  * If we find clauses that are mutually contradictory, or contradictory with
  * the partitioning constraint, or a pseudoconstant clause that contains
- * false, we set context->contradictory to true and return NIL (that is, no
+ * false, we set context->contradictory to true and return NULL (that is, no
  * pruning steps).  Caller should consider all partitions as pruned in that
  * case.
  */
@@ -975,7 +974,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 		predicate_refuted_by(context->rel->partition_qual, clauses, false))
 	{
 		context->contradictory = true;
-		return NIL;
+		return NULL;
 	}
 
 	memset(keyclauses, 0, sizeof(keyclauses));
@@ -994,7 +993,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 			 !DatumGetBool(((Const *) clause)->constvalue)))
 		{
 			context->contradictory = true;
-			return NIL;
+			return NULL;
 		}
 
 		/* Get the BoolExpr's out of the way. */
@@ -1046,11 +1045,14 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 
 					if (argsteps != NIL)
 					{
-						PartitionPruneStep *step;
+						/*
+						 * Because of the way gen_partprune_steps_internal()
+						 * generates its result list, it suffices to only add
+						 * the last of the steps we got from it.
+						 */
+						PartitionPruneStep *last = llast(argsteps);
 
-						Assert(list_length(argsteps) == 1);
-						step = (PartitionPruneStep *) linitial(argsteps);
-						arg_stepids = lappend_int(arg_stepids, step->step_id);
+						arg_stepids = lappend_int(arg_stepids, last->step_id);
 					}
 					else
 					{
@@ -1073,7 +1075,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 				if (all_args_contradictory)
 				{
 					context->contradictory = true;
-					return NIL;
+					return NULL;
 				}
 
 				if (arg_stepids != NIL)
@@ -1089,9 +1091,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 			else if (is_andclause(clause))
 			{
 				List	   *args = ((BoolExpr *) clause)->args;
-				List	   *argsteps,
-						   *arg_stepids = NIL;
-				ListCell   *lc1;
+				List	   *argsteps;
 
 				/*
 				 * args may itself contain clauses of arbitrary type, so just
@@ -1102,23 +1102,16 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 
 				/* If any AND arm is contradictory, we can stop immediately */
 				if (context->contradictory)
-					return NIL;
-
-				foreach(lc1, argsteps)
-				{
-					PartitionPruneStep *step = lfirst(lc1);
-
-					arg_stepids = lappend_int(arg_stepids, step->step_id);
-				}
+					return NULL;
 
-				if (arg_stepids != NIL)
-				{
-					PartitionPruneStep *step;
+				/*
+				 * Because of the way gen_partprune_steps_internal() generates
+				 * its result list, it suffices to only add the last of the
+				 * steps we got from it.
+				 */
+				if (argsteps != NIL)
+					result = lappend(result, llast(argsteps));
 
-					step = gen_prune_step_combine(context, arg_stepids,
-												  PARTPRUNE_COMBINE_INTERSECT);
-					result = lappend(result, step);
-				}
 				continue;
 			}
 
@@ -1155,7 +1148,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 					if (bms_is_member(i, nullkeys))
 					{
 						context->contradictory = true;
-						return NIL;
+						return NULL;
 					}
 					generate_opsteps = true;
 					keyclauses[i] = lappend(keyclauses[i], pc);
@@ -1172,7 +1165,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 							keyclauses[i] != NIL)
 						{
 							context->contradictory = true;
-							return NIL;
+							return NULL;
 						}
 						nullkeys = bms_add_member(nullkeys, i);
 					}
@@ -1182,7 +1175,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 						if (bms_is_member(i, nullkeys))
 						{
 							context->contradictory = true;
-							return NIL;
+							return NULL;
 						}
 						notnullkeys = bms_add_member(notnullkeys, i);
 					}
@@ -1196,7 +1189,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 				case PARTCLAUSE_MATCH_CONTRADICT:
 					/* We've nothing more to do if a contradiction was found. */
 					context->contradictory = true;
-					return NIL;
+					return NULL;
 
 				case PARTCLAUSE_NOMATCH:
 
@@ -1253,12 +1246,11 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 	}
 	else if (generate_opsteps)
 	{
-		PartitionPruneStep *step;
+		List   *opsteps;
 
 		/* Strategy 2 */
-		step = gen_prune_steps_from_opexps(context, keyclauses, nullkeys);
-		if (step != NULL)
-			result = lappend(result, step);
+		opsteps = gen_prune_steps_from_opexps(context, keyclauses, nullkeys);
+		result = list_concat(result, opsteps);
 	}
 	else if (bms_num_members(notnullkeys) == part_scheme->partnatts)
 	{
@@ -1271,12 +1263,14 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 	}
 
 	/*
-	 * Finally, results from all entries appearing in result should be
-	 * combined using an INTERSECT combine step, if more than one.
+	 * Finally, if there are multiple steps, add an INTERSECT step to combine
+	 * the partition sets resulting from them and append it to the result
+	 * list.
 	 */
 	if (list_length(result) > 1)
 	{
 		List	   *step_ids = NIL;
+		PartitionPruneStep *final;
 
 		foreach(lc, result)
 		{
@@ -1285,14 +1279,9 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 			step_ids = lappend_int(step_ids, step->step_id);
 		}
 
-		if (step_ids != NIL)
-		{
-			PartitionPruneStep *step;
-
-			step = gen_prune_step_combine(context, step_ids,
-										  PARTPRUNE_COMBINE_INTERSECT);
-			result = lappend(result, step);
-		}
+		final = gen_prune_step_combine(context, step_ids,
+									   PARTPRUNE_COMBINE_INTERSECT);
+		result = lappend(result, final);
 	}
 
 	return result;
@@ -1356,15 +1345,28 @@ gen_prune_step_combine(GeneratePruningStepsContext *context,
 
 /*
  * gen_prune_steps_from_opexps
- *		Generate pruning steps based on clauses for partition keys
+ *		Generate pruning "op" steps (PartitionPruneStepOp) based on OpExpr
+ *		clauses that have been matched to partition keys
  *
- * 'keyclauses' contains one list of clauses per partition key.  We check here
- * if we have found clauses for a valid subset of the partition key. In some
- * cases, (depending on the type of partitioning being used) if we didn't
- * find clauses for a given key, we discard clauses that may have been
+ * 'keyclauses' contains a list of clauses for each partition key.  We check
+ * here if we have found clauses for a valid subset of the partition keys. In
+ * some cases, depending on the type of partitioning being used, if we don't
+ * see clauses for a given key, we discard clauses that may have been
  * found for any subsequent keys; see specific notes below.
+ *
+ * Each "op" step generated here will contain a vector of expressions whose
+ * values will constitute a tuple to be used as the look up key to search
+ * partitions by comparing it with partition bound tuples in the
+ * PartitionBoundInfo of the parent table, along with the effective operator
+ * strategy to use in those comparisons (=, >, <, etc.), and a vector of
+ * possibly cross-type comparison functions.
+ *
+ * Note that if multiple steps are returned, the partition set satisfying all
+ * the clauses is the one obtained by intersecting the partition sets of those
+ * individual steps, which the caller should do by adding a relevant "combine"
+ * step.
  */
-static PartitionPruneStep *
+static List *
 gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
 							List **keyclauses, Bitmapset *nullkeys)
 {
@@ -1397,7 +1399,7 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
 		 */
 		if (part_scheme->strategy == PARTITION_STRATEGY_HASH &&
 			clauselist == NIL && !bms_is_member(i, nullkeys))
-			return NULL;
+			return NIL;
 
 		foreach(lc, clauselist)
 		{
@@ -1728,27 +1730,7 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
 			break;
 	}
 
-	/* Lastly, add a combine step to mutually AND these op steps, if needed */
-	if (list_length(opsteps) > 1)
-	{
-		List	   *opstep_ids = NIL;
-
-		foreach(lc, opsteps)
-		{
-			PartitionPruneStep *step = lfirst(lc);
-
-			opstep_ids = lappend_int(opstep_ids, step->step_id);
-		}
-
-		if (opstep_ids != NIL)
-			return gen_prune_step_combine(context, opstep_ids,
-										  PARTPRUNE_COMBINE_INTERSECT);
-		return NULL;
-	}
-	else if (opsteps != NIL)
-		return linitial(opsteps);
-
-	return NULL;
+	return opsteps;
 }
 
 /*
@@ -1782,8 +1764,8 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
  *   true otherwise.
  *
  * * PARTCLAUSE_MATCH_STEPS if there is a match.
- *   Output arguments: *clause_steps is set to a list of PartitionPruneStep
- *   generated for the clause.
+ *   Output arguments: *clause_steps is set to the list of recursively
+ *   generated steps for the clause.
  *
  * * PARTCLAUSE_MATCH_CONTRADICT if the clause is self-contradictory, ie
  *   it provably returns FALSE or NULL.
#18Amit Langote
amitlangote09@gmail.com
In reply to: David Rowley (#15)
Re: Wired if-statement in gen_partprune_steps_internal

On Wed, Apr 7, 2021 at 6:53 PM David Rowley <dgrowleyml@gmail.com> wrote:

On Wed, 7 Apr 2021 at 21:04, Amit Langote <amitlangote09@gmail.com> wrote:

On Wed, Apr 7, 2021 at 4:43 PM David Rowley <dgrowleyml@gmail.com> wrote:

However, it does change the meaning of what PARTCLAUSE_MATCH_STEPS
does. If we ever needed to expand what PARTCLAUSE_MATCH_STEPS does,
then we'll have less flexibility with the newly updated code. For
example if we needed to return multiple steps and only combine them at
the top level then we now can't. I feel there's a good possibility
that we'll never need to do that, but I'm not certain of that.

I'm keen to hear your opinion on this.

That's a good point. So maybe gen_partprune_steps_internal() should
continue to return a list of steps, the last of which would be an
intersect step to combine the results of the earlier multiple steps.
We should still fix the originally reported issue that
gen_prune_steps_from_opexps() seems to needlessly add an intersect
step.

I was hoping you'd just say that we'll likely not need to do that and
if we ever did we could adapt the code at that time. :)

Thinking more about it, these steps we're talking about are generated
from a recursive call to gen_partprune_steps_internal(). I'm finding
it very hard to imagine that we'd want to combine steps generated in
some recursive call with steps from outside that same call. Right now
we recuse into AND BoolExprs OR BoolExprs. I'm struggling to think of
why we'd want to combine a set of steps we generated processing some
of those with steps from outside that BoolExpr. If we did, we might
want to consider teaching canonicalize_qual() to fix it beforehand.

e.g.

postgres=# explain select * from ab where (a = 1 and b = 1) or (a = 1
and b = 2);
QUERY PLAN
---------------------------------------------------
Seq Scan on ab (cost=0.00..49.55 rows=1 width=8)
Filter: ((a = 1) AND ((b = 1) OR (b = 2)))
(2 rows)

If canonicalize_qual() had been unable to rewrite that WHERE clause
then I could see that we might want to combine steps from other
recursive quals. I'm thinking right now that I'm glad
canonicalize_qual() does that hard work for us.
(I think partprune.c
could handle the original WHERE clause as-is in this example
anyway...)

Actually, I am not sure that canonicalization always makes things
better for partprune.c. I can show examples where canonicalization
causes partprune.c as it is today to not be able to prune as optimally
as it could have with the original ones.

create table ab (a int, b int) partition by range (a, b);
create table ab0 partition of ab for values from (1, 1) to (1, 2);
create table ab1 partition of ab for values from (1, 2) to (1, 3);
create table ab2 partition of ab for values from (1, 3) to (1, 4);
create table ab3 partition of ab for values from (2, 1) to (2, 2);

explain select * from ab where (a = 1 and b = 1) or (a = 1 and b = 2);
QUERY PLAN
---------------------------------------------------------------
Append (cost=0.00..148.66 rows=3 width=8)
-> Seq Scan on ab0 ab_1 (cost=0.00..49.55 rows=1 width=8)
Filter: ((a = 1) AND ((b = 1) OR (b = 2)))
-> Seq Scan on ab1 ab_2 (cost=0.00..49.55 rows=1 width=8)
Filter: ((a = 1) AND ((b = 1) OR (b = 2)))
-> Seq Scan on ab2 ab_3 (cost=0.00..49.55 rows=1 width=8)
Filter: ((a = 1) AND ((b = 1) OR (b = 2)))
(7 rows)

explain select * from ab where (a = 1 and b = 1) or (a = 1 and b = 3);
QUERY PLAN
---------------------------------------------------------------
Append (cost=0.00..148.66 rows=3 width=8)
-> Seq Scan on ab0 ab_1 (cost=0.00..49.55 rows=1 width=8)
Filter: ((a = 1) AND ((b = 1) OR (b = 3)))
-> Seq Scan on ab1 ab_2 (cost=0.00..49.55 rows=1 width=8)
Filter: ((a = 1) AND ((b = 1) OR (b = 3)))
-> Seq Scan on ab2 ab_3 (cost=0.00..49.55 rows=1 width=8)
Filter: ((a = 1) AND ((b = 1) OR (b = 3)))
(7 rows)

I would've expected the 1st query to scan ab0 and ab1, whereas the 2nd
query to scan ab0 and ab2. But in the canonicalized version, the
AND's 2nd arm is useless for multi-column range pruning, because it
only provides clauses for the 2nd key. With the original version,
both arms of the OR have ANDed clauses covering both keys, so pruning
with that would have produced the desired result.

So, if I am not entirely wrong, maybe it is exactly because of
canonicalization that partprune.c should be looking to peek across
BoolExprs.

--
Amit Langote
EDB: http://www.enterprisedb.com

#19David Rowley
dgrowleyml@gmail.com
In reply to: Amit Langote (#17)
1 attachment(s)
Re: Wired if-statement in gen_partprune_steps_internal

On Thu, 8 Apr 2021 at 00:49, Amit Langote <amitlangote09@gmail.com> wrote:

Thanks David. Actually, I was busy updating the patch to revert to
gen_partprune_steps_internal() returning a list and was almost done
with it when I saw your message.

I read through v3 and can say that it certainly looks better than v2.
If you are happy with gen_partprune_steps_internal() no longer
returning a list, I would not object if you wanted to go ahead and
commit the v3.

I've attached the patch I had ended up with and was about to post as
v3, just in case you wanted to glance.

Thanks. I've made a pass over that and just fixed up the places that
were mixing up NIL and NULL.

I applied most of my comments from my last version after adapting them
to account for the variation in the functions return value. I also did
a bit more explaining about op steps and combine steps in the header
comment for gen_partprune_steps_internal.

Patch attached.

David

Attachments:

v5_fixup_partprune_dgr.patchapplication/octet-stream; name=v5_fixup_partprune_dgr.patchDownload
diff --git a/src/backend/partitioning/partbounds.c b/src/backend/partitioning/partbounds.c
index 2b2b1dc1ad..bfa6e27e9b 100644
--- a/src/backend/partitioning/partbounds.c
+++ b/src/backend/partitioning/partbounds.c
@@ -4067,7 +4067,7 @@ get_qual_for_list(Relation parent, PartitionBoundSpec *spec)
 	if (!list_has_null)
 	{
 		/*
-		 * Gin up a "col IS NOT NULL" test that will be AND'd with the main
+		 * Gin up a "col IS NOT NULL" test that will be ANDed with the main
 		 * expression.  This might seem redundant, but the partition routing
 		 * machinery needs it.
 		 */
diff --git a/src/backend/partitioning/partprune.c b/src/backend/partitioning/partprune.c
index d08739127b..802f616e38 100644
--- a/src/backend/partitioning/partprune.c
+++ b/src/backend/partitioning/partprune.c
@@ -156,8 +156,8 @@ static PartitionPruneStep *gen_prune_step_op(GeneratePruningStepsContext *contex
 static PartitionPruneStep *gen_prune_step_combine(GeneratePruningStepsContext *context,
 												  List *source_stepids,
 												  PartitionPruneCombineOp combineOp);
-static PartitionPruneStep *gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
-													   List **keyclauses, Bitmapset *nullkeys);
+static List *gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
+										 List **keyclauses, Bitmapset *nullkeys);
 static PartClauseMatchStatus match_clause_to_partition_key(GeneratePruningStepsContext *context,
 														   Expr *clause, Expr *partkey, int partkeyidx,
 														   bool *clause_is_not_null,
@@ -924,22 +924,34 @@ get_matching_partitions(PartitionPruneContext *context, List *pruning_steps)
 
 /*
  * gen_partprune_steps_internal
- *		Processes 'clauses' to generate partition pruning steps.
- *
- * From OpExpr clauses that are mutually AND'd, we find combinations of those
- * that match to the partition key columns and for every such combination,
- * we emit a PartitionPruneStepOp containing a vector of expressions whose
- * values are used as a look up key to search partitions by comparing the
- * values with partition bounds.  Relevant details of the operator and a
- * vector of (possibly cross-type) comparison functions is also included with
- * each step.
- *
- * For BoolExpr clauses, we recursively generate steps for each argument, and
- * return a PartitionPruneStepCombine of their results.
- *
- * The return value is a list of the steps generated, which are also added to
- * the context's steps list.  Each step is assigned a step identifier, unique
- * even across recursive calls.
+ *		Processes 'clauses' to generate a List of partition pruning steps.  We
+ *		return NIL when no steps were generated.
+ *
+ * These partition pruning steps come in 2 forms; operation steps and combine
+ * steps.
+ *
+ * Operation steps (PartitionPruneStepOp) contain details of clauses that we
+ * determined that we can use for partition pruning.  These contain details of
+ * the expression which is being compared to the partition key and the
+ * comparison function.
+ *
+ * Combine steps (PartitionPruneStepCombine) instruct the partition pruning
+ * code how it should produce a single set of partitions from multiple input
+ * operation steps.  A PARTPRUNE_COMBINE_INTERSECT type combine step will
+ * merge its input steps to produce a result which only contains the
+ * partitions which are present in all of the input operation steps.  A
+ * PARTPRUNE_COMBINE_UNION combine step will produce a result that has all of
+ * the partitions from each of the input operation steps.
+ *
+ * For BoolExpr clauses, each argument is processed recursively. Steps
+ * generated from processing an OR BoolExpr will be combined using
+ * PARTPRUNE_COMBINE_UNION.  AND BoolExprs get combined using
+ * PARTPRUNE_COMBINE_INTERSECT.
+ *
+ * Otherwise, the list of clauses we receive we assume to be mutually ANDed.
+ * We generate all of the pruning steps we can based on these clauses and then
+ * at the end, if we have more than 1 step, we combine each step with a
+ * PARTPRUNE_COMBINE_INTERSECT combine step.  Single steps are returned as-is.
  *
  * If we find clauses that are mutually contradictory, or contradictory with
  * the partitioning constraint, or a pseudoconstant clause that contains
@@ -1046,11 +1058,16 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 
 					if (argsteps != NIL)
 					{
-						PartitionPruneStep *step;
+						/*
+						 * gen_partprune_steps_internal() always adds a single
+						 * combine step when it generates multiple steps, so
+						 * here we can just pay attention to the last one in
+						 * the list.  If it just generated one, then the last
+						 * one in the list is still the one we want.
+						 */
+						PartitionPruneStep *last = llast(argsteps);
 
-						Assert(list_length(argsteps) == 1);
-						step = (PartitionPruneStep *) linitial(argsteps);
-						arg_stepids = lappend_int(arg_stepids, step->step_id);
+						arg_stepids = lappend_int(arg_stepids, last->step_id);
 					}
 					else
 					{
@@ -1089,9 +1106,7 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 			else if (is_andclause(clause))
 			{
 				List	   *args = ((BoolExpr *) clause)->args;
-				List	   *argsteps,
-						   *arg_stepids = NIL;
-				ListCell   *lc1;
+				List	   *argsteps;
 
 				/*
 				 * args may itself contain clauses of arbitrary type, so just
@@ -1104,21 +1119,16 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 				if (context->contradictory)
 					return NIL;
 
-				foreach(lc1, argsteps)
-				{
-					PartitionPruneStep *step = lfirst(lc1);
-
-					arg_stepids = lappend_int(arg_stepids, step->step_id);
-				}
-
-				if (arg_stepids != NIL)
-				{
-					PartitionPruneStep *step;
+				/*
+				 * gen_partprune_steps_internal() always adds a single combine
+				 * step when it generates multiple steps, so here we can just
+				 * pay attention to the last one in the list.  If it just
+				 * generated one, then the last one in the list is still the
+				 * one we want.
+				 */
+				if (argsteps != NIL)
+					result = lappend(result, llast(argsteps));
 
-					step = gen_prune_step_combine(context, arg_stepids,
-												  PARTPRUNE_COMBINE_INTERSECT);
-					result = lappend(result, step);
-				}
 				continue;
 			}
 
@@ -1253,12 +1263,11 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 	}
 	else if (generate_opsteps)
 	{
-		PartitionPruneStep *step;
+		List	   *opsteps;
 
 		/* Strategy 2 */
-		step = gen_prune_steps_from_opexps(context, keyclauses, nullkeys);
-		if (step != NULL)
-			result = lappend(result, step);
+		opsteps = gen_prune_steps_from_opexps(context, keyclauses, nullkeys);
+		result = list_concat(result, opsteps);
 	}
 	else if (bms_num_members(notnullkeys) == part_scheme->partnatts)
 	{
@@ -1271,12 +1280,14 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 	}
 
 	/*
-	 * Finally, results from all entries appearing in result should be
-	 * combined using an INTERSECT combine step, if more than one.
+	 * Finally, if there are multiple steps, since the 'clauses' are mutually
+	 * ANDed, add an INTERSECT step to combine the partition sets resulting
+	 * from them and append it to the result list.
 	 */
 	if (list_length(result) > 1)
 	{
 		List	   *step_ids = NIL;
+		PartitionPruneStep *final;
 
 		foreach(lc, result)
 		{
@@ -1285,14 +1296,9 @@ gen_partprune_steps_internal(GeneratePruningStepsContext *context,
 			step_ids = lappend_int(step_ids, step->step_id);
 		}
 
-		if (step_ids != NIL)
-		{
-			PartitionPruneStep *step;
-
-			step = gen_prune_step_combine(context, step_ids,
-										  PARTPRUNE_COMBINE_INTERSECT);
-			result = lappend(result, step);
-		}
+		final = gen_prune_step_combine(context, step_ids,
+									   PARTPRUNE_COMBINE_INTERSECT);
+		result = lappend(result, final);
 	}
 
 	return result;
@@ -1356,15 +1362,26 @@ gen_prune_step_combine(GeneratePruningStepsContext *context,
 
 /*
  * gen_prune_steps_from_opexps
- *		Generate pruning steps based on clauses for partition keys
- *
- * 'keyclauses' contains one list of clauses per partition key.  We check here
- * if we have found clauses for a valid subset of the partition key. In some
- * cases, (depending on the type of partitioning being used) if we didn't
- * find clauses for a given key, we discard clauses that may have been
- * found for any subsequent keys; see specific notes below.
+ *		Generate and return a list of PartitionPruneStepOp that are based on
+ *		OpExpr and BooleanTest clauses that have been matched to the partition
+ *		key.
+ *
+ * 'keyclauses' is an array of List pointers, indexed by the partition key's
+ * index.  Each List element in the array can contain clauses that match to
+ * the corresponding partition key column.  Partition key columns without any
+ * matched clauses will have an empty List.
+ *
+ * Some partitioning strategies allow pruning to still occur when we only have
+ * clauses for a prefix of the partition key columns, for example, RANGE
+ * partitioning.  Other strategies, such as HASH partitioning, require clauses
+ * for all partition key columns.
+ *
+ * When we return multiple pruning steps here, it's up to the caller to add a
+ * relevant "combine" step to combine the returned steps.  This is not done
+ * here as callers may wish to include additional pruning steps before
+ * combining them all.
  */
-static PartitionPruneStep *
+static List *
 gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
 							List **keyclauses, Bitmapset *nullkeys)
 {
@@ -1397,7 +1414,7 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
 		 */
 		if (part_scheme->strategy == PARTITION_STRATEGY_HASH &&
 			clauselist == NIL && !bms_is_member(i, nullkeys))
-			return NULL;
+			return NIL;
 
 		foreach(lc, clauselist)
 		{
@@ -1728,27 +1745,7 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
 			break;
 	}
 
-	/* Lastly, add a combine step to mutually AND these op steps, if needed */
-	if (list_length(opsteps) > 1)
-	{
-		List	   *opstep_ids = NIL;
-
-		foreach(lc, opsteps)
-		{
-			PartitionPruneStep *step = lfirst(lc);
-
-			opstep_ids = lappend_int(opstep_ids, step->step_id);
-		}
-
-		if (opstep_ids != NIL)
-			return gen_prune_step_combine(context, opstep_ids,
-										  PARTPRUNE_COMBINE_INTERSECT);
-		return NULL;
-	}
-	else if (opsteps != NIL)
-		return linitial(opsteps);
-
-	return NULL;
+	return opsteps;
 }
 
 /*
@@ -1782,8 +1779,8 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
  *   true otherwise.
  *
  * * PARTCLAUSE_MATCH_STEPS if there is a match.
- *   Output arguments: *clause_steps is set to a list of PartitionPruneStep
- *   generated for the clause.
+ *   Output arguments: *clause_steps is set to the list of recursively
+ *   generated steps for the clause.
  *
  * * PARTCLAUSE_MATCH_CONTRADICT if the clause is self-contradictory, ie
  *   it provably returns FALSE or NULL.
diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h
index 1678bd66fe..d671328dfd 100644
--- a/src/include/nodes/plannodes.h
+++ b/src/include/nodes/plannodes.h
@@ -1215,7 +1215,7 @@ typedef struct PartitionPruneStep
 } PartitionPruneStep;
 
 /*
- * PartitionPruneStepOp - Information to prune using a set of mutually AND'd
+ * PartitionPruneStepOp - Information to prune using a set of mutually ANDed
  *							OpExpr clauses
  *
  * This contains information extracted from up to partnatts OpExpr clauses,
#20Amit Langote
amitlangote09@gmail.com
In reply to: David Rowley (#19)
Re: Wired if-statement in gen_partprune_steps_internal

On Thu, Apr 8, 2021 at 5:34 PM David Rowley <dgrowleyml@gmail.com> wrote:

On Thu, 8 Apr 2021 at 00:49, Amit Langote <amitlangote09@gmail.com> wrote:

Thanks David. Actually, I was busy updating the patch to revert to
gen_partprune_steps_internal() returning a list and was almost done
with it when I saw your message.

I read through v3 and can say that it certainly looks better than v2.
If you are happy with gen_partprune_steps_internal() no longer
returning a list, I would not object if you wanted to go ahead and
commit the v3.

I've attached the patch I had ended up with and was about to post as
v3, just in case you wanted to glance.

Thanks. I've made a pass over that and just fixed up the places that
were mixing up NIL and NULL.

I applied most of my comments from my last version after adapting them
to account for the variation in the functions return value. I also did
a bit more explaining about op steps and combine steps in the header
comment for gen_partprune_steps_internal.

Thanks for updating the patch.

+ * These partition pruning steps come in 2 forms; operation steps and combine
+ * steps.

Maybe you meant "operator" steps? IIRC, the reason why we named it
PartitionPruneStepOp is that an op step is built to prune based on the
semantics of the operators that were involved in the matched clause.
Although, they're abused for pruning based on nullness clauses too.
Maybe, we should also updated the description of node struct as
follows to consider that last point:

* PartitionPruneStepOp - Information to prune using a set of mutually ANDed
* OpExpr and any IS [ NOT ] NULL clauses

+ * Combine steps (PartitionPruneStepCombine) instruct the partition pruning
+ * code how it should produce a single set of partitions from multiple input
+ * operation steps.

I think the last part should be: ...from multiple operation/operator
and [ other ] combine steps.

If that sounds fine, likewise adjust the following sentences in the
same paragraph.

--
Amit Langote
EDB: http://www.enterprisedb.com

#21David Rowley
dgrowleyml@gmail.com
In reply to: Amit Langote (#20)
Re: Wired if-statement in gen_partprune_steps_internal

On Thu, 8 Apr 2021 at 21:04, Amit Langote <amitlangote09@gmail.com> wrote:

+ * These partition pruning steps come in 2 forms; operation steps and combine
+ * steps.

Maybe you meant "operator" steps? IIRC, the reason why we named it
PartitionPruneStepOp is that an op step is built to prune based on the
semantics of the operators that were involved in the matched clause.
Although, they're abused for pruning based on nullness clauses too.
Maybe, we should also updated the description of node struct as
follows to consider that last point:

Oh right. Thanks. I fixed that.

* PartitionPruneStepOp - Information to prune using a set of mutually ANDed
* OpExpr and any IS [ NOT ] NULL clauses

+ * Combine steps (PartitionPruneStepCombine) instruct the partition pruning
+ * code how it should produce a single set of partitions from multiple input
+ * operation steps.

I didn't add that. I wasn't really sure if I understood why we'd talk
about PartitionPruneStepCombine in the PartitionPruneStepOp. I thought
the overview in gen_partprune_steps_internal was ok to link the two
together and explain why they're both needed.

I think the last part should be: ...from multiple operation/operator
and [ other ] combine steps.

Change that and pushed.

David

#22Amit Langote
amitlangote09@gmail.com
In reply to: David Rowley (#21)
Re: Wired if-statement in gen_partprune_steps_internal

On Thu, Apr 8, 2021 at 7:41 PM David Rowley <dgrowleyml@gmail.com> wrote:

On Thu, 8 Apr 2021 at 21:04, Amit Langote <amitlangote09@gmail.com> wrote:

Maybe, we should also updated the description of node struct as
follows to consider that last point:

* PartitionPruneStepOp - Information to prune using a set of mutually ANDed
* OpExpr and any IS [ NOT ] NULL clauses

I didn't add that. I wasn't really sure if I understood why we'd talk
about PartitionPruneStepCombine in the PartitionPruneStepOp. I thought
the overview in gen_partprune_steps_internal was ok to link the two
together and explain why they're both needed.

Sorry, maybe the way I wrote it was a bit confusing, but I meant to
suggest that we do what I have quoted above from my last email. That
is, we should clarify in the description of PartitionPruneStepOp that
it contains information derived from OpExprs and in some cases also IS
[ NOT ] NULL clauses.

Thanks for the commit.

--
Amit Langote
EDB: http://www.enterprisedb.com

#23Andy Fan
zhihui.fan1213@gmail.com
In reply to: Amit Langote (#22)
2 attachment(s)
Re: Wired if-statement in gen_partprune_steps_internal

On Thu, Apr 8, 2021 at 7:59 PM Amit Langote <amitlangote09@gmail.com> wrote:

On Thu, Apr 8, 2021 at 7:41 PM David Rowley <dgrowleyml@gmail.com> wrote:

On Thu, 8 Apr 2021 at 21:04, Amit Langote <amitlangote09@gmail.com>

wrote:

Maybe, we should also updated the description of node struct as
follows to consider that last point:

* PartitionPruneStepOp - Information to prune using a set of mutually

ANDed

* OpExpr and any IS [ NOT ] NULL clauses

I didn't add that. I wasn't really sure if I understood why we'd talk
about PartitionPruneStepCombine in the PartitionPruneStepOp. I thought
the overview in gen_partprune_steps_internal was ok to link the two
together and explain why they're both needed.

Sorry, maybe the way I wrote it was a bit confusing, but I meant to
suggest that we do what I have quoted above from my last email. That
is, we should clarify in the description of PartitionPruneStepOp that
it contains information derived from OpExprs and in some cases also IS
[ NOT ] NULL clauses.

Thanks for the commit.

--
Amit Langote
EDB: http://www.enterprisedb.com

Thanks for the patch.

Recently I am reading the partition prune code again, and want to
propose some tiny changes. That is helpful for me and hope it is
helpful for others as well, especially for the people who are not familiar
with these codes.

-- v1-0001-Document-enhancement-for-RelOptInfo.partexprs-nul.patch

Just add comments for RelOptInfo.partexprs & nullable_partexprs to
remind the reader nullable_partexprs is just for partition wise join. and
use bms_add_member(relinfo->all_partrels, childRTindex); instead of
bms_add_members(relinfo->all_partrels, childrelinfo->relids); which
would be more explicit to say add the child rt index to all_partrels.

-- v1-0002-Split-gen_prune_steps_from_exprs-into-some-smalle.patch

Just split the gen_prune_steps_from_opexps into some smaller chunks.
The benefits are the same as smaller functions.

--
Best Regards
Andy Fan (https://www.aliyun.com/)

Attachments:

v1-0001-Document-enhancement-for-RelOptInfo.partexprs-nul.patchapplication/octet-stream; name=v1-0001-Document-enhancement-for-RelOptInfo.partexprs-nul.patchDownload
From 896221df304acf788e8f970bd7af4295ee20a50b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E4=B8=80=E6=8C=83?= <yizhi.fzh@alibaba-inc.com>
Date: Mon, 12 Apr 2021 10:37:43 +0800
Subject: [PATCH v1 1/2] Document enhancement for RelOptInfo.partexprs &
 nullable_partexprs.

Also use more explicit way to add RelOptInfo.all_partrels member in
expand_partitioned_rtentry.
---
 src/backend/optimizer/util/inherit.c |  4 ++--
 src/include/nodes/pathnodes.h        | 11 +++++++++--
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c
index 13f67ab744..0dac3ea3ee 100644
--- a/src/backend/optimizer/util/inherit.c
+++ b/src/backend/optimizer/util/inherit.c
@@ -380,8 +380,8 @@ expand_partitioned_rtentry(PlannerInfo *root, RelOptInfo *relinfo,
 		/* Create the otherrel RelOptInfo too. */
 		childrelinfo = build_simple_rel(root, childRTindex, relinfo);
 		relinfo->part_rels[i] = childrelinfo;
-		relinfo->all_partrels = bms_add_members(relinfo->all_partrels,
-												childrelinfo->relids);
+		relinfo->all_partrels = bms_add_member(relinfo->all_partrels,
+											   childRTindex);
 
 		/* If this child is itself partitioned, recurse */
 		if (childrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h
index a65bda7e3c..06c74fe5e9 100644
--- a/src/include/nodes/pathnodes.h
+++ b/src/include/nodes/pathnodes.h
@@ -763,8 +763,15 @@ typedef struct RelOptInfo
 	struct RelOptInfo **part_rels;	/* Array of RelOptInfos of partitions,
 									 * stored in the same order as bounds */
 	Relids		all_partrels;	/* Relids set of all partition relids */
-	List	  **partexprs;		/* Non-nullable partition key expressions */
-	List	  **nullable_partexprs; /* Nullable partition key expressions */
+	List	  **partexprs;		/* Non-nullable partition key expressions,
+								 * Base Relations have a single expression per key,
+								 */
+	List	  **nullable_partexprs; /* Nullable partition key expressions,
+									 * Base Relations doesn't have such exprs
+									 * since no outer join is involved.
+									 * Check set_baserel_partition_key_exprs for
+									 * more information.
+									 */
 } RelOptInfo;
 
 /*
-- 
2.21.0

v1-0002-Split-gen_prune_steps_from_exprs-into-some-smalle.patchapplication/octet-stream; name=v1-0002-Split-gen_prune_steps_from_exprs-into-some-smalle.patchDownload
From eadd00e5e1b48de16797344f8d5fa368312e4a89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E4=B8=80=E6=8C=83?= <yizhi.fzh@alibaba-inc.com>
Date: Mon, 12 Apr 2021 15:17:10 +0800
Subject: [PATCH v1 2/2] Split gen_prune_steps_from_exprs into some smaller
 chunks

---
 src/backend/partitioning/partprune.c | 719 ++++++++++++++-------------
 1 file changed, 384 insertions(+), 335 deletions(-)

diff --git a/src/backend/partitioning/partprune.c b/src/backend/partitioning/partprune.c
index c79374265c..2c44c28ebd 100644
--- a/src/backend/partitioning/partprune.c
+++ b/src/backend/partitioning/partprune.c
@@ -205,6 +205,16 @@ static PartClauseMatchStatus match_boolean_partition_clause(Oid partopfamily,
 static void partkey_datum_from_expr(PartitionPruneContext *context,
 									Expr *expr, int stateidx,
 									Datum *value, bool *isnull);
+static bool divide_keyclauses_by_opstrategy(PartitionScheme part_scheme,
+											List **keyclauses,
+											Bitmapset *nullkeys,
+											List **btree_clauses,
+											List **hash_clauses);
+static List *gen_prune_steps_opexprs_btree(GeneratePruningStepsContext *context,
+												 List **btree_clauses);
+
+static List *gen_prune_steps_from_opexps_hash(GeneratePruningStepsContext *context,
+											  List **hash_clauses, Bitmapset *nullkeys);
 
 
 /*
@@ -1386,89 +1396,15 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
 							List **keyclauses, Bitmapset *nullkeys)
 {
 	PartitionScheme part_scheme = context->rel->part_scheme;
-	List	   *opsteps = NIL;
 	List	   *btree_clauses[BTMaxStrategyNumber + 1],
 			   *hash_clauses[HTMaxStrategyNumber + 1];
-	int			i;
-	ListCell   *lc;
 
 	memset(btree_clauses, 0, sizeof(btree_clauses));
 	memset(hash_clauses, 0, sizeof(hash_clauses));
-	for (i = 0; i < part_scheme->partnatts; i++)
-	{
-		List	   *clauselist = keyclauses[i];
-		bool		consider_next_key = true;
-
-		/*
-		 * For range partitioning, if we have no clauses for the current key,
-		 * we can't consider any later keys either, so we can stop here.
-		 */
-		if (part_scheme->strategy == PARTITION_STRATEGY_RANGE &&
-			clauselist == NIL)
-			break;
-
-		/*
-		 * For hash partitioning, if a column doesn't have the necessary
-		 * equality clause, there should be an IS NULL clause, otherwise
-		 * pruning is not possible.
-		 */
-		if (part_scheme->strategy == PARTITION_STRATEGY_HASH &&
-			clauselist == NIL && !bms_is_member(i, nullkeys))
-			return NIL;
-
-		foreach(lc, clauselist)
-		{
-			PartClauseInfo *pc = (PartClauseInfo *) lfirst(lc);
-			Oid			lefttype,
-						righttype;
-
-			/* Look up the operator's btree/hash strategy number. */
-			if (pc->op_strategy == InvalidStrategy)
-				get_op_opfamily_properties(pc->opno,
-										   part_scheme->partopfamily[i],
-										   false,
-										   &pc->op_strategy,
-										   &lefttype,
-										   &righttype);
-
-			switch (part_scheme->strategy)
-			{
-				case PARTITION_STRATEGY_LIST:
-				case PARTITION_STRATEGY_RANGE:
-					btree_clauses[pc->op_strategy] =
-						lappend(btree_clauses[pc->op_strategy], pc);
 
-					/*
-					 * We can't consider subsequent partition keys if the
-					 * clause for the current key contains a non-inclusive
-					 * operator.
-					 */
-					if (pc->op_strategy == BTLessStrategyNumber ||
-						pc->op_strategy == BTGreaterStrategyNumber)
-						consider_next_key = false;
-					break;
-
-				case PARTITION_STRATEGY_HASH:
-					if (pc->op_strategy != HTEqualStrategyNumber)
-						elog(ERROR, "invalid clause for hash partitioning");
-					hash_clauses[pc->op_strategy] =
-						lappend(hash_clauses[pc->op_strategy], pc);
-					break;
-
-				default:
-					elog(ERROR, "invalid partition strategy: %c",
-						 part_scheme->strategy);
-					break;
-			}
-		}
-
-		/*
-		 * If we've decided that clauses for subsequent partition keys
-		 * wouldn't be useful for pruning, don't search any further.
-		 */
-		if (!consider_next_key)
-			break;
-	}
+	if (!divide_keyclauses_by_opstrategy(part_scheme, keyclauses, nullkeys,
+										 btree_clauses, hash_clauses))
+		return NIL;
 
 	/*
 	 * Now, we have divided clauses according to their operator strategies.
@@ -1481,271 +1417,17 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context,
 	{
 		case PARTITION_STRATEGY_LIST:
 		case PARTITION_STRATEGY_RANGE:
-			{
-				List	   *eq_clauses = btree_clauses[BTEqualStrategyNumber];
-				List	   *le_clauses = btree_clauses[BTLessEqualStrategyNumber];
-				List	   *ge_clauses = btree_clauses[BTGreaterEqualStrategyNumber];
-				int			strat;
-
-				/*
-				 * For each clause under consideration for a given strategy,
-				 * we collect expressions from clauses for earlier keys, whose
-				 * operator strategy is inclusive, into a list called
-				 * 'prefix'. By appending the clause's own expression to the
-				 * 'prefix', we'll generate one step using the so generated
-				 * vector and assign the current strategy to it.  Actually,
-				 * 'prefix' might contain multiple clauses for the same key,
-				 * in which case, we must generate steps for various
-				 * combinations of expressions of different keys, which
-				 * get_steps_using_prefix takes care of for us.
-				 */
-				for (strat = 1; strat <= BTMaxStrategyNumber; strat++)
-				{
-					foreach(lc, btree_clauses[strat])
-					{
-						PartClauseInfo *pc = lfirst(lc);
-						ListCell   *eq_start;
-						ListCell   *le_start;
-						ListCell   *ge_start;
-						ListCell   *lc1;
-						List	   *prefix = NIL;
-						List	   *pc_steps;
-						bool		prefix_valid = true;
-						bool		pk_has_clauses;
-						int			keyno;
-
-						/*
-						 * If this is a clause for the first partition key,
-						 * there are no preceding expressions; generate a
-						 * pruning step without a prefix.
-						 *
-						 * Note that we pass NULL for step_nullkeys, because
-						 * we don't search list/range partition bounds where
-						 * some keys are NULL.
-						 */
-						if (pc->keyno == 0)
-						{
-							Assert(pc->op_strategy == strat);
-							pc_steps = get_steps_using_prefix(context, strat,
-															  pc->op_is_ne,
-															  pc->expr,
-															  pc->cmpfn,
-															  0,
-															  NULL,
-															  NIL);
-							opsteps = list_concat(opsteps, pc_steps);
-							continue;
-						}
-
-						eq_start = list_head(eq_clauses);
-						le_start = list_head(le_clauses);
-						ge_start = list_head(ge_clauses);
-
-						/*
-						 * We arrange clauses into prefix in ascending order
-						 * of their partition key numbers.
-						 */
-						for (keyno = 0; keyno < pc->keyno; keyno++)
-						{
-							pk_has_clauses = false;
-
-							/*
-							 * Expressions from = clauses can always be in the
-							 * prefix, provided they're from an earlier key.
-							 */
-							for_each_cell(lc1, eq_clauses, eq_start)
-							{
-								PartClauseInfo *eqpc = lfirst(lc1);
-
-								if (eqpc->keyno == keyno)
-								{
-									prefix = lappend(prefix, eqpc);
-									pk_has_clauses = true;
-								}
-								else
-								{
-									Assert(eqpc->keyno > keyno);
-									break;
-								}
-							}
-							eq_start = lc1;
-
-							/*
-							 * If we're generating steps for </<= strategy, we
-							 * can add other <= clauses to the prefix,
-							 * provided they're from an earlier key.
-							 */
-							if (strat == BTLessStrategyNumber ||
-								strat == BTLessEqualStrategyNumber)
-							{
-								for_each_cell(lc1, le_clauses, le_start)
-								{
-									PartClauseInfo *lepc = lfirst(lc1);
-
-									if (lepc->keyno == keyno)
-									{
-										prefix = lappend(prefix, lepc);
-										pk_has_clauses = true;
-									}
-									else
-									{
-										Assert(lepc->keyno > keyno);
-										break;
-									}
-								}
-								le_start = lc1;
-							}
-
-							/*
-							 * If we're generating steps for >/>= strategy, we
-							 * can add other >= clauses to the prefix,
-							 * provided they're from an earlier key.
-							 */
-							if (strat == BTGreaterStrategyNumber ||
-								strat == BTGreaterEqualStrategyNumber)
-							{
-								for_each_cell(lc1, ge_clauses, ge_start)
-								{
-									PartClauseInfo *gepc = lfirst(lc1);
-
-									if (gepc->keyno == keyno)
-									{
-										prefix = lappend(prefix, gepc);
-										pk_has_clauses = true;
-									}
-									else
-									{
-										Assert(gepc->keyno > keyno);
-										break;
-									}
-								}
-								ge_start = lc1;
-							}
-
-							/*
-							 * If this key has no clauses, prefix is not valid
-							 * anymore.
-							 */
-							if (!pk_has_clauses)
-							{
-								prefix_valid = false;
-								break;
-							}
-						}
-
-						/*
-						 * If prefix_valid, generate PartitionPruneStepOps.
-						 * Otherwise, we would not find clauses for a valid
-						 * subset of the partition keys anymore for the
-						 * strategy; give up on generating partition pruning
-						 * steps further for the strategy.
-						 *
-						 * As mentioned above, if 'prefix' contains multiple
-						 * expressions for the same key, the following will
-						 * generate multiple steps, one for each combination
-						 * of the expressions for different keys.
-						 *
-						 * Note that we pass NULL for step_nullkeys, because
-						 * we don't search list/range partition bounds where
-						 * some keys are NULL.
-						 */
-						if (prefix_valid)
-						{
-							Assert(pc->op_strategy == strat);
-							pc_steps = get_steps_using_prefix(context, strat,
-															  pc->op_is_ne,
-															  pc->expr,
-															  pc->cmpfn,
-															  pc->keyno,
-															  NULL,
-															  prefix);
-							opsteps = list_concat(opsteps, pc_steps);
-						}
-						else
-							break;
-					}
-				}
-				break;
-			}
-
+			return gen_prune_steps_opexprs_btree(context, btree_clauses);
 		case PARTITION_STRATEGY_HASH:
-			{
-				List	   *eq_clauses = hash_clauses[HTEqualStrategyNumber];
-
-				/* For hash partitioning, we have just the = strategy. */
-				if (eq_clauses != NIL)
-				{
-					PartClauseInfo *pc;
-					List	   *pc_steps;
-					List	   *prefix = NIL;
-					int			last_keyno;
-					ListCell   *lc1;
-
-					/*
-					 * Locate the clause for the greatest column.  This may
-					 * not belong to the last partition key, but it is the
-					 * clause belonging to the last partition key we found a
-					 * clause for above.
-					 */
-					pc = llast(eq_clauses);
-
-					/*
-					 * There might be multiple clauses which matched to that
-					 * partition key; find the first such clause.  While at
-					 * it, add all the clauses before that one to 'prefix'.
-					 */
-					last_keyno = pc->keyno;
-					foreach(lc, eq_clauses)
-					{
-						pc = lfirst(lc);
-						if (pc->keyno == last_keyno)
-							break;
-						prefix = lappend(prefix, pc);
-					}
-
-					/*
-					 * For each clause for the "last" column, after appending
-					 * the clause's own expression to the 'prefix', we'll
-					 * generate one step using the so generated vector and
-					 * assign = as its strategy.  Actually, 'prefix' might
-					 * contain multiple clauses for the same key, in which
-					 * case, we must generate steps for various combinations
-					 * of expressions of different keys, which
-					 * get_steps_using_prefix will take care of for us.
-					 */
-					for_each_cell(lc1, eq_clauses, lc)
-					{
-						pc = lfirst(lc1);
-
-						/*
-						 * Note that we pass nullkeys for step_nullkeys,
-						 * because we need to tell hash partition bound search
-						 * function which of the keys we found IS NULL clauses
-						 * for.
-						 */
-						Assert(pc->op_strategy == HTEqualStrategyNumber);
-						pc_steps =
-							get_steps_using_prefix(context,
-												   HTEqualStrategyNumber,
-												   false,
-												   pc->expr,
-												   pc->cmpfn,
-												   pc->keyno,
-												   nullkeys,
-												   prefix);
-						opsteps = list_concat(opsteps, pc_steps);
-					}
-				}
-				break;
-			}
-
+			return gen_prune_steps_from_opexps_hash(context, hash_clauses, nullkeys);
 		default:
 			elog(ERROR, "invalid partition strategy: %c",
 				 part_scheme->strategy);
 			break;
 	}
 
-	return opsteps;
+	/* Impossible to get here */
+	return NIL;
 }
 
 /*
@@ -3688,3 +3370,370 @@ partkey_datum_from_expr(PartitionPruneContext *context,
 		*value = ExecEvalExprSwitchContext(exprstate, ectx, isnull);
 	}
 }
+
+
+/*
+ * divide_keyclauses_by_opstrategy
+ *	output are btree_clauses and hash_clauses. Return false if we don't need
+ * to do that.
+ *
+ */
+static bool
+divide_keyclauses_by_opstrategy(PartitionScheme part_scheme,
+								List **keyclauses,
+								Bitmapset *nullkeys,
+								List **btree_clauses,
+								List **hash_clauses)
+{
+	int i;
+	ListCell   *lc;
+	for (i = 0; i < part_scheme->partnatts; i++)
+	{
+		List	   *clauselist = keyclauses[i];
+		bool		consider_next_key = true;
+
+		/*
+		 * For range partitioning, if we have no clauses for the current key,
+		 * we can't consider any later keys either, so we can stop here.
+		 */
+		if (part_scheme->strategy == PARTITION_STRATEGY_RANGE &&
+			clauselist == NIL)
+			break;
+
+		/*
+		 * For hash partitioning, if a column doesn't have the necessary
+		 * equality clause, there should be an IS NULL clause, otherwise
+		 * pruning is not possible.
+		 */
+		if (part_scheme->strategy == PARTITION_STRATEGY_HASH &&
+			clauselist == NIL && !bms_is_member(i, nullkeys))
+			return false;
+
+		foreach(lc, clauselist)
+		{
+			PartClauseInfo *pc = (PartClauseInfo *) lfirst(lc);
+			Oid			lefttype,
+						righttype;
+
+			/* Look up the operator's btree/hash strategy number. */
+			if (pc->op_strategy == InvalidStrategy)
+				get_op_opfamily_properties(pc->opno,
+										   part_scheme->partopfamily[i],
+										   false,
+										   &pc->op_strategy,
+										   &lefttype,
+										   &righttype);
+
+			switch (part_scheme->strategy)
+			{
+				case PARTITION_STRATEGY_LIST:
+				case PARTITION_STRATEGY_RANGE:
+					btree_clauses[pc->op_strategy] =
+						lappend(btree_clauses[pc->op_strategy], pc);
+
+					/*
+					 * We can't consider subsequent partition keys if the
+					 * clause for the current key contains a non-inclusive
+					 * operator.
+					 */
+					if (pc->op_strategy == BTLessStrategyNumber ||
+						pc->op_strategy == BTGreaterStrategyNumber)
+						consider_next_key = false;
+					break;
+
+				case PARTITION_STRATEGY_HASH:
+					if (pc->op_strategy != HTEqualStrategyNumber)
+						elog(ERROR, "invalid clause for hash partitioning");
+					hash_clauses[pc->op_strategy] =
+						lappend(hash_clauses[pc->op_strategy], pc);
+					break;
+
+				default:
+					elog(ERROR, "invalid partition strategy: %c",
+						 part_scheme->strategy);
+					break;
+			}
+		}
+
+		/*
+		 * If we've decided that clauses for subsequent partition keys
+		 * wouldn't be useful for pruning, don't search any further.
+		 */
+		if (!consider_next_key)
+			break;
+	}
+	return true;
+}
+
+/*
+ * gen_prune_steps_from_opexps_btree
+ *
+ */
+static List *
+gen_prune_steps_opexprs_btree(GeneratePruningStepsContext *context,
+								   List **btree_clauses)
+{
+	List	   *eq_clauses = btree_clauses[BTEqualStrategyNumber];
+	List	   *le_clauses = btree_clauses[BTLessEqualStrategyNumber];
+	List	   *ge_clauses = btree_clauses[BTGreaterEqualStrategyNumber];
+	int			strat;
+	List	*opsteps = NIL;
+	ListCell	*lc;
+
+	/*
+	 * For each clause under consideration for a given strategy,
+	 * we collect expressions from clauses for earlier keys, whose
+	 * operator strategy is inclusive, into a list called
+	 * 'prefix'. By appending the clause's own expression to the
+	 * 'prefix', we'll generate one step using the so generated
+	 * vector and assign the current strategy to it.  Actually,
+	 * 'prefix' might contain multiple clauses for the same key,
+	 * in which case, we must generate steps for various
+	 * combinations of expressions of different keys, which
+	 * get_steps_using_prefix takes care of for us.
+	 */
+	for (strat = 1; strat <= BTMaxStrategyNumber; strat++)
+	{
+		foreach(lc, btree_clauses[strat])
+		{
+			PartClauseInfo *pc = lfirst(lc);
+			ListCell   *eq_start;
+			ListCell   *le_start;
+			ListCell   *ge_start;
+			ListCell   *lc1;
+			List	   *prefix = NIL;
+			List	   *pc_steps;
+			bool		prefix_valid = true;
+			bool		pk_has_clauses;
+			int			keyno;
+
+			/*
+			 * If this is a clause for the first partition key,
+			 * there are no preceding expressions; generate a
+			 * pruning step without a prefix.
+			 *
+			 * Note that we pass NULL for step_nullkeys, because
+			 * we don't search list/range partition bounds where
+			 * some keys are NULL.
+			 */
+			if (pc->keyno == 0)
+			{
+				Assert(pc->op_strategy == strat);
+				pc_steps = get_steps_using_prefix(context, strat,
+												  pc->op_is_ne,
+												  pc->expr,
+												  pc->cmpfn,
+												  0,
+												  NULL,
+												  NIL);
+				opsteps = list_concat(opsteps, pc_steps);
+				continue;
+			}
+
+			eq_start = list_head(eq_clauses);
+			le_start = list_head(le_clauses);
+			ge_start = list_head(ge_clauses);
+
+			/*
+			 * We arrange clauses into prefix in ascending order
+			 * of their partition key numbers.
+			 */
+			for (keyno = 0; keyno < pc->keyno; keyno++)
+			{
+				pk_has_clauses = false;
+
+				/*
+				 * Expressions from = clauses can always be in the
+				 * prefix, provided they're from an earlier key.
+				 */
+				for_each_cell(lc1, eq_clauses, eq_start)
+				{
+					PartClauseInfo *eqpc = lfirst(lc1);
+
+					if (eqpc->keyno == keyno)
+					{
+						prefix = lappend(prefix, eqpc);
+						pk_has_clauses = true;
+					}
+					else
+					{
+						Assert(eqpc->keyno > keyno);
+						break;
+					}
+				}
+				eq_start = lc1;
+
+				/*
+				 * If we're generating steps for </<= strategy, we
+				 * can add other <= clauses to the prefix,
+				 * provided they're from an earlier key.
+				 */
+				if (strat == BTLessStrategyNumber ||
+					strat == BTLessEqualStrategyNumber)
+				{
+					for_each_cell(lc1, le_clauses, le_start)
+					{
+						PartClauseInfo *lepc = lfirst(lc1);
+
+						if (lepc->keyno == keyno)
+						{
+							prefix = lappend(prefix, lepc);
+							pk_has_clauses = true;
+						}
+						else
+						{
+							Assert(lepc->keyno > keyno);
+							break;
+						}
+					}
+					le_start = lc1;
+				}
+
+				/*
+				 * If we're generating steps for >/>= strategy, we
+				 * can add other >= clauses to the prefix,
+				 * provided they're from an earlier key.
+				 */
+				if (strat == BTGreaterStrategyNumber ||
+					strat == BTGreaterEqualStrategyNumber)
+				{
+					for_each_cell(lc1, ge_clauses, ge_start)
+					{
+						PartClauseInfo *gepc = lfirst(lc1);
+
+						if (gepc->keyno == keyno)
+						{
+							prefix = lappend(prefix, gepc);
+							pk_has_clauses = true;
+						}
+						else
+						{
+							Assert(gepc->keyno > keyno);
+							break;
+						}
+					}
+					ge_start = lc1;
+				}
+
+				/*
+				 * If this key has no clauses, prefix is not valid
+				 * anymore.
+				 */
+				if (!pk_has_clauses)
+				{
+					prefix_valid = false;
+					break;
+				}
+			}
+
+			/*
+			 * If prefix_valid, generate PartitionPruneStepOps.
+			 * Otherwise, we would not find clauses for a valid
+			 * subset of the partition keys anymore for the
+			 * strategy; give up on generating partition pruning
+			 * steps further for the strategy.
+			 *
+			 * As mentioned above, if 'prefix' contains multiple
+			 * expressions for the same key, the following will
+			 * generate multiple steps, one for each combination
+			 * of the expressions for different keys.
+			 *
+			 * Note that we pass NULL for step_nullkeys, because
+			 * we don't search list/range partition bounds where
+			 * some keys are NULL.
+			 */
+			if (prefix_valid)
+			{
+				Assert(pc->op_strategy == strat);
+				pc_steps = get_steps_using_prefix(context, strat,
+												  pc->op_is_ne,
+												  pc->expr,
+												  pc->cmpfn,
+												  pc->keyno,
+												  NULL,
+												  prefix);
+				opsteps = list_concat(opsteps, pc_steps);
+			}
+			else
+				break;
+		}
+	}
+	return opsteps;
+}
+
+
+static List *
+gen_prune_steps_from_opexps_hash(GeneratePruningStepsContext *context,
+								 List **hash_clauses, Bitmapset *nullkeys)
+
+{
+	List	*opsteps = NIL;
+	ListCell	*lc;
+	List	   *eq_clauses = hash_clauses[HTEqualStrategyNumber];
+
+	/* For hash partitioning, we have just the = strategy. */
+	if (eq_clauses != NIL)
+	{
+		PartClauseInfo *pc;
+		List	   *pc_steps;
+		List	   *prefix = NIL;
+		int			last_keyno;
+		ListCell   *lc1;
+
+		/*
+		 * Locate the clause for the greatest column.  This may
+		 * not belong to the last partition key, but it is the
+		 * clause belonging to the last partition key we found a
+		 * clause for above.
+		 */
+		pc = llast(eq_clauses);
+
+		/*
+		 * There might be multiple clauses which matched to that
+		 * partition key; find the first such clause.  While at
+		 * it, add all the clauses before that one to 'prefix'.
+		 */
+		last_keyno = pc->keyno;
+		foreach(lc, eq_clauses)
+		{
+			pc = lfirst(lc);
+			if (pc->keyno == last_keyno)
+				break;
+			prefix = lappend(prefix, pc);
+		}
+
+		/*
+		 * For each clause for the "last" column, after appending
+		 * the clause's own expression to the 'prefix', we'll
+		 * generate one step using the so generated vector and
+		 * assign = as its strategy.  Actually, 'prefix' might
+		 * contain multiple clauses for the same key, in which
+		 * case, we must generate steps for various combinations
+		 * of expressions of different keys, which
+		 * get_steps_using_prefix will take care of for us.
+		 */
+		for_each_cell(lc1, eq_clauses, lc)
+		{
+			pc = lfirst(lc1);
+
+			/*
+			 * Note that we pass nullkeys for step_nullkeys,
+			 * because we need to tell hash partition bound search
+			 * function which of the keys we found IS NULL clauses
+			 * for.
+			 */
+			Assert(pc->op_strategy == HTEqualStrategyNumber);
+			pc_steps =
+				get_steps_using_prefix(context,
+									   HTEqualStrategyNumber,
+									   false,
+									   pc->expr,
+									   pc->cmpfn,
+									   pc->keyno,
+									   nullkeys,
+									   prefix);
+			opsteps = list_concat(opsteps, pc_steps);
+		}
+	}
+	return opsteps;
+}
-- 
2.21.0