diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index 370cc36..1e855d6 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -117,10 +117,12 @@ PG_FUNCTION_INFO_V1(file_fdw_validator); */ static void fileGetForeignRelSize(PlannerInfo *root, RelOptInfo *baserel, - Oid foreigntableid); + Oid foreigntableid, + bool grouped); static void fileGetForeignPaths(PlannerInfo *root, RelOptInfo *baserel, - Oid foreigntableid); + Oid foreigntableid, + bool grouped); static ForeignScan *fileGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid, @@ -487,11 +489,19 @@ get_file_fdw_attribute_options(Oid relid) static void fileGetForeignRelSize(PlannerInfo *root, RelOptInfo *baserel, - Oid foreigntableid) + Oid foreigntableid, + bool grouped) { FileFdwPlanState *fdw_private; /* + * XXX Grouping at relation level is possible but this FDW does not + * implement it yet. + */ + if (grouped) + return; + + /* * Fetch options. We only need filename (or program) at this point, but * we might as well get everything and not need to re-fetch it later in * planning. @@ -518,7 +528,8 @@ fileGetForeignRelSize(PlannerInfo *root, static void fileGetForeignPaths(PlannerInfo *root, RelOptInfo *baserel, - Oid foreigntableid) + Oid foreigntableid, + bool grouped) { FileFdwPlanState *fdw_private = (FileFdwPlanState *) baserel->fdw_private; Cost startup_cost; @@ -526,6 +537,13 @@ fileGetForeignPaths(PlannerInfo *root, List *columns; List *coptions = NIL; + /* + * XXX Grouping at relation level is possible but this FDW does not + * implement it yet. + */ + if (grouped) + return; + /* Decide whether to selectively perform binary conversion */ if (check_selective_binary_conversion(baserel, foreigntableid, @@ -551,7 +569,7 @@ fileGetForeignPaths(PlannerInfo *root, NIL, /* no pathkeys */ NULL, /* no outer rel either */ NULL, /* no extra plan */ - coptions)); + coptions), false); /* * If data file was sorted, and we knew it somehow, we could insert diff --git a/contrib/postgres_fdw/deparse.c b/contrib/postgres_fdw/deparse.c index 0876589..b67f6fd 100644 --- a/contrib/postgres_fdw/deparse.c +++ b/contrib/postgres_fdw/deparse.c @@ -103,6 +103,7 @@ typedef struct deparse_expr_cxt * a base relation. */ StringInfo buf; /* output buffer to append to */ List **params_list; /* exprs that will become remote Params */ + List *tlist; } deparse_expr_cxt; #define REL_ALIAS_PREFIX "r" @@ -133,6 +134,7 @@ static void deparseTargetList(StringInfo buf, bool qualify_col, List **retrieved_attrs); static void deparseExplicitTargetList(List *tlist, List **retrieved_attrs, + bool grouped, deparse_expr_cxt *context); static void deparseSubqueryTargetList(deparse_expr_cxt *context); static void deparseReturningList(StringInfo buf, PlannerInfo *root, @@ -163,9 +165,10 @@ static void printRemoteParam(int paramindex, Oid paramtype, int32 paramtypmod, static void printRemotePlaceholder(Oid paramtype, int32 paramtypmod, deparse_expr_cxt *context); static void deparseSelectSql(List *tlist, bool is_subquery, List **retrieved_attrs, - deparse_expr_cxt *context); + deparse_expr_cxt *context, bool grouped); static void deparseLockingClause(deparse_expr_cxt *context); -static void appendOrderByClause(List *pathkeys, deparse_expr_cxt *context); +static void appendOrderByClause(List *pathkeys, deparse_expr_cxt *context, + bool grouped); static void appendConditions(List *exprs, deparse_expr_cxt *context); static void deparseFromExprForRel(StringInfo buf, PlannerInfo *root, RelOptInfo *joinrel, bool use_alias, List **params_list); @@ -177,9 +180,10 @@ static void deparseAggref(Aggref *node, deparse_expr_cxt *context); static void appendGroupByClause(List *tlist, deparse_expr_cxt *context); static void appendAggOrderBy(List *orderList, List *targetList, deparse_expr_cxt *context); -static void appendFunctionName(Oid funcid, deparse_expr_cxt *context); +static void appendFunctionName(Oid funcid, StringInfo buf, char *schemaname); static Node *deparseSortGroupClause(Index ref, List *tlist, deparse_expr_cxt *context); +static void deparseSortGroupExpr(Expr *expr, deparse_expr_cxt *context); /* * Helper functions @@ -365,6 +369,14 @@ foreign_expr_walker(Node *node, } } break; + case T_GroupedVar: + { + GroupedVar *gvar = (GroupedVar *) node; + + Assert(!IsA(gvar->gvexpr, Aggref)); + return foreign_expr_walker((Node *) gvar->gvexpr, glob_cxt, + outer_cxt); + } case T_Const: { Const *c = (Const *) node; @@ -676,12 +688,15 @@ foreign_expr_walker(Node *node, Aggref *agg = (Aggref *) node; ListCell *lc; - /* Not safe to pushdown when not in grouping context */ - if (!IS_UPPER_REL(glob_cxt->foreignrel)) - return false; - - /* Only non-split aggregates are pushable. */ - if (agg->aggsplit != AGGSPLIT_SIMPLE) + /* + * Only non-split aggregates are pushable. + * + * XXX Like above, AGGSPLIT_SIMPLE shouldn't appear here if + * the aggregation of the whole upper relation gets abandoned. + * (Check all occurrences of AGGSPLIT_SIMPLE.) + */ + if (agg->aggsplit != AGGSPLIT_SIMPLE && + agg->aggsplit != AGGSPLIT_INITIAL_SERIAL) return false; /* As usual, it must be shippable. */ @@ -925,11 +940,12 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root, RelOptInfo *rel, List *tlist, List *remote_conds, List *pathkeys, bool is_subquery, List **retrieved_attrs, - List **params_list) + List **params_list, bool grouping) { deparse_expr_cxt context; PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) rel->fdw_private; List *quals; + bool grouped; /* * We handle relations for foreign tables, joins between those and upper @@ -943,9 +959,10 @@ deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root, RelOptInfo *rel, context.foreignrel = rel; context.scanrel = IS_UPPER_REL(rel) ? fpinfo->outerrel : rel; context.params_list = params_list; + context.tlist = tlist; /* Construct SELECT clause */ - deparseSelectSql(tlist, is_subquery, retrieved_attrs, &context); + deparseSelectSql(tlist, is_subquery, retrieved_attrs, &context, grouping); /* * For upper relations, the WHERE clause is built from the remote @@ -965,13 +982,30 @@ deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root, RelOptInfo *rel, /* Construct FROM and WHERE clauses */ deparseFromExpr(quals, &context); - if (IS_UPPER_REL(rel)) + grouped = IS_UPPER_REL(rel) || grouping; + if (grouped) { /* Append GROUP BY clause */ appendGroupByClause(tlist, &context); - /* Append HAVING clause */ - if (remote_conds) + /* + * Append HAVING clause. + * + * XXX As the "partition-wise join" patch will probably be committed + * to PG core earlier than the "aggregate push-down" patch, we don't + * add HAVING clause to query that represents remote simple relation + * or join. The point is that the HAVING clause can't be evaluated + * below the final aggregation node, and the final aggregation is + * essentially local. + * + * To surpass this limitation, the FDW would have to know whether the + * relation / join is a partition or the whole table. In case it's + * partition, HAVING is only legal if the query does not use any other + * partition. If we eventually can add the HAVING clause to scan / + * join remote queries, we also have to change the API so that WHERE / + * ON conditions are passed separate from the HAVING expression. + */ + if (IS_UPPER_REL(rel) && remote_conds) { appendStringInfoString(buf, " HAVING "); appendConditions(remote_conds, &context); @@ -980,7 +1014,7 @@ deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root, RelOptInfo *rel, /* Add ORDER BY clause if we found any useful pathkeys */ if (pathkeys) - appendOrderByClause(pathkeys, &context); + appendOrderByClause(pathkeys, &context, grouped); /* Add any necessary FOR UPDATE/SHARE. */ deparseLockingClause(&context); @@ -1001,7 +1035,7 @@ deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root, RelOptInfo *rel, */ static void deparseSelectSql(List *tlist, bool is_subquery, List **retrieved_attrs, - deparse_expr_cxt *context) + deparse_expr_cxt *context, bool grouping) { StringInfo buf = context->buf; RelOptInfo *foreignrel = context->foreignrel; @@ -1028,7 +1062,16 @@ deparseSelectSql(List *tlist, bool is_subquery, List **retrieved_attrs, * For a join or upper relation the input tlist gives the list of * columns required to be fetched from the foreign server. */ - deparseExplicitTargetList(tlist, retrieved_attrs, context); + deparseExplicitTargetList(tlist, retrieved_attrs, grouping, context); + } + else if (grouping && IS_SIMPLE_REL(foreignrel) && foreignrel->gpi != NULL) + { + /* + * An explicit targetlist is also passed if aggregation should take + * place in the remote database. + */ + deparseExplicitTargetList(fpinfo->grouped_tlist, retrieved_attrs, + grouping, context); } else { @@ -1342,7 +1385,7 @@ get_jointype_name(JoinType jointype) * from 1. It has same number of entries as tlist. */ static void -deparseExplicitTargetList(List *tlist, List **retrieved_attrs, +deparseExplicitTargetList(List *tlist, List **retrieved_attrs, bool grouping, deparse_expr_cxt *context) { ListCell *lc; @@ -1505,7 +1548,7 @@ deparseRangeTblRef(StringInfo buf, PlannerInfo *root, RelOptInfo *foreignrel, appendStringInfoChar(buf, '('); deparseSelectStmtForRel(buf, root, foreignrel, NIL, fpinfo->remote_conds, NIL, true, - &retrieved_attrs, params_list); + &retrieved_attrs, params_list, false); appendStringInfoChar(buf, ')'); /* Append the relation alias. */ @@ -2126,6 +2169,9 @@ deparseExpr(Expr *node, deparse_expr_cxt *context) case T_Var: deparseVar((Var *) node, context); break; + case T_GroupedVar: + deparseExpr((Expr *) ((GroupedVar *) node)->gvexpr, context); + break; case T_Const: deparseConst((Const *) node, context, 0); break; @@ -2469,7 +2515,7 @@ deparseFuncExpr(FuncExpr *node, deparse_expr_cxt *context) /* * Normal function: display as proname(args). */ - appendFunctionName(node->funcid, context); + appendFunctionName(node->funcid, context->buf, NULL); appendStringInfoChar(buf, '('); /* ... and all the arguments */ @@ -2747,15 +2793,84 @@ deparseAggref(Aggref *node, deparse_expr_cxt *context) { StringInfo buf = context->buf; bool use_variadic; + ListCell *lc; + bool use_core_aggregate = true; + char *schemaname = NULL; /* Only basic, non-split aggregation accepted. */ - Assert(node->aggsplit == AGGSPLIT_SIMPLE); + Assert(node->aggsplit == AGGSPLIT_SIMPLE || + node->aggsplit == AGGSPLIT_INITIAL_SERIAL); /* Check if need to print VARIADIC (cf. ruleutils.c) */ use_variadic = node->aggvariadic; + if (context->root->grouped_var_list != NIL) + { + /* + * The core aggregates (i.e. those in postgres catalog) do or do not + * perform finalization on both remote and local side. But sometimes + * we need the same aggregate to run w/o finalization on the remote + * node and with the finalization locally. + * + * Since the current Aggref is partial, it essentially returns the + * transient type. We need to find the original aggregate. + */ + foreach(lc, context->root->grouped_var_list) + { + GroupedVarInfo *gvi; + Aggref *orig; + + gvi = lfirst_node(GroupedVarInfo, lc); + /* GroupedVar currently represents only aggregates. */ + orig = castNode(Aggref, gvi->gvexpr); + + if (orig->aggfnoid == node->aggfnoid) + { + /* + * The core aggregate can only be useful if it does not + * perform finalization. + */ + if (!OidIsValid(orig->aggfinalfn)) + { + use_core_aggregate = true; + + /* + * If there's no finalization, the aggregate should return + * value of the internal type. + */ + Assert(orig->aggtype == orig->aggtranstype); + } + else + use_core_aggregate = false; + + /* + * If there are multiple aggregates of the same aggfnoid in + * the query, they should all have the same values of the + * fields we're going to check. So just use the first one we + * find. + */ + break; + } + } + + /* Must have found the original aggregate in the list. */ + Assert(lc != NULL); + } + + /* + * TODO + * + * 1. Make the schema name configurable as FDW option. + * + * 2. Implement the partial aggregates. These don't fit into postgres_fdw + * because there's no reason to install postgres_fdw on the remote node. + * Separate extension (contrib module?) is probably needed. + */ + if (!use_core_aggregate) + schemaname = "partial"; + /* Find aggregate name from aggfnoid which is a pg_proc entry */ - appendFunctionName(node->aggfnoid, context); + appendFunctionName(node->aggfnoid, context->buf, schemaname); appendStringInfoChar(buf, '('); /* Add DISTINCT */ @@ -2829,6 +2944,22 @@ deparseAggref(Aggref *node, deparse_expr_cxt *context) } appendStringInfoChar(buf, ')'); + + /* + * If special (non-core) aggregate is needed on the remote node, it can + * return data type for which postgres does not have textual format. cast + * to bytea seems to be an universal solution in such a case. + * + * TODO + * + * Consider checking if aggserialfn exists --- aggtranstype is used now + * because the info is easier to access. Also consider a requirement that + * the "remote aggregates" must return bytea, unless they return a + * primitive type. Thus we wouldn't have to deal with the cast here at + * all. + */ + if (schemaname != NULL && node->aggtranstype == INTERNALOID) + appendStringInfoString(buf, "::bytea"); } /* @@ -2952,15 +3083,28 @@ appendGroupByClause(List *tlist, deparse_expr_cxt *context) */ Assert(!query->groupingSets); - foreach(lc, query->groupClause) + /* + * Due to aggregation push-down (which includes join aggregation on the + * remote server) we should expect multiple TLEs containing different + * expressions but having the same sortgroupref. The typical case is that + * an equality join clause receives the input value (var) from each side + * of a join, and each of these vars is needed above the join for a + * different reason. That's why we don't use query->groupClause here. + * + * TODO Consider using get_grouping_expressions(). + */ + foreach(lc, tlist) { - SortGroupClause *grp = (SortGroupClause *) lfirst(lc); + TargetEntry *te = lfirst_node(TargetEntry, lc); - if (!first) - appendStringInfoString(buf, ", "); - first = false; + if (te->ressortgroupref > 0) + { + if (!first) + appendStringInfoString(buf, ", "); + first = false; - deparseSortGroupClause(grp->tleSortGroupRef, tlist, context); + deparseSortGroupExpr(te->expr, context); + } } } @@ -2970,7 +3114,7 @@ appendGroupByClause(List *tlist, deparse_expr_cxt *context) * base relation are obtained and deparsed. */ static void -appendOrderByClause(List *pathkeys, deparse_expr_cxt *context) +appendOrderByClause(List *pathkeys, deparse_expr_cxt *context, bool grouped) { ListCell *lcell; int nestlevel; @@ -2981,13 +3125,91 @@ appendOrderByClause(List *pathkeys, deparse_expr_cxt *context) /* Make sure any constants in the exprs are printed portably */ nestlevel = set_transmission_modes(); + /* + * If there's no GROUP BY clause, the query cannot be sorted by aggregates. + */ + if (!grouped) + { + ListCell *l1; + + foreach(l1, pathkeys) + { + PathKey *pathkey = lfirst_node(PathKey, l1); + EquivalenceClass *ec = pathkey->pk_eclass; + ListCell *l2; + + foreach(l2, ec->ec_members) + { + EquivalenceMember *em = lfirst_node(EquivalenceMember, l2); + + if (IsA(em->em_expr, Aggref)) + return; + } + } + } + appendStringInfoString(buf, " ORDER BY"); foreach(lcell, pathkeys) { PathKey *pathkey = lfirst(lcell); - Expr *em_expr; + Expr *em_expr = NULL; - em_expr = find_em_expr_for_rel(pathkey->pk_eclass, baserel); + if (grouped) + { + ListCell *l1; + + /* + * Since the targetlist can contain EC-derived expressions, extra + * effort is needed to ensure that ORDER BY clause references + * expression actually contained in the targetlist. + */ + foreach(l1, pathkey->pk_eclass->ec_members) + { + EquivalenceMember *em = lfirst_node(EquivalenceMember, l1); + + if (bms_is_subset(em->em_relids, baserel->relids)) + { + ListCell *l2; + + /* + * If there is more than one equivalence member whose Vars + * are taken entirely from this relation, we only accept + * one that exists in the targetlist. + */ + Assert(context->tlist != NIL); + foreach(l2, context->tlist) + { + TargetEntry *te = lfirst_node(TargetEntry, l2); + Expr *expr = te->expr; + + if (IsA(expr, Aggref) && IsA(em->em_expr, Aggref)) + { + Aggref *aggref = (Aggref *) copyObject(expr); + Aggref *aggref_em = (Aggref *) em->em_expr; + + /* + * The EC members are simple aggregates, so adjust + * the target expression to make match possible. + */ + Assert(aggref->aggsplit == AGGSPLIT_INITIAL_SERIAL); + aggref->aggsplit = AGGSPLIT_SIMPLE; + aggref->aggtype = aggref_em->aggtype; + expr = (Expr *) aggref; + } + + if (equal(expr, em->em_expr)) + { + em_expr = em->em_expr; + break; + } + } + if (em_expr != NULL) + break; + } + } + } + else + em_expr = find_em_expr_for_rel(pathkey->pk_eclass, baserel); Assert(em_expr != NULL); appendStringInfoString(buf, delim); @@ -3012,9 +3234,8 @@ appendOrderByClause(List *pathkeys, deparse_expr_cxt *context) * Deparses function name from given function oid. */ static void -appendFunctionName(Oid funcid, deparse_expr_cxt *context) +appendFunctionName(Oid funcid, StringInfo buf, char *schemaname) { - StringInfo buf = context->buf; HeapTuple proctup; Form_pg_proc procform; const char *proname; @@ -3025,11 +3246,11 @@ appendFunctionName(Oid funcid, deparse_expr_cxt *context) procform = (Form_pg_proc) GETSTRUCT(proctup); /* Print schema name only if it's not pg_catalog */ - if (procform->pronamespace != PG_CATALOG_NAMESPACE) + if (procform->pronamespace != PG_CATALOG_NAMESPACE || + schemaname != NULL) { - const char *schemaname; - - schemaname = get_namespace_name(procform->pronamespace); + if (schemaname == NULL) + schemaname = get_namespace_name(procform->pronamespace); appendStringInfo(buf, "%s.", quote_identifier(schemaname)); } @@ -3049,13 +3270,22 @@ appendFunctionName(Oid funcid, deparse_expr_cxt *context) static Node * deparseSortGroupClause(Index ref, List *tlist, deparse_expr_cxt *context) { - StringInfo buf = context->buf; TargetEntry *tle; Expr *expr; tle = get_sortgroupref_tle(ref, tlist); expr = tle->expr; + deparseSortGroupExpr(expr, context); + + return (Node *) expr; +} + +static void +deparseSortGroupExpr(Expr *expr, deparse_expr_cxt *context) +{ + StringInfo buf = context->buf; + if (expr && IsA(expr, Const)) { /* @@ -3074,8 +3304,6 @@ deparseSortGroupClause(Index ref, List *tlist, deparse_expr_cxt *context) deparseExpr(expr, context); appendStringInfoChar(buf, ')'); } - - return (Node *) expr; } diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index bce3348..6a7e7fb 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -626,12 +626,12 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 IS NULL; -- Nu Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" WHERE (("C 1" IS NULL)) (3 rows) -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 IS NOT NULL; -- NullTest - QUERY PLAN ------------------------------------------------------------------------------------------------------ +EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 IS NOT NULL and c3 is not null; -- NullTest + QUERY PLAN +-------------------------------------------------------------------------------------------------- Foreign Scan on public.ft1 t1 Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" WHERE (("C 1" IS NOT NULL)) + Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" WHERE ((c3 IS NOT NULL)) (3 rows) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE round(abs(c1), 0) = 1; -- FuncExpr diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index fb65e2e..a434fe9 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -273,10 +273,11 @@ PG_FUNCTION_INFO_V1(postgres_fdw_handler); */ static void postgresGetForeignRelSize(PlannerInfo *root, RelOptInfo *baserel, - Oid foreigntableid); + Oid foreigntableid, bool grouped); static void postgresGetForeignPaths(PlannerInfo *root, RelOptInfo *baserel, - Oid foreigntableid); + Oid foreigntableid, + bool grouped); static ForeignScan *postgresGetForeignPlan(PlannerInfo *root, RelOptInfo *foreignrel, Oid foreigntableid, @@ -341,7 +342,8 @@ static void postgresGetForeignJoinPaths(PlannerInfo *root, RelOptInfo *outerrel, RelOptInfo *innerrel, JoinType jointype, - JoinPathExtraData *extra); + JoinPathExtraData *extra, + bool grouped); static bool postgresRecheckForeignScan(ForeignScanState *node, TupleTableSlot *slot); static void postgresGetForeignUpperPaths(PlannerInfo *root, @@ -352,12 +354,14 @@ static void postgresGetForeignUpperPaths(PlannerInfo *root, /* * Helper functions */ +static void init_pgfdw_baserel_info(PlannerInfo *root, RelOptInfo *rel, + Oid foreigntableid, bool grouped); static void estimate_path_cost_size(PlannerInfo *root, RelOptInfo *baserel, List *join_conds, List *pathkeys, - double *p_rows, int *p_width, - Cost *p_startup_cost, Cost *p_total_cost); + EstimateInfo * estimates, + bool grouped); static void get_remote_estimate(const char *sql, PGconn *conn, double *rows, @@ -404,13 +408,16 @@ static HeapTuple make_tuple_from_result_row(PGresult *res, static void conversion_error_callback(void *arg); static bool foreign_join_ok(PlannerInfo *root, RelOptInfo *joinrel, JoinType jointype, RelOptInfo *outerrel, RelOptInfo *innerrel, - JoinPathExtraData *extra); + JoinPathExtraData *extra, bool grouped); static bool foreign_grouping_ok(PlannerInfo *root, RelOptInfo *grouped_rel); +static List *create_remote_agg_tlist(PlannerInfo *root, + RelOptInfo *grouped_rel, + PathTarget *grouping_target); static List *get_useful_pathkeys_for_relation(PlannerInfo *root, - RelOptInfo *rel); + RelOptInfo *rel, bool grouped); static List *get_useful_ecs_for_relation(PlannerInfo *root, RelOptInfo *rel); static void add_paths_with_pathkeys_for_rel(PlannerInfo *root, RelOptInfo *rel, - Path *epq_path); + Path *epq_path, bool grouped); static void add_foreign_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel, RelOptInfo *grouped_rel); @@ -485,83 +492,51 @@ postgres_fdw_handler(PG_FUNCTION_ARGS) static void postgresGetForeignRelSize(PlannerInfo *root, RelOptInfo *baserel, - Oid foreigntableid) + Oid foreigntableid, + bool grouped) { PgFdwRelationInfo *fpinfo; - ListCell *lc; - RangeTblEntry *rte = planner_rt_fetch(baserel->relid, root); - const char *namespace; - const char *relname; - const char *refname; + EstimateInfo *estimates; /* * We use PgFdwRelationInfo to pass various information to subsequent * functions. + * + * This function can be called twice: first time for plain scan, second + * time for the grouped one. fpinfo should only be initialized once. */ - fpinfo = (PgFdwRelationInfo *) palloc0(sizeof(PgFdwRelationInfo)); - baserel->fdw_private = (void *) fpinfo; - - /* Base foreign tables need to be pushed down always. */ - fpinfo->pushdown_safe = true; - - /* Look up foreign-table catalog info. */ - fpinfo->table = GetForeignTable(foreigntableid); - fpinfo->server = GetForeignServer(fpinfo->table->serverid); - - /* - * Extract user-settable option values. Note that per-table setting of - * use_remote_estimate overrides per-server setting. - */ - fpinfo->use_remote_estimate = false; - fpinfo->fdw_startup_cost = DEFAULT_FDW_STARTUP_COST; - fpinfo->fdw_tuple_cost = DEFAULT_FDW_TUPLE_COST; - fpinfo->shippable_extensions = NIL; - fpinfo->fetch_size = 100; - - apply_server_options(fpinfo); - apply_table_options(fpinfo); - - /* - * If the table or the server is configured to use remote estimates, - * identify which user to do remote access as during planning. This - * should match what ExecCheckRTEPerms() does. If we fail due to lack of - * permissions, the query would have failed at runtime anyway. - */ - if (fpinfo->use_remote_estimate) + if (baserel->fdw_private == NULL) { - Oid userid = rte->checkAsUser ? rte->checkAsUser : GetUserId(); - - fpinfo->user = GetUserMapping(userid, fpinfo->server->serverid); + fpinfo = (PgFdwRelationInfo *) palloc0(sizeof(PgFdwRelationInfo)); + baserel->fdw_private = fpinfo; + init_pgfdw_baserel_info(root, baserel, foreigntableid, grouped); } - else - fpinfo->user = NULL; - /* - * Identify which baserestrictinfo clauses can be sent to the remote - * server and which can't. - */ - classifyConditions(root, baserel, baserel->baserestrictinfo, - &fpinfo->remote_conds, &fpinfo->local_conds); + fpinfo = (PgFdwRelationInfo *) baserel->fdw_private; - /* - * Identify which attributes will need to be retrieved from the remote - * server. These include all attrs needed for joins or final output, plus - * all attrs used in the local_conds. (Note: if we end up using a - * parameterized scan, it's possible that some of the join clauses will be - * sent to the remote and thus we wouldn't really need to retrieve the - * columns used in them. Doesn't seem worth detecting that case though.) - */ - fpinfo->attrs_used = NULL; - pull_varattnos((Node *) baserel->reltarget->exprs, baserel->relid, - &fpinfo->attrs_used); - foreach(lc, fpinfo->local_conds) + if (grouped) { - RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + /* + * baserestrictinfo must be evaluated below the aggregation, so all + * its expressions must be remote. + */ + if (fpinfo->local_conds != NIL) + return; - pull_varattnos((Node *) rinfo->clause, baserel->relid, - &fpinfo->attrs_used); + /* + * Given that there are no local_conds and remote_conds are evaluated + * on the foreign server, baserel->gpi->target is the only thing left + * to check. + * + * XXX Do we need to verify that the target contains no + * PlaceHolderVars? + */ + if (!foreign_grouping_ok(root, baserel)) + return; } + estimates = !grouped ? &fpinfo->est_plain : &fpinfo->est_grouped; + /* * Compute the selectivity and cost of the local_conds, so we don't have * to do it over again for each path. The best we can do for these @@ -580,8 +555,8 @@ postgresGetForeignRelSize(PlannerInfo *root, * when they are set to some sensible costs during one (usually the first) * of the calls to estimate_path_cost_size(). */ - fpinfo->rel_startup_cost = -1; - fpinfo->rel_total_cost = -1; + estimates->rel_startup_cost = -1; + estimates->rel_total_cost = -1; /* * If the table or the server is configured to use remote estimates, @@ -597,13 +572,16 @@ postgresGetForeignRelSize(PlannerInfo *root, * values in fpinfo so we don't need to do it again to generate the * basic foreign path. */ - estimate_path_cost_size(root, baserel, NIL, NIL, - &fpinfo->rows, &fpinfo->width, - &fpinfo->startup_cost, &fpinfo->total_cost); + estimate_path_cost_size(root, baserel, NIL, NIL, estimates, grouped); /* Report estimated baserel size to planner. */ - baserel->rows = fpinfo->rows; - baserel->reltarget->width = fpinfo->width; + + /* + * TODO If grouped, make sure baserel->gpi exists and store the values + * there. + */ + baserel->rows = estimates->rows; + baserel->reltarget->width = estimates->width; } else { @@ -628,34 +606,8 @@ postgresGetForeignRelSize(PlannerInfo *root, set_baserel_size_estimates(root, baserel); /* Fill in basically-bogus cost estimates for use later. */ - estimate_path_cost_size(root, baserel, NIL, NIL, - &fpinfo->rows, &fpinfo->width, - &fpinfo->startup_cost, &fpinfo->total_cost); + estimate_path_cost_size(root, baserel, NIL, NIL, estimates, grouped); } - - /* - * Set the name of relation in fpinfo, while we are constructing it here. - * It will be used to build the string describing the join relation in - * EXPLAIN output. We can't know whether VERBOSE option is specified or - * not, so always schema-qualify the foreign table name. - */ - fpinfo->relation_name = makeStringInfo(); - namespace = get_namespace_name(get_rel_namespace(foreigntableid)); - relname = get_rel_name(foreigntableid); - refname = rte->eref->aliasname; - appendStringInfo(fpinfo->relation_name, "%s.%s", - quote_identifier(namespace), - quote_identifier(relname)); - if (*refname && strcmp(refname, relname) != 0) - appendStringInfo(fpinfo->relation_name, " %s", - quote_identifier(rte->eref->aliasname)); - - /* No outer and inner relations. */ - fpinfo->make_outerrel_subquery = false; - fpinfo->make_innerrel_subquery = false; - fpinfo->lower_subquery_rels = NULL; - /* Set the relation index. */ - fpinfo->relation_index = baserel->relid; } /* @@ -775,7 +727,8 @@ get_useful_ecs_for_relation(PlannerInfo *root, RelOptInfo *rel) * to figure out which pathkeys to consider. */ static List * -get_useful_pathkeys_for_relation(PlannerInfo *root, RelOptInfo *rel) +get_useful_pathkeys_for_relation(PlannerInfo *root, RelOptInfo *rel, + bool grouped) { List *useful_pathkeys_list = NIL; List *useful_eclass_list; @@ -790,12 +743,15 @@ get_useful_pathkeys_for_relation(PlannerInfo *root, RelOptInfo *rel) if (root->query_pathkeys) { bool query_pathkeys_ok = true; + bool sort_pathkeys_ok = true; + PathKey *pathkey; + EquivalenceClass *pathkey_ec; + Expr *em_expr; foreach(lc, root->query_pathkeys) { - PathKey *pathkey = (PathKey *) lfirst(lc); - EquivalenceClass *pathkey_ec = pathkey->pk_eclass; - Expr *em_expr; + pathkey = lfirst_node(PathKey, lc); + pathkey_ec = pathkey->pk_eclass; /* * The planner and executor don't have any clever strategy for @@ -815,9 +771,31 @@ get_useful_pathkeys_for_relation(PlannerInfo *root, RelOptInfo *rel) break; } } - if (query_pathkeys_ok) useful_pathkeys_list = list_make1(list_copy(root->query_pathkeys)); + + /* + * If the path is grouped, then it can be sorted by aggregates. These + * are only present in sort_pathkeys. + */ + if (grouped) + { + foreach(lc, root->sort_pathkeys) + { + pathkey = lfirst_node(PathKey, lc); + pathkey_ec = pathkey->pk_eclass; + + if (pathkey_ec->ec_has_volatile || + !(em_expr = find_em_expr_for_rel(pathkey_ec, rel)) || + !is_foreign_expr(root, rel, em_expr)) + { + sort_pathkeys_ok = false; + break; + } + } + if (sort_pathkeys_ok) + useful_pathkeys_list = list_make1(list_copy(root->sort_pathkeys)); + } } /* @@ -884,12 +862,44 @@ get_useful_pathkeys_for_relation(PlannerInfo *root, RelOptInfo *rel) static void postgresGetForeignPaths(PlannerInfo *root, RelOptInfo *baserel, - Oid foreigntableid) + Oid foreigntableid, + bool grouped) { PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) baserel->fdw_private; ForeignPath *path; + PathTarget *target = NULL; List *ppi_list; ListCell *lc; + List *fdw_private; + EstimateInfo *estimates; + + if (grouped) + { + /* + * See postgresGetForeignRelSize(). + */ + if (fpinfo->local_conds != NIL) + return; + + /* + * Wasn't it possible to construct remote query? + */ + if (fpinfo->grouped_tlist == NIL) + return; + + /* + * The target must contain aggregates. + */ + Assert(baserel->gpi != NULL); + target = baserel->gpi->target; + } + + estimates = !grouped ? &fpinfo->est_plain : &fpinfo->est_grouped; + + /* + * Add information to the paths whether they are grouped or not. + */ + fdw_private = list_make1(makeInteger(grouped ? 1 : 0)); /* * Create simplest ForeignScan path node and add it to baserel. This path @@ -899,18 +909,26 @@ postgresGetForeignPaths(PlannerInfo *root, * to estimate cost and size of this path. */ path = create_foreignscan_path(root, baserel, - NULL, /* default pathtarget */ - fpinfo->rows, - fpinfo->startup_cost, - fpinfo->total_cost, + target, + estimates->rows, + estimates->startup_cost, + estimates->total_cost, NIL, /* no pathkeys */ NULL, /* no outer rel either */ NULL, /* no extra plan */ - NIL); /* no fdw_private list */ - add_path(baserel, (Path *) path); + fdw_private); + + /* + * Grouped path essentially produces an unique set of grouping + * keys. + */ + if (grouped) + make_uniquekeys_for_agg_path((Path *) path); + + add_path(baserel, (Path *) path, grouped); /* Add paths with pathkeys */ - add_paths_with_pathkeys_for_rel(root, baserel, NULL); + add_paths_with_pathkeys_for_rel(root, baserel, NULL, grouped); /* * If we're not using remote estimates, stop here. We have no way to @@ -921,6 +939,14 @@ postgresGetForeignPaths(PlannerInfo *root, return; /* + * Grouped parameterized path would lead to repeated aggregation, which is + * not too interesting. Since the rest deals only with parameterized + * paths, return now. + */ + if (grouped) + return; + + /* * Thumb through all join clauses for the rel to identify which outer * relations could supply one or more safe-to-send-to-remote join clauses. * We'll build a parameterized path for each such outer relation. @@ -1052,34 +1078,32 @@ postgresGetForeignPaths(PlannerInfo *root, foreach(lc, ppi_list) { ParamPathInfo *param_info = (ParamPathInfo *) lfirst(lc); - double rows; - int width; - Cost startup_cost; - Cost total_cost; + EstimateInfo est_param; /* Get a cost estimate from the remote */ + memset(&est_param, 0, sizeof(EstimateInfo)); estimate_path_cost_size(root, baserel, - param_info->ppi_clauses, NIL, - &rows, &width, - &startup_cost, &total_cost); + param_info->ppi_clauses, NIL, &est_param, + grouped); /* * ppi_rows currently won't get looked at by anything, but still we * may as well ensure that it matches our idea of the rowcount. */ - param_info->ppi_rows = rows; + param_info->ppi_rows = est_param.rows; /* Make the path */ path = create_foreignscan_path(root, baserel, NULL, /* default pathtarget */ - rows, - startup_cost, - total_cost, + est_param.rows, + est_param.startup_cost, + est_param.total_cost, NIL, /* no pathkeys */ param_info->ppi_req_outer, NULL, - NIL); /* no fdw_private list */ - add_path(baserel, (Path *) path); + fdw_private); + + add_path(baserel, (Path *) path, grouped); } } @@ -1107,13 +1131,39 @@ postgresGetForeignPlan(PlannerInfo *root, List *retrieved_attrs; StringInfoData sql; ListCell *lc; + Value *grouped_val; + bool grouped; + + Assert(best_path->fdw_private != NIL); + grouped_val = (Value *) linitial(best_path->fdw_private); + Assert(IsA(grouped_val, Integer)); + grouped = intVal(grouped_val) == 1; if (IS_SIMPLE_REL(foreignrel)) { - /* - * For base relations, set scan_relid as the relid of the relation. - */ - scan_relid = foreignrel->relid; + if (!grouped) + { + /* + * For base relations, set scan_relid as the relid of the + * relation. + */ + scan_relid = foreignrel->relid; + } + else + { + /* + * XXX Something seems to be inconsistent here: deparseSelectSql() + * would create the targetlist from foreignrel->gpi->target, but + * not appendGroupByClause(). Try to refactor. + */ + fdw_scan_tlist = fpinfo->grouped_tlist; + + /* + * The scan tuple descriptor should be constructed from + * fdw_scan_tlist rather than from that of the remote relation. + */ + scan_relid = 0; + } /* * In a base-relation scan, we must apply the given scan_clauses. @@ -1191,7 +1241,10 @@ postgresGetForeignPlan(PlannerInfo *root, */ /* Build the list of columns to be fetched from the foreign server. */ - fdw_scan_tlist = build_tlist_to_deparse(foreignrel); + if (!grouped) + fdw_scan_tlist = build_tlist_to_deparse(foreignrel); + else + fdw_scan_tlist = fpinfo->grouped_tlist; /* * Ensure that the outer plan produces a tuple whose descriptor @@ -1239,7 +1292,8 @@ postgresGetForeignPlan(PlannerInfo *root, initStringInfo(&sql); deparseSelectStmtForRel(&sql, root, foreignrel, fdw_scan_tlist, remote_exprs, best_path->path.pathkeys, - false, &retrieved_attrs, ¶ms_list); + false, &retrieved_attrs, ¶ms_list, + grouped); /* Remember remote_exprs for possible use by postgresPlanDirectModify */ fpinfo->final_remote_exprs = remote_exprs; @@ -2481,6 +2535,124 @@ postgresExplainDirectModify(ForeignScanState *node, ExplainState *es) } } +/* + * init_pgfdw_rel_info + * Initialize PgFdwRelationInfo and assign it to the base relation. That + * includes estimates, as well as separation of local expressions from + * remote ones. + */ +static void +init_pgfdw_baserel_info(PlannerInfo *root, RelOptInfo *rel, + Oid foreigntableid, bool grouped) + +{ + const char *namespace; + const char *relname; + const char *refname; + PgFdwRelationInfo *fpinfo; + ListCell *lc; + RangeTblEntry *rte; + + rte = planner_rt_fetch(rel->relid, root); + fpinfo = (PgFdwRelationInfo *) rel->fdw_private; + + /* + * Identify which baserestrictinfo clauses can be sent to the remote + * server and which can't. + */ + classifyConditions(root, rel, rel->baserestrictinfo, + &fpinfo->remote_conds, &fpinfo->local_conds); + + /* + * Identify which attributes will need to be retrieved from the remote + * server. These include all attrs needed for joins or final output, plus + * all attrs used in the local_conds. (Note: if we end up using a + * parameterized scan, it's possible that some of the join clauses will be + * sent to the remote and thus we wouldn't really need to retrieve the + * columns used in them. Doesn't seem worth detecting that case though.) + */ + fpinfo->attrs_used = NULL; + if (!grouped) + { + pull_varattnos((Node *) rel->reltarget->exprs, rel->relid, + &fpinfo->attrs_used); + foreach(lc, fpinfo->local_conds) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + + pull_varattnos((Node *) rinfo->clause, rel->relid, + &fpinfo->attrs_used); + } + } + else + { + /* + * deparseExplicitTargetList() is used for the grouped relation, thus + * no need to retrieve attrs_used. + */ + } + + /* Base foreign tables need to be pushed down always. */ + fpinfo->pushdown_safe = true; + + /* Look up foreign-table catalog info. */ + fpinfo->table = GetForeignTable(foreigntableid); + fpinfo->server = GetForeignServer(fpinfo->table->serverid); + + /* + * Extract user-settable option values. Note that per-table setting of + * use_remote_estimate overrides per-server setting. + */ + fpinfo->use_remote_estimate = false; + fpinfo->fdw_startup_cost = DEFAULT_FDW_STARTUP_COST; + fpinfo->fdw_tuple_cost = DEFAULT_FDW_TUPLE_COST; + fpinfo->shippable_extensions = NIL; + fpinfo->fetch_size = 100; + + apply_server_options(fpinfo); + apply_table_options(fpinfo); + + /* + * If the table or the server is configured to use remote estimates, + * identify which user to do remote access as during planning. This + * should match what ExecCheckRTEPerms() does. If we fail due to lack of + * permissions, the query would have failed at runtime anyway. + */ + if (fpinfo->use_remote_estimate) + { + Oid userid = rte->checkAsUser ? rte->checkAsUser : + GetUserId(); + + fpinfo->user = GetUserMapping(userid, fpinfo->server->serverid); + } + else + fpinfo->user = NULL; + + /* + * Set the name of relation in fpinfo, while we are constructing it here. + * It will be used to build the string describing the join relation in + * EXPLAIN output. We can't know whether VERBOSE option is specified or + * not, so always schema-qualify the foreign table name. + */ + fpinfo->relation_name = makeStringInfo(); + namespace = get_namespace_name(get_rel_namespace(foreigntableid)); + relname = get_rel_name(foreigntableid); + refname = rte->eref->aliasname; + appendStringInfo(fpinfo->relation_name, "%s.%s", + quote_identifier(namespace), + quote_identifier(relname)); + if (*refname && strcmp(refname, relname) != 0) + appendStringInfo(fpinfo->relation_name, " %s", + quote_identifier(rte->eref->aliasname)); + + /* No outer and inner relations. */ + fpinfo->make_outerrel_subquery = false; + fpinfo->make_innerrel_subquery = false; + fpinfo->lower_subquery_rels = NULL; + /* Set the relation index. */ + fpinfo->relation_index = rel->relid; +} + /* * estimate_path_cost_size @@ -2499,8 +2671,8 @@ estimate_path_cost_size(PlannerInfo *root, RelOptInfo *foreignrel, List *param_join_conds, List *pathkeys, - double *p_rows, int *p_width, - Cost *p_startup_cost, Cost *p_total_cost) + EstimateInfo * estimates, + bool grouped) { PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) foreignrel->fdw_private; double rows; @@ -2539,8 +2711,13 @@ estimate_path_cost_size(PlannerInfo *root, &remote_param_join_conds, &local_param_join_conds); /* Build the list of columns to be fetched from the foreign server. */ - if (IS_JOIN_REL(foreignrel) || IS_UPPER_REL(foreignrel)) + if ((IS_JOIN_REL(foreignrel) && !grouped) || IS_UPPER_REL(foreignrel)) fdw_scan_tlist = build_tlist_to_deparse(foreignrel); + else if (grouped) + { + Assert(fpinfo->grouped_tlist != NIL); + fdw_scan_tlist = fpinfo->grouped_tlist; + } else fdw_scan_tlist = NIL; @@ -2561,7 +2738,7 @@ estimate_path_cost_size(PlannerInfo *root, appendStringInfoString(&sql, "EXPLAIN "); deparseSelectStmtForRel(&sql, root, foreignrel, fdw_scan_tlist, remote_conds, pathkeys, false, - &retrieved_attrs, NULL); + &retrieved_attrs, NULL, grouped); /* Get the remote estimate */ conn = GetConnection(fpinfo->user, false); @@ -2615,15 +2792,17 @@ estimate_path_cost_size(PlannerInfo *root, * bare scan each time. Instead, use the costs if we have cached them * already. */ - if (fpinfo->rel_startup_cost > 0 && fpinfo->rel_total_cost > 0) + if (estimates->rel_startup_cost > 0 && estimates->rel_total_cost > 0) { - startup_cost = fpinfo->rel_startup_cost; - run_cost = fpinfo->rel_total_cost - fpinfo->rel_startup_cost; + startup_cost = estimates->rel_startup_cost; + run_cost = estimates->rel_total_cost - estimates->rel_startup_cost; } else if (IS_JOIN_REL(foreignrel)) { - PgFdwRelationInfo *fpinfo_i; - PgFdwRelationInfo *fpinfo_o; + PgFdwRelationInfo *fpinfo_i, + *fpinfo_o; + EstimateInfo *est_i, + *est_o; QualCost join_cost; QualCost remote_conds_cost; double nrows; @@ -2632,10 +2811,12 @@ estimate_path_cost_size(PlannerInfo *root, Assert(fpinfo->innerrel && fpinfo->outerrel); fpinfo_i = (PgFdwRelationInfo *) fpinfo->innerrel->fdw_private; + est_i = !grouped ? &fpinfo_i->est_plain : &fpinfo_i->est_grouped; fpinfo_o = (PgFdwRelationInfo *) fpinfo->outerrel->fdw_private; + est_o = !grouped ? &fpinfo_o->est_plain : &fpinfo_o->est_grouped; /* Estimate of number of rows in cross product */ - nrows = fpinfo_i->rows * fpinfo_o->rows; + nrows = est_i->rows * est_o->rows; /* Clamp retrieved rows estimate to at most size of cross product */ retrieved_rows = Min(retrieved_rows, nrows); @@ -2659,7 +2840,7 @@ estimate_path_cost_size(PlannerInfo *root, * tables) since we do not know what strategy the foreign server * is going to use. */ - startup_cost = fpinfo_i->rel_startup_cost + fpinfo_o->rel_startup_cost; + startup_cost = est_i->rel_startup_cost + est_o->rel_startup_cost; startup_cost += join_cost.startup; startup_cost += remote_conds_cost.startup; startup_cost += fpinfo->local_conds_cost.startup; @@ -2679,17 +2860,29 @@ estimate_path_cost_size(PlannerInfo *root, * 4. Run time cost of applying nonpushable other clauses locally * on the result fetched from the foreign server. */ - run_cost = fpinfo_i->rel_total_cost - fpinfo_i->rel_startup_cost; - run_cost += fpinfo_o->rel_total_cost - fpinfo_o->rel_startup_cost; + run_cost = est_i->rel_total_cost - est_i->rel_startup_cost; + run_cost += est_o->rel_total_cost - est_o->rel_startup_cost; run_cost += nrows * join_cost.per_tuple; nrows = clamp_row_est(nrows * fpinfo->joinclause_sel); run_cost += nrows * remote_conds_cost.per_tuple; run_cost += fpinfo->local_conds_cost.per_tuple * retrieved_rows; } - else if (IS_UPPER_REL(foreignrel)) + + /* + * TODO Consider a separate branch for + * + * (IS_SIMPLE_REL(foreignrel) && * grouped) + * + * and use foreignrel->gpi->target in it instead of + * fpinfo->grouped_tlist. + */ + else if (IS_UPPER_REL(foreignrel) || + (IS_SIMPLE_REL(foreignrel) && grouped) + ) { PgFdwRelationInfo *ofpinfo; - PathTarget *ptarget = root->upper_targets[UPPERREL_GROUP_AGG]; + EstimateInfo *est; + PathTarget *ptarget; AggClauseCosts aggcosts; double input_rows; int numGroupCols; @@ -2707,11 +2900,13 @@ estimate_path_cost_size(PlannerInfo *root, * considering remote and local conditions for costing. */ - ofpinfo = (PgFdwRelationInfo *) fpinfo->outerrel->fdw_private; + ofpinfo = IS_UPPER_REL(foreignrel) ? + (PgFdwRelationInfo *) fpinfo->outerrel->fdw_private : fpinfo; + est = !grouped ? &ofpinfo->est_plain : &ofpinfo->est_grouped; /* Get rows and width from input rel */ - input_rows = ofpinfo->rows; - width = ofpinfo->width; + input_rows = est->rows; + width = est->width; /* Collect statistics about aggregates for estimating costs. */ MemSet(&aggcosts, 0, sizeof(AggClauseCosts)); @@ -2736,6 +2931,14 @@ estimate_path_cost_size(PlannerInfo *root, */ rows = retrieved_rows = numGroups; + if (IS_UPPER_REL(foreignrel)) + ptarget = root->upper_targets[UPPERREL_GROUP_AGG]; + else + { + Assert(foreignrel->gpi != NULL); + ptarget = foreignrel->gpi->target; + } + /*----- * Startup cost includes: * 1. Startup cost for underneath input * relation @@ -2743,7 +2946,7 @@ estimate_path_cost_size(PlannerInfo *root, * 3. Startup cost for PathTarget eval *----- */ - startup_cost = ofpinfo->rel_startup_cost; + startup_cost = est->rel_startup_cost; startup_cost += aggcosts.transCost.startup; startup_cost += aggcosts.transCost.per_tuple * input_rows; startup_cost += (cpu_operator_cost * numGroupCols) * input_rows; @@ -2756,7 +2959,7 @@ estimate_path_cost_size(PlannerInfo *root, * 3. PathTarget eval cost for each output row *----- */ - run_cost = ofpinfo->rel_total_cost - ofpinfo->rel_startup_cost; + run_cost = est->rel_total_cost - est->rel_startup_cost; run_cost += aggcosts.finalCost * numGroups; run_cost += cpu_tuple_cost * numGroups; run_cost += ptarget->cost.per_tuple * numGroups; @@ -2809,8 +3012,8 @@ estimate_path_cost_size(PlannerInfo *root, */ if (pathkeys == NIL && param_join_conds == NIL) { - fpinfo->rel_startup_cost = startup_cost; - fpinfo->rel_total_cost = total_cost; + estimates->rel_startup_cost = startup_cost; + estimates->rel_total_cost = total_cost; } /* @@ -2825,10 +3028,10 @@ estimate_path_cost_size(PlannerInfo *root, total_cost += cpu_tuple_cost * retrieved_rows; /* Return results. */ - *p_rows = rows; - *p_width = width; - *p_startup_cost = startup_cost; - *p_total_cost = total_cost; + estimates->rows = rows; + estimates->width = width; + estimates->startup_cost = startup_cost; + estimates->total_cost = total_cost; } /* @@ -4057,19 +4260,30 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid) * Assess whether the join between inner and outer relations can be pushed down * to the foreign server. As a side effect, save information we obtain in this * function to PgFdwRelationInfo passed in. + * + * Caller expects that the "grouped" argument does not affect the result, so + * it's enough to call the function only once per relation. XXX As that + * argument is only used in Assert() statement, shouldn't it be removed? */ static bool foreign_join_ok(PlannerInfo *root, RelOptInfo *joinrel, JoinType jointype, RelOptInfo *outerrel, RelOptInfo *innerrel, - JoinPathExtraData *extra) + JoinPathExtraData *extra, bool grouped) { - PgFdwRelationInfo *fpinfo; - PgFdwRelationInfo *fpinfo_o; - PgFdwRelationInfo *fpinfo_i; + PgFdwRelationInfo *fpinfo, + *fpinfo_o, + *fpinfo_i; ListCell *lc; List *joinclauses; /* + * TODO + * + * Put the (join-relevant) initial checks of foreign_grouping_ok into a + * function and call it here. + */ + + /* * We support pushing down INNER, LEFT, RIGHT and FULL OUTER joins. * Constructing queries representing SEMI and ANTI joins is hard, hence * not considered right now. @@ -4139,7 +4353,17 @@ foreign_join_ok(PlannerInfo *root, RelOptInfo *joinrel, JoinType jointype, if (is_remote_clause) fpinfo->remote_conds = lappend(fpinfo->remote_conds, rinfo); else + { + /* + * We could return in the grouped case because all clauses + * must be evaluated below the grouping plan (Thus if the + * grouping is remote, those clauses must be remote too.) + * However the same PgFdwRelationInfo is used for both grouped + * and non-grouped case and we don't want to force caller to + * process the non-grouped case first. + */ fpinfo->local_conds = lappend(fpinfo->local_conds, rinfo); + } } } @@ -4158,9 +4382,26 @@ foreign_join_ok(PlannerInfo *root, RelOptInfo *joinrel, JoinType jointype, PlaceHolderInfo *phinfo = lfirst(lc); Relids relids = joinrel->relids; - if (bms_is_subset(phinfo->ph_eval_at, relids) && - bms_nonempty_difference(relids, phinfo->ph_eval_at)) - return false; + if (bms_is_subset(phinfo->ph_eval_at, relids)) + { + if (bms_nonempty_difference(relids, phinfo->ph_eval_at)) + { +#ifdef USE_ASSERT_CHECKING + if (grouped) + { + /* + * Grouped join shouldn't have PlaceHolderVar even in its + * own targetlist. It'd mean that the output of partial + * aggregation can be set to NULL before it gets to the + * input of the final aggregation. + */ + Assert(false); + } +#endif /* USE_ASSERT_CHECKING */ + + return false; + } + } } /* Save the join clauses, for later use. */ @@ -4283,14 +4524,6 @@ foreign_join_ok(PlannerInfo *root, RelOptInfo *joinrel, JoinType jointype, fpinfo->user = NULL; /* - * Set cached relation costs to some negative value, so that we can detect - * when they are set to some sensible costs, during one (usually the - * first) of the calls to estimate_path_cost_size(). - */ - fpinfo->rel_startup_cost = -1; - fpinfo->rel_total_cost = -1; - - /* * Set the string describing this join relation to be used in EXPLAIN * output of corresponding ForeignScan. */ @@ -4315,35 +4548,60 @@ foreign_join_ok(PlannerInfo *root, RelOptInfo *joinrel, JoinType jointype, static void add_paths_with_pathkeys_for_rel(PlannerInfo *root, RelOptInfo *rel, - Path *epq_path) + Path *epq_path, bool grouped) { List *useful_pathkeys_list = NIL; /* List of all pathkeys */ ListCell *lc; + List *fdw_private; + PathTarget *target = NULL; - useful_pathkeys_list = get_useful_pathkeys_for_relation(root, rel); + useful_pathkeys_list = get_useful_pathkeys_for_relation(root, rel, + grouped); + + /* + * Add information to the paths whether they are grouped or not. + */ + fdw_private = list_make1(makeInteger(grouped ? 1 : 0)); + + /* + * Grouped foreign scan should emit tuples according to the grouped + * target, i.e. the one containing GroupedVars. + */ + if (grouped) + { + Assert(rel->gpi != NULL); + target = rel->gpi->target; + } /* Create one path for each set of pathkeys we found above. */ foreach(lc, useful_pathkeys_list) { - double rows; - int width; - Cost startup_cost; - Cost total_cost; List *useful_pathkeys = lfirst(lc); + EstimateInfo estimates; + Path *path; estimate_path_cost_size(root, rel, NIL, useful_pathkeys, - &rows, &width, &startup_cost, &total_cost); - - add_path(rel, (Path *) - create_foreignscan_path(root, rel, - NULL, - rows, - startup_cost, - total_cost, - useful_pathkeys, - NULL, - epq_path, - NIL)); + &estimates, grouped); + + + path = (Path *) create_foreignscan_path(root, rel, + target, + estimates.rows, + estimates.startup_cost, + estimates.total_cost, + useful_pathkeys, + NULL, + epq_path, + fdw_private); + + /* + * Grouped path essentially produces an unique set of grouping + * keys. + */ + if (grouped) + make_uniquekeys_for_agg_path(path); + + add_path(rel, path, grouped); } } @@ -4461,35 +4719,51 @@ postgresGetForeignJoinPaths(PlannerInfo *root, RelOptInfo *outerrel, RelOptInfo *innerrel, JoinType jointype, - JoinPathExtraData *extra) + JoinPathExtraData *extra, + bool grouped) { PgFdwRelationInfo *fpinfo; + EstimateInfo *estimates; ForeignPath *joinpath; - double rows; - int width; - Cost startup_cost; - Cost total_cost; Path *epq_path; /* Path to create plan to be executed when * EvalPlanQual gets triggered. */ + bool first_time; + PathTarget *target = NULL; + List *fdw_private; - /* - * Skip if this join combination has been considered already. - */ - if (joinrel->fdw_private) - return; + if (joinrel->fdw_private == NULL) + { + /* + * First time through. + * + * Create unfinished PgFdwRelationInfo entry which is used to indicate + * that the join relation is already considered, so that we won't + * waste time in judging safety of join pushdown and adding the same + * paths again if found safe. Once we know that this join can be + * pushed down, we fill the entry. + */ + fpinfo = (PgFdwRelationInfo *) palloc0(sizeof(PgFdwRelationInfo)); + fpinfo->pushdown_safe = false; + joinrel->fdw_private = fpinfo; + /* attrs_used is only for base relations. */ + fpinfo->attrs_used = NULL; - /* - * Create unfinished PgFdwRelationInfo entry which is used to indicate - * that the join relation is already considered, so that we won't waste - * time in judging safety of join pushdown and adding the same paths again - * if found safe. Once we know that this join can be pushed down, we fill - * the entry. - */ - fpinfo = (PgFdwRelationInfo *) palloc0(sizeof(PgFdwRelationInfo)); - fpinfo->pushdown_safe = false; - joinrel->fdw_private = fpinfo; - /* attrs_used is only for base relations. */ - fpinfo->attrs_used = NULL; + first_time = true; + } + else + { + fpinfo = (PgFdwRelationInfo *) joinrel->fdw_private; + first_time = false; + + if (!fpinfo->pushdown_safe) + { + /* + * The function was already called but some test failed. No reason + * to see that failure again. + */ + return; + } + } /* * If there is a possibility that EvalPlanQual will be executed, we need @@ -4500,27 +4774,78 @@ postgresGetForeignJoinPaths(PlannerInfo *root, * dominate the only suitable local path available. We also do it before * calling foreign_join_ok(), since that function updates fpinfo and marks * it as pushable if the join is found to be pushable. + * + * The following tests should only be performed once --- repeated, the are + * supposed to yield the same result. (If anything failed initially, + * pushdown_safe should have caused return above.) */ - if (root->parse->commandType == CMD_DELETE || - root->parse->commandType == CMD_UPDATE || - root->rowMarks) + if (first_time) { - epq_path = GetExistingLocalJoinPath(joinrel); - if (!epq_path) + if (root->parse->commandType == CMD_DELETE || + root->parse->commandType == CMD_UPDATE || + root->rowMarks) + { + epq_path = GetExistingLocalJoinPath(joinrel); + if (!epq_path) + { + elog(DEBUG3, "could not push down foreign join because a local path suitable for EPQ checks was not found"); + return; + } + } + else + epq_path = NULL; + + if (!foreign_join_ok(root, joinrel, jointype, outerrel, innerrel, extra, + grouped)) { - elog(DEBUG3, "could not push down foreign join because a local path suitable for EPQ checks was not found"); + /* + * Free path required for EPQ if we copied one; we don't need it + * now + */ + if (epq_path) + pfree(epq_path); return; } } - else - epq_path = NULL; - if (!foreign_join_ok(root, joinrel, jointype, outerrel, innerrel, extra)) - { - /* Free path required for EPQ if we copied one; we don't need it now */ - if (epq_path) - pfree(epq_path); + /* + * Local expressions would be evaluated above the grouping plan, which + * will be remote. + */ + if (grouped && fpinfo->local_conds != NIL) return; + + /* + * This function should not be called repeatedly wit the same value of + * "grouped", but if it happened, make sure the "upper conditions" do not + * get duplicated or used at all if !grouped. + */ + if (fpinfo->remote_conds_upper) + { + list_free(fpinfo->remote_conds_upper); + fpinfo->remote_conds_upper = NIL; + } + if (fpinfo->local_conds_upper) + { + list_free(fpinfo->local_conds_upper); + fpinfo->local_conds_upper = NIL; + } + + if (grouped) + { + /* + * Create targelist to be used for the remote grouping. + */ + Assert(joinrel->gpi != NULL); + fpinfo->grouped_tlist = create_remote_agg_tlist(root, joinrel, + joinrel->gpi->target); + + /* + * Couldn't push any GROUP BY expression or aggregate to the remote + * server? + */ + if (fpinfo->grouped_tlist == NIL) + return; } /* @@ -4547,16 +4872,44 @@ postgresGetForeignJoinPaths(PlannerInfo *root, 0, fpinfo->jointype, extra->sjinfo); + /* + * Set cached relation costs to some negative value, so that we can detect + * when they are set to some sensible costs, during one (usually the + * first) of the calls to estimate_path_cost_size(). + */ + estimates = !grouped ? &fpinfo->est_plain : &fpinfo->est_grouped; + estimates->rel_startup_cost = -1; + estimates->rel_total_cost = -1; + /* Estimate costs for bare join relation */ - estimate_path_cost_size(root, joinrel, NIL, NIL, &rows, - &width, &startup_cost, &total_cost); + + /* + * TODO Include remote_conds_upper and local_conds_upper into the + * estimates. + */ + estimate_path_cost_size(root, joinrel, NIL, NIL, estimates, grouped); /* Now update this information in the joinrel */ - joinrel->rows = rows; - joinrel->reltarget->width = width; - fpinfo->rows = rows; - fpinfo->width = width; - fpinfo->startup_cost = startup_cost; - fpinfo->total_cost = total_cost; + + /* + * TODO If grouped, make sure baserel->gpi exists and store the values + * there. + */ + joinrel->rows = estimates->rows; + joinrel->reltarget->width = estimates->width; + + if (grouped) + { + /* + * The target must contain aggregates. + */ + Assert(joinrel->gpi != NULL); + target = joinrel->gpi->target; + } + + /* + * Add information to the paths whether they are grouped or not. + */ + fdw_private = list_make1(makeInteger(grouped ? 1 : 0)); /* * Create a new join path and add it to the joinrel which represents a @@ -4564,20 +4917,27 @@ postgresGetForeignJoinPaths(PlannerInfo *root, */ joinpath = create_foreignscan_path(root, joinrel, - NULL, /* default pathtarget */ - rows, - startup_cost, - total_cost, + target, /* default pathtarget */ + estimates->rows, + estimates->startup_cost, + estimates->total_cost, NIL, /* no pathkeys */ NULL, /* no required_outer */ epq_path, - NIL); /* no fdw_private */ + fdw_private); /* no fdw_private */ + + /* + * Grouped path essentially produces an unique set of grouping + * keys. + */ + if (grouped) + make_uniquekeys_for_agg_path((Path *) joinpath); /* Add generated path into joinrel by add_path(). */ - add_path(joinrel, (Path *) joinpath); + add_path(joinrel, (Path *) joinpath, grouped); /* Consider pathkeys for the join relation */ - add_paths_with_pathkeys_for_rel(root, joinrel, epq_path); + add_paths_with_pathkeys_for_rel(root, joinrel, epq_path, grouped); /* XXX Consider parameterized paths for the join relation */ } @@ -4593,25 +4953,23 @@ foreign_grouping_ok(PlannerInfo *root, RelOptInfo *grouped_rel) Query *query = root->parse; PathTarget *grouping_target; PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) grouped_rel->fdw_private; - PgFdwRelationInfo *ofpinfo; - List *aggvars; - ListCell *lc; - int i; - List *tlist = NIL; + PgFdwRelationInfo *ofpinfo = NULL; /* Grouping Sets are not pushable */ if (query->groupingSets) return false; /* Get the fpinfo of the underlying scan relation. */ - ofpinfo = (PgFdwRelationInfo *) fpinfo->outerrel->fdw_private; + if (IS_UPPER_REL(grouped_rel)) + ofpinfo = (PgFdwRelationInfo *) fpinfo->outerrel->fdw_private; /* * If underneath input relation has any local conditions, those conditions * are required to be applied before performing aggregation. Hence the * aggregate cannot be pushed down. */ - if (ofpinfo->local_conds) + if ((IS_UPPER_REL(grouped_rel) && ofpinfo->local_conds) || + fpinfo->local_conds) return false; /* @@ -4622,15 +4980,69 @@ foreign_grouping_ok(PlannerInfo *root, RelOptInfo *grouped_rel) * different from those in the plan's targetlist. Use a copy of path * target to record the new sortgrouprefs. */ - grouping_target = copy_pathtarget(root->upper_targets[UPPERREL_GROUP_AGG]); + if (IS_UPPER_REL(grouped_rel)) + grouping_target = root->upper_targets[UPPERREL_GROUP_AGG]; + else + { + /* + * FDW should not be asked to generate grouped paths if the simple or + * join relation has no relevant target. + */ + Assert(grouped_rel->gpi != NULL); + Assert(grouped_rel->gpi->target != NULL); + + grouping_target = grouped_rel->gpi->target; + } + + /* + * Create targelist to be sent to the remote server. + */ + fpinfo->grouped_tlist = create_remote_agg_tlist(root, grouped_rel, + grouping_target); + if (fpinfo->grouped_tlist == NIL) + return false; + + /* Safe to pushdown */ + fpinfo->pushdown_safe = true; /* - * Evaluate grouping targets and check whether they are safe to push down - * to the foreign side. All GROUP BY expressions will be part of the - * grouping target and thus there is no need to evaluate it separately. - * While doing so, add required expressions into target list which can - * then be used to pass to foreign server. + * Set the string describing this grouped relation to be used in EXPLAIN + * output of corresponding ForeignScan. */ + if (ofpinfo != NULL) + { + fpinfo->relation_name = makeStringInfo(); + appendStringInfo(fpinfo->relation_name, "Aggregate on (%s)", + ofpinfo->relation_name->data); + } + + return true; +} + +/* + * Evaluate grouping targets and check whether they are safe to push down to + * the foreign side. All GROUP BY expressions will be part of the grouping + * target and thus there is no need to evaluate it separately. While doing + * so, add required expressions into target list which can then be used to + * pass to foreign server. + * + * NIL is returned if the remote aggregation appears to be impossible. + * + * Note that the function can add items to both remote_conds_upper and + * local_conds_upper lists of PgFdwRelationInfo. + */ +static List * +create_remote_agg_tlist(PlannerInfo *root, RelOptInfo *grouped_rel, + PathTarget *grouping_target) +{ + List *tlist = NIL; + ListCell *lc; + int i; + Query *query = root->parse; + PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) grouped_rel->fdw_private; + + grouping_target = copy_pathtarget(grouping_target); + i = 0; foreach(lc, grouping_target->exprs) { @@ -4646,13 +5058,34 @@ foreign_grouping_ok(PlannerInfo *root, RelOptInfo *grouped_rel) * push down aggregation to the foreign server. */ if (!is_foreign_expr(root, grouped_rel, expr)) - return false; + return NIL; /* Pushable, add to tlist */ tlist = add_to_flat_tlist(tlist, list_make1(expr)); } else { + /* + * Aggregate can sometimes be wrapped by GroupedVar. + * + */ + if ((IS_SIMPLE_REL(grouped_rel) || IS_JOIN_REL(grouped_rel)) && + IsA(expr, GroupedVar)) + { + GroupedVar *gvar = castNode(GroupedVar, expr); + + /* + * GroupedVar is currently used only to handle Aggrefs. + * + * XXX These may include aggregates that appear in the HAVING + * clause. Although the HAVING clause is currently not sent to + * the remote server (see deparseSelectStmtForRel), we still + * need to send the aggregates the HAVING clause contains. + */ + Assert(IsA(gvar->agg_partial, Aggref)); + expr = (Expr *) gvar->agg_partial; + } + /* Check entire expression whether it is pushable or not */ if (is_foreign_expr(root, grouped_rel, expr)) { @@ -4661,6 +5094,8 @@ foreign_grouping_ok(PlannerInfo *root, RelOptInfo *grouped_rel) } else { + List *aggvars; + /* * If we have sortgroupref set, then it means that we have an * ORDER BY entry pointing to this expression. Since we are @@ -4669,12 +5104,14 @@ foreign_grouping_ok(PlannerInfo *root, RelOptInfo *grouped_rel) if (sgref) grouping_target->sortgrouprefs[i] = 0; - /* Not matched exactly, pull the var with aggregates then */ + /* + * Not matched exactly, pull the var with aggregates then + */ aggvars = pull_var_clause((Node *) expr, PVC_INCLUDE_AGGREGATES); if (!is_foreign_expr(root, grouped_rel, (Expr *) aggvars)) - return false; + return NIL; /* * Add aggregates, if any, into the targetlist. Plain var @@ -4702,12 +5139,17 @@ foreign_grouping_ok(PlannerInfo *root, RelOptInfo *grouped_rel) /* * Classify the pushable and non-pushable having clauses and save them in - * remote_conds and local_conds of the grouped rel's fpinfo. + * remote_conds_upper and local_conds_upper of the grouped rel's fpinfo. + */ + + /* + * TODO For non-upper grouped_rel, only pick those expressions for which + * grouped_rel provides all input vars. (And Assert() that grouped_rel + * does not emit just some of them --- in such a case grouped_rel + * shouldn't be subject to partial aggregation at all.) */ if (root->hasHavingQual && query->havingQual) { - ListCell *lc; - foreach(lc, (List *) query->havingQual) { Expr *expr = (Expr *) lfirst(lc); @@ -4727,9 +5169,11 @@ foreign_grouping_ok(PlannerInfo *root, RelOptInfo *grouped_rel) NULL, NULL); if (is_foreign_expr(root, grouped_rel, expr)) - fpinfo->remote_conds = lappend(fpinfo->remote_conds, rinfo); + fpinfo->remote_conds_upper = + lappend(fpinfo->remote_conds_upper, rinfo); else - fpinfo->local_conds = lappend(fpinfo->local_conds, rinfo); + fpinfo->local_conds_upper = + lappend(fpinfo->local_conds_upper, rinfo); } } @@ -4737,12 +5181,11 @@ foreign_grouping_ok(PlannerInfo *root, RelOptInfo *grouped_rel) * If there are any local conditions, pull Vars and aggregates from it and * check whether they are safe to pushdown or not. */ - if (fpinfo->local_conds) + if (fpinfo->local_conds_upper) { List *aggvars = NIL; - ListCell *lc; - foreach(lc, fpinfo->local_conds) + foreach(lc, fpinfo->local_conds_upper) { RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); @@ -4764,7 +5207,7 @@ foreign_grouping_ok(PlannerInfo *root, RelOptInfo *grouped_rel) if (IsA(expr, Aggref)) { if (!is_foreign_expr(root, grouped_rel, expr)) - return false; + return NIL; tlist = add_to_flat_tlist(tlist, list_make1(expr)); } @@ -4774,29 +5217,7 @@ foreign_grouping_ok(PlannerInfo *root, RelOptInfo *grouped_rel) /* Transfer any sortgroupref data to the replacement tlist */ apply_pathtarget_labeling_to_tlist(tlist, grouping_target); - /* Store generated targetlist */ - fpinfo->grouped_tlist = tlist; - - /* Safe to pushdown */ - fpinfo->pushdown_safe = true; - - /* - * Set cached relation costs to some negative value, so that we can detect - * when they are set to some sensible costs, during one (usually the - * first) of the calls to estimate_path_cost_size(). - */ - fpinfo->rel_startup_cost = -1; - fpinfo->rel_total_cost = -1; - - /* - * Set the string describing this grouped relation to be used in EXPLAIN - * output of corresponding ForeignScan. - */ - fpinfo->relation_name = makeStringInfo(); - appendStringInfo(fpinfo->relation_name, "Aggregate on (%s)", - ofpinfo->relation_name->data); - - return true; + return tlist; } /* @@ -4845,12 +5266,10 @@ add_foreign_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel, Query *parse = root->parse; PgFdwRelationInfo *ifpinfo = input_rel->fdw_private; PgFdwRelationInfo *fpinfo = grouped_rel->fdw_private; + EstimateInfo *estimates; ForeignPath *grouppath; PathTarget *grouping_target; - double rows; - int width; - Cost startup_cost; - Cost total_cost; + List *fdw_private; /* Nothing to be done, if there is no grouping or aggregation required. */ if (!parse->groupClause && !parse->groupingSets && !parse->hasAggs && @@ -4876,29 +5295,29 @@ add_foreign_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel, return; /* Estimate the cost of push down */ - estimate_path_cost_size(root, grouped_rel, NIL, NIL, &rows, - &width, &startup_cost, &total_cost); + estimates = &fpinfo->est_grouped; + estimate_path_cost_size(root, grouped_rel, NIL, NIL, estimates, true); - /* Now update this information in the fpinfo */ - fpinfo->rows = rows; - fpinfo->width = width; - fpinfo->startup_cost = startup_cost; - fpinfo->total_cost = total_cost; + /* + * The path is considered grouped when it comes to plan creation and + * deparsing. + */ + fdw_private = list_make1(makeInteger(1)); /* Create and add foreign path to the grouping relation. */ grouppath = create_foreignscan_path(root, grouped_rel, grouping_target, - rows, - startup_cost, - total_cost, + estimates->rows, + estimates->startup_cost, + estimates->total_cost, NIL, /* no pathkeys */ NULL, /* no required_outer */ NULL, - NIL); /* no fdw_private */ + fdw_private); /* Add generated path into grouped_rel by add_path(). */ - add_path(grouped_rel, (Path *) grouppath); + add_path(grouped_rel, (Path *) grouppath, false); } /* diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h index 788b003..e796236 100644 --- a/contrib/postgres_fdw/postgres_fdw.h +++ b/contrib/postgres_fdw/postgres_fdw.h @@ -20,6 +20,18 @@ #include "libpq-fe.h" +typedef struct EstimateInfo +{ + /* Estimated size and cost for a scan or join. */ + double rows; + int width; + Cost startup_cost; + Cost total_cost; + /* Costs excluding costs for transferring data from the foreign server */ + Cost rel_startup_cost; + Cost rel_total_cost; +} EstimateInfo; + /* * FDW-specific planner information kept in RelOptInfo.fdw_private for a * postgres_fdw foreign table. For a baserel, this struct is created by @@ -43,6 +55,14 @@ typedef struct PgFdwRelationInfo List *remote_conds; List *local_conds; + /* + * HAVING clauses. + * + * TODO Use these where appropriate. + */ + List *remote_conds_upper; + List *local_conds_upper; + /* Actual remote restriction clauses for scan (sans RestrictInfos) */ List *final_remote_exprs; @@ -56,14 +76,9 @@ typedef struct PgFdwRelationInfo /* Selectivity of join conditions */ Selectivity joinclause_sel; - /* Estimated size and cost for a scan or join. */ - double rows; - int width; - Cost startup_cost; - Cost total_cost; - /* Costs excluding costs for transferring data from the foreign server */ - Cost rel_startup_cost; - Cost rel_total_cost; + /* Estimates for both plain and grouped relation. */ + EstimateInfo est_plain; + EstimateInfo est_grouped; /* Options extracted from catalogs. */ bool use_remote_estimate; @@ -92,7 +107,9 @@ typedef struct PgFdwRelationInfo /* joinclauses contains only JOIN/ON conditions for an outer join */ List *joinclauses; /* List of RestrictInfo */ - /* Grouping information */ + /* + * The targetlist used to construct the remote query. + */ List *grouped_tlist; /* Subquery information */ @@ -121,6 +138,7 @@ extern unsigned int GetCursorNumber(PGconn *conn); extern unsigned int GetPrepStmtNumber(PGconn *conn); extern PGresult *pgfdw_get_result(PGconn *conn, const char *query); extern PGresult *pgfdw_exec_query(PGconn *conn, const char *query); +extern void pgfdw_send_query(PGconn *conn, const char *query); extern void pgfdw_report_error(int elevel, PGresult *res, PGconn *conn, bool clear, const char *sql); @@ -175,7 +193,8 @@ extern List *build_tlist_to_deparse(RelOptInfo *foreignrel); extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root, RelOptInfo *foreignrel, List *tlist, List *remote_conds, List *pathkeys, bool is_subquery, - List **retrieved_attrs, List **params_list); + List **retrieved_attrs, List **params_list, + bool grouping); extern const char *get_jointype_name(JoinType jointype); /* in shippable.c */ diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index 1df1e3a..c421530 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -292,7 +292,7 @@ RESET enable_nestloop; EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE t1.c1 = 1; -- Var, OpExpr(b), Const EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE t1.c1 = 100 AND t1.c2 = 0; -- BoolExpr EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 IS NULL; -- NullTest -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 IS NOT NULL; -- NullTest +EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 IS NOT NULL and c3 is not null; -- NullTest EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE round(abs(c1), 0) = 1; -- FuncExpr EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = -c1; -- OpExpr(l) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE 1 = c1!; -- OpExpr(r) diff --git a/src/backend/executor/execExpr.c b/src/backend/executor/execExpr.c index e083961..6be6b70 100644 --- a/src/backend/executor/execExpr.c +++ b/src/backend/executor/execExpr.c @@ -723,6 +723,37 @@ ExecInitExprRec(Expr *node, PlanState *parent, ExprState *state, break; } + case T_GroupedVar: + + /* + * If GroupedVar appears in targetlist of Agg node, it can + * represent either Aggref or grouping expression. + */ + if (parent && (IsA(parent, AggState))) + { + GroupedVar *gvar = (GroupedVar *) node; + + /* + * TODO Assert() that all Aggrefs of the AggState are partial: + * GroupedVar shouldn't find its way to the query targetlist. + */ + if (IsA(gvar->gvexpr, Aggref)) + ExecInitExprRec((Expr *) gvar->agg_partial, parent, state, + resv, resnull); + else + ExecInitExprRec((Expr *) gvar->gvexpr, parent, state, + resv, resnull); + break; + } + else + { + /* + * set_plan_refs should have replaced GroupedVar in the + * targetlist with an ordinary Var. + */ + elog(ERROR, "parent of GroupedVar is not Agg node"); + } + case T_GroupingFunc: { GroupingFunc *grp_node = (GroupingFunc *) node; diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c index da6ef1a..2dabe4c 100644 --- a/src/backend/executor/nodeAgg.c +++ b/src/backend/executor/nodeAgg.c @@ -1875,6 +1875,13 @@ find_unaggregated_cols_walker(Node *node, Bitmapset **colnos) /* do not descend into aggregate exprs */ return false; } + if (IsA(node, GroupedVar)) + { + GroupedVar *gvar = (GroupedVar *) node; + + if (IsA(gvar->gvexpr, Aggref)) + return false; + } return expression_tree_walker(node, find_unaggregated_cols_walker, (void *) colnos); } diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index aff9a62..371dda1 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -1359,6 +1359,8 @@ _copyAggref(const Aggref *from) COPY_SCALAR_FIELD(aggcollid); COPY_SCALAR_FIELD(inputcollid); COPY_SCALAR_FIELD(aggtranstype); + COPY_SCALAR_FIELD(aggcombinefn); + COPY_SCALAR_FIELD(aggfinalfn); COPY_NODE_FIELD(aggargtypes); COPY_NODE_FIELD(aggdirectargs); COPY_NODE_FIELD(args); @@ -2211,6 +2213,22 @@ _copyPlaceHolderVar(const PlaceHolderVar *from) } /* + * _copyGroupedVar + */ +static GroupedVar * +_copyGroupedVar(const GroupedVar *from) +{ + GroupedVar *newnode = makeNode(GroupedVar); + + COPY_NODE_FIELD(gvexpr); + COPY_NODE_FIELD(agg_partial); + COPY_SCALAR_FIELD(sortgroupref); + COPY_SCALAR_FIELD(gvid); + + return newnode; +} + +/* * _copySpecialJoinInfo */ static SpecialJoinInfo * @@ -2283,6 +2301,21 @@ _copyPlaceHolderInfo(const PlaceHolderInfo *from) return newnode; } +static GroupedVarInfo * +_copyGroupedVarInfo(const GroupedVarInfo *from) +{ + GroupedVarInfo *newnode = makeNode(GroupedVarInfo); + + COPY_SCALAR_FIELD(gvid); + COPY_NODE_FIELD(gvexpr); + COPY_NODE_FIELD(agg_partial); + COPY_SCALAR_FIELD(sortgroupref); + COPY_SCALAR_FIELD(gv_eval_at); + COPY_SCALAR_FIELD(gv_width); + + return newnode; +} + /* **************************************************************** * parsenodes.h copy functions * **************************************************************** @@ -5018,6 +5051,9 @@ copyObjectImpl(const void *from) case T_PlaceHolderVar: retval = _copyPlaceHolderVar(from); break; + case T_GroupedVar: + retval = _copyGroupedVar(from); + break; case T_SpecialJoinInfo: retval = _copySpecialJoinInfo(from); break; @@ -5030,6 +5066,9 @@ copyObjectImpl(const void *from) case T_PlaceHolderInfo: retval = _copyPlaceHolderInfo(from); break; + case T_GroupedVarInfo: + retval = _copyGroupedVarInfo(from); + break; /* * VALUE NODES diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 2e869a9..3d735a2 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -873,6 +873,14 @@ _equalPlaceHolderVar(const PlaceHolderVar *a, const PlaceHolderVar *b) } static bool +_equalGroupedVar(const GroupedVar *a, const GroupedVar *b) +{ + COMPARE_SCALAR_FIELD(gvid); + + return true; +} + +static bool _equalSpecialJoinInfo(const SpecialJoinInfo *a, const SpecialJoinInfo *b) { COMPARE_BITMAPSET_FIELD(min_lefthand); @@ -3171,6 +3179,9 @@ equal(const void *a, const void *b) case T_PlaceHolderVar: retval = _equalPlaceHolderVar(a, b); break; + case T_GroupedVar: + retval = _equalGroupedVar(a, b); + break; case T_SpecialJoinInfo: retval = _equalSpecialJoinInfo(a, b); break; diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c index c2a93b2..9a53aa7 100644 --- a/src/backend/nodes/nodeFuncs.c +++ b/src/backend/nodes/nodeFuncs.c @@ -259,6 +259,12 @@ exprType(const Node *expr) case T_PlaceHolderVar: type = exprType((Node *) ((const PlaceHolderVar *) expr)->phexpr); break; + case T_GroupedVar: + if (IsA(((const GroupedVar *) expr)->gvexpr, Aggref)) + type = exprType((Node *) ((const GroupedVar *) expr)->agg_partial); + else + type = exprType((Node *) ((const GroupedVar *) expr)->gvexpr); + break; default: elog(ERROR, "unrecognized node type: %d", (int) nodeTag(expr)); type = InvalidOid; /* keep compiler quiet */ @@ -492,6 +498,11 @@ exprTypmod(const Node *expr) return ((const SetToDefault *) expr)->typeMod; case T_PlaceHolderVar: return exprTypmod((Node *) ((const PlaceHolderVar *) expr)->phexpr); + case T_GroupedVar: + if (IsA(((const GroupedVar *) expr)->gvexpr, Aggref)) + return exprTypmod((Node *) ((const GroupedVar *) expr)->agg_partial); + else + return exprTypmod((Node *) ((const GroupedVar *) expr)->gvexpr); default: break; } @@ -903,6 +914,12 @@ exprCollation(const Node *expr) case T_PlaceHolderVar: coll = exprCollation((Node *) ((const PlaceHolderVar *) expr)->phexpr); break; + case T_GroupedVar: + if (IsA(((const GroupedVar *) expr)->gvexpr, Aggref)) + coll = exprCollation((Node *) ((const GroupedVar *) expr)->agg_partial); + else + coll = exprCollation((Node *) ((const GroupedVar *) expr)->gvexpr); + break; default: elog(ERROR, "unrecognized node type: %d", (int) nodeTag(expr)); coll = InvalidOid; /* keep compiler quiet */ @@ -2176,6 +2193,8 @@ expression_tree_walker(Node *node, break; case T_PlaceHolderVar: return walker(((PlaceHolderVar *) node)->phexpr, context); + case T_GroupedVar: + return walker(((GroupedVar *) node)->gvexpr, context); case T_InferenceElem: return walker(((InferenceElem *) node)->expr, context); case T_AppendRelInfo: @@ -2968,6 +2987,16 @@ expression_tree_mutator(Node *node, return (Node *) newnode; } break; + case T_GroupedVar: + { + GroupedVar *gv = (GroupedVar *) node; + GroupedVar *newnode; + + FLATCOPY(newnode, gv, GroupedVar); + MUTATE(newnode->gvexpr, gv->gvexpr, Expr *); + MUTATE(newnode->agg_partial, gv->agg_partial, Aggref *); + return (Node *) newnode; + } case T_InferenceElem: { InferenceElem *inferenceelemdexpr = (InferenceElem *) node; diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index c97ee24..2c0ceb8 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -1134,6 +1134,8 @@ _outAggref(StringInfo str, const Aggref *node) WRITE_OID_FIELD(aggcollid); WRITE_OID_FIELD(inputcollid); WRITE_OID_FIELD(aggtranstype); + WRITE_OID_FIELD(aggcombinefn); + WRITE_OID_FIELD(aggfinalfn); WRITE_NODE_FIELD(aggargtypes); WRITE_NODE_FIELD(aggdirectargs); WRITE_NODE_FIELD(args); @@ -2223,6 +2225,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(pcinfo_list); WRITE_NODE_FIELD(rowMarks); WRITE_NODE_FIELD(placeholder_list); + WRITE_NODE_FIELD(grouped_var_list); WRITE_NODE_FIELD(fkey_list); WRITE_NODE_FIELD(query_pathkeys); WRITE_NODE_FIELD(group_pathkeys); @@ -2230,6 +2233,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(distinct_pathkeys); WRITE_NODE_FIELD(sort_pathkeys); WRITE_NODE_FIELD(processed_tlist); + WRITE_INT_FIELD(max_sortgroupref); WRITE_NODE_FIELD(minmax_aggs); WRITE_FLOAT_FIELD(total_table_pages, "%.0f"); WRITE_FLOAT_FIELD(tuple_fraction, "%.4f"); @@ -2269,6 +2273,7 @@ _outRelOptInfo(StringInfo str, const RelOptInfo *node) WRITE_NODE_FIELD(cheapest_parameterized_paths); WRITE_BITMAPSET_FIELD(direct_lateral_relids); WRITE_BITMAPSET_FIELD(lateral_relids); + WRITE_NODE_FIELD(gpi); WRITE_UINT_FIELD(relid); WRITE_OID_FIELD(reltablespace); WRITE_ENUM_FIELD(rtekind, RTEKind); @@ -2445,6 +2450,18 @@ _outParamPathInfo(StringInfo str, const ParamPathInfo *node) } static void +_outGroupedPathInfo(StringInfo str, const GroupedPathInfo *node) +{ + WRITE_NODE_TYPE("GROUPEDPATHINFO"); + + WRITE_NODE_FIELD(target); + WRITE_NODE_FIELD(group_exprs); + WRITE_NODE_FIELD(pathlist); + WRITE_NODE_FIELD(partial_pathlist); + WRITE_NODE_FIELD(sortgroupclauses); +} + +static void _outRestrictInfo(StringInfo str, const RestrictInfo *node) { WRITE_NODE_TYPE("RESTRICTINFO"); @@ -2488,6 +2505,17 @@ _outPlaceHolderVar(StringInfo str, const PlaceHolderVar *node) } static void +_outGroupedVar(StringInfo str, const GroupedVar *node) +{ + WRITE_NODE_TYPE("GROUPEDVAR"); + + WRITE_NODE_FIELD(gvexpr); + WRITE_NODE_FIELD(agg_partial); + WRITE_UINT_FIELD(sortgroupref); + WRITE_UINT_FIELD(gvid); +} + +static void _outSpecialJoinInfo(StringInfo str, const SpecialJoinInfo *node) { WRITE_NODE_TYPE("SPECIALJOININFO"); @@ -2541,6 +2569,19 @@ _outPlaceHolderInfo(StringInfo str, const PlaceHolderInfo *node) } static void +_outGroupedVarInfo(StringInfo str, const GroupedVarInfo *node) +{ + WRITE_NODE_TYPE("GROUPEDVARINFO"); + + WRITE_UINT_FIELD(gvid); + WRITE_NODE_FIELD(gvexpr); + WRITE_NODE_FIELD(agg_partial); + WRITE_UINT_FIELD(sortgroupref); + WRITE_BITMAPSET_FIELD(gv_eval_at); + WRITE_INT_FIELD(gv_width); +} + +static void _outMinMaxAggInfo(StringInfo str, const MinMaxAggInfo *node) { WRITE_NODE_TYPE("MINMAXAGGINFO"); @@ -4043,12 +4084,18 @@ outNode(StringInfo str, const void *obj) case T_ParamPathInfo: _outParamPathInfo(str, obj); break; + case T_GroupedPathInfo: + _outGroupedPathInfo(str, obj); + break; case T_RestrictInfo: _outRestrictInfo(str, obj); break; case T_PlaceHolderVar: _outPlaceHolderVar(str, obj); break; + case T_GroupedVar: + _outGroupedVar(str, obj); + break; case T_SpecialJoinInfo: _outSpecialJoinInfo(str, obj); break; @@ -4061,6 +4108,9 @@ outNode(StringInfo str, const void *obj) case T_PlaceHolderInfo: _outPlaceHolderInfo(str, obj); break; + case T_GroupedVarInfo: + _outGroupedVarInfo(str, obj); + break; case T_MinMaxAggInfo: _outMinMaxAggInfo(str, obj); break; diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 7eb67fc0..74bfb5e 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -529,6 +529,22 @@ _readVar(void) } /* + * _readGroupedVar + */ +static GroupedVar * +_readGroupedVar(void) +{ + READ_LOCALS(GroupedVar); + + READ_NODE_FIELD(gvexpr); + READ_NODE_FIELD(agg_partial); + READ_UINT_FIELD(sortgroupref); + READ_UINT_FIELD(gvid); + + READ_DONE(); +} + +/* * _readConst */ static Const * @@ -584,6 +600,8 @@ _readAggref(void) READ_OID_FIELD(aggcollid); READ_OID_FIELD(inputcollid); READ_OID_FIELD(aggtranstype); + READ_OID_FIELD(aggcombinefn); + READ_OID_FIELD(aggfinalfn); READ_NODE_FIELD(aggargtypes); READ_NODE_FIELD(aggdirectargs); READ_NODE_FIELD(args); @@ -2470,6 +2488,8 @@ parseNodeString(void) return_value = _readTableFunc(); else if (MATCH("VAR", 3)) return_value = _readVar(); + else if (MATCH("GROUPEDVAR", 10)) + return_value = _readGroupedVar(); else if (MATCH("CONST", 5)) return_value = _readConst(); else if (MATCH("PARAM", 5)) diff --git a/src/backend/optimizer/README b/src/backend/optimizer/README index 1e4084d..3994394 100644 --- a/src/backend/optimizer/README +++ b/src/backend/optimizer/README @@ -1076,6 +1076,41 @@ plan as possible. Expanding the range of cases in which more work can be pushed below the Gather (and costing them accurately) is likely to keep us busy for a long time to come. +Grouped Paths +------------- + +If both query semantics and path type allow, an existing path can be subject +to partial aggregation (ie aggregation without calling the aggfinalfn +function) and stored as a "grouped path" in a separate pathlist. Both base +relation and join paths can be turned into a grouped path this way. If the +source path was partial (in terms of parallel processing, see above), then the +grouped path is called "partial grouped path". + +If a grouped path appears at the top of the join tree, the transient state +values are aggregated and the aggfinalfn function is called for each +aggregate. Likewise, if the top-level join produces a partial grouped path, we +first apply Gather plan on it and then we finalize the aggregation in the same +way. + +Besides being a result of aggregation, (partial) grouped path can be created +by joining an existing grouped path to an ordinary (non-grouped) path. This +technique can result in duplication of grouping keys, which are essentially +unique in the output of aggregation. The final aggregation takes these +duplicities into account. + +In contrast, join of two grouped paths is not supported as there doesn't seem +to be an interesting use case for it. If we want to support this kind of join +in the future, additional attention needs to be paid to the duplication of +grouping keys mentioned above. For example, if scan of table A is joined to a +grouped scan of table B, then the cardinality of grouping keys of table B +present in (non-grouped) table A determines how many times does each +aggregation transient state value of B appear on the input of the final +aggregation path. However if A is grouped too, the number of those key values +on the A side can get lower and thus affect the input of the final aggregation +path. When compensating for this effect, we'd have to count input values of +each group during the aggregation of A and "multiply" the corresponding +transient state values of B accordingly. + Partition-wise joins -------------------- A join between two similarly partitioned tables can be broken down into joins diff --git a/src/backend/optimizer/geqo/geqo_eval.c b/src/backend/optimizer/geqo/geqo_eval.c index 3cf268c..30ac459 100644 --- a/src/backend/optimizer/geqo/geqo_eval.c +++ b/src/backend/optimizer/geqo/geqo_eval.c @@ -268,7 +268,8 @@ merge_clump(PlannerInfo *root, List *clumps, Clump *new_clump, bool force) generate_partition_wise_join_paths(root, joinrel); /* Create GatherPaths for any useful partial paths for rel */ - generate_gather_paths(root, joinrel); + generate_gather_paths(root, joinrel, false); + generate_gather_paths(root, joinrel, true); /* Find and save the cheapest paths for this joinrel */ set_cheapest(joinrel); diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 44f6b03..9cd5228 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -97,7 +97,8 @@ static void set_append_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, static void generate_mergeappend_paths(PlannerInfo *root, RelOptInfo *rel, List *live_childrels, List *all_child_pathkeys, - List *partitioned_rels); + List *partitioned_rels, + bool grouped); static Path *get_cheapest_parameterized_child_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer); @@ -345,6 +346,7 @@ set_rel_size(PlannerInfo *root, RelOptInfo *rel, switch (rel->rtekind) { case RTE_RELATION: + remove_restrictions_implied_by_constraints(root, rel, rte); if (rte->relkind == RELKIND_FOREIGN_TABLE) { /* Foreign table */ @@ -487,7 +489,10 @@ set_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, * we'll consider gathering partial paths for the parent appendrel.) */ if (rel->reloptkind == RELOPT_BASEREL) - generate_gather_paths(root, rel); + { + generate_gather_paths(root, rel, false); + generate_gather_paths(root, rel, true); + } /* * Allow a plugin to editorialize on the set of Paths for this base @@ -688,6 +693,7 @@ static void set_plain_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) { Relids required_outer; + Path *seq_path; /* * We don't support pushing join clauses into the quals of a seqscan, but @@ -696,15 +702,40 @@ set_plain_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) */ required_outer = rel->lateral_relids; - /* Consider sequential scan */ - add_path(rel, create_seqscan_path(root, rel, required_outer, 0)); + /* Consider sequential scan, both plain and grouped. */ + seq_path = create_seqscan_path(root, rel, required_outer, 0); + + /* Try to compute unique keys. */ + make_uniquekeys(root, seq_path); + + add_path(rel, seq_path, false); + + if (rel->gpi != NULL && rel->gpi->target != NULL && + required_outer == NULL) + { + /* + * Only AGG_HASHED is suitable here as it does not expect the input + + * set to be sorted. + */ + create_grouped_path(root, rel, seq_path, false, false, AGG_HASHED); + } - /* If appropriate, consider parallel sequential scan */ + /* If appropriate, consider parallel sequential scan (plain or grouped) */ if (rel->consider_parallel && required_outer == NULL) create_plain_partial_paths(root, rel); /* Consider index scans */ - create_index_paths(root, rel); + create_index_paths(root, rel, false); + if (rel->gpi != NULL) + { + /* + * TODO Instead of calling the whole clause-matching machinery twice + * (there should be no difference between plain and grouped paths from + * this point of view), consider returning a separate list of paths + * usable as grouped ones. + */ + create_index_paths(root, rel, true); + } /* Consider TID scans */ create_tidscan_paths(root, rel); @@ -718,6 +749,7 @@ static void create_plain_partial_paths(PlannerInfo *root, RelOptInfo *rel) { int parallel_workers; + Path *path; parallel_workers = compute_parallel_worker(rel, rel->pages, -1); @@ -726,7 +758,130 @@ create_plain_partial_paths(PlannerInfo *root, RelOptInfo *rel) return; /* Add an unordered partial path based on a parallel sequential scan. */ - add_partial_path(rel, create_seqscan_path(root, rel, NULL, parallel_workers)); + path = create_seqscan_path(root, rel, NULL, parallel_workers); + add_partial_path(rel, path, false); + + /* + * Do partial aggregation at base relation level if the relation is + * eligible for it. Only AGG_HASHED is suitable here as it does not expect + * the input set to be sorted. + */ + if (rel->gpi != NULL) + create_grouped_path(root, rel, path, false, true, AGG_HASHED); +} + +/* + * Apply partial aggregation to a subpath and add the AggPath to the + * appropriate pathlist. + * + * "precheck" tells whether the aggregation path should first be checked using + * add_path_precheck(). + * + * If "partial" is true, the resulting path is considered partial in terms of + * parallel execution. + * + * The path we create here shouldn't be parameterized because of supposedly + * high startup cost of aggregation (whether due to build of hash table for + * AGG_HASHED strategy or due to explicit sort for AGG_SORTED). + * + * XXX IndexPath as an input for AGG_SORTED seems to be an exception --- + * consider implementing parameterized AGG_SORTED unless the IndexPath is + * partial. + */ +void +create_grouped_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, + bool precheck, bool partial, AggStrategy aggstrategy) +{ + List *group_clauses = NIL; + List *group_exprs = NIL; + List *agg_exprs = NIL; + Path *agg_path; + + /* + * If the AggPath should be partial, the subpath must be too, and + * therefore the subpath is essentially parallel_safe. + */ + Assert(subpath->parallel_safe || !partial); + + /* + * Grouped path should never be parameterized, so we're not supposed to + * receive parameterized subpath. + */ + Assert(subpath->param_info == NULL || aggstrategy != AGG_HASHED); + + /* + * Non-var grouping expressions will eventually require projection using + * Result plan, but that does not work with SRFs. + */ + Assert(rel->gpi != NULL); + if (rel->gpi->group_exprs != NULL) + { + ListCell *lc; + + foreach(lc, rel->gpi->group_exprs->exprs) + { + GroupedVar *gvar = lfirst_node(GroupedVar, lc); + + if (IsA(gvar->gvexpr, FuncExpr)) + { + FuncExpr *fexpr = (FuncExpr *) gvar->gvexpr; + + if (fexpr->funcretset) + return; + } + } + } + + /* + * Note that "partial" in the following function names refers to 2-stage + * aggregation, not to parallel processing. + */ + if (aggstrategy == AGG_HASHED) + agg_path = (Path *) create_partial_agg_hashed_path(root, subpath, + true, + &group_clauses, + &group_exprs, + &agg_exprs, + subpath->rows, + partial); + else if (aggstrategy == AGG_SORTED) + agg_path = (Path *) create_partial_agg_sorted_path(root, subpath, + true, + &group_clauses, + &group_exprs, + &agg_exprs, + subpath->rows, + partial); + else + elog(ERROR, "unexpected strategy %d", aggstrategy); + + /* Add the grouped path to the list of grouped base paths. */ + if (agg_path != NULL) + { + if (precheck) + { + List *pathkeys; + + /* AGG_HASH is not supposed to generate sorted output. */ + pathkeys = aggstrategy == AGG_SORTED ? subpath->pathkeys : NIL; + + if (!partial && + !add_path_precheck(rel, agg_path->startup_cost, + agg_path->total_cost, pathkeys, NULL, + true)) + return; + + if (partial && + !add_partial_path_precheck(rel, agg_path->total_cost, pathkeys, + true)) + return; + } + + if (!partial) + add_path(rel, (Path *) agg_path, true); + else + add_partial_path(rel, (Path *) agg_path, true); + } } /* @@ -812,7 +967,7 @@ set_tablesample_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry * path = (Path *) create_material_path(rel, path); } - add_path(rel, path); + add_path(rel, path, false); /* For the moment, at least, there are no other paths to consider */ } @@ -824,11 +979,29 @@ set_tablesample_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry * static void set_foreign_size(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) { + Relids required_outer = rel->lateral_relids; + /* Mark rel with estimated output rows, width, etc */ - set_foreign_size_estimates(root, rel); + set_foreign_size_estimates(root, rel, false); + + /* + * Let FDW adjust the size estimates, if it can. + * + * For the grouped relation it only makes sense if set_foreign_pathlist is + * supposed to handle it, see below. + */ + rel->fdwroutine->GetForeignRelSize(root, rel, rte->relid, false); + if (rel->gpi != NULL && rel->gpi->target != NULL && + required_outer == NULL) + { + rel->fdwroutine->GetForeignRelSize(root, rel, rte->relid, true); - /* Let FDW adjust the size estimates, if it can */ - rel->fdwroutine->GetForeignRelSize(root, rel, rte->relid); + /* + * XXX Should cost_qual_eval be called separate so we don't repeat it + * here for essentially identical baserestrictinfo? + */ + set_foreign_size_estimates(root, rel, true); + } /* ... but do not let it set the rows estimate to zero */ rel->rows = clamp_row_est(rel->rows); @@ -841,8 +1014,18 @@ set_foreign_size(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) static void set_foreign_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) { + Relids required_outer = rel->lateral_relids; + /* Call the FDW's GetForeignPaths function to generate path(s) */ - rel->fdwroutine->GetForeignPaths(root, rel, rte->relid); + rel->fdwroutine->GetForeignPaths(root, rel, rte->relid, false); + + /* + * Create grouped paths if grouped target exists. Do nothing if outer + * relations are needed --- that could cause parameterized (and therefore + * repeated) grouping. + */ + if (rel->gpi != NULL && rel->gpi->target && required_outer == NULL) + rel->fdwroutine->GetForeignPaths(root, rel, rte->relid, true); } /* @@ -1130,12 +1313,44 @@ set_append_rel_size(PlannerInfo *root, RelOptInfo *rel, set_dummy_rel_pathlist(childrel); continue; } + remove_restrictions_implied_by_constraints(root, childrel, childRTE); /* CE failed, so finish copying/modifying join quals. */ childrel->joininfo = (List *) adjust_appendrel_attrs(root, (Node *) rel->joininfo, 1, &appinfo); + childrel->reltarget->exprs = (List *) + adjust_appendrel_attrs(root, + (Node *) rel->reltarget->exprs, + 1, &appinfo); + + /* + * Setup GroupedPathInfo for the child relation if the parent has + * some. + */ + if (rel->gpi != NULL) + build_chiid_rel_gpi(root, childrel, rel, 1, &appinfo); + + /* + * We have to make child entries in the EquivalenceClass data + * structures as well. This is needed either if the parent + * participates in some eclass joins (because we will want to consider + * inner-indexscan joins on the individual children) or if the parent + * has useful pathkeys (because we should try to build MergeAppend + * paths that produce those sort orderings). + */ + if (rel->has_eclass_joins || has_useful_pathkeys(root, rel)) + add_child_rel_equivalences(root, appinfo, rel, childrel); + childrel->has_eclass_joins = rel->has_eclass_joins; + + /* + * Note: we could compute appropriate attr_needed data for the child's + * variables, by transforming the parent's attr_needed through the + * translated_vars mapping. However, currently there's no need + * because attr_needed is only examined for base relations not + * otherrels. So we just leave the child's attr_needed empty. + */ /* * If parallelism is allowable for this query in general, see whether @@ -1332,6 +1547,8 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, bool subpaths_valid = true; List *partial_subpaths = NIL; bool partial_subpaths_valid = true; + List *grouped_subpaths = NIL; + bool grouped_subpaths_valid = true; List *all_child_pathkeys = NIL; List *all_child_outers = NIL; ListCell *l; @@ -1387,6 +1604,7 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, foreach(l, live_childrels) { RelOptInfo *childrel = lfirst(l); + List *childpaths; ListCell *lcp; /* @@ -1421,12 +1639,45 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, partial_subpaths_valid = false; /* + * For grouped paths, use only the unparameterized subpaths. + * + * XXX Consider if the parameterized subpaths should be processed + * below. It's probably not useful for sequential scans (due to + * repeated aggregation), but might be worthwhile for other child + * nodes. + */ + if (childrel->gpi != NULL && childrel->gpi->pathlist != NIL) + { + Path *path; + + path = (Path *) linitial(childrel->gpi->pathlist); + if (path->param_info == NULL) + grouped_subpaths = accumulate_append_subpath(grouped_subpaths, + path); + else + grouped_subpaths_valid = false; + } + else + grouped_subpaths_valid = false; + + + /* * Collect lists of all the available path orderings and * parameterizations for all the children. We use these as a * heuristic to indicate which sort orderings and parameterizations we * should build Append and MergeAppend paths for. */ - foreach(lcp, childrel->pathlist) + childpaths = childrel->pathlist; + + /* + * Extra orderings may be available for grouped paths, i.e. ordered by + * aggregate. (At least ForeignPath can generate these.) + */ + if (childrel->gpi != NULL && childrel->gpi->pathlist != NIL) + childpaths = list_concat(list_copy(childpaths), + childrel->gpi->pathlist); + + foreach(lcp, childpaths) { Path *childpath = (Path *) lfirst(lcp); List *childkeys = childpath->pathkeys; @@ -1492,7 +1743,8 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, */ if (subpaths_valid) add_path(rel, (Path *) create_append_path(rel, subpaths, NULL, 0, - partitioned_rels)); + partitioned_rels), + false); /* * Consider an append of partial unordered, unparameterized partial paths. @@ -1519,8 +1771,25 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, /* Generate a partial append path. */ appendpath = create_append_path(rel, partial_subpaths, NULL, - parallel_workers, partitioned_rels); - add_partial_path(rel, (Path *) appendpath); + parallel_workers, + partitioned_rels); + add_partial_path(rel, (Path *) appendpath, false); + } + + /* TODO Also partial grouped paths? */ + if (grouped_subpaths_valid) + { + Path *path; + + path = (Path *) create_append_path(rel, grouped_subpaths, NULL, 0, + partitioned_rels); + /* pathtarget will produce the grouped relation.. */ + path->pathtarget = rel->gpi->target; + + /* Try to compute unique keys. */ + make_uniquekeys(root, path); + + add_path(rel, path, true); } /* @@ -1530,7 +1799,11 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, if (subpaths_valid) generate_mergeappend_paths(root, rel, live_childrels, all_child_pathkeys, - partitioned_rels); + partitioned_rels, false); + if (grouped_subpaths_valid) + generate_mergeappend_paths(root, rel, live_childrels, + all_child_pathkeys, + partitioned_rels, true); /* * Build Append paths for each parameterization seen among the child rels. @@ -1573,7 +1846,8 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, if (subpaths_valid) add_path(rel, (Path *) create_append_path(rel, subpaths, required_outer, 0, - partitioned_rels)); + partitioned_rels), + false); } } @@ -1604,9 +1878,11 @@ static void generate_mergeappend_paths(PlannerInfo *root, RelOptInfo *rel, List *live_childrels, List *all_child_pathkeys, - List *partitioned_rels) + List *partitioned_rels, + bool grouped) { ListCell *lcp; + PathTarget *target = NULL; foreach(lcp, all_child_pathkeys) { @@ -1615,23 +1891,32 @@ generate_mergeappend_paths(PlannerInfo *root, RelOptInfo *rel, List *total_subpaths = NIL; bool startup_neq_total = false; ListCell *lcr; + Path *path; /* Select the child paths for this ordering... */ foreach(lcr, live_childrels) { RelOptInfo *childrel = (RelOptInfo *) lfirst(lcr); + List *pathlist; Path *cheapest_startup, *cheapest_total; + if (grouped && + (childrel->gpi == NULL || childrel->gpi->pathlist == NIL)) + return; + + pathlist = !grouped ? childrel->pathlist : + childrel->gpi->pathlist; + /* Locate the right paths, if they are available. */ cheapest_startup = - get_cheapest_path_for_pathkeys(childrel->pathlist, + get_cheapest_path_for_pathkeys(pathlist, pathkeys, NULL, STARTUP_COST, false); cheapest_total = - get_cheapest_path_for_pathkeys(childrel->pathlist, + get_cheapest_path_for_pathkeys(pathlist, pathkeys, NULL, TOTAL_COST, @@ -1663,20 +1948,51 @@ generate_mergeappend_paths(PlannerInfo *root, RelOptInfo *rel, accumulate_append_subpath(total_subpaths, cheapest_total); } + /* + * Special target is needed if the path output should be grouped. + */ + if (grouped) + { + Assert(rel->gpi != NULL); + target = rel->gpi->target; + } + /* ... and build the MergeAppend paths */ - add_path(rel, (Path *) create_merge_append_path(root, - rel, - startup_subpaths, - pathkeys, - NULL, - partitioned_rels)); + path = (Path *) create_merge_append_path(root, + rel, + target, + startup_subpaths, + pathkeys, + NULL, + partitioned_rels); + + /* Try to compute unique keys. */ + make_uniquekeys(root, path); + + /* pathtarget will produce the grouped relation.. */ + if (grouped) + { + Assert(rel->gpi != NULL && rel->gpi->target != NULL); + path->pathtarget = rel->gpi->target; + } + + add_path(rel, path, grouped); + if (startup_neq_total) - add_path(rel, (Path *) create_merge_append_path(root, - rel, - total_subpaths, - pathkeys, - NULL, - partitioned_rels)); + { + path = (Path *) create_merge_append_path(root, + rel, + target, + total_subpaths, + pathkeys, + NULL, + partitioned_rels); + make_uniquekeys(root, path); + if (grouped) + path->pathtarget = rel->gpi->target; + add_path(rel, path, grouped); + } + } } @@ -1809,7 +2125,7 @@ set_dummy_rel_pathlist(RelOptInfo *rel) rel->pathlist = NIL; rel->partial_pathlist = NIL; - add_path(rel, (Path *) create_append_path(rel, NIL, NULL, 0, NIL)); + add_path(rel, (Path *) create_append_path(rel, NIL, NULL, 0, NIL), false); /* * We set the cheapest path immediately, to ensure that IS_DUMMY_REL() @@ -2023,7 +2339,7 @@ set_subquery_pathlist(PlannerInfo *root, RelOptInfo *rel, /* Generate outer path using this subpath */ add_path(rel, (Path *) create_subqueryscan_path(root, rel, subpath, - pathkeys, required_outer)); + pathkeys, required_outer), false); } } @@ -2092,7 +2408,7 @@ set_function_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) /* Generate appropriate path */ add_path(rel, create_functionscan_path(root, rel, - pathkeys, required_outer)); + pathkeys, required_outer), false); } /* @@ -2112,7 +2428,7 @@ set_values_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) required_outer = rel->lateral_relids; /* Generate appropriate path */ - add_path(rel, create_valuesscan_path(root, rel, required_outer)); + add_path(rel, create_valuesscan_path(root, rel, required_outer), false); } /* @@ -2133,7 +2449,7 @@ set_tablefunc_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) /* Generate appropriate path */ add_path(rel, create_tablefuncscan_path(root, rel, - required_outer)); + required_outer), false); } /* @@ -2199,7 +2515,7 @@ set_cte_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) required_outer = rel->lateral_relids; /* Generate appropriate path */ - add_path(rel, create_ctescan_path(root, rel, required_outer)); + add_path(rel, create_ctescan_path(root, rel, required_outer), false); } /* @@ -2226,7 +2542,8 @@ set_namedtuplestore_pathlist(PlannerInfo *root, RelOptInfo *rel, required_outer = rel->lateral_relids; /* Generate appropriate path */ - add_path(rel, create_namedtuplestorescan_path(root, rel, required_outer)); + add_path(rel, create_namedtuplestorescan_path(root, rel, required_outer), + false); /* Select cheapest path (pretty easy in this case...) */ set_cheapest(rel); @@ -2279,7 +2596,8 @@ set_worktable_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) required_outer = rel->lateral_relids; /* Generate appropriate path */ - add_path(rel, create_worktablescan_path(root, rel, required_outer)); + add_path(rel, create_worktablescan_path(root, rel, required_outer), + false); } /* @@ -2292,14 +2610,21 @@ set_worktable_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) * path that some GatherPath or GatherMergePath has a reference to.) */ void -generate_gather_paths(PlannerInfo *root, RelOptInfo *rel) +generate_gather_paths(PlannerInfo *root, RelOptInfo *rel, bool grouped) { Path *cheapest_partial_path; Path *simple_gather_path; + List *pathlist = NIL; + PathTarget *partial_target; ListCell *lc; + if (!grouped) + pathlist = rel->partial_pathlist; + else if (rel->gpi != NULL) + pathlist = rel->gpi->partial_pathlist; + /* If there are no partial paths, there's nothing to do here. */ - if (rel->partial_pathlist == NIL) + if (pathlist == NIL) return; /* @@ -2307,17 +2632,23 @@ generate_gather_paths(PlannerInfo *root, RelOptInfo *rel) * path of interest: the cheapest one. That will be the one at the front * of partial_pathlist because of the way add_partial_path works. */ - cheapest_partial_path = linitial(rel->partial_pathlist); + cheapest_partial_path = linitial(pathlist); + + if (!grouped) + partial_target = rel->reltarget; + else if (rel->gpi != NULL && rel->gpi->target != NULL) + partial_target = rel->gpi->target; + simple_gather_path = (Path *) - create_gather_path(root, rel, cheapest_partial_path, rel->reltarget, + create_gather_path(root, rel, cheapest_partial_path, partial_target, NULL, NULL); - add_path(rel, simple_gather_path); + add_path(rel, simple_gather_path, grouped); /* * For each useful ordering, we can consider an order-preserving Gather * Merge. */ - foreach(lc, rel->partial_pathlist) + foreach(lc, pathlist) { Path *subpath = (Path *) lfirst(lc); GatherMergePath *path; @@ -2325,9 +2656,9 @@ generate_gather_paths(PlannerInfo *root, RelOptInfo *rel) if (subpath->pathkeys == NIL) continue; - path = create_gather_merge_path(root, rel, subpath, rel->reltarget, + path = create_gather_merge_path(root, rel, subpath, partial_target, subpath->pathkeys, NULL, NULL); - add_path(rel, &path->path); + add_path(rel, &path->path, grouped); } } @@ -2499,7 +2830,8 @@ standard_join_search(PlannerInfo *root, int levels_needed, List *initial_rels) generate_partition_wise_join_paths(root, rel); /* Create GatherPaths for any useful partial paths for rel */ - generate_gather_paths(root, rel); + generate_gather_paths(root, rel, false); + generate_gather_paths(root, rel, true); /* Find and save the cheapest paths for this rel */ set_cheapest(rel); @@ -3152,7 +3484,8 @@ create_partial_bitmap_paths(PlannerInfo *root, RelOptInfo *rel, return; add_partial_path(rel, (Path *) create_bitmap_heap_path(root, rel, - bitmapqual, rel->lateral_relids, 1.0, parallel_workers)); + bitmapqual, rel->lateral_relids, 1.0, parallel_workers), + false); } /* diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index d11bf19..c580e49 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -91,6 +91,7 @@ #include "optimizer/plancat.h" #include "optimizer/planmain.h" #include "optimizer/restrictinfo.h" +#include "optimizer/var.h" #include "parser/parsetree.h" #include "utils/lsyscache.h" #include "utils/selfuncs.h" @@ -128,6 +129,7 @@ bool enable_mergejoin = true; bool enable_hashjoin = true; bool enable_gathermerge = true; bool enable_partition_wise_join = false; +bool enable_agg_pushdown = false; typedef struct { @@ -160,7 +162,8 @@ static Selectivity get_foreign_key_join_selectivity(PlannerInfo *root, Relids inner_relids, SpecialJoinInfo *sjinfo, List **restrictlist); -static void set_rel_width(PlannerInfo *root, RelOptInfo *rel); +static void set_rel_width(PlannerInfo *root, RelOptInfo *rel, + PathTarget *reltarget); static double relation_byte_size(double tuples, int width); static double page_size(double tuples, int width); static double get_parallel_divisor(Path *path); @@ -4100,7 +4103,14 @@ set_baserel_size_estimates(PlannerInfo *root, RelOptInfo *rel) cost_qual_eval(&rel->baserestrictcost, rel->baserestrictinfo, root); - set_rel_width(root, rel); + set_rel_width(root, rel, rel->reltarget); + + /* + * If the relation can produce grouped path, estimate width and costs of + * the corresponding target. + */ + if (rel->gpi) + set_rel_width(root, rel, rel->gpi->target); } /* @@ -4842,7 +4852,7 @@ set_namedtuplestore_size_estimates(PlannerInfo *root, RelOptInfo *rel) * already. */ void -set_foreign_size_estimates(PlannerInfo *root, RelOptInfo *rel) +set_foreign_size_estimates(PlannerInfo *root, RelOptInfo *rel, bool grouped) { /* Should only be applied to base relations */ Assert(rel->relid > 0); @@ -4851,7 +4861,19 @@ set_foreign_size_estimates(PlannerInfo *root, RelOptInfo *rel) cost_qual_eval(&rel->baserestrictcost, rel->baserestrictinfo, root); - set_rel_width(root, rel); + if (!grouped) + set_rel_width(root, rel, rel->reltarget); + else + { + Assert(rel->gpi != NULL); + + /* + * If the relation can produce grouped path, estimate width and costs + * of the corresponding target. + */ + if (rel->gpi->target) + set_rel_width(root, rel, rel->gpi->target); + } } @@ -4877,7 +4899,7 @@ set_foreign_size_estimates(PlannerInfo *root, RelOptInfo *rel) * building join relations or post-scan/join pathtargets. */ static void -set_rel_width(PlannerInfo *root, RelOptInfo *rel) +set_rel_width(PlannerInfo *root, RelOptInfo *rel, PathTarget *reltarget) { Oid reloid = planner_rt_fetch(rel->relid, root)->relid; int32 tuple_width = 0; @@ -4885,10 +4907,10 @@ set_rel_width(PlannerInfo *root, RelOptInfo *rel) ListCell *lc; /* Vars are assumed to have cost zero, but other exprs do not */ - rel->reltarget->cost.startup = 0; - rel->reltarget->cost.per_tuple = 0; + reltarget->cost.startup = 0; + reltarget->cost.per_tuple = 0; - foreach(lc, rel->reltarget->exprs) + foreach(lc, reltarget->exprs) { Node *node = (Node *) lfirst(lc); @@ -4963,8 +4985,19 @@ set_rel_width(PlannerInfo *root, RelOptInfo *rel) tuple_width += phinfo->ph_width; cost_qual_eval_node(&cost, (Node *) phv->phexpr, root); - rel->reltarget->cost.startup += cost.startup; - rel->reltarget->cost.per_tuple += cost.per_tuple; + reltarget->cost.startup += cost.startup; + reltarget->cost.per_tuple += cost.per_tuple; + } + else if (IsA(node, GroupedVar)) + { + GroupedVar *gvar = (GroupedVar *) node; + GroupedVarInfo *gvinfo = find_grouped_var_info(root, gvar); + QualCost cost; + + tuple_width += gvinfo->gv_width; + cost_qual_eval_node(&cost, (Node *) gvar->gvexpr, root); + reltarget->cost.startup += cost.startup; + reltarget->cost.per_tuple += cost.per_tuple; } else { @@ -4981,8 +5014,8 @@ set_rel_width(PlannerInfo *root, RelOptInfo *rel) tuple_width += item_width; /* Not entirely clear if we need to account for cost, but do so */ cost_qual_eval_node(&cost, node, root); - rel->reltarget->cost.startup += cost.startup; - rel->reltarget->cost.per_tuple += cost.per_tuple; + reltarget->cost.startup += cost.startup; + reltarget->cost.per_tuple += cost.per_tuple; } } @@ -5019,7 +5052,7 @@ set_rel_width(PlannerInfo *root, RelOptInfo *rel) } Assert(tuple_width >= 0); - rel->reltarget->width = tuple_width; + reltarget->width = tuple_width; } /* diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 45a6889..772b780 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -65,6 +65,19 @@ static bool reconsider_outer_join_clause(PlannerInfo *root, static bool reconsider_full_join_clause(PlannerInfo *root, RestrictInfo *rinfo); +typedef struct translate_expr_context +{ + Var **keys; /* Dictionary keys. */ + Var **values; /* Dictionary values */ + int nitems; /* Number of dictionary items. */ + Relids *gv_eval_at_p; /* See GroupedVarInfo. */ + Relids relids; /* Translate into these relids. */ +} translate_expr_context; + +static Node *translate_expression_to_rels_mutator(Node *node, + translate_expr_context * context); +static int var_dictionary_comparator(const void *a, const void *b); + /* * process_equivalence @@ -2510,3 +2523,322 @@ is_redundant_derived_clause(RestrictInfo *rinfo, List *clauselist) return false; } + +/* + * translate_expression_to_rels + * + * If the appropriate equivalence classes exist, replace vars in + * gvi->gvexpr with vars whose varno is contained in relids. + */ +GroupedVarInfo * +translate_expression_to_rels(PlannerInfo *root, GroupedVarInfo *gvi, + Relids relids) +{ + List *vars; + ListCell *l1; + int i, + j; + int nkeys, + nkeys_resolved; + Var **keys, + **values, + **keys_tmp; + Var *key, + *key_prev; + translate_expr_context context; + GroupedVarInfo *result; + + /* Can't do anything w/o equivalence classes. */ + if (root->eq_classes == NIL) + return NULL; + + /* + * Before actually trying to modify the expression tree, find out if all + * vars can be translated. + */ + vars = pull_var_clause((Node *) gvi->gvexpr, 0); + + /* No vars to translate? */ + if (vars == NIL) + return NULL; + + /* + * Search for individual replacement vars as well as the actual expression + * translation will be more efficient if we use a dictionary with the keys + * (i.e. the "source vars") unique and sorted. + */ + nkeys = list_length(vars); + keys = (Var **) palloc(nkeys * sizeof(Var *)); + i = 0; + foreach(l1, vars) + { + key = lfirst_node(Var, l1); + keys[i++] = key; + } + + /* + * Sort the keys by varno. varattno decides where varnos are equal. + */ + pg_qsort(keys, nkeys, sizeof(Var *), var_dictionary_comparator); + + /* + * Pick unique values and get rid of the vars that need no translation. + */ + keys_tmp = (Var **) palloc(nkeys * sizeof(Var *)); + key_prev = NULL; + j = 0; + for (i = 0; i < nkeys; i++) + { + key = keys[i]; + + if ((key_prev == NULL || (key->varno != key_prev->varno && + key->varattno != key_prev->varattno)) && + !bms_is_member(key->varno, relids)) + keys_tmp[j++] = key; + + key_prev = key; + } + pfree(keys); + keys = keys_tmp; + nkeys = j; + + /* + * Is there actually nothing to be translated? + */ + if (nkeys == 0) + { + pfree(keys); + return NULL; + } + + nkeys_resolved = 0; + + /* + * Find the replacement vars. + */ + values = (Var **) palloc0(nkeys * sizeof(Var *)); + foreach(l1, root->eq_classes) + { + EquivalenceClass *ec = lfirst_node(EquivalenceClass, l1); + Relids ec_var_relids; + Var **ec_vars; + int ec_nvars; + ListCell *l2; + + /* TODO Re-check if any other EC kind should be ignored. */ + if (ec->ec_has_volatile || ec->ec_below_outer_join || ec->ec_broken) + continue; + + /* Single-element EC can hardly help in translations. */ + if (list_length(ec->ec_members) == 1) + continue; + + /* + * Collect all vars of this EC and their varnos. + * + * ec->ec_relids does not help because we're only interested in a + * subset of EC members. + */ + ec_vars = (Var **) palloc(list_length(ec->ec_members) * sizeof(Var *)); + ec_nvars = 0; + ec_var_relids = NULL; + foreach(l2, ec->ec_members) + { + EquivalenceMember *em = lfirst_node(EquivalenceMember, l2); + Var *ec_var; + + if (!IsA(em->em_expr, Var)) + continue; + + ec_var = castNode(Var, em->em_expr); + ec_vars[ec_nvars++] = ec_var; + ec_var_relids = bms_add_member(ec_var_relids, ec_var->varno); + } + + /* + * At least two vars are needed so that the EC is usable for + * translation. + */ + if (ec_nvars <= 1) + { + pfree(ec_vars); + bms_free(ec_var_relids); + continue; + } + + /* + * Now check where this EC can help. + */ + for (i = 0; i < nkeys; i++) + { + Relids ec_rest; + bool relids_ok, + key_found; + Var *key = keys[i]; + Var *value = values[i]; + + /* Skip this item if it's already resolved. */ + if (value != NULL) + continue; + + /* + * Can't translate if the EC does not mention key->varno. + */ + if (!bms_is_member(key->varno, ec_var_relids)) + continue; + + /* + * Besides key, at least one EC member must belong to the relation + * we're translating our expression to. + */ + ec_rest = bms_copy(ec_var_relids); + ec_rest = bms_del_member(ec_rest, key->varno); + relids_ok = bms_overlap(ec_rest, relids); + bms_free(ec_rest); + if (!relids_ok) + continue; + + /* + * The preliminary checks passed, so try to find the exact vars. + */ + key_found = false; + for (j = 0; j < ec_nvars; j++) + { + Var *ec_var = ec_vars[j]; + + if (!key_found && key->varno == ec_var->varno && + key->varattno == ec_var->varattno) + key_found = true; + + /* + * If relids contains multiple members, it shouldn't matter to + * which one we translate our key. Simply use the first one. + * + * XXX Shouldn't ec_var be copied? + */ + if (value == NULL && bms_is_member(ec_var->varno, relids)) + value = ec_var; + + if (key_found && value != NULL) + break; + } + + if (key_found && value != NULL) + { + values[i] = value; + nkeys_resolved++; + + if (nkeys_resolved == nkeys) + break; + } + } + + pfree(ec_vars); + bms_free(ec_var_relids); + + /* Don't need to check the remaining ECs? */ + if (nkeys_resolved == nkeys) + break; + } + + /* Couldn't compose usable dictionary? */ + if (nkeys_resolved < nkeys) + { + pfree(keys); + pfree(values); + return NULL; + } + + result = makeNode(GroupedVarInfo); + memcpy(result, gvi, sizeof(GroupedVarInfo)); + + /* + * translate_expression_to_rels_mutator updates gv_eval_at. + */ + result->gv_eval_at = bms_copy(result->gv_eval_at); + + /* The dictionary is ready, so perform the translation. */ + context.keys = keys; + context.values = values; + context.nitems = nkeys; + context.gv_eval_at_p = &result->gv_eval_at; + context.relids = relids; + result->gvexpr = (Expr *) + translate_expression_to_rels_mutator((Node *) gvi->gvexpr, &context); + + pfree(keys); + pfree(values); + return result; +} + +static Node * +translate_expression_to_rels_mutator(Node *node, + translate_expr_context * context) +{ + if (node == NULL) + return NULL; + + if (IsA(node, Var)) + { + Var *var = castNode(Var, node); + Var **key_p; + Var *value; + int index; + + /* + * Simply return the existing variable if already belongs to the + * relation we're adjusting the expression to. + */ + if (bms_is_member(var->varno, context->relids)) + return (Node *) var; + + key_p = bsearch(&var, context->keys, context->nitems, sizeof(Var *), + var_dictionary_comparator); + + /* We shouldn't have omitted any var from the dictionary. */ + Assert(key_p != NULL); + + index = key_p - context->keys; + Assert(index >= 0 && index < context->nitems); + value = context->values[index]; + + /* All values should be present in the dictionary. */ + Assert(value != NULL); + + /* Update gv_eval_at accordingly. */ + bms_del_member(*context->gv_eval_at_p, var->varno); + *context->gv_eval_at_p = bms_add_member(*context->gv_eval_at_p, + value->varno); + + return (Node *) value; + } + + return expression_tree_mutator(node, translate_expression_to_rels_mutator, + (void *) context); +} + +static int +var_dictionary_comparator(const void *a, const void *b) +{ + Var **var1_p, + **var2_p; + Var *var1, + *var2; + + var1_p = (Var **) a; + var1 = castNode(Var, *var1_p); + var2_p = (Var **) b; + var2 = castNode(Var, *var2_p); + + if (var1->varno < var2->varno) + return -1; + else if (var1->varno > var2->varno) + return 1; + + if (var1->varattno < var2->varattno) + return -1; + else if (var1->varattno > var2->varattno) + return 1; + + return 0; +} diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index 18f6baf..fb41c82 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -32,6 +32,7 @@ #include "optimizer/predtest.h" #include "optimizer/prep.h" #include "optimizer/restrictinfo.h" +#include "optimizer/tlist.h" #include "optimizer/var.h" #include "utils/builtins.h" #include "utils/bytea.h" @@ -107,13 +108,14 @@ static bool eclass_already_used(EquivalenceClass *parent_ec, Relids oldrelids, static bool bms_equal_any(Relids relids, List *relids_list); static void get_index_paths(PlannerInfo *root, RelOptInfo *rel, IndexOptInfo *index, IndexClauseSet *clauses, - List **bitindexpaths); + List **bitindexpaths, bool grouped); static List *build_index_paths(PlannerInfo *root, RelOptInfo *rel, IndexOptInfo *index, IndexClauseSet *clauses, bool useful_predicate, ScanTypeControl scantype, bool *skip_nonnative_saop, - bool *skip_lower_saop); + bool *skip_lower_saop, + bool grouped); static List *build_paths_for_OR(PlannerInfo *root, RelOptInfo *rel, List *clauses, List *other_clauses); static List *generate_bitmap_or_paths(PlannerInfo *root, RelOptInfo *rel, @@ -229,7 +231,7 @@ static Const *string_to_const(const char *str, Oid datatype); * as meaning "unparameterized so far as the indexquals are concerned". */ void -create_index_paths(PlannerInfo *root, RelOptInfo *rel) +create_index_paths(PlannerInfo *root, RelOptInfo *rel, bool grouped) { List *indexpaths; List *bitindexpaths; @@ -274,8 +276,8 @@ create_index_paths(PlannerInfo *root, RelOptInfo *rel) * non-parameterized paths. Plain paths go directly to add_path(), * bitmap paths are added to bitindexpaths to be handled below. */ - get_index_paths(root, rel, index, &rclauseset, - &bitindexpaths); + get_index_paths(root, rel, index, &rclauseset, &bitindexpaths, + grouped); /* * Identify the join clauses that can match the index. For the moment @@ -338,7 +340,7 @@ create_index_paths(PlannerInfo *root, RelOptInfo *rel) bitmapqual = choose_bitmap_and(root, rel, bitindexpaths); bpath = create_bitmap_heap_path(root, rel, bitmapqual, rel->lateral_relids, 1.0, 0); - add_path(rel, (Path *) bpath); + add_path(rel, (Path *) bpath, false); /* create a partial bitmap heap path */ if (rel->consider_parallel && rel->lateral_relids == NULL) @@ -415,7 +417,7 @@ create_index_paths(PlannerInfo *root, RelOptInfo *rel) loop_count = get_loop_count(root, rel->relid, required_outer); bpath = create_bitmap_heap_path(root, rel, bitmapqual, required_outer, loop_count, 0); - add_path(rel, (Path *) bpath); + add_path(rel, (Path *) bpath, false); } } } @@ -667,7 +669,7 @@ get_join_index_paths(PlannerInfo *root, RelOptInfo *rel, Assert(clauseset.nonempty); /* Build index path(s) using the collected set of clauses */ - get_index_paths(root, rel, index, &clauseset, bitindexpaths); + get_index_paths(root, rel, index, &clauseset, bitindexpaths, false); /* * Remember we considered paths for this set of relids. We use lcons not @@ -736,7 +738,7 @@ bms_equal_any(Relids relids, List *relids_list) static void get_index_paths(PlannerInfo *root, RelOptInfo *rel, IndexOptInfo *index, IndexClauseSet *clauses, - List **bitindexpaths) + List **bitindexpaths, bool grouped) { List *indexpaths; bool skip_nonnative_saop = false; @@ -754,7 +756,7 @@ get_index_paths(PlannerInfo *root, RelOptInfo *rel, index->predOK, ST_ANYSCAN, &skip_nonnative_saop, - &skip_lower_saop); + &skip_lower_saop, grouped); /* * If we skipped any lower-order ScalarArrayOpExprs on an index with an AM @@ -769,7 +771,7 @@ get_index_paths(PlannerInfo *root, RelOptInfo *rel, index->predOK, ST_ANYSCAN, &skip_nonnative_saop, - NULL)); + NULL, grouped)); } /* @@ -789,9 +791,18 @@ get_index_paths(PlannerInfo *root, RelOptInfo *rel, IndexPath *ipath = (IndexPath *) lfirst(lc); if (index->amhasgettuple) - add_path(rel, (Path *) ipath); + { + /* + * In case grouping may be pushed down, try to identify unique + * keys to possibly avoid final aggregation. + */ + if (root->grouped_var_list != NIL) + make_uniquekeys(root, (Path *) ipath); - if (index->amhasgetbitmap && + add_path(rel, (Path *) ipath, grouped); + } + + if (!grouped && index->amhasgetbitmap && (ipath->path.pathkeys == NIL || ipath->indexselectivity < 1.0)) *bitindexpaths = lappend(*bitindexpaths, ipath); @@ -802,14 +813,15 @@ get_index_paths(PlannerInfo *root, RelOptInfo *rel, * natively, generate bitmap scan paths relying on executor-managed * ScalarArrayOpExpr. */ - if (skip_nonnative_saop) + if (!grouped && skip_nonnative_saop) { indexpaths = build_index_paths(root, rel, index, clauses, false, ST_BITMAPSCAN, NULL, - NULL); + NULL, + false); *bitindexpaths = list_concat(*bitindexpaths, indexpaths); } } @@ -861,7 +873,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, bool useful_predicate, ScanTypeControl scantype, bool *skip_nonnative_saop, - bool *skip_lower_saop) + bool *skip_lower_saop, bool grouped) { List *result = NIL; IndexPath *ipath; @@ -878,6 +890,12 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, bool index_is_ordered; bool index_only_scan; int indexcol; + bool can_agg_sorted; + List *group_clauses, + *group_exprs, + *agg_exprs; + AggPath *agg_path; + double agg_input_rows; /* * Check that index supports the desired scan type(s) @@ -891,6 +909,9 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, case ST_BITMAPSCAN: if (!index->amhasgetbitmap) return NIL; + + if (grouped) + return NIL; break; case ST_ANYSCAN: /* either or both are OK */ @@ -1032,6 +1053,10 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, * later merging or final output ordering, OR the index has a useful * predicate, OR an index-only scan is possible. */ + can_agg_sorted = true; + group_clauses = NIL; + group_exprs = NIL; + agg_exprs = NIL; if (index_clauses != NIL || useful_pathkeys != NIL || useful_predicate || index_only_scan) { @@ -1048,7 +1073,26 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, outer_relids, loop_count, false); - result = lappend(result, ipath); + if (!grouped) + result = lappend(result, ipath); + else if (rel->gpi != NULL && rel->gpi->target != NULL) + { + /* TODO Double-check if this is the correct input value. */ + agg_input_rows = rel->rows * ipath->indexselectivity; + + agg_path = create_partial_agg_sorted_path(root, (Path *) ipath, + true, + &group_clauses, + &group_exprs, + &agg_exprs, + agg_input_rows, + false); + + if (agg_path != NULL) + result = lappend(result, agg_path); + else + can_agg_sorted = false; + } /* * If appropriate, consider parallel index scan. We don't allow @@ -1077,7 +1121,33 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, * parallel workers, just free it. */ if (ipath->path.parallel_workers > 0) - add_partial_path(rel, (Path *) ipath); + { + if (!grouped) + add_partial_path(rel, (Path *) ipath, grouped); + else if (can_agg_sorted && outer_relids == NULL && + rel->gpi != NULL && rel->gpi->target != NULL) + { + /* TODO Double-check if this is the correct input value. */ + agg_input_rows = rel->rows * ipath->indexselectivity; + + agg_path = create_partial_agg_sorted_path(root, + (Path *) ipath, + false, + &group_clauses, + &group_exprs, + &agg_exprs, + agg_input_rows, + true); + + /* + * If create_agg_sorted_path succeeded once, it should + * always do. + */ + Assert(agg_path != NULL); + + add_partial_path(rel, (Path *) agg_path, grouped); + } + } else pfree(ipath); } @@ -1105,7 +1175,27 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, outer_relids, loop_count, false); - result = lappend(result, ipath); + + if (!grouped) + result = lappend(result, ipath); + else if (can_agg_sorted && + rel->gpi != NULL && rel->gpi->target != NULL) + { + /* TODO Double-check if this is the correct input value. */ + agg_input_rows = rel->rows * ipath->indexselectivity; + + agg_path = create_partial_agg_sorted_path(root, + (Path *) ipath, + true, + &group_clauses, + &group_exprs, + &agg_exprs, + agg_input_rows, + false); + + Assert(agg_path != NULL); + result = lappend(result, agg_path); + } /* If appropriate, consider parallel index scan */ if (index->amcanparallel && @@ -1129,7 +1219,30 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, * using parallel workers, just free it. */ if (ipath->path.parallel_workers > 0) - add_partial_path(rel, (Path *) ipath); + { + if (!grouped) + add_partial_path(rel, (Path *) ipath, grouped); + else if (can_agg_sorted && outer_relids == NULL && + rel->gpi != NULL && rel->gpi->target != NULL) + { + /* + * TODO Double-check if this is the correct input + * value. + */ + agg_input_rows = rel->rows * ipath->indexselectivity; + + agg_path = create_partial_agg_sorted_path(root, + (Path *) ipath, + false, + &group_clauses, + &group_exprs, + &agg_exprs, + agg_input_rows, + true); + Assert(agg_path != NULL); + add_partial_path(rel, (Path *) agg_path, grouped); + } + } else pfree(ipath); } @@ -1244,7 +1357,8 @@ build_paths_for_OR(PlannerInfo *root, RelOptInfo *rel, useful_predicate, ST_BITMAPSCAN, NULL, - NULL); + NULL, + false); result = list_concat(result, indexpaths); } diff --git a/src/backend/optimizer/path/joinpath.c b/src/backend/optimizer/path/joinpath.c index 02a6302..aa3362b 100644 --- a/src/backend/optimizer/path/joinpath.c +++ b/src/backend/optimizer/path/joinpath.c @@ -22,6 +22,7 @@ #include "optimizer/pathnode.h" #include "optimizer/paths.h" #include "optimizer/planmain.h" +#include "optimizer/tlist.h" /* Hook for plugins to get control in add_paths_to_joinrel() */ set_join_pathlist_hook_type set_join_pathlist_hook = NULL; @@ -39,6 +40,10 @@ set_join_pathlist_hook_type set_join_pathlist_hook = NULL; #define PATH_PARAM_BY_REL(path, rel) \ (PATH_PARAM_BY_REL_SELF(path, rel) || PATH_PARAM_BY_PARENT(path, rel)) +#define REL_HAS_GROUPED_PATHS(rel) ((rel)->gpi && (rel)->gpi->pathlist) +#define REL_HAS_PARTIAL_GROUPED_PATHS(rel) \ + ((rel)->gpi && (rel)->gpi->partial_pathlist) + static void try_partial_mergejoin_path(PlannerInfo *root, RelOptInfo *joinrel, Path *outer_path, @@ -48,10 +53,21 @@ static void try_partial_mergejoin_path(PlannerInfo *root, List *outersortkeys, List *innersortkeys, JoinType jointype, - JoinPathExtraData *extra); + JoinPathExtraData *extra, + bool grouped, + bool do_aggregate); static void sort_inner_and_outer(PlannerInfo *root, RelOptInfo *joinrel, RelOptInfo *outerrel, RelOptInfo *innerrel, JoinType jointype, JoinPathExtraData *extra); +static void sort_inner_and_outer_common(PlannerInfo *root, + RelOptInfo *joinrel, + RelOptInfo *outerrel, + RelOptInfo *innerrel, + JoinType jointype, + JoinPathExtraData *extra, + bool grouped_outer, + bool grouped_inner, + bool do_aggregate); static void match_unsorted_outer(PlannerInfo *root, RelOptInfo *joinrel, RelOptInfo *outerrel, RelOptInfo *innerrel, JoinType jointype, JoinPathExtraData *extra); @@ -60,14 +76,17 @@ static void consider_parallel_nestloop(PlannerInfo *root, RelOptInfo *outerrel, RelOptInfo *innerrel, JoinType jointype, - JoinPathExtraData *extra); + JoinPathExtraData *extra, + bool grouped, bool do_aggregate); static void consider_parallel_mergejoin(PlannerInfo *root, RelOptInfo *joinrel, RelOptInfo *outerrel, RelOptInfo *innerrel, JoinType jointype, JoinPathExtraData *extra, - Path *inner_cheapest_total); + Path *inner_cheapest_total, + bool grouped_outer, + bool do_aggregate); static void hash_inner_and_outer(PlannerInfo *root, RelOptInfo *joinrel, RelOptInfo *outerrel, RelOptInfo *innerrel, JoinType jointype, JoinPathExtraData *extra); @@ -87,7 +106,10 @@ static void generate_mergejoin_paths(PlannerInfo *root, bool useallclauses, Path *inner_cheapest_total, List *merge_pathkeys, - bool is_partial); + bool is_partial, + bool grouped_outer, + bool grouped_inner, + bool do_aggregate); /* @@ -302,8 +324,8 @@ add_paths_to_joinrel(PlannerInfo *root, * joins, because there may be no other alternative. */ if (enable_hashjoin || jointype == JOIN_FULL) - hash_inner_and_outer(root, joinrel, outerrel, innerrel, - jointype, &extra); + hash_inner_and_outer(root, joinrel, outerrel, innerrel, jointype, + &extra); /* * 5. If inner and outer relations are foreign tables (or joins) belonging @@ -312,9 +334,15 @@ add_paths_to_joinrel(PlannerInfo *root, */ if (joinrel->fdwroutine && joinrel->fdwroutine->GetForeignJoinPaths) + { joinrel->fdwroutine->GetForeignJoinPaths(root, joinrel, outerrel, innerrel, - jointype, &extra); + jointype, &extra, false); + if (joinrel->gpi != NULL && joinrel->gpi->target != NULL) + joinrel->fdwroutine->GetForeignJoinPaths(root, joinrel, + outerrel, innerrel, + jointype, &extra, true); + } /* * 6. Finally, give extensions a chance to manipulate the path list. @@ -364,7 +392,9 @@ try_nestloop_path(PlannerInfo *root, Path *inner_path, List *pathkeys, JoinType jointype, - JoinPathExtraData *extra) + JoinPathExtraData *extra, + bool grouped, + bool do_aggregate) { Relids required_outer; JoinCostWorkspace workspace; @@ -374,6 +404,20 @@ try_nestloop_path(PlannerInfo *root, Relids outerrelids; Relids inner_paramrels = PATH_REQ_OUTER(inner_path); Relids outer_paramrels = PATH_REQ_OUTER(outer_path); + Path *join_path; + PathTarget *join_target; + + /* Caller should not request aggregation w/o grouped output. */ + Assert(!do_aggregate || grouped); + + /* GroupedPathInfo is necessary for us to produce a grouped set. */ + Assert(joinrel->gpi || !grouped); + + /* + * Aggregation target is necessary to produce grouped join output. + */ + Assert((joinrel->gpi && joinrel->gpi->target) || !grouped || + !do_aggregate); /* * Paths are parameterized by top-level parents, so run parameterization @@ -420,42 +464,62 @@ try_nestloop_path(PlannerInfo *root, initial_cost_nestloop(root, &workspace, jointype, outer_path, inner_path, extra); - if (add_path_precheck(joinrel, - workspace.startup_cost, workspace.total_cost, - pathkeys, required_outer)) + /* + * Determine which target the join should produce. + * + * In the case of explicit aggregation, output of the join itself is + * plain. + */ + if (!grouped || do_aggregate) + join_target = joinrel->reltarget; + else + join_target = joinrel->gpi->target; + + /* + * If the inner path is parameterized, it is parameterized by the topmost + * parent of the outer rel, not the outer rel itself. Fix that. + */ + if (PATH_PARAM_BY_PARENT(inner_path, outer_path->parent)) { + inner_path = reparameterize_path_by_child(root, inner_path, + outer_path->parent); + /* - * If the inner path is parameterized, it is parameterized by the - * topmost parent of the outer rel, not the outer rel itself. Fix - * that. + * If we could not translate the path, we can't create nest loop path. */ - if (PATH_PARAM_BY_PARENT(inner_path, outer_path->parent)) + if (!inner_path) { - inner_path = reparameterize_path_by_child(root, inner_path, - outer_path->parent); - - /* - * If we could not translate the path, we can't create nest loop - * path. - */ - if (!inner_path) - { - bms_free(required_outer); - return; - } + bms_free(required_outer); + return; } + } - add_path(joinrel, (Path *) - create_nestloop_path(root, - joinrel, - jointype, - &workspace, - extra, - outer_path, - inner_path, - extra->restrictlist, - pathkeys, - required_outer)); + join_path = (Path *) create_nestloop_path(root, joinrel, jointype, + &workspace, extra, + outer_path, inner_path, + extra->restrictlist, pathkeys, + required_outer, join_target); + + /* Do partial aggregation if needed. */ + if (do_aggregate && required_outer == NULL) + { + create_grouped_path(root, joinrel, join_path, true, false, + AGG_HASHED); + create_grouped_path(root, joinrel, join_path, true, false, + AGG_SORTED); + } + else if (add_path_precheck(joinrel, + workspace.startup_cost, workspace.total_cost, + pathkeys, required_outer, grouped)) + { + /* + * For a grouped path try to identify unique keys, to possibly avoid + * final aggregation. + */ + if (grouped) + make_uniquekeys(root, (Path *) join_path); + + add_path(joinrel, join_path, grouped); } else { @@ -476,9 +540,19 @@ try_partial_nestloop_path(PlannerInfo *root, Path *inner_path, List *pathkeys, JoinType jointype, - JoinPathExtraData *extra) + JoinPathExtraData *extra, + bool grouped, + bool do_aggregate) { JoinCostWorkspace workspace; + Path *join_path; + PathTarget *join_target; + + /* The same checks we do in try_nestloop_path. */ + Assert(!do_aggregate || grouped); + Assert(joinrel->gpi || !grouped); + Assert((joinrel->gpi && joinrel->gpi->target) || !grouped || + !do_aggregate); /* * If the inner path is parameterized, the parameterization must be fully @@ -513,7 +587,62 @@ try_partial_nestloop_path(PlannerInfo *root, */ initial_cost_nestloop(root, &workspace, jointype, outer_path, inner_path, extra); - if (!add_partial_path_precheck(joinrel, workspace.total_cost, pathkeys)) + + /* + * Determine which target the join should produce. + * + * In the case of explicit aggregation, output of the join itself is + * plain. + */ + if (!grouped || do_aggregate) + join_target = joinrel->reltarget; + else + { + Assert(joinrel->gpi != NULL); + join_target = joinrel->gpi->target; + } + + join_path = (Path *) create_nestloop_path(root, joinrel, jointype, + &workspace, extra, + outer_path, inner_path, + extra->restrictlist, pathkeys, + NULL, join_target); + + if (do_aggregate) + { + create_grouped_path(root, joinrel, join_path, true, true, AGG_HASHED); + create_grouped_path(root, joinrel, join_path, true, true, AGG_SORTED); + } + else if (add_partial_path_precheck(joinrel, workspace.total_cost, + pathkeys, grouped)) + { + /* Might be good enough to be worth trying, so let's try it. */ + add_partial_path(joinrel, (Path *) join_path, grouped); + } +} + +static void +try_grouped_nestloop_path(PlannerInfo *root, + RelOptInfo *joinrel, + Path *outer_path, + Path *inner_path, + List *pathkeys, + JoinType jointype, + JoinPathExtraData *extra, + bool partial, + bool do_aggregate) +{ + /* + * Missing GroupedPathInfo indicates that we should not try to create a + * grouped join. + */ + if (joinrel->gpi == NULL) + return; + + /* + * Can't preform explicit aggregation w/o the appropriate target. + */ + if (do_aggregate && joinrel->gpi->target == NULL) return; /* @@ -532,18 +661,75 @@ try_partial_nestloop_path(PlannerInfo *root, return; } - /* Might be good enough to be worth trying, so let's try it. */ - add_partial_path(joinrel, (Path *) - create_nestloop_path(root, - joinrel, - jointype, - &workspace, - extra, - outer_path, - inner_path, - extra->restrictlist, - pathkeys, - NULL)); + if (!partial) + try_nestloop_path(root, joinrel, outer_path, inner_path, pathkeys, + jointype, extra, true, do_aggregate); + else + try_partial_nestloop_path(root, joinrel, outer_path, inner_path, + pathkeys, jointype, extra, true, + do_aggregate); +} + +static void +try_nestloop_path_common(PlannerInfo *root, + RelOptInfo *joinrel, + Path *outer_path, + Path *inner_path, + List *pathkeys, + JoinType jointype, + JoinPathExtraData *extra, + bool partial, + bool grouped_outer, + bool grouped_inner, + bool do_aggregate) +{ + bool grouped_join; + + grouped_join = grouped_outer || grouped_inner || do_aggregate; + + /* Join of two grouped paths is not supported. */ + Assert(!(grouped_outer && grouped_inner)); + + if (!grouped_join) + { + /* Only join plain paths. */ + if (!partial) + try_nestloop_path(root, + joinrel, + outer_path, + inner_path, + pathkeys, + jointype, + extra, + grouped_join, + do_aggregate); + else + try_partial_nestloop_path(root, + joinrel, + outer_path, + inner_path, + pathkeys, + jointype, + extra, + grouped_join, + do_aggregate); + } + else + { + /* + * Either exactly one of the input paths should be grouped, or no one + * is grouped and do_aggregate is true. + */ + try_grouped_nestloop_path(root, + joinrel, + outer_path, + inner_path, + pathkeys, + jointype, + extra, + partial, + do_aggregate); + } } /* @@ -562,38 +748,39 @@ try_mergejoin_path(PlannerInfo *root, List *innersortkeys, JoinType jointype, JoinPathExtraData *extra, - bool is_partial) + bool grouped, + bool do_aggregate) { Relids required_outer; JoinCostWorkspace workspace; + Path *join_path; + PathTarget *join_target; - if (is_partial) - { - try_partial_mergejoin_path(root, - joinrel, - outer_path, - inner_path, - pathkeys, - mergeclauses, - outersortkeys, - innersortkeys, - jointype, - extra); - return; - } + /* Caller should not request aggregation w/o grouped output. */ + Assert(!do_aggregate || grouped); + + /* GroupedPathInfo is necessary for us to produce a grouped set. */ + Assert(joinrel->gpi || !grouped); + + /* + * Aggregation target is necessary to produce grouped join output. + */ + Assert((joinrel->gpi && joinrel->gpi->target) || !grouped || + !do_aggregate); /* * Check to see if proposed path is still parameterized, and reject if the * parameterization wouldn't be sensible. */ - required_outer = calc_non_nestloop_required_outer(outer_path, - inner_path); - if (required_outer && - !bms_overlap(required_outer, extra->param_source_rels)) + required_outer = calc_non_nestloop_required_outer(outer_path, inner_path); + if (required_outer) { - /* Waste no memory when we reject a path here */ - bms_free(required_outer); - return; + if (!bms_overlap(required_outer, extra->param_source_rels)) + { + /* Waste no memory when we reject a path here */ + bms_free(required_outer); + return; + } } /* @@ -612,27 +799,54 @@ try_mergejoin_path(PlannerInfo *root, */ initial_cost_mergejoin(root, &workspace, jointype, mergeclauses, outer_path, inner_path, - outersortkeys, innersortkeys, - extra); + outersortkeys, innersortkeys, extra); - if (add_path_precheck(joinrel, - workspace.startup_cost, workspace.total_cost, - pathkeys, required_outer)) + /* + * Determine which target the join should produce. + * + * In the case of explicit aggregation, output of the join itself is + * plain. + */ + if (!grouped || do_aggregate) + join_target = joinrel->reltarget; + else + join_target = joinrel->gpi->target; + + join_path = (Path *) create_mergejoin_path(root, + joinrel, + jointype, + &workspace, + extra, + outer_path, + inner_path, + extra->restrictlist, + pathkeys, + required_outer, + mergeclauses, + outersortkeys, + innersortkeys, + join_target); + + /* Do partial aggregation if needed. */ + if (do_aggregate) { - add_path(joinrel, (Path *) - create_mergejoin_path(root, - joinrel, - jointype, - &workspace, - extra, - outer_path, - inner_path, - extra->restrictlist, - pathkeys, - required_outer, - mergeclauses, - outersortkeys, - innersortkeys)); + create_grouped_path(root, joinrel, join_path, true, false, + AGG_HASHED); + create_grouped_path(root, joinrel, join_path, true, false, + AGG_SORTED); + } + else if (add_path_precheck(joinrel, + workspace.startup_cost, workspace.total_cost, + pathkeys, required_outer, grouped)) + { + /* + * For a grouped path try to identify unique keys, to possibly avoid + * final aggregation. + */ + if (grouped) + make_uniquekeys(root, (Path *) join_path); + + add_path(joinrel, (Path *) join_path, grouped); } else { @@ -656,9 +870,19 @@ try_partial_mergejoin_path(PlannerInfo *root, List *outersortkeys, List *innersortkeys, JoinType jointype, - JoinPathExtraData *extra) + JoinPathExtraData *extra, + bool grouped, + bool do_aggregate) { JoinCostWorkspace workspace; + Path *join_path; + PathTarget *join_target; + + /* The same checks we do in try_mergejoin_path. */ + Assert(!do_aggregate || grouped); + Assert(joinrel->gpi || !grouped); + Assert((joinrel->gpi && joinrel->gpi->target) || !grouped || + !do_aggregate); /* * See comments in try_partial_hashjoin_path(). @@ -688,27 +912,162 @@ try_partial_mergejoin_path(PlannerInfo *root, */ initial_cost_mergejoin(root, &workspace, jointype, mergeclauses, outer_path, inner_path, - outersortkeys, innersortkeys, - extra); + outersortkeys, innersortkeys, extra); + + /* + * Determine which target the join should produce. + * + * In the case of explicit aggregation, output of the join itself is + * plain. + */ + if (!grouped || do_aggregate) + join_target = joinrel->reltarget; + else + { + Assert(joinrel->gpi != NULL); + join_target = joinrel->gpi->target; + } + + join_path = (Path *) create_mergejoin_path(root, + joinrel, + jointype, + &workspace, + extra, + outer_path, + inner_path, + extra->restrictlist, + pathkeys, + NULL, + mergeclauses, + outersortkeys, + innersortkeys, + join_target); + + if (do_aggregate) + { + create_grouped_path(root, joinrel, join_path, true, true, AGG_HASHED); + create_grouped_path(root, joinrel, join_path, true, true, AGG_SORTED); + } + else if (add_partial_path_precheck(joinrel, workspace.total_cost, + pathkeys, grouped)) + { + /* Might be good enough to be worth trying, so let's try it. */ + add_partial_path(joinrel, (Path *) join_path, grouped); + } +} + +static void +try_grouped_mergejoin_path(PlannerInfo *root, + RelOptInfo *joinrel, + Path *outer_path, + Path *inner_path, + List *pathkeys, + List *mergeclauses, + List *outersortkeys, + List *innersortkeys, + JoinType jointype, + JoinPathExtraData *extra, + bool partial, + bool do_aggregate) +{ + /* + * Missing GroupedPathInfo indicates that we should not try to create a + * grouped join. + */ + if (joinrel->gpi == NULL) + return; - if (!add_partial_path_precheck(joinrel, workspace.total_cost, pathkeys)) + /* + * Can't preform explicit aggregation w/o the appropriate target. + */ + if (do_aggregate && joinrel->gpi->target == NULL) return; - /* Might be good enough to be worth trying, so let's try it. */ - add_partial_path(joinrel, (Path *) - create_mergejoin_path(root, - joinrel, - jointype, - &workspace, - extra, - outer_path, - inner_path, - extra->restrictlist, - pathkeys, - NULL, - mergeclauses, - outersortkeys, - innersortkeys)); + if (!partial) + try_mergejoin_path(root, joinrel, outer_path, inner_path, pathkeys, + mergeclauses, outersortkeys, innersortkeys, + jointype, extra, true, do_aggregate); + else + try_partial_mergejoin_path(root, joinrel, outer_path, inner_path, + pathkeys, + mergeclauses, outersortkeys, innersortkeys, + jointype, extra, true, do_aggregate); +} + +/* + * Generic front-end to create combinations of full/partial and grouped/plain + * merge joins. + */ +static void +try_mergejoin_path_common(PlannerInfo *root, + RelOptInfo *joinrel, + Path *outer_path, + Path *inner_path, + List *pathkeys, + List *mergeclauses, + List *outersortkeys, + List *innersortkeys, + JoinType jointype, + JoinPathExtraData *extra, + bool partial, + bool grouped_outer, + bool grouped_inner, + bool do_aggregate) +{ + bool grouped_join; + + grouped_join = grouped_outer || grouped_inner || do_aggregate; + + /* Join of two grouped paths is not supported. */ + Assert(!(grouped_outer && grouped_inner)); + + if (!grouped_join) + { + /* Only join plain paths. */ + if (!partial) + try_mergejoin_path(root, + joinrel, + outer_path, + inner_path, + pathkeys, + mergeclauses, + outersortkeys, + innersortkeys, + jointype, + extra, + false, false); + else + try_partial_mergejoin_path(root, + joinrel, + outer_path, + inner_path, + pathkeys, + mergeclauses, + outersortkeys, + innersortkeys, + jointype, + extra, + false, false); + } + else + { + /* + * Either exactly one of the input paths should be grouped, or no one + * is grouped and do_aggregate is true. + */ + try_grouped_mergejoin_path(root, + joinrel, + outer_path, + inner_path, + pathkeys, + mergeclauses, + outersortkeys, + innersortkeys, + jointype, + extra, + partial, + do_aggregate); + } } /* @@ -723,47 +1082,92 @@ try_hashjoin_path(PlannerInfo *root, Path *inner_path, List *hashclauses, JoinType jointype, - JoinPathExtraData *extra) + JoinPathExtraData *extra, + bool grouped, + bool do_aggregate) { Relids required_outer; JoinCostWorkspace workspace; + Path *join_path; + PathTarget *join_target; + + /* Caller should not request aggregation w/o grouped output. */ + Assert(!do_aggregate || grouped); + + /* GroupedPathInfo is necessary for us to produce a grouped set. */ + Assert(joinrel->gpi || !grouped); + + /* + * Aggregation target is necessary to produce grouped join output. + */ + Assert((joinrel->gpi && joinrel->gpi->target) || !grouped || + !do_aggregate); /* * Check to see if proposed path is still parameterized, and reject if the * parameterization wouldn't be sensible. */ - required_outer = calc_non_nestloop_required_outer(outer_path, - inner_path); - if (required_outer && - !bms_overlap(required_outer, extra->param_source_rels)) + required_outer = calc_non_nestloop_required_outer(outer_path, inner_path); + if (required_outer) { - /* Waste no memory when we reject a path here */ - bms_free(required_outer); - return; + if (!bms_overlap(required_outer, extra->param_source_rels)) + { + /* Waste no memory when we reject a path here */ + bms_free(required_outer); + return; + } } /* * See comments in try_nestloop_path(). Also note that hashjoin paths * never have any output pathkeys, per comments in create_hashjoin_path. + * + * TODO Need to consider aggregation here? */ initial_cost_hashjoin(root, &workspace, jointype, hashclauses, outer_path, inner_path, extra); - if (add_path_precheck(joinrel, - workspace.startup_cost, workspace.total_cost, - NIL, required_outer)) + /* + * Determine which target the join should produce. + * + * In the case of explicit aggregation, output of the join itself is + * plain. + */ + if (!grouped || do_aggregate) + join_target = joinrel->reltarget; + else + join_target = joinrel->gpi->target; + + join_path = (Path *) create_hashjoin_path(root, joinrel, jointype, + &workspace, + extra, + outer_path, inner_path, + extra->restrictlist, + required_outer, hashclauses, + join_target); + + /* Do partial aggregation if needed. */ + if (do_aggregate) { - add_path(joinrel, (Path *) - create_hashjoin_path(root, - joinrel, - jointype, - &workspace, - extra, - outer_path, - inner_path, - extra->restrictlist, - required_outer, - hashclauses)); + /* + * Only AGG_HASHED is suitable here as it does not expect the input + * set to be sorted. + */ + create_grouped_path(root, joinrel, join_path, true, false, + AGG_HASHED); + } + else if (add_path_precheck(joinrel, + workspace.startup_cost, workspace.total_cost, + NIL, required_outer, grouped)) + { + /* + * For a grouped path try to identify unique keys, to possibly avoid + * final aggregation. + */ + if (grouped) + make_uniquekeys(root, (Path *) join_path); + + add_path(joinrel, (Path *) join_path, grouped); } else { @@ -784,9 +1188,19 @@ try_partial_hashjoin_path(PlannerInfo *root, Path *inner_path, List *hashclauses, JoinType jointype, - JoinPathExtraData *extra) + JoinPathExtraData *extra, + bool grouped, + bool do_aggregate) { JoinCostWorkspace workspace; + Path *join_path; + PathTarget *join_target; + + /* The same checks we do in try_hashjoin_path. */ + Assert(!do_aggregate || grouped); + Assert(joinrel->gpi || !grouped); + Assert((joinrel->gpi && joinrel->gpi->target) || !grouped || + !do_aggregate); /* * If the inner path is parameterized, the parameterization must be fully @@ -809,23 +1223,154 @@ try_partial_hashjoin_path(PlannerInfo *root, */ initial_cost_hashjoin(root, &workspace, jointype, hashclauses, outer_path, inner_path, extra); - if (!add_partial_path_precheck(joinrel, workspace.total_cost, NIL)) + + /* + * Determine which target the join should produce. + * + * In the case of explicit aggregation, output of the join itself is + * plain. + */ + if (!grouped || do_aggregate) + join_target = joinrel->reltarget; + else + { + Assert(joinrel->gpi != NULL); + join_target = joinrel->gpi->target; + } + + join_path = (Path *) create_hashjoin_path(root, joinrel, jointype, + &workspace, + extra, + outer_path, inner_path, + extra->restrictlist, NULL, + hashclauses, join_target); + + /* Do partial aggregation if needed. */ + if (do_aggregate) + { + create_grouped_path(root, joinrel, join_path, true, true, AGG_HASHED); + } + else if (add_partial_path_precheck(joinrel, workspace.total_cost, + NIL, grouped)) + { + add_partial_path(joinrel, (Path *) join_path, grouped); + } +} + +/* + * Create a new grouped hash join path by joining a grouped path to plain + * (non-grouped) one, or by joining 2 plain relations and applying grouping on + * the result. + * + * Joining of 2 grouped paths is not supported. If a grouped relation A was + * joined to grouped relation B, then the grouping of B reduces the number of + * times each group of A is appears in the join output. This makes difference + * for some aggregates, e.g. sum(). + * + * If do_aggregate is true, neither input rel is grouped so we need to + * aggregate the join result explicitly. + * + * partial argument tells whether the join path should be considered partial. + */ +static void +try_grouped_hashjoin_path(PlannerInfo *root, + RelOptInfo *joinrel, + Path *outer_path, + Path *inner_path, + List *hashclauses, + JoinType jointype, + JoinPathExtraData *extra, + bool partial, + bool do_aggregate +) +{ + /* + * Missing GroupedPathInfo indicates that we should not try to create a + * grouped join. + */ + if (joinrel->gpi == NULL) return; - /* Might be good enough to be worth trying, so let's try it. */ - add_partial_path(joinrel, (Path *) - create_hashjoin_path(root, - joinrel, - jointype, - &workspace, - extra, - outer_path, - inner_path, - extra->restrictlist, - NULL, - hashclauses)); + /* + * Can't preform explicit aggregation w/o the appropriate target. + */ + if (do_aggregate && joinrel->gpi->target == NULL) + return; + + if (!partial) + try_hashjoin_path(root, joinrel, outer_path, inner_path, hashclauses, + jointype, extra, true, do_aggregate); + else + try_partial_hashjoin_path(root, joinrel, outer_path, inner_path, + hashclauses, jointype, extra, true, + do_aggregate); +} + +/* + * Generic front-end to create combinations of full/partial and grouped/plain + * hash joins. + */ +static void +try_hashjoin_path_common(PlannerInfo *root, + RelOptInfo *joinrel, + Path *outer_path, + Path *inner_path, + List *hashclauses, + JoinType jointype, + JoinPathExtraData *extra, + bool partial, + bool grouped_outer, + bool grouped_inner, + bool do_aggregate) +{ + bool grouped_join; + + grouped_join = grouped_outer || grouped_inner || do_aggregate; + + /* Join of two grouped paths is not supported. */ + Assert(!(grouped_outer && grouped_inner)); + + if (!grouped_join) + { + /* Only join plain paths. */ + if (!partial) + try_hashjoin_path(root, + joinrel, + outer_path, + inner_path, + hashclauses, + jointype, + extra, + false, false); + else + try_partial_hashjoin_path(root, + joinrel, + outer_path, + inner_path, + hashclauses, + jointype, + extra, + false, false); + } + else + { + /* + * Either exactly one of the input paths should be grouped, or no one + * is grouped and do_aggregate is true. + */ + try_grouped_hashjoin_path(root, + joinrel, + outer_path, + inner_path, + hashclauses, + jointype, + extra, + partial, + do_aggregate); + } } + /* * clause_sides_match_join * Determine whether a join clause is of the right form to use in this join. @@ -876,6 +1421,34 @@ sort_inner_and_outer(PlannerInfo *root, JoinType jointype, JoinPathExtraData *extra) { + /* Plain (non-grouped) join. */ + sort_inner_and_outer_common(root, joinrel, outerrel, innerrel, + jointype, extra, false, false, false); + + /* Use all the supported strategies to generate grouped join. */ + sort_inner_and_outer_common(root, joinrel, outerrel, innerrel, + jointype, extra, true, false, false); + sort_inner_and_outer_common(root, joinrel, outerrel, innerrel, + jointype, extra, false, true, false); + sort_inner_and_outer_common(root, joinrel, outerrel, innerrel, + jointype, extra, false, false, true); +} + +/* + * TODO As merge_pathkeys shouldn't differ across calls, use a separate + * function to derive them and pass them here in a list. + */ +static void +sort_inner_and_outer_common(PlannerInfo *root, + RelOptInfo *joinrel, + RelOptInfo *outerrel, + RelOptInfo *innerrel, + JoinType jointype, + JoinPathExtraData *extra, + bool grouped_outer, + bool grouped_inner, + bool do_aggregate) +{ JoinType save_jointype = jointype; Path *outer_path; Path *inner_path; @@ -897,8 +1470,25 @@ sort_inner_and_outer(PlannerInfo *root, * against mergejoins with parameterized inputs; see comments in * src/backend/optimizer/README. */ - outer_path = outerrel->cheapest_total_path; - inner_path = innerrel->cheapest_total_path; + if (grouped_outer) + { + if (REL_HAS_GROUPED_PATHS(outerrel)) + outer_path = linitial(outerrel->gpi->pathlist); + else + return; + } + else + outer_path = outerrel->cheapest_total_path; + + if (grouped_inner) + { + if (REL_HAS_GROUPED_PATHS(innerrel)) + inner_path = linitial(innerrel->gpi->pathlist); + else + return; + } + else + inner_path = innerrel->cheapest_total_path; /* * If either cheapest-total path is parameterized by the other rel, we @@ -916,6 +1506,16 @@ sort_inner_and_outer(PlannerInfo *root, */ if (jointype == JOIN_UNIQUE_OUTER) { + /* + * TODO This is just a temporary limitation. Before lifting it, make + * sure that the UniquePath does emit GroupedVars. Also try to avoid + * the unique-ification if outer_path comes directly from AggPath + * (i.e. it's not grouped path combined with plain one) and the + * grouping keys guaranteed the uniqueness. + */ + if (grouped_outer) + return; + outer_path = (Path *) create_unique_path(root, outerrel, outer_path, extra->sjinfo); Assert(outer_path); @@ -923,6 +1523,10 @@ sort_inner_and_outer(PlannerInfo *root, } else if (jointype == JOIN_UNIQUE_INNER) { + /* TODO Temporary limitation, like above. */ + if (grouped_inner) + return; + inner_path = (Path *) create_unique_path(root, innerrel, inner_path, extra->sjinfo); Assert(inner_path); @@ -944,13 +1548,50 @@ sort_inner_and_outer(PlannerInfo *root, outerrel->partial_pathlist != NIL && bms_is_empty(joinrel->lateral_relids)) { - cheapest_partial_outer = (Path *) linitial(outerrel->partial_pathlist); + if (grouped_outer) + { + if (REL_HAS_PARTIAL_GROUPED_PATHS(outerrel)) + cheapest_partial_outer = (Path *) + linitial(outerrel->gpi->partial_pathlist); + else + return; + } + else + cheapest_partial_outer = (Path *) + linitial(outerrel->partial_pathlist); + + if (grouped_inner) + { + if (REL_HAS_GROUPED_PATHS(innerrel)) + inner_path = linitial(innerrel->gpi->pathlist); + else + return; + } + else + inner_path = innerrel->cheapest_total_path; if (inner_path->parallel_safe) cheapest_safe_inner = inner_path; else if (save_jointype != JOIN_UNIQUE_INNER) + { + List *inner_pathlist; + + if (!grouped_inner) + inner_pathlist = innerrel->pathlist; + else + { + Assert(innerrel->gpi != NULL); + inner_pathlist = innerrel->gpi->pathlist; + } + + /* + * All the grouped paths should be unparameterized, so the + * function is overly stringent in the grouped_inner case, but + * still useful. + */ cheapest_safe_inner = - get_cheapest_parallel_safe_total_inner(innerrel->pathlist); + get_cheapest_parallel_safe_total_inner(inner_pathlist); + } } /* @@ -1026,33 +1667,24 @@ sort_inner_and_outer(PlannerInfo *root, * properly. try_mergejoin_path will detect that case and suppress an * explicit sort step, so we needn't do so here. */ - try_mergejoin_path(root, - joinrel, - outer_path, - inner_path, - merge_pathkeys, - cur_mergeclauses, - outerkeys, - innerkeys, - jointype, - extra, - false); + try_mergejoin_path_common(root, joinrel, outer_path, inner_path, + merge_pathkeys, cur_mergeclauses, + outerkeys, innerkeys, jointype, extra, + false, grouped_outer, grouped_inner, + do_aggregate); /* * If we have partial outer and parallel safe inner path then try * partial mergejoin path. */ if (cheapest_partial_outer && cheapest_safe_inner) - try_partial_mergejoin_path(root, - joinrel, - cheapest_partial_outer, - cheapest_safe_inner, - merge_pathkeys, - cur_mergeclauses, - outerkeys, - innerkeys, - jointype, - extra); + try_mergejoin_path_common(root, joinrel, + cheapest_partial_outer, + cheapest_safe_inner, + merge_pathkeys, cur_mergeclauses, + outerkeys, innerkeys, jointype, extra, + true, grouped_outer, grouped_inner, + do_aggregate); } } @@ -1069,6 +1701,14 @@ sort_inner_and_outer(PlannerInfo *root, * some sort key requirements). So, we consider truncations of the * mergeclause list as well as the full list. (Ideally we'd consider all * subsets of the mergeclause list, but that seems way too expensive.) + * + * grouped_outer - is outerpath grouped? + * grouped_inner - use grouped paths of innerrel? + * do_aggregate - apply (partial) aggregation to the output? + * + * TODO If subsequent calls often differ only by the 3 arguments above, + * consider a workspace structure to share useful info (eg merge clauses) + * across calls. */ static void generate_mergejoin_paths(PlannerInfo *root, @@ -1080,7 +1720,10 @@ generate_mergejoin_paths(PlannerInfo *root, bool useallclauses, Path *inner_cheapest_total, List *merge_pathkeys, - bool is_partial) + bool is_partial, + bool grouped_outer, + bool grouped_inner, + bool do_aggregate) { List *mergeclauses; List *innersortkeys; @@ -1131,17 +1774,18 @@ generate_mergejoin_paths(PlannerInfo *root, * try_mergejoin_path will do the right thing if inner_cheapest_total is * already correctly sorted.) */ - try_mergejoin_path(root, - joinrel, - outerpath, - inner_cheapest_total, - merge_pathkeys, - mergeclauses, - NIL, - innersortkeys, - jointype, - extra, - is_partial); + try_mergejoin_path_common(root, + joinrel, + outerpath, + inner_cheapest_total, + merge_pathkeys, + mergeclauses, + NIL, + innersortkeys, + jointype, + extra, + is_partial, + grouped_outer, grouped_inner, do_aggregate); /* Can't do anything else if inner path needs to be unique'd */ if (save_jointype == JOIN_UNIQUE_INNER) @@ -1197,16 +1841,22 @@ generate_mergejoin_paths(PlannerInfo *root, for (sortkeycnt = num_sortkeys; sortkeycnt > 0; sortkeycnt--) { + List *inner_pathlist = NIL; Path *innerpath; List *newclauses = NIL; + if (!grouped_inner) + inner_pathlist = innerrel->pathlist; + else if (innerrel->gpi != NULL) + inner_pathlist = innerrel->gpi->pathlist; + /* * Look for an inner path ordered well enough for the first * 'sortkeycnt' innersortkeys. NB: trialsortkeys list is modified * destructively, which is why we made a copy... */ trialsortkeys = list_truncate(trialsortkeys, sortkeycnt); - innerpath = get_cheapest_path_for_pathkeys(innerrel->pathlist, + innerpath = get_cheapest_path_for_pathkeys(inner_pathlist, trialsortkeys, NULL, TOTAL_COST, @@ -1229,21 +1879,25 @@ generate_mergejoin_paths(PlannerInfo *root, } else newclauses = mergeclauses; - try_mergejoin_path(root, - joinrel, - outerpath, - innerpath, - merge_pathkeys, - newclauses, - NIL, - NIL, - jointype, - extra, - is_partial); + + try_mergejoin_path_common(root, + joinrel, + outerpath, + innerpath, + merge_pathkeys, + newclauses, + NIL, + NIL, + jointype, + extra, + is_partial, + grouped_outer, grouped_inner, + do_aggregate); + cheapest_total_inner = innerpath; } /* Same on the basis of cheapest startup cost ... */ - innerpath = get_cheapest_path_for_pathkeys(innerrel->pathlist, + innerpath = get_cheapest_path_for_pathkeys(inner_pathlist, trialsortkeys, NULL, STARTUP_COST, @@ -1274,17 +1928,19 @@ generate_mergejoin_paths(PlannerInfo *root, else newclauses = mergeclauses; } - try_mergejoin_path(root, - joinrel, - outerpath, - innerpath, - merge_pathkeys, - newclauses, - NIL, - NIL, - jointype, - extra, - is_partial); + try_mergejoin_path_common(root, + joinrel, + outerpath, + innerpath, + merge_pathkeys, + newclauses, + NIL, + NIL, + jointype, + extra, + is_partial, + grouped_outer, grouped_inner, + do_aggregate); } cheapest_startup_inner = innerpath; } @@ -1297,43 +1953,32 @@ generate_mergejoin_paths(PlannerInfo *root, } } + /* - * match_unsorted_outer - * Creates possible join paths for processing a single join relation - * 'joinrel' by employing either iterative substitution or - * mergejoining on each of its possible outer paths (considering - * only outer paths that are already ordered well enough for merging). - * - * We always generate a nestloop path for each available outer path. - * In fact we may generate as many as five: one on the cheapest-total-cost - * inner path, one on the same with materialization, one on the - * cheapest-startup-cost inner path (if different), one on the - * cheapest-total inner-indexscan path (if any), and one on the - * cheapest-startup inner-indexscan path (if different). - * - * We also consider mergejoins if mergejoin clauses are available. See - * detailed comments in generate_mergejoin_paths. - * - * 'joinrel' is the join relation - * 'outerrel' is the outer join relation - * 'innerrel' is the inner join relation - * 'jointype' is the type of join to do - * 'extra' contains additional input values + * TODO As merge_pathkeys shouldn't differ across calls, use a separate + * function to derive them and pass them here in a list. */ static void -match_unsorted_outer(PlannerInfo *root, - RelOptInfo *joinrel, - RelOptInfo *outerrel, - RelOptInfo *innerrel, - JoinType jointype, - JoinPathExtraData *extra) +match_unsorted_outer_common(PlannerInfo *root, + RelOptInfo *joinrel, + RelOptInfo *outerrel, + RelOptInfo *innerrel, + JoinType jointype, + JoinPathExtraData *extra, + bool grouped_outer, + bool grouped_inner, + bool do_aggregate) { JoinType save_jointype = jointype; bool nestjoinOK; bool useallclauses; - Path *inner_cheapest_total = innerrel->cheapest_total_path; + Path *inner_cheapest_total; Path *matpath = NULL; ListCell *lc1; + List *outer_pathlist; + + /* At most one input path may be grouped. */ + Assert(!(grouped_outer && grouped_inner)); /* * Nestloop only supports inner, left, semi, and anti joins. Also, if we @@ -1370,13 +2015,38 @@ match_unsorted_outer(PlannerInfo *root, break; } + if (grouped_outer) + { + if (REL_HAS_GROUPED_PATHS(outerrel)) + outer_pathlist = outerrel->gpi->pathlist; + else + return; + } + else + outer_pathlist = outerrel->pathlist; + + if (grouped_inner) + { + if (REL_HAS_GROUPED_PATHS(innerrel)) + inner_cheapest_total = linitial(innerrel->gpi->pathlist); + else + return; + } + else + inner_cheapest_total = innerrel->cheapest_total_path; + /* * If inner_cheapest_total is parameterized by the outer rel, ignore it; * we will consider it below as a member of cheapest_parameterized_paths, * but the other possibilities considered in this routine aren't usable. */ if (PATH_PARAM_BY_REL(inner_cheapest_total, outerrel)) + { + /* Grouped path should not be parameterized at all. */ + Assert(!grouped_inner); + inner_cheapest_total = NULL; + } /* * If we need to unique-ify the inner path, we will consider only the @@ -1387,16 +2057,31 @@ match_unsorted_outer(PlannerInfo *root, /* No way to do this with an inner path parameterized by outer rel */ if (inner_cheapest_total == NULL) return; + + /* + * TODO This is just a temporary limitation. Before lifting it, make + * sure that the UniquePath does emit GroupedVars. Also try to avoid + * the unique-ification if outer_path comes directly from AggPath + * (i.e. it's not grouped path combined with plain one) and the + * grouping keys guaranteed the uniqueness. + */ + if (grouped_inner) + return; + inner_cheapest_total = (Path *) create_unique_path(root, innerrel, inner_cheapest_total, extra->sjinfo); Assert(inner_cheapest_total); } - else if (nestjoinOK) + else if (nestjoinOK && !grouped_inner) { /* * Consider materializing the cheapest inner path, unless * enable_material is off or the path in question materializes its * output anyway. + * + * TODO Verify that this is ok even if grouped_inner is true (at least + * make sure that mathpath does contain GroupedVars) and remove + * grouped_inner from the condition above. */ if (enable_material && inner_cheapest_total != NULL && !ExecMaterializesOutput(inner_cheapest_total->pathtype)) @@ -1404,7 +2089,7 @@ match_unsorted_outer(PlannerInfo *root, create_material_path(innerrel, inner_cheapest_total); } - foreach(lc1, outerrel->pathlist) + foreach(lc1, outer_pathlist) { Path *outerpath = (Path *) lfirst(lc1); List *merge_pathkeys; @@ -1424,6 +2109,11 @@ match_unsorted_outer(PlannerInfo *root, { if (outerpath != outerrel->cheapest_total_path) continue; + + /* TODO Temporary restriction, see above. */ + if (grouped_outer) + continue; + outerpath = (Path *) create_unique_path(root, outerrel, outerpath, extra->sjinfo); Assert(outerpath); @@ -1439,17 +2129,23 @@ match_unsorted_outer(PlannerInfo *root, if (save_jointype == JOIN_UNIQUE_INNER) { + /* TODO Temporary restriction, see above. */ + if (grouped_inner) + continue; + /* * Consider nestloop join, but only with the unique-ified cheapest * inner path */ - try_nestloop_path(root, - joinrel, - outerpath, - inner_cheapest_total, - merge_pathkeys, - jointype, - extra); + try_nestloop_path_common(root, + joinrel, + outerpath, + inner_cheapest_total, + merge_pathkeys, + jointype, + extra, + false, grouped_outer, grouped_inner, + do_aggregate); } else if (nestjoinOK) { @@ -1461,28 +2157,40 @@ match_unsorted_outer(PlannerInfo *root, */ ListCell *lc2; - foreach(lc2, innerrel->cheapest_parameterized_paths) + /* + * There are no grouped paths in cheapest_parameterized_paths. + */ + if (!grouped_inner) { - Path *innerpath = (Path *) lfirst(lc2); + foreach(lc2, innerrel->cheapest_parameterized_paths) + { + Path *innerpath = (Path *) lfirst(lc2); - try_nestloop_path(root, - joinrel, - outerpath, - innerpath, - merge_pathkeys, - jointype, - extra); + try_nestloop_path_common(root, + joinrel, + outerpath, + innerpath, + merge_pathkeys, + jointype, + extra, + false, grouped_outer, grouped_inner, + do_aggregate); + } } - /* Also consider materialized form of the cheapest inner path */ + /* + * Also consider materialized form of the cheapest inner path. + */ if (matpath != NULL) - try_nestloop_path(root, - joinrel, - outerpath, - matpath, - merge_pathkeys, - jointype, - extra); + try_nestloop_path_common(root, + joinrel, + outerpath, + matpath, + merge_pathkeys, + jointype, + extra, + false, grouped_outer, grouped_inner, + do_aggregate); } /* Can't do anything else if outer path needs to be unique'd */ @@ -1497,7 +2205,8 @@ match_unsorted_outer(PlannerInfo *root, generate_mergejoin_paths(root, joinrel, innerrel, outerpath, save_jointype, extra, useallclauses, inner_cheapest_total, merge_pathkeys, - false); + false, grouped_outer, grouped_inner, + do_aggregate); } /* @@ -1508,17 +2217,23 @@ match_unsorted_outer(PlannerInfo *root, * we handle extra_lateral_rels, since partial paths must not be * parameterized. Similarly, we can't handle JOIN_FULL and JOIN_RIGHT, * because they can produce false null extended rows. + * + * grouped_inner must be false because we're not interested in nest loop + * joins with the grouped path on the inner side (i.e. repeated + * aggregation). */ if (joinrel->consider_parallel && save_jointype != JOIN_UNIQUE_OUTER && save_jointype != JOIN_FULL && save_jointype != JOIN_RIGHT && outerrel->partial_pathlist != NIL && - bms_is_empty(joinrel->lateral_relids)) + bms_is_empty(joinrel->lateral_relids) && + !grouped_inner) { if (nestjoinOK) consider_parallel_nestloop(root, joinrel, outerrel, innerrel, - save_jointype, extra); + save_jointype, extra, + grouped_outer, do_aggregate); /* * If inner_cheapest_total is NULL or non parallel-safe then find the @@ -1538,11 +2253,58 @@ match_unsorted_outer(PlannerInfo *root, if (inner_cheapest_total) consider_parallel_mergejoin(root, joinrel, outerrel, innerrel, save_jointype, extra, - inner_cheapest_total); + inner_cheapest_total, + grouped_outer, do_aggregate); } } /* + * match_unsorted_outer + * Creates possible join paths for processing a single join relation + * 'joinrel' by employing either iterative substitution or + * mergejoining on each of its possible outer paths (considering + * only outer paths that are already ordered well enough for merging). + * + * We always generate a nestloop path for each available outer path. + * In fact we may generate as many as five: one on the cheapest-total-cost + * inner path, one on the same with materialization, one on the + * cheapest-startup-cost inner path (if different), one on the + * cheapest-total inner-indexscan path (if any), and one on the + * cheapest-startup inner-indexscan path (if different). + * + * We also consider mergejoins if mergejoin clauses are available. See + * detailed comments in generate_mergejoin_paths. + * + * 'joinrel' is the join relation + * 'outerrel' is the outer join relation + * 'innerrel' is the inner join relation + * 'jointype' is the type of join to do + * 'extra' contains additional input values + * 'grouped' indicates that the at least one relation in the join has been + * aggregated. + */ +static void +match_unsorted_outer(PlannerInfo *root, + RelOptInfo *joinrel, + RelOptInfo *outerrel, + RelOptInfo *innerrel, + JoinType jointype, + JoinPathExtraData *extra) +{ + /* Plain (non-grouped) join. */ + match_unsorted_outer_common(root, joinrel, outerrel, innerrel, + jointype, extra, false, false, false); + + /* Use all the supported strategies to generate grouped join. */ + match_unsorted_outer_common(root, joinrel, outerrel, innerrel, + jointype, extra, true, false, false); + match_unsorted_outer_common(root, joinrel, outerrel, innerrel, + jointype, extra, false, true, false); + match_unsorted_outer_common(root, joinrel, outerrel, innerrel, + jointype, extra, false, false, true); +} + +/* * consider_parallel_mergejoin * Try to build partial paths for a joinrel by joining a partial path * for the outer relation to a complete path for the inner relation. @@ -1554,6 +2316,10 @@ match_unsorted_outer(PlannerInfo *root, * 'extra' contains additional input values * 'inner_cheapest_total' cheapest total path for innerrel */ +/* + * TODO Store merge_pathkeys across calls if these (the calls) only differ in + * grouped_outer or do_aggregate. + */ static void consider_parallel_mergejoin(PlannerInfo *root, RelOptInfo *joinrel, @@ -1561,12 +2327,36 @@ consider_parallel_mergejoin(PlannerInfo *root, RelOptInfo *innerrel, JoinType jointype, JoinPathExtraData *extra, - Path *inner_cheapest_total) + Path *inner_cheapest_total, + bool grouped_outer, + bool do_aggregate) { ListCell *lc1; + List *outer_pathlist; + bool grouped = grouped_outer || do_aggregate; + + if (!grouped || do_aggregate) + { + /* + * If creating grouped paths by explicit aggregation, the input paths + * must be plain. + */ + outer_pathlist = outerrel->partial_pathlist; + } + else if (outerrel->gpi != NULL) + { + /* + * Only the outer paths are accepted as grouped when we try to combine + * grouped and plain ones. Grouped inner path implies repeated + * aggregation, which doesn't sound as a good idea. + */ + outer_pathlist = outerrel->gpi->partial_pathlist; + } + else + return; /* generate merge join path for each partial outer path */ - foreach(lc1, outerrel->partial_pathlist) + foreach(lc1, outer_pathlist) { Path *outerpath = (Path *) lfirst(lc1); List *merge_pathkeys; @@ -1577,9 +2367,10 @@ consider_parallel_mergejoin(PlannerInfo *root, merge_pathkeys = build_join_pathkeys(root, joinrel, jointype, outerpath->pathkeys); - generate_mergejoin_paths(root, joinrel, innerrel, outerpath, jointype, - extra, false, inner_cheapest_total, - merge_pathkeys, true); + generate_mergejoin_paths(root, joinrel, innerrel, outerpath, + jointype, extra, false, + inner_cheapest_total, merge_pathkeys, + true, grouped_outer, false, do_aggregate); } } @@ -1594,21 +2385,48 @@ consider_parallel_mergejoin(PlannerInfo *root, * 'jointype' is the type of join to do * 'extra' contains additional input values */ +/* + * TODO Store pathkeys across calls if these (the calls) only differ in + * grouped_outer or do_aggregate. + */ static void consider_parallel_nestloop(PlannerInfo *root, RelOptInfo *joinrel, RelOptInfo *outerrel, RelOptInfo *innerrel, JoinType jointype, - JoinPathExtraData *extra) + JoinPathExtraData *extra, + bool grouped_outer, bool do_aggregate) { JoinType save_jointype = jointype; + List *outer_pathlist; ListCell *lc1; + bool grouped = grouped_outer || do_aggregate; if (jointype == JOIN_UNIQUE_INNER) jointype = JOIN_INNER; - foreach(lc1, outerrel->partial_pathlist) + if (!grouped || do_aggregate) + { + /* + * If creating grouped paths by explicit aggregation, the input paths + * must be plain. + */ + outer_pathlist = outerrel->partial_pathlist; + } + else if (outerrel->gpi != NULL) + { + /* + * Only the outer paths are accepted as grouped when we try to combine + * grouped and plain ones. Grouped inner path implies repeated + * aggregation, which doesn't sound as a good idea. + */ + outer_pathlist = outerrel->gpi->partial_pathlist; + } + else + return; + + foreach(lc1, outer_pathlist) { Path *outerpath = (Path *) lfirst(lc1); List *pathkeys; @@ -1639,7 +2457,7 @@ consider_parallel_nestloop(PlannerInfo *root, * inner paths, but right now create_unique_path is not on board * with that.) */ - if (save_jointype == JOIN_UNIQUE_INNER) + if (save_jointype == JOIN_UNIQUE_INNER && !grouped) { if (innerpath != innerrel->cheapest_total_path) continue; @@ -1649,30 +2467,29 @@ consider_parallel_nestloop(PlannerInfo *root, Assert(innerpath); } - try_partial_nestloop_path(root, joinrel, outerpath, innerpath, - pathkeys, jointype, extra); + try_nestloop_path_common(root, joinrel, outerpath, innerpath, + pathkeys, jointype, extra, + true, grouped_outer, false, + do_aggregate); } } } /* - * hash_inner_and_outer - * Create hashjoin join paths by explicitly hashing both the outer and - * inner keys of each available hash clause. - * - * 'joinrel' is the join relation - * 'outerrel' is the outer join relation - * 'innerrel' is the inner join relation - * 'jointype' is the type of join to do - * 'extra' contains additional input values + * TODO hashclauses (and mabye some other info) should be shared across calls + * if only some of the following arguments change: partial, grouped_outer, + * grouped_inner, do_aggregate. */ static void -hash_inner_and_outer(PlannerInfo *root, - RelOptInfo *joinrel, - RelOptInfo *outerrel, - RelOptInfo *innerrel, - JoinType jointype, - JoinPathExtraData *extra) +hash_inner_and_outer_common(PlannerInfo *root, + RelOptInfo *joinrel, + RelOptInfo *outerrel, + RelOptInfo *innerrel, + JoinType jointype, + JoinPathExtraData *extra, + bool grouped_outer, + bool grouped_inner, + bool do_aggregate) { JoinType save_jointype = jointype; bool isouterjoin = IS_OUTER_JOIN(jointype); @@ -1719,9 +2536,36 @@ hash_inner_and_outer(PlannerInfo *root, * outer paths. There's no need to consider any but the * cheapest-total-cost inner path, however. */ - Path *cheapest_startup_outer = outerrel->cheapest_startup_path; - Path *cheapest_total_outer = outerrel->cheapest_total_path; - Path *cheapest_total_inner = innerrel->cheapest_total_path; + Path *cheapest_startup_outer, + *cheapest_total_outer, + *cheapest_total_inner; + + if (grouped_outer) + { + if (REL_HAS_GROUPED_PATHS(outerrel)) + { + cheapest_total_outer = linitial(outerrel->gpi->pathlist); + /* No parameterized grouped paths. */ + cheapest_startup_outer = NULL; + } + else + return; + } + else + { + cheapest_total_outer = outerrel->cheapest_total_path; + cheapest_startup_outer = outerrel->cheapest_startup_path; + } + + if (grouped_inner) + { + if (REL_HAS_GROUPED_PATHS(innerrel)) + cheapest_total_inner = linitial(innerrel->gpi->pathlist); + else + return; + } + else + cheapest_total_inner = innerrel->cheapest_total_path; /* * If either cheapest-total path is parameterized by the other rel, we @@ -1736,43 +2580,66 @@ hash_inner_and_outer(PlannerInfo *root, /* Unique-ify if need be; we ignore parameterized possibilities */ if (jointype == JOIN_UNIQUE_OUTER) { + /* + * TODO This is just a temporary limitation. Before lifting it, + * make sure that the UniquePath does emit GroupedVars. Also try + * to avoid the unique-ification if the outer path comes directly + * from AggPath (i.e. it's not grouped path combined with plain + * one) and the grouping keys guaranteed the uniqueness. + */ + if (grouped_outer) + return; + cheapest_total_outer = (Path *) create_unique_path(root, outerrel, cheapest_total_outer, extra->sjinfo); Assert(cheapest_total_outer); jointype = JOIN_INNER; - try_hashjoin_path(root, - joinrel, - cheapest_total_outer, - cheapest_total_inner, - hashclauses, - jointype, - extra); + try_hashjoin_path_common(root, + joinrel, + cheapest_total_outer, + cheapest_total_inner, + hashclauses, + jointype, + extra, + false, grouped_outer, grouped_inner, + do_aggregate); /* no possibility of cheap startup here */ } else if (jointype == JOIN_UNIQUE_INNER) { + /* TODO Temporary restriction, see above. */ + if (grouped_inner) + return; + cheapest_total_inner = (Path *) create_unique_path(root, innerrel, cheapest_total_inner, extra->sjinfo); Assert(cheapest_total_inner); jointype = JOIN_INNER; - try_hashjoin_path(root, - joinrel, - cheapest_total_outer, - cheapest_total_inner, - hashclauses, - jointype, - extra); + try_hashjoin_path_common(root, + joinrel, + cheapest_total_outer, + cheapest_total_inner, + hashclauses, + jointype, + extra, + false, + grouped_outer, grouped_inner, + do_aggregate); + if (cheapest_startup_outer != NULL && cheapest_startup_outer != cheapest_total_outer) - try_hashjoin_path(root, - joinrel, - cheapest_startup_outer, - cheapest_total_inner, - hashclauses, - jointype, - extra); + try_hashjoin_path_common(root, + joinrel, + cheapest_startup_outer, + cheapest_total_inner, + hashclauses, + jointype, + extra, + false, + grouped_outer, grouped_inner, + do_aggregate); } else { @@ -1782,22 +2649,35 @@ hash_inner_and_outer(PlannerInfo *root, * pairings of cheapest-total paths including parameterized ones. * There is no use in generating parameterized paths on the basis * of possibly cheap startup cost, so this is sufficient. + * + * For if either side of the join is grouped, we simply use + * rel->gpi->pathlist. (cheapest_startup_outer should be NULL if + * grouped_outer.) */ ListCell *lc1; ListCell *lc2; + List *outer_pathlist; if (cheapest_startup_outer != NULL) - try_hashjoin_path(root, - joinrel, - cheapest_startup_outer, - cheapest_total_inner, - hashclauses, - jointype, - extra); - - foreach(lc1, outerrel->cheapest_parameterized_paths) + try_hashjoin_path_common(root, + joinrel, + cheapest_startup_outer, + cheapest_total_inner, + hashclauses, + jointype, + extra, + false, + grouped_outer, grouped_inner, + do_aggregate); + + outer_pathlist = !grouped_outer ? + outerrel->cheapest_parameterized_paths : + outerrel->gpi->pathlist; + + foreach(lc1, outer_pathlist) { Path *outerpath = (Path *) lfirst(lc1); + List *inner_pathlist; /* * We cannot use an outer path that is parameterized by the @@ -1806,7 +2686,11 @@ hash_inner_and_outer(PlannerInfo *root, if (PATH_PARAM_BY_REL(outerpath, innerrel)) continue; - foreach(lc2, innerrel->cheapest_parameterized_paths) + inner_pathlist = !grouped_inner ? + innerrel->cheapest_parameterized_paths : + innerrel->gpi->pathlist; + + foreach(lc2, inner_pathlist) { Path *innerpath = (Path *) lfirst(lc2); @@ -1821,13 +2705,16 @@ hash_inner_and_outer(PlannerInfo *root, innerpath == cheapest_total_inner) continue; /* already tried it */ - try_hashjoin_path(root, - joinrel, - outerpath, - innerpath, - hashclauses, - jointype, - extra); + try_hashjoin_path_common(root, + joinrel, + outerpath, + innerpath, + hashclauses, + jointype, + extra, + false, + grouped_outer, grouped_inner, + do_aggregate); } } } @@ -1850,32 +2737,96 @@ hash_inner_and_outer(PlannerInfo *root, Path *cheapest_partial_outer; Path *cheapest_safe_inner = NULL; - cheapest_partial_outer = - (Path *) linitial(outerrel->partial_pathlist); + if (grouped_outer) + { + if (REL_HAS_PARTIAL_GROUPED_PATHS(outerrel)) + { + cheapest_partial_outer = + (Path *) linitial(outerrel->gpi->partial_pathlist); + } + else + return; + } + else + cheapest_partial_outer = + (Path *) linitial(outerrel->partial_pathlist); /* * Normally, given that the joinrel is parallel-safe, the cheapest * total inner path will also be parallel-safe, but if not, we'll - * have to search for the cheapest safe, unparameterized inner - * path. If doing JOIN_UNIQUE_INNER, we can't use any alternative - * inner path. + * have to search cheapest_parameterized_paths for the cheapest + * safe, unparameterized inner path. If doing JOIN_UNIQUE_INNER, + * we can't use any alternative inner path. */ if (cheapest_total_inner->parallel_safe) cheapest_safe_inner = cheapest_total_inner; else if (save_jointype != JOIN_UNIQUE_INNER) - cheapest_safe_inner = - get_cheapest_parallel_safe_total_inner(innerrel->pathlist); + { + ListCell *lc; + List *inner_pathlist; + + inner_pathlist = !grouped_inner ? + innerrel->cheapest_parameterized_paths : + innerrel->gpi->pathlist; + + foreach(lc, inner_pathlist) + { + Path *innerpath = (Path *) lfirst(lc); + + if (innerpath->parallel_safe && + bms_is_empty(PATH_REQ_OUTER(innerpath))) + { + cheapest_safe_inner = innerpath; + break; + } + } + } if (cheapest_safe_inner != NULL) - try_partial_hashjoin_path(root, joinrel, - cheapest_partial_outer, - cheapest_safe_inner, - hashclauses, jointype, extra); + try_hashjoin_path_common(root, joinrel, + cheapest_partial_outer, + cheapest_safe_inner, + hashclauses, jointype, extra, + true, + grouped_outer, grouped_inner, + do_aggregate); } } } /* + * hash_inner_and_outer + * Create hashjoin join paths by explicitly hashing both the outer and + * inner keys of each available hash clause. + * + * 'joinrel' is the join relation + * 'outerrel' is the outer join relation + * 'innerrel' is the inner join relation + * 'jointype' is the type of join to do + * 'extra' contains additional input values + */ +static void +hash_inner_and_outer(PlannerInfo *root, + RelOptInfo *joinrel, + RelOptInfo *outerrel, + RelOptInfo *innerrel, + JoinType jointype, + JoinPathExtraData *extra) +{ + /* Plain (non-grouped) join. */ + hash_inner_and_outer_common(root, joinrel, outerrel, innerrel, + jointype, extra, false, false, false); + + /* Use all the supported strategies to generate grouped join. */ + hash_inner_and_outer_common(root, joinrel, outerrel, innerrel, + jointype, extra, true, false, false); + hash_inner_and_outer_common(root, joinrel, outerrel, innerrel, + jointype, extra, false, true, false); + hash_inner_and_outer_common(root, joinrel, outerrel, innerrel, + jointype, extra, false, false, true); +} + +/* * select_mergejoin_clauses * Select mergejoin clauses that are usable for a particular join. * Returns a list of RestrictInfo nodes for those clauses. diff --git a/src/backend/optimizer/path/joinrels.c b/src/backend/optimizer/path/joinrels.c index 453f259..cdd6d18 100644 --- a/src/backend/optimizer/path/joinrels.c +++ b/src/backend/optimizer/path/joinrels.c @@ -1232,7 +1232,7 @@ mark_dummy_rel(RelOptInfo *rel) rel->partial_pathlist = NIL; /* Set up the dummy path */ - add_path(rel, (Path *) create_append_path(rel, NIL, NULL, 0, NIL)); + add_path(rel, (Path *) create_append_path(rel, NIL, NULL, 0, NIL), false); /* Set or update cheapest_total_path and related fields */ set_cheapest(rel); @@ -1403,7 +1403,6 @@ try_partition_wise_join(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2, (List *) adjust_appendrel_attrs(root, (Node *) parent_restrictlist, nappinfos, appinfos); - pfree(appinfos); child_joinrel = joinrel->part_rels[cnt_parts]; if (!child_joinrel) @@ -1413,7 +1412,15 @@ try_partition_wise_join(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2, child_sjinfo, child_sjinfo->jointype); joinrel->part_rels[cnt_parts] = child_joinrel; + + /* + * If the parent join can be grouped, so can be its children. + */ + if (joinrel->gpi != NULL) + build_chiid_rel_gpi(root, child_joinrel, joinrel, nappinfos, + appinfos); } + pfree(appinfos); Assert(bms_equal(child_joinrel->relids, child_joinrelids)); diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index c6870d3..c267d48 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -1563,3 +1563,157 @@ has_useful_pathkeys(PlannerInfo *root, RelOptInfo *rel) return true; /* might be able to use them for ordering */ return false; /* definitely useless */ } + +/* + * Add a new set of unique keys to path's list of unique key sets. If an + * identical set is already there --- free new_set instead of adding it. + */ +void +add_uniquekeys_to_path(Path *path, Bitmapset *new_set) +{ + ListCell *lc; + + foreach(lc, path->uniquekeys) + { + Bitmapset *set = (Bitmapset *) lfirst(lc); + + if (bms_equal(new_set, set)) + break; + } + if (lc == NULL) + path->uniquekeys = lappend(path->uniquekeys, new_set); + else + bms_free(new_set); +} + +/* + * Return true if path output is grouped in a way described by + * root->group_pathkeys. AGGSPLIT_FINAL_DESERIAL can be skipped in such a + * case. + */ +bool +match_path_to_group_pathkeys(PlannerInfo *root, Path *path) +{ + Bitmapset *uniquekeys_all = NULL; + ListCell *l1; + int i; + bool *is_group_expr; + + /* + * group_pathkeys are essential for this function. + */ + if (root->group_pathkeys == NIL) + return false; + + /* + * The path is not aware of being unique. + */ + if (path->uniquekeys == NIL) + return false; + + /* + * There can be multiple known unique key sets. Gather pathkeys of all the + * unique expressions the sets may reference. + */ + foreach(l1, path->uniquekeys) + { + Bitmapset *set = (Bitmapset *) lfirst(l1); + + uniquekeys_all = bms_union(uniquekeys_all, set); + } + + /* + * Find pathkeys for the expressions. + */ + is_group_expr = (bool *) + palloc0(list_length(path->pathtarget->exprs) * sizeof(bool)); + + i = 0; + foreach(l1, path->pathtarget->exprs) + { + Expr *expr = (Expr *) lfirst(l1); + + if (bms_is_member(i, uniquekeys_all)) + { + ListCell *l2; + bool found = false; + + /* + * This is an unique expression, so find its pathkey. + */ + foreach(l2, root->group_pathkeys) + { + PathKey *pk = lfirst_node(PathKey, l2); + EquivalenceClass *ec = pk->pk_eclass; + ListCell *l3; + EquivalenceMember *em = NULL; + + if (ec->ec_below_outer_join) + continue; + if (ec->ec_has_volatile) + continue; + + foreach(l3, ec->ec_members) + { + em = lfirst_node(EquivalenceMember, l3); + + if (em->em_nullable_relids) + continue; + + if (equal(em->em_expr, expr)) + { + found = true; + break; + } + } + if (found) + break; + + } + is_group_expr[i] = found; + } + + i++; + } + + /* + * Now check the unique key sets and see if any one matches all items of + * group_pathkeys. + */ + foreach(l1, path->uniquekeys) + { + Bitmapset *set = (Bitmapset *) lfirst(l1); + bool found = false; + + /* + * Collect PKs associated with this set. + */ + for (i = 0; i < list_length(path->pathtarget->exprs); i++) + { + if (bms_is_member(i, set)) + { + /* + * If the set misses a single grouping path key, at least one + * expression of the unique key is outside the grouping + * expressions, and thus the grouping expressions are not + * guaranteed to be unique. Thus the avoidance of final + * aggregation is not justified. + */ + if (!is_group_expr[i]) + { + found = true; + break; + } + } + } + + /* + * No problem with this set. No need to check the other ones. + */ + if (!found) + return true; + } + + /* No match found. */ + return false; +} diff --git a/src/backend/optimizer/path/tidpath.c b/src/backend/optimizer/path/tidpath.c index a2fe661..91d855c 100644 --- a/src/backend/optimizer/path/tidpath.c +++ b/src/backend/optimizer/path/tidpath.c @@ -266,5 +266,5 @@ create_tidscan_paths(PlannerInfo *root, RelOptInfo *rel) if (tidquals) add_path(rel, (Path *) create_tidscan_path(root, rel, tidquals, - required_outer)); + required_outer), false); } diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index d445477..f123fbb 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -250,6 +250,7 @@ static Plan *prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, static EquivalenceMember *find_ec_member_for_tle(EquivalenceClass *ec, TargetEntry *tle, Relids relids); +static TargetEntry *get_aggref_from_tle(TargetEntry *tle); static Sort *make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids); static Sort *make_sort_from_groupcols(List *groupcls, @@ -808,6 +809,12 @@ use_physical_tlist(PlannerInfo *root, Path *path, int flags) return false; /* + * Grouped relation's target list contains GroupedVars. + */ + if (rel->gpi != NULL) + return false; + + /* * If a bitmap scan's tlist is empty, keep it as-is. This may allow the * executor to skip heap page fetches, and in any case, the benefit of * using a physical tlist instead would be minimal. @@ -1585,8 +1592,9 @@ create_projection_plan(PlannerInfo *root, ProjectionPath *best_path) * creation, but that would add expense to creating Paths we might end up * not using.) */ - if (is_projection_capable_path(best_path->subpath) || - tlist_same_exprs(tlist, subplan->targetlist)) + if (!best_path->force_result && + (is_projection_capable_path(best_path->subpath) || + tlist_same_exprs(tlist, subplan->targetlist))) { /* Don't need a separate Result, just assign tlist to subplan */ plan = subplan; @@ -5633,6 +5641,9 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, tle = get_tle_by_resno(tlist, reqColIdx[numsortkeys]); if (tle) { + /* Handle special cases of sorting by aggregate. */ + tle = get_aggref_from_tle(tle); + em = find_ec_member_for_tle(ec, tle, relids); if (em) { @@ -5664,6 +5675,10 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, foreach(j, tlist) { tle = (TargetEntry *) lfirst(j); + + /* Handle special cases of sorting by aggregate. */ + tle = get_aggref_from_tle(tle); + em = find_ec_member_for_tle(ec, tle, relids); if (em) { @@ -5837,6 +5852,49 @@ find_ec_member_for_tle(EquivalenceClass *ec, } /* + * Get Aggref from target entry if it's wrapped in GroupedVar. + */ +static TargetEntry * +get_aggref_from_tle(TargetEntry *tle) +{ + /* + * Does this happen to be an aggregate final function referencing the + * partial aggregate? This can replace the final aggregation if the path + * generates an unique set of grouping keys. + */ + if (IS_AGGFINALFN_STANDALONE(tle->expr)) + { + FuncExpr *fexpr = castNode(FuncExpr, tle->expr); + GroupedVar *gvar = linitial_node(GroupedVar, fexpr->args); + Aggref *aggref = castNode(Aggref, gvar->gvexpr); + + Assert(fexpr->funcid == aggref->aggfinalfn); + + tle = flatCopyTargetEntry(tle); + tle->expr = (Expr *) aggref; + } + + /* + * Aggregate w/o aggfinalfn? + */ + else if (IsA(tle->expr, GroupedVar)) + { + GroupedVar *gvar = castNode(GroupedVar, tle->expr); + + if (IsA(gvar->gvexpr, Aggref)) + { + Aggref *aggref = castNode(Aggref, gvar->gvexpr); + + Assert(aggref->aggfinalfn == InvalidOid); + tle = flatCopyTargetEntry(tle); + tle->expr = (Expr *) aggref; + } + } + + return tle; +} + +/* * make_sort_from_pathkeys * Create sort plan to sort according to given pathkeys * diff --git a/src/backend/optimizer/plan/initsplan.c b/src/backend/optimizer/plan/initsplan.c index 448cb73..102791b 100644 --- a/src/backend/optimizer/plan/initsplan.c +++ b/src/backend/optimizer/plan/initsplan.c @@ -14,8 +14,10 @@ */ #include "postgres.h" +#include "access/sysattr.h" #include "catalog/pg_type.h" #include "catalog/pg_class.h" +#include "catalog/pg_constraint_fn.h" #include "nodes/nodeFuncs.h" #include "optimizer/clauses.h" #include "optimizer/cost.h" @@ -27,6 +29,7 @@ #include "optimizer/planner.h" #include "optimizer/prep.h" #include "optimizer/restrictinfo.h" +#include "optimizer/tlist.h" #include "optimizer/var.h" #include "parser/analyze.h" #include "rewrite/rewriteManip.h" @@ -46,6 +49,8 @@ typedef struct PostponedQual } PostponedQual; +static void create_aggregate_grouped_var_infos(PlannerInfo *root); +static void create_grouping_expr_grouped_var_infos(PlannerInfo *root); static void extract_lateral_references(PlannerInfo *root, RelOptInfo *brel, Index rtindex); static List *deconstruct_recurse(PlannerInfo *root, Node *jtnode, @@ -241,6 +246,558 @@ add_vars_to_targetlist(PlannerInfo *root, List *vars, } } +/* + * Add GroupedVarInfo to grouped_var_list for each aggregate as well as for + * each possible grouping expression and setup GroupedPathInfo for each base + * relation that can product grouped paths. + * + * root->group_pathkeys must be setup before this function is called. + */ +extern void +add_grouping_info_to_base_rels(PlannerInfo *root) +{ + int i; + ListCell *lc; + + /* + * Isn't user interested in the aggregate push-down feature? + */ + if (!enable_agg_pushdown) + return; + + /* No grouping in the query? */ + if (!root->parse->groupClause) + return; + + /* + * Grouping sets require multiple different groupings but the base + * relation can only generate one. + */ + if (root->parse->groupingSets) + return; + + /* + * TODO Consider if this is a real limitation. + */ + if (root->parse->hasWindowFuncs) + return; + + /* Create GroupedVarInfo per (distinct) aggregate. */ + create_aggregate_grouped_var_infos(root); + + /* Is no grouping is possible below the top-level join? */ + if (root->grouped_var_list == NIL) + return; + + /* Create GroupedVarInfo per grouping expression. */ + create_grouping_expr_grouped_var_infos(root); + + /* Process the individual base relations. */ + for (i = 1; i < root->simple_rel_array_size; i++) + { + RelOptInfo *rel = root->simple_rel_array[i]; + + /* + * "other rels" will have their targets built later, by translation of + * the target of the parent rel - see set_append_rel_size. If we + * wanted to prepare the child rels here, we'd need another iteration + * of simple_rel_array_size. + */ + if (rel != NULL && rel->reloptkind == RELOPT_BASEREL) + prepare_rel_for_grouping(root, rel); + } + + /* + * Now that we know that grouping can be pushed down, search for the + * maximum sortgroupref. The base relations may need it if extra grouping + * expressions get added to them. + */ + Assert(root->max_sortgroupref == 0); + foreach(lc, root->processed_tlist) + { + TargetEntry *te = lfirst_node(TargetEntry, lc); + + if (te->ressortgroupref > root->max_sortgroupref) + root->max_sortgroupref = te->ressortgroupref; + } +} + +/* + * Create GroupedVarInfo for each distinct aggregate. + * + * If any aggregate is not suitable, set root->grouped_var_list to NIL and + * return. + */ +static void +create_aggregate_grouped_var_infos(PlannerInfo *root) +{ + List *tlist_exprs; + ListCell *lc; + + Assert(root->grouped_var_list == NIL); + + tlist_exprs = pull_var_clause((Node *) root->processed_tlist, + PVC_INCLUDE_AGGREGATES); + + /* + * Although GroupingFunc is related to root->parse->groupingSets, this + * field does not necessarily reflect its presence. + */ + foreach(lc, tlist_exprs) + { + Expr *expr = (Expr *) lfirst(lc); + + if (IsA(expr, GroupingFunc)) + return; + } + + /* + * Aggregates within the HAVING clause need to be processed in the same + * way as those in the main targetlist. + */ + if (root->parse->havingQual != NULL) + { + List *having_exprs; + + having_exprs = pull_var_clause((Node *) root->parse->havingQual, + PVC_INCLUDE_AGGREGATES); + if (having_exprs != NIL) + tlist_exprs = list_concat(tlist_exprs, having_exprs); + } + + if (tlist_exprs == NIL) + return; + + /* tlist_exprs may also contain Vars, but we only need Aggrefs. */ + foreach(lc, tlist_exprs) + { + Expr *expr = (Expr *) lfirst(lc); + Aggref *aggref; + ListCell *lc2; + GroupedVarInfo *gvi; + bool exists; + + if (IsA(expr, Var)) + continue; + + aggref = castNode(Aggref, expr); + + /* TODO Think if (some of) these can be handled. */ + if (aggref->aggvariadic || + aggref->aggdirectargs || aggref->aggorder || + aggref->aggdistinct || aggref->aggfilter) + { + /* + * Partial aggregation is not useful if at least one aggregate + * cannot be evaluated below the top-level join. + * + * XXX Is it worth freeing the GroupedVarInfos and their subtrees? + */ + root->grouped_var_list = NIL; + break; + } + + /* + * Aggregation push-down does not work w/o aggcombinefn. This field is + * not mandatory, so check if this particular aggregate can handle + * partial aggregation. + */ + if (!OidIsValid(aggref->aggcombinefn)) + { + root->grouped_var_list = NIL; + break; + } + + /* Does GroupedVarInfo for this aggregate already exist? */ + exists = false; + foreach(lc2, root->grouped_var_list) + { + gvi = lfirst_node(GroupedVarInfo, lc2); + + if (equal(expr, gvi->gvexpr)) + { + exists = true; + break; + } + } + + /* Construct a new GroupedVarInfo if does not exist yet. */ + if (!exists) + { + Relids relids; + + /* TODO Initialize gv_width. */ + gvi = makeNode(GroupedVarInfo); + + gvi->gvid = list_length(root->grouped_var_list); + gvi->gvexpr = (Expr *) copyObject(aggref); + gvi->agg_partial = copyObject(aggref); + mark_partial_aggref(gvi->agg_partial, AGGSPLIT_INITIAL_SERIAL); + + /* Find out where the aggregate should be evaluated. */ + relids = pull_varnos((Node *) aggref); + if (!bms_is_empty(relids)) + gvi->gv_eval_at = relids; + else + gvi->gv_eval_at = NULL; + + gvi->gv_width = get_typavgwidth(exprType((Node *) gvi->gvexpr), + exprTypmod((Node *) gvi->gvexpr)); + + root->grouped_var_list = lappend(root->grouped_var_list, gvi); + } + } + + list_free(tlist_exprs); +} + +/* + * Create GroupedVarInfo for each expression usable as grouping key. + * + * In addition to the expressions of the query targetlist, group_pathkeys is + * also considered the source of grouping expressions. That increases the + * chance to get the relation output grouped. + */ +static void +create_grouping_expr_grouped_var_infos(PlannerInfo *root) +{ + ListCell *l1, + *l2; + List *exprs = NIL; + List *sortgrouprefs = NIL; + + /* + * Make sure GroupedVar exists for each expression usable as grouping key. + */ + foreach(l1, root->processed_tlist) + { + TargetEntry *te = lfirst_node(TargetEntry, l1); + Index sortgroupref = te->ressortgroupref; + + if (sortgroupref == 0) + continue; + + /* + * Non-zero sortgroupref does not necessarily imply grouping + * expression: data can also be sorted by aggregate. + */ + if (IsA(te->expr, Aggref)) + continue; + + exprs = lappend(exprs, te->expr); + sortgrouprefs = lappend_int(sortgrouprefs, sortgroupref); + } + + /* + * Try to derive additional grouping expressions from group_pathkeys. This + * increases the chance that relation can be grouped even if its columns + * are not mentioned in the grouping clause. + * + * It's important that we have processed the explicit grouping columns + * first. If the grouping clause contains multiple expressions belonging + * to the same EC, the original (i.e. not derived) one should be preferred + * when we build grouping target for a relation. Otherwise we have a + * problem when trying to match target entries to grouping clauses during + * plan creation, see extract_grouping_cols. + */ + foreach(l1, root->group_pathkeys) + { + PathKey *pk = lfirst_node(PathKey, l1); + EquivalenceClass *ec = pk->pk_eclass; + EquivalenceMember *em; + Index sortgroupref = 0; + + /* We need equality anywhere in the join tree. */ + if (ec->ec_below_outer_join) + continue; + + /* + * TODO Reconsider this restriction. As the grouping expression is + * only evaluated at the relation level (and only the result will be + * propagated to the final targetlist), volatile function might be + * o.k. Need to think what volatile EC exactly means. + */ + if (ec->ec_has_volatile) + continue; + + foreach(l2, ec->ec_members) + { + ListCell *l3; + + em = lfirst_node(EquivalenceMember, l2); + + /* + * We search for grouping expressions now. + */ + if (IsA(em->em_expr, Aggref)) + continue; + + if (em->em_nullable_relids) + continue; + + /* + * If target list contains this expression, it should provide us + * with the sortgroupref. + */ + foreach(l3, root->processed_tlist) + { + TargetEntry *te = lfirst_node(TargetEntry, l3); + + if (equal(em->em_expr, te->expr)) + { + /* + * XXX Not sure if the same expression exist in the + * targetlist multiple times and if some occurrences can + * miss the ressortgroupref. Maybe we just need to + * Assert() here that ressortgroupref is non-zero. + */ + if (te->ressortgroupref > 0) + { + sortgroupref = te->ressortgroupref; + break; + } + } + } + + /* + * If a single EC member matches, no need to check the rest of the + * EC. + */ + if (sortgroupref > 0) + break; + } + + if (sortgroupref == 0) + /* Go for the next EC. */ + continue; + + /* + * The EC contains a grouping expression, so each member of that EC + * should be usable as grouping key. Remember all the expressions of + * the EC as well as their (supposedly common) sortgroupref for the + * final processing. + */ + foreach(l2, ec->ec_members) + { + em = lfirst_node(EquivalenceMember, l2); + exprs = lappend(exprs, em->em_expr); + sortgrouprefs = lappend_int(sortgrouprefs, sortgroupref); + } + } + + /* + * Finally construct GroupedVarInfo for each expression. + */ + forboth(l1, exprs, l2, sortgrouprefs) + { + Expr *expr = (Expr *) lfirst(l1); + int sortgroupref = lfirst_int(l2); + GroupedVarInfo *gvi = makeNode(GroupedVarInfo); + + gvi->gvid = list_length(root->grouped_var_list); + gvi->gvexpr = (Expr *) copyObject(expr); + gvi->sortgroupref = sortgroupref; + + /* Find out where the aggregate should be evaluated. */ + gvi->gv_eval_at = pull_varnos((Node *) expr); + + /* + * If the grouping column is a plain Var, it has GroupedVarInfo (XXX + * should it be so?) but the actual Var will appear in the target. + * Thus set_rel_width should get better estimate from statistics. Do + * not waste cycles here to get less accurate value. + */ + if (!IsA(gvi->gvexpr, Var)) + gvi->gv_width = get_typavgwidth(exprType((Node *) gvi->gvexpr), + exprTypmod((Node *) gvi->gvexpr)); + + root->grouped_var_list = lappend(root->grouped_var_list, gvi); + } +} + +/* + * Check if rel->reltarget allows the relation to be grouped and initialize + * the grouping target(s). + * + * target_agg is a target that we'll eventually used to aggregate the output + * of relation paths. + * + * If we succeed to create the grouping target, also replace rel->reltarget + * with a new one that has sortgrouprefs initialized --- this is necessary for + * create_agg_plan to match the grouping clauses against the input target + * expressions. (Thus the scan / join that provides the AggPath with input + * does not have to care whether the source relation does have grouped target + * or not.) + * + * The *group_exprs_extra_p list may receive additional grouping expressions + * that the query does not have. These can make the aggregation of base + * relation / join less efficient, but it may still be better than not trying + * at all. + * + * TODO Make sure cost / width of both "result" and "plain" are correct. + */ +void +initialize_grouped_target(PlannerInfo *root, RelOptInfo *rel, + PathTarget *target_agg, + List **group_exprs_extra_p) +{ + PathTarget *target_plain; + ListCell *lc1; + List *vars_unresolved = NIL; + + /* The target to replace rel->reltarget. */ + target_plain = create_empty_pathtarget(); + + foreach(lc1, rel->reltarget->exprs) + { + Var *tvar; + GroupedVar *gvar; + + /* + * Given that PlaceHolderVar currently prevents us from doing + * aggregation push-down, the source target cannot contain anything + * more complex than a Var. (As for generic grouping expressions, + * add_grouped_vars_to_target will retrieve them from the query + * targetlist and add them to target_agg outside this function.) + */ + tvar = lfirst_node(Var, lc1); + + gvar = get_grouping_expression(root, (Expr *) tvar); + if (gvar != NULL) + { + /* + * It's o.k. to use the target expression for grouping. + * + * The actual Var is added to the target. If we used the + * containing GroupedVar, references from various clauses (e.g. + * join quals) wouldn't work. + */ + add_column_to_pathtarget(target_agg, (Expr *) gvar->gvexpr, + gvar->sortgroupref); + + /* + * As for the plain target, add the original expression but set + * sortgroupref in addition. + */ + add_column_to_pathtarget(target_plain, gvar->gvexpr, + gvar->sortgroupref); + + /* Process the next expression. */ + continue; + } + + /* + * Further investigation involves dependency check, for which we need + * to have all the plain-var grouping expressions gathered. So far + * only store the var in a list. + */ + vars_unresolved = lappend(vars_unresolved, tvar); + } + + /* + * Check for other possible reasons for the var to be in the plain target. + */ + foreach(lc1, vars_unresolved) + { + Var *var; + RangeTblEntry *rte; + List *deps = NIL; + Relids relids_subtract; + int ndx; + RelOptInfo *baserel; + + var = lfirst_node(Var, lc1); + rte = root->simple_rte_array[var->varno]; + + /* + * Dependent var is almost the same as one that has sortgroupref. + */ + if (check_functional_grouping(rte->relid, var->varno, + var->varlevelsup, + target_agg->exprs, &deps)) + { + + Index sortgroupref = 0; + + add_column_to_pathtarget(target_agg, (Expr *) var, sortgroupref); + + /* + * The var shouldn't be actually used as a grouping key (instead, + * the one this depends on will be), so sortgroupref should not be + * important. But once we have it ... + */ + add_column_to_pathtarget(target_plain, (Expr *) var, + sortgroupref); + + /* + * The var may or may not be present in generic grouping + * expression(s) or aggregate arguments, but we already have it in + * the targets, so don't care. + */ + continue; + } + + /* + * Isn't the expression needed by joins above the current rel? + * + * The relids we're not interested in do include 0, which is the + * top-level targetlist. The only reason for relids to contain 0 + * should be that arg_var is referenced either by aggregate or by + * grouping expression, but right now we're interested in the *other* + * reasons. (As soon as GroupedVars are installed, the top level + * aggregates / grouping expressions no longer need direct reference + * to arg_var anyway.) + */ + relids_subtract = bms_copy(rel->relids); + bms_add_member(relids_subtract, 0); + + baserel = find_base_rel(root, var->varno); + ndx = var->varattno - baserel->min_attr; + if (bms_nonempty_difference(baserel->attr_needed[ndx], + relids_subtract)) + { + /* + * The variable is needed by upper join. This includes one that is + * referenced by a generic grouping expression but couldn't be + * recognized as grouping expression on its own at the top of the + * loop. + * + * The only way to bring this var to the aggregation output is to + * add it to the grouping expressions too. + * + * Since root->parse->groupClause is not supposed to contain this + * expression, we need construct special SortGroupClause. Its + * tleSortGroupRef needs to be unique within target_agg, so + * postpone creation of the SortGroupRefs until we're done with + * the iteration of rel->reltarget->exprs. + */ + *group_exprs_extra_p = lappend(*group_exprs_extra_p, var); + } + else + { + /* + * As long as the query is semantically correct, arriving here + * means that the var is referenced either by aggregate argument + * or by generic grouping expression. The aggregation target + * should not contain it, as it only provides input for the final + * aggregation. + */ + } + + /* + * The var is not suitable for grouping, but the plain target ought to + * stay complete. + */ + add_column_to_pathtarget(target_plain, (Expr *) var, 0); + } + + /* + * Apply the adjusted input target as the replacement is complete now. + */ + /* TODO Free the old one. */ + rel->reltarget = target_plain; +} + /***************************************************************************** * diff --git a/src/backend/optimizer/plan/planagg.c b/src/backend/optimizer/plan/planagg.c index 889e8af..73bccee 100644 --- a/src/backend/optimizer/plan/planagg.c +++ b/src/backend/optimizer/plan/planagg.c @@ -223,7 +223,7 @@ preprocess_minmax_aggregates(PlannerInfo *root, List *tlist) create_minmaxagg_path(root, grouped_rel, create_pathtarget(root, tlist), aggs_list, - (List *) parse->havingQual)); + (List *) parse->havingQual), false); } /* diff --git a/src/backend/optimizer/plan/planmain.c b/src/backend/optimizer/plan/planmain.c index f4e0a6e..2bf4c8b 100644 --- a/src/backend/optimizer/plan/planmain.c +++ b/src/backend/optimizer/plan/planmain.c @@ -83,7 +83,7 @@ query_planner(PlannerInfo *root, List *tlist, add_path(final_rel, (Path *) create_result_path(root, final_rel, final_rel->reltarget, - (List *) parse->jointree->quals)); + (List *) parse->jointree->quals), false); /* Select cheapest path (pretty easy in this case...) */ set_cheapest(final_rel); @@ -114,6 +114,7 @@ query_planner(PlannerInfo *root, List *tlist, root->full_join_clauses = NIL; root->join_info_list = NIL; root->placeholder_list = NIL; + root->grouped_var_list = NIL; root->fkey_list = NIL; root->initial_rels = NIL; @@ -177,6 +178,14 @@ query_planner(PlannerInfo *root, List *tlist, (*qp_callback) (root, qp_extra); /* + * If the query result can be grouped, check if any grouping can be + * performed below the top-level join. If so, Initialize GroupedPathInfo + * of base relations capable to do the grouping and setup + * root->grouped_var_list. + */ + add_grouping_info_to_base_rels(root); + + /* * Examine any "placeholder" expressions generated during subquery pullup. * Make sure that the Vars they need are marked as needed at the relevant * join level. This must be done before join removal because it might diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index ef2eaea..ec99718 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -131,9 +131,6 @@ static void standard_qp_callback(PlannerInfo *root, void *extra); static double get_number_of_groups(PlannerInfo *root, double path_rows, grouping_sets_data *gd); -static Size estimate_hashagg_tablesize(Path *path, - const AggClauseCosts *agg_costs, - double dNumGroups); static RelOptInfo *create_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel, PathTarget *target, @@ -148,6 +145,18 @@ static void consider_groupingsets_paths(PlannerInfo *root, grouping_sets_data *gd, const AggClauseCosts *agg_costs, double dNumGroups); +static void sort_agg_partial_grouped_paths(PlannerInfo *root, List *pathlist, + AggClauseCosts *agg_final_costs, + double dNumGroups, + RelOptInfo *grouped_rel, + PathTarget *gather_target, + PathTarget *group_target); +static void hash_agg_partial_grouped_path(PlannerInfo *root, Path *path, + AggClauseCosts *agg_final_costs, + double dNumGroups, + RelOptInfo *grouped_rel, + PathTarget *gather_target, + PathTarget *group_target); static RelOptInfo *create_window_paths(PlannerInfo *root, RelOptInfo *input_rel, PathTarget *input_target, @@ -185,6 +194,8 @@ static PathTarget *make_sort_input_target(PlannerInfo *root, bool *have_postponed_srfs); static void adjust_paths_for_srfs(PlannerInfo *root, RelOptInfo *rel, List *targets, List *targets_contain_srfs); +static Path *adjust_path_for_unique_group_keys(PlannerInfo *root, Path *path, + PathTarget *target); /***************************************************************************** @@ -544,6 +555,7 @@ subquery_planner(PlannerGlobal *glob, Query *parse, memset(root->upper_rels, 0, sizeof(root->upper_rels)); memset(root->upper_targets, 0, sizeof(root->upper_targets)); root->processed_tlist = NIL; + root->max_sortgroupref = 0; root->grouping_map = NULL; root->minmax_aggs = NIL; root->qual_security_level = 0; @@ -1519,7 +1531,7 @@ inheritance_planner(PlannerInfo *root) returningLists, rowMarks, NULL, - SS_assign_special_param(root))); + SS_assign_special_param(root)), false); } /*-------------------- @@ -1922,7 +1934,8 @@ grouping_planner(PlannerInfo *root, bool inheritance_update, newpath = (Path *) create_projection_path(root, current_rel, subpath, - scanjoin_target); + scanjoin_target, + false); lfirst(lc) = newpath; } } @@ -1966,6 +1979,7 @@ grouping_planner(PlannerInfo *root, bool inheritance_update, grouping_target, &agg_costs, gset_data); + /* Fix things up if grouping_target contains SRFs */ if (parse->hasTargetSRFs) adjust_paths_for_srfs(root, current_rel, @@ -2134,7 +2148,7 @@ grouping_planner(PlannerInfo *root, bool inheritance_update, } /* And shove it into final_rel */ - add_path(final_rel, path); + add_path(final_rel, path, false); } /* @@ -3540,40 +3554,6 @@ get_number_of_groups(PlannerInfo *root, } /* - * estimate_hashagg_tablesize - * estimate the number of bytes that a hash aggregate hashtable will - * require based on the agg_costs, path width and dNumGroups. - * - * XXX this may be over-estimating the size now that hashagg knows to omit - * unneeded columns from the hashtable. Also for mixed-mode grouping sets, - * grouping columns not in the hashed set are counted here even though hashagg - * won't store them. Is this a problem? - */ -static Size -estimate_hashagg_tablesize(Path *path, const AggClauseCosts *agg_costs, - double dNumGroups) -{ - Size hashentrysize; - - /* Estimate per-hash-entry space at tuple width... */ - hashentrysize = MAXALIGN(path->pathtarget->width) + - MAXALIGN(SizeofMinimalTupleHeader); - - /* plus space for pass-by-ref transition values... */ - hashentrysize += agg_costs->transitionSpace; - /* plus the per-hash-entry overhead */ - hashentrysize += hash_agg_entry_size(agg_costs->numAggs); - - /* - * Note that this disregards the effect of fill-factor and growth policy - * of the hash-table. That's probably ok, given default the default - * fill-factor is relatively high. It'd be hard to meaningfully factor in - * "double-in-size" growth policies here. - */ - return hashentrysize * dNumGroups; -} - -/* * create_grouping_paths * * Build a new upperrel containing Paths for grouping and/or aggregation. @@ -3604,8 +3584,8 @@ create_grouping_paths(PlannerInfo *root, Path *cheapest_path = input_rel->cheapest_total_path; RelOptInfo *grouped_rel; PathTarget *partial_grouping_target = NULL; - AggClauseCosts agg_partial_costs; /* parallel only */ - AggClauseCosts agg_final_costs; /* parallel only */ + AggClauseCosts agg_final_costs; + bool agg_final_costs_known = false; Size hashaggtablesize; double dNumGroups; double dNumPartialGroups = 0; @@ -3694,7 +3674,7 @@ create_grouping_paths(PlannerInfo *root, (List *) parse->havingQual); } - add_path(grouped_rel, path); + add_path(grouped_rel, path, false); /* No need to consider any other alternatives. */ set_cheapest(grouped_rel); @@ -3787,6 +3767,13 @@ create_grouping_paths(PlannerInfo *root, } /* + * Collect statistics about aggregates for estimating costs of performing + * aggregation in parallel. + */ + MemSet(&agg_final_costs, 0, sizeof(AggClauseCosts)); + agg_final_costs_known = false; + + /* * Before generating paths for grouped_rel, we first generate any possible * partial paths; that way, later code can easily consider both parallel * and non-parallel approaches to grouping. Note that the partial paths @@ -3797,6 +3784,9 @@ create_grouping_paths(PlannerInfo *root, if (try_parallel_aggregation) { Path *cheapest_partial_path = linitial(input_rel->partial_pathlist); + AggClauseCosts agg_partial_costs; + + MemSet(&agg_partial_costs, 0, sizeof(AggClauseCosts)); /* * Build target list for partial aggregate paths. These paths cannot @@ -3812,26 +3802,12 @@ create_grouping_paths(PlannerInfo *root, cheapest_partial_path->rows, gd); - /* - * Collect statistics about aggregates for estimating costs of - * performing aggregation in parallel. - */ - MemSet(&agg_partial_costs, 0, sizeof(AggClauseCosts)); - MemSet(&agg_final_costs, 0, sizeof(AggClauseCosts)); if (parse->hasAggs) { /* partial phase */ get_agg_clause_costs(root, (Node *) partial_grouping_target->exprs, AGGSPLIT_INITIAL_SERIAL, &agg_partial_costs); - - /* final phase */ - get_agg_clause_costs(root, (Node *) target->exprs, - AGGSPLIT_FINAL_DESERIAL, - &agg_final_costs); - get_agg_clause_costs(root, parse->havingQual, - AGGSPLIT_FINAL_DESERIAL, - &agg_final_costs); } if (can_sort) @@ -3871,7 +3847,8 @@ create_grouping_paths(PlannerInfo *root, parse->groupClause, NIL, &agg_partial_costs, - dNumPartialGroups)); + dNumPartialGroups), + false); else add_partial_path(grouped_rel, (Path *) create_group_path(root, @@ -3880,7 +3857,8 @@ create_grouping_paths(PlannerInfo *root, partial_grouping_target, parse->groupClause, NIL, - dNumPartialGroups)); + dNumPartialGroups), + false); } } } @@ -3911,7 +3889,8 @@ create_grouping_paths(PlannerInfo *root, parse->groupClause, NIL, &agg_partial_costs, - dNumPartialGroups)); + dNumPartialGroups), + false); } } } @@ -3919,17 +3898,36 @@ create_grouping_paths(PlannerInfo *root, /* Build final grouping paths */ if (can_sort) { + List *pathlist = input_rel->pathlist; + int nplain = list_length(input_rel->pathlist); + int i; + + /* + * The grouped paths created out of grouped base relations or joins + * can be treated almost identically here (the major difference is + * that their targetlists contain GroupedVars instead aggregate input + * vars, but this is handled by using the appropriate value of + * AggSplit), so process them in the same loop. + */ + if (input_rel->gpi != NULL && input_rel->gpi->pathlist != NIL) + pathlist = list_concat(list_copy(pathlist), + input_rel->gpi->pathlist); + /* * Use any available suitably-sorted path as input, and also consider * sorting the cheapest-total path. */ - foreach(lc, input_rel->pathlist) + i = 0; + foreach(lc, pathlist) { Path *path = (Path *) lfirst(lc); bool is_sorted; + bool is_grouped; is_sorted = pathkeys_contained_in(root->group_pathkeys, path->pathkeys); + is_grouped = i >= nplain; + if (path == cheapest_path || is_sorted) { /* Sort the cheapest-total path if it isn't already sorted */ @@ -3943,12 +3941,39 @@ create_grouping_paths(PlannerInfo *root, /* Now decide what to stick atop it */ if (parse->groupingSets) { + /* + * No grouping should have taken place at base relation / + * join level, i.e. pathlist should be equal to + * input_rel->pathlist. + */ + Assert(!is_grouped); + consider_groupingsets_paths(root, grouped_rel, path, true, can_hash, target, gd, agg_costs, dNumGroups); } else if (parse->hasAggs) { + AggStrategy aggstrategy; + AggSplit aggsplit; + + if (!is_grouped) + { + aggstrategy = parse->groupClause ? AGG_SORTED : AGG_PLAIN; + aggsplit = AGGSPLIT_SIMPLE; + } + else + { + /* + * Like above, no grouping of base relation is not + * possible w/o this. + */ + Assert(parse->groupClause); + + aggstrategy = AGG_SORTED; + aggsplit = AGGSPLIT_FINAL_DESERIAL; + } + /* * We have aggregation, possibly with plain GROUP BY. Make * an AggPath. @@ -3958,12 +3983,12 @@ create_grouping_paths(PlannerInfo *root, grouped_rel, path, target, - parse->groupClause ? AGG_SORTED : AGG_PLAIN, - AGGSPLIT_SIMPLE, + aggstrategy, + aggsplit, parse->groupClause, (List *) parse->havingQual, agg_costs, - dNumGroups)); + dNumGroups), false); } else if (parse->groupClause) { @@ -3978,7 +4003,7 @@ create_grouping_paths(PlannerInfo *root, target, parse->groupClause, (List *) parse->havingQual, - dNumGroups)); + dNumGroups), false); } else { @@ -3986,122 +4011,50 @@ create_grouping_paths(PlannerInfo *root, Assert(false); } } + + i++; } /* - * Now generate a complete GroupAgg Path atop of the cheapest partial - * path. We can do this using either Gather or Gather Merge. + * Compute agg_final_costs iff the aggregation should take place in 2 + * steps. */ - if (grouped_rel->partial_pathlist) + if (parse->hasAggs && + (grouped_rel->partial_pathlist || + (input_rel->gpi != NULL && input_rel->gpi->partial_pathlist))) { - Path *path = (Path *) linitial(grouped_rel->partial_pathlist); - double total_groups = path->rows * path->parallel_workers; + get_agg_clause_costs(root, (Node *) target->exprs, + AGGSPLIT_FINAL_DESERIAL, + &agg_final_costs); + get_agg_clause_costs(root, parse->havingQual, + AGGSPLIT_FINAL_DESERIAL, + &agg_final_costs); - path = (Path *) create_gather_path(root, - grouped_rel, - path, - partial_grouping_target, - NULL, - &total_groups); + agg_final_costs_known = true; + } - /* - * Since Gather's output is always unsorted, we'll need to sort, - * unless there's no GROUP BY clause or a degenerate (constant) - * one, in which case there will only be a single group. - */ - if (root->group_pathkeys) - path = (Path *) create_sort_path(root, - grouped_rel, - path, - root->group_pathkeys, - -1.0); + /* + * Gather grouped partial paths and apply AGG_SORTED to them. + */ + if (grouped_rel->partial_pathlist) + sort_agg_partial_grouped_paths(root, + grouped_rel->partial_pathlist, + &agg_final_costs, + dNumGroups, grouped_rel, + partial_grouping_target, target); - if (parse->hasAggs) - add_path(grouped_rel, (Path *) - create_agg_path(root, - grouped_rel, - path, - target, - parse->groupClause ? AGG_SORTED : AGG_PLAIN, - AGGSPLIT_FINAL_DESERIAL, - parse->groupClause, - (List *) parse->havingQual, - &agg_final_costs, - dNumGroups)); - else - add_path(grouped_rel, (Path *) - create_group_path(root, + /* + * The same for the grouped partial paths involving base relation / + * join grouping. + */ + if (input_rel->gpi != NULL && input_rel->gpi->target != NULL && + input_rel->gpi->partial_pathlist) + sort_agg_partial_grouped_paths(root, + input_rel->gpi->partial_pathlist, + &agg_final_costs, dNumGroups, grouped_rel, - path, - target, - parse->groupClause, - (List *) parse->havingQual, - dNumGroups)); - - /* - * The point of using Gather Merge rather than Gather is that it - * can preserve the ordering of the input path, so there's no - * reason to try it unless (1) it's possible to produce more than - * one output row and (2) we want the output path to be ordered. - */ - if (parse->groupClause != NIL && root->group_pathkeys != NIL) - { - foreach(lc, grouped_rel->partial_pathlist) - { - Path *subpath = (Path *) lfirst(lc); - Path *gmpath; - double total_groups; - - /* - * It's useful to consider paths that are already properly - * ordered for Gather Merge, because those don't need a - * sort. It's also useful to consider the cheapest path, - * because sorting it in parallel and then doing Gather - * Merge may be better than doing an unordered Gather - * followed by a sort. But there's no point in - * considering non-cheapest paths that aren't already - * sorted correctly. - */ - if (path != subpath && - !pathkeys_contained_in(root->group_pathkeys, - subpath->pathkeys)) - continue; - - total_groups = subpath->rows * subpath->parallel_workers; - - gmpath = (Path *) - create_gather_merge_path(root, - grouped_rel, - subpath, - partial_grouping_target, - root->group_pathkeys, - NULL, - &total_groups); - - if (parse->hasAggs) - add_path(grouped_rel, (Path *) - create_agg_path(root, - grouped_rel, - gmpath, - target, - parse->groupClause ? AGG_SORTED : AGG_PLAIN, - AGGSPLIT_FINAL_DESERIAL, - parse->groupClause, - (List *) parse->havingQual, - &agg_final_costs, - dNumGroups)); - else - add_path(grouped_rel, (Path *) - create_group_path(root, - grouped_rel, - gmpath, - target, - parse->groupClause, - (List *) parse->havingQual, - dNumGroups)); - } - } - } + input_rel->gpi->target, + target); } if (can_hash) @@ -4143,11 +4096,27 @@ create_grouping_paths(PlannerInfo *root, parse->groupClause, (List *) parse->havingQual, agg_costs, - dNumGroups)); + dNumGroups), false); } } /* + * Compute agg_final_costs if needed below and if not done above. + */ + if (parse->hasAggs && !agg_final_costs_known && + (grouped_rel->partial_pathlist || + (input_rel->gpi != NULL && + (input_rel->gpi->pathlist || input_rel->gpi->partial_pathlist)))) + { + get_agg_clause_costs(root, (Node *) target->exprs, + AGGSPLIT_FINAL_DESERIAL, + &agg_final_costs); + get_agg_clause_costs(root, parse->havingQual, + AGGSPLIT_FINAL_DESERIAL, + &agg_final_costs); + } + + /* * Generate a HashAgg Path atop of the cheapest partial path. Once * again, we'll only do this if it looks as though the hash table * won't exceed work_mem. @@ -4156,33 +4125,82 @@ create_grouping_paths(PlannerInfo *root, { Path *path = (Path *) linitial(grouped_rel->partial_pathlist); + hash_agg_partial_grouped_path(root, path, &agg_final_costs, + dNumGroups, grouped_rel, + partial_grouping_target, target); + } + + /* + * If input_rel has partially aggregated paths, perform the final + * aggregation. + */ + if (input_rel->gpi != NULL && input_rel->gpi->pathlist != NIL) + { + Path *path = (Path *) linitial(input_rel->gpi->pathlist); + hashaggtablesize = estimate_hashagg_tablesize(path, &agg_final_costs, dNumGroups); if (hashaggtablesize < work_mem * 1024L) { - double total_groups = path->rows * path->parallel_workers; + /* + * The top-level grouped_rel needs to receive the path into + * regular pathlist, as opposed grouped_rel->gpi->pathlist. So + * pass FALSE for grouped. + */ + add_path(grouped_rel, + (Path *) create_agg_path(root, grouped_rel, + path, + target, + AGG_HASHED, + AGGSPLIT_FINAL_DESERIAL, + parse->groupClause, + (List *) parse->havingQual, + &agg_final_costs, + dNumGroups), + false); + } + } - path = (Path *) create_gather_path(root, - grouped_rel, - path, - partial_grouping_target, - NULL, - &total_groups); + /* + * If input_rel has partially aggregated partial paths, gather them + * and perform the final aggregation. + */ + if (input_rel->gpi != NULL && input_rel->gpi->target != NULL && + input_rel->gpi->partial_pathlist != NIL) + { + Path *path = (Path *) linitial(input_rel->gpi->partial_pathlist); - add_path(grouped_rel, (Path *) - create_agg_path(root, - grouped_rel, - path, - target, - AGG_HASHED, - AGGSPLIT_FINAL_DESERIAL, - parse->groupClause, - (List *) parse->havingQual, - &agg_final_costs, - dNumGroups)); - } + hash_agg_partial_grouped_path(root, path, &agg_final_costs, dNumGroups, + grouped_rel, input_rel->gpi->target, + target); + } + } + + /* + * If input_rel has partially aggregated paths which emit an unique set of + * grouping keys, we only need to call aggfinalfn on each aggregate state + * instead of doing the final aggregation + */ + if (input_rel->gpi != NULL && !parse->groupingSets) + { + foreach(lc, input_rel->gpi->pathlist) + { + Path *path = (Path *) lfirst(lc); + + /* + * If aggregation could have been pushed down and if the path + * generates unique grouping keys, replace aggregates with the + * aggregate final functions alone. + */ + if ((parse->groupClause || parse->hasAggs || root->hasHavingQual) + && root->grouped_var_list != NIL && + match_path_to_group_pathkeys(root, path)) + add_path(grouped_rel, + adjust_path_for_unique_group_keys(root, path, + target), + false); } } @@ -4383,7 +4401,7 @@ consider_groupingsets_paths(PlannerInfo *root, strat, new_rollups, agg_costs, - dNumGroups)); + dNumGroups), false); return; } @@ -4541,7 +4559,7 @@ consider_groupingsets_paths(PlannerInfo *root, AGG_MIXED, rollups, agg_costs, - dNumGroups)); + dNumGroups), false); } } @@ -4558,7 +4576,174 @@ consider_groupingsets_paths(PlannerInfo *root, AGG_SORTED, gd->rollups, agg_costs, - dNumGroups)); + dNumGroups), false); +} + +/* + * Subroutine of create_grouping_paths() to apply GatherPath or + * GatherMergePath and AGG_SORTED AggPath to cases which only differ in the + * GatherPath / GatherMergePath target. + */ +static void +sort_agg_partial_grouped_paths(PlannerInfo *root, List *pathlist, + AggClauseCosts *agg_final_costs, + double dNumGroups, RelOptInfo *grouped_rel, + PathTarget *gather_target, + PathTarget *group_target) +{ + Path *path = (Path *) linitial(pathlist); + double total_groups = path->rows * path->parallel_workers; + Query *parse = root->parse; + ListCell *lc; + + /* + * Generate a complete GroupAgg Path atop of the cheapest partial path. We + * can do this using either Gather or Gather Merge. + */ + path = (Path *) create_gather_path(root, + grouped_rel, + path, + gather_target, + NULL, + &total_groups); + + /* + * Since Gather's output is always unsorted, we'll need to sort, unless + * there's no GROUP BY clause or a degenerate (constant) one, in which + * case there will only be a single group. + */ + if (root->group_pathkeys) + path = (Path *) create_sort_path(root, + grouped_rel, + path, + root->group_pathkeys, + -1.0); + + if (parse->hasAggs) + add_path(grouped_rel, (Path *) + create_agg_path(root, + grouped_rel, + path, + group_target, + parse->groupClause ? AGG_SORTED : AGG_PLAIN, + AGGSPLIT_FINAL_DESERIAL, + parse->groupClause, + (List *) parse->havingQual, + agg_final_costs, + dNumGroups), false); + else + add_path(grouped_rel, (Path *) + create_group_path(root, + grouped_rel, + path, + group_target, + parse->groupClause, + (List *) parse->havingQual, + dNumGroups), false); + + /* + * The point of using Gather Merge rather than Gather is that it can + * preserve the ordering of the input path, so there's no reason to try it + * unless (1) it's possible to produce more than one output row and (2) we + * want the output path to be ordered. + */ + if (parse->groupClause != NIL && root->group_pathkeys != NIL) + { + foreach(lc, pathlist) + { + Path *subpath = (Path *) lfirst(lc); + Path *gmpath; + double total_groups; + + /* + * It's useful to consider paths that are already properly ordered + * for Gather Merge, because those don't need a sort. It's also + * useful to consider the cheapest path, because sorting it in + * parallel and then doing Gather Merge may be better than doing + * an unordered Gather followed by a sort. But there's no point + * in considering non-cheapest paths that aren't already sorted + * correctly. + */ + if (path != subpath && + !pathkeys_contained_in(root->group_pathkeys, + subpath->pathkeys)) + continue; + + total_groups = subpath->rows * subpath->parallel_workers; + + gmpath = (Path *) + create_gather_merge_path(root, + grouped_rel, + subpath, + gather_target, + root->group_pathkeys, + NULL, + &total_groups); + + if (parse->hasAggs) + add_path(grouped_rel, (Path *) + create_agg_path(root, + grouped_rel, + gmpath, + group_target, + parse->groupClause ? AGG_SORTED : AGG_PLAIN, + AGGSPLIT_FINAL_DESERIAL, + parse->groupClause, + (List *) parse->havingQual, + agg_final_costs, + dNumGroups), false); + else + add_path(grouped_rel, (Path *) + create_group_path(root, + grouped_rel, + gmpath, + group_target, + parse->groupClause, + (List *) parse->havingQual, + dNumGroups), false); + } + } +} + +/* + * Subroutine of create_grouping_paths() to apply GatherPath and AGG_HASHED + * AggPath to cases which only differ in the GatherPath target. + */ +static void +hash_agg_partial_grouped_path(PlannerInfo *root, Path *path, + AggClauseCosts *agg_final_costs, + double dNumGroups, RelOptInfo *grouped_rel, + PathTarget *gather_target, + PathTarget *group_target) +{ + Size hashaggtablesize = estimate_hashagg_tablesize(path, agg_final_costs, + dNumGroups); + Query *parse = root->parse; + double total_groups; + + if (hashaggtablesize >= work_mem * 1024L) + return; + + total_groups = path->rows * path->parallel_workers; + + path = (Path *) create_gather_path(root, + grouped_rel, + path, + gather_target, + NULL, + &total_groups); + + add_path(grouped_rel, (Path *) + create_agg_path(root, + grouped_rel, + path, + group_target, + AGG_HASHED, + AGGSPLIT_FINAL_DESERIAL, + parse->groupClause, + (List *) parse->havingQual, + agg_final_costs, + dNumGroups), false); } /* @@ -4743,7 +4928,7 @@ create_one_window_path(PlannerInfo *root, window_pathkeys); } - add_path(window_rel, path); + add_path(window_rel, path, false); } /* @@ -4849,7 +5034,8 @@ create_distinct_paths(PlannerInfo *root, create_upper_unique_path(root, distinct_rel, path, list_length(root->distinct_pathkeys), - numDistinctRows)); + numDistinctRows), + false); } } @@ -4876,7 +5062,8 @@ create_distinct_paths(PlannerInfo *root, create_upper_unique_path(root, distinct_rel, path, list_length(root->distinct_pathkeys), - numDistinctRows)); + numDistinctRows), + false); } /* @@ -4923,7 +5110,7 @@ create_distinct_paths(PlannerInfo *root, parse->distinctClause, NIL, NULL, - numDistinctRows)); + numDistinctRows), false); } /* Give a helpful error if we failed to find any implementation */ @@ -5021,7 +5208,7 @@ create_ordered_paths(PlannerInfo *root, path = apply_projection_to_path(root, ordered_rel, path, target); - add_path(ordered_rel, path); + add_path(ordered_rel, path, false); } } @@ -5072,7 +5259,7 @@ create_ordered_paths(PlannerInfo *root, path = apply_projection_to_path(root, ordered_rel, path, target); - add_path(ordered_rel, path); + add_path(ordered_rel, path, false); } } @@ -5990,7 +6177,8 @@ adjust_paths_for_srfs(PlannerInfo *root, RelOptInfo *rel, newpath = (Path *) create_projection_path(root, rel, newpath, - thistarget); + thistarget, + false); } } lfirst(lc) = newpath; @@ -5998,6 +6186,86 @@ adjust_paths_for_srfs(PlannerInfo *root, RelOptInfo *rel, } /* + * Project a path that generates unique grouping keys to a new one whose + * target, instead of the (partial) aggregates, contains the aggregates' final + * functions, each referencing the corresponding partial aggregate. + */ +static Path * +adjust_path_for_unique_group_keys(PlannerInfo *root, Path *path, + PathTarget *target) +{ + ListCell *lc1; + PathTarget *target_new; + List *exprs = NIL; + + target_new = copy_pathtarget(target); + + /* + * Replace (partial) aggregates with aggfinalfn function calls. + */ + foreach(lc1, target_new->exprs) + { + Expr *expr = (Expr *) lfirst(lc1); + + if (IsA(expr, Aggref)) + { + ListCell *lc2; + Aggref *aggref = castNode(Aggref, expr); + GroupedVar *gvar = NULL; + + /* + * Find GroupedVar for this aggregate --- the input path should + * contain it too. Put it to the target for set_upper_references + * to be able to match the output and input target. + */ + foreach(lc2, path->pathtarget->exprs) + { + Expr *texpr = (Expr *) lfirst(lc2); + + if (IsA(texpr, GroupedVar)) + { + gvar = castNode(GroupedVar, texpr); + + if (equal(gvar->gvexpr, aggref)) + break; + } + } + /* The GroupedVar should have been found. */ + Assert(lc2 != NULL); + + /* + * Arrange for the call of aggfinalfn or let the transient state + * appear on the output unprocessed. + */ + if (OidIsValid(aggref->aggfinalfn)) + { + FuncExpr *fexpr = makeNode(FuncExpr); + + fexpr = makeNode(FuncExpr); + fexpr->funcid = aggref->aggfinalfn; + fexpr->funcresulttype = aggref->aggtype; + fexpr->funcretset = false; + fexpr->funcvariadic = aggref->aggvariadic; + fexpr->funcformat = COERCE_EXPLICIT_CALL; + fexpr->funccollid = aggref->aggcollid; + fexpr->inputcollid = aggref->inputcollid; + fexpr->args = list_make1(gvar); + fexpr->location = -1; + expr = (Expr *) fexpr; + } + else + expr = (Expr *) gvar; + } + + exprs = lappend(exprs, expr); + } + target_new->exprs = exprs; + + return (Path *) create_projection_path(root, path->parent, path, + target_new, false); +} + +/* * expression_planner * Perform planner's transformations on a standalone expression. * diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index b5c4124..9c08ac0 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -40,6 +40,7 @@ typedef struct List *tlist; /* underlying target list */ int num_vars; /* number of plain Var tlist entries */ bool has_ph_vars; /* are there PlaceHolderVar entries? */ + bool has_grp_vars; /* are there GroupedVar entries? */ bool has_non_vars; /* are there other entries? */ bool has_conv_whole_rows; /* are there ConvertRowtypeExpr * entries encapsulating a whole-row @@ -1196,33 +1197,50 @@ set_foreignscan_references(PlannerInfo *root, if (fscan->fdw_scan_tlist != NIL || fscan->scan.scanrelid == 0) { + List *tlist_tmp; + indexed_tlist *itlist = build_tlist_index(fscan->fdw_scan_tlist); + /* * Adjust tlist, qual, fdw_exprs, fdw_recheck_quals to reference - * foreign scan tuple + * foreign scan tuple. + * + * Since fdw_scan_tlist contains Aggrefs instead of GroupVars (this is + * convenient for deparsing), the Aggrefs also need to be restored in + * the referencing lists. + * + * XXX Consider thoroughly where tlist_tmp really needs to be used, + * i.e. which of the lists can / cannot contain aggregates. */ - indexed_tlist *itlist = build_tlist_index(fscan->fdw_scan_tlist); - + tlist_tmp = restore_grouping_expressions(root, + fscan->scan.plan.targetlist, + true); fscan->scan.plan.targetlist = (List *) fix_upper_expr(root, - (Node *) fscan->scan.plan.targetlist, + (Node *) tlist_tmp, itlist, INDEX_VAR, rtoffset); + fscan->scan.plan.qual = (List *) fix_upper_expr(root, (Node *) fscan->scan.plan.qual, itlist, INDEX_VAR, rtoffset); + fscan->fdw_exprs = (List *) fix_upper_expr(root, (Node *) fscan->fdw_exprs, itlist, INDEX_VAR, rtoffset); + + tlist_tmp = restore_grouping_expressions(root, + fscan->fdw_recheck_quals, + true); fscan->fdw_recheck_quals = (List *) fix_upper_expr(root, - (Node *) fscan->fdw_recheck_quals, + (Node *) tlist_tmp, itlist, INDEX_VAR, rtoffset); @@ -1739,9 +1757,81 @@ set_upper_references(PlannerInfo *root, Plan *plan, int rtoffset) indexed_tlist *subplan_itlist; List *output_targetlist; ListCell *l; + List *sub_tlist_save = NIL; + + if (root->grouped_var_list != NIL) + { + if (IsA(plan, Agg)) + { + Agg *agg = (Agg *) plan; + + if (agg->aggsplit == AGGSPLIT_FINAL_DESERIAL) + { + /* + * convert_combining_aggrefs could have replaced some vars + * with Aggref expressions representing the partial + * aggregation. We need to restore the same Aggrefs in the + * subplan targetlist, but this would break the subplan if + * it's something else than the partial aggregation (i.e. the + * partial aggregation takes place lower in the plan tree). So + * we'll eventually need to restore the current + * subplan->targetlist. + */ + if (!IsA(subplan, Agg)) + sub_tlist_save = subplan->targetlist; +#ifdef USE_ASSERT_CHECKING + else + Assert(((Agg *) subplan)->aggsplit == AGGSPLIT_INITIAL_SERIAL); +#endif /* USE_ASSERT_CHECKING */ + + /* + * Restore the aggregate expressions that we might have + * removed when planning for aggregation at base relation + * level. + */ + subplan->targetlist = + restore_grouping_expressions(root, subplan->targetlist, + true); + } + else if (agg->aggsplit == AGGSPLIT_INITIAL_SERIAL) + { + /* + * Partial aggregation node can have GroupedVar's on the input + * if those represent generic (non-Var) grouping expressions. + * Unlike above, the restored expressions should stay there. + */ + subplan->targetlist = + restore_grouping_expressions(root, subplan->targetlist, + true); + } + } + else if (IsA(plan, Result)) + { + /* + * If standalone aggfinalfn function is used (e.g. in the target + * of a Sort node) instead of AGGSPLIT_FINAL_DESERIAL aggregate, + * projection to the aggregate might have been added. Replace the + * function with the aggregate. + * + * TODO Find out if the Result plan is in parallel worker. In that + * case we'd better pass "true" for agg_partial. + */ + sub_tlist_save = subplan->targetlist; + subplan->targetlist = + restore_grouping_expressions(root, subplan->targetlist, + false); + } + } subplan_itlist = build_tlist_index(subplan->targetlist); + /* + * The replacement of GroupVars by Aggrefs was only needed for the index + * build. + */ + if (sub_tlist_save != NIL) + subplan->targetlist = sub_tlist_save; + output_targetlist = NIL; foreach(l, plan->targetlist) { @@ -1996,6 +2086,7 @@ build_tlist_index(List *tlist) itlist->tlist = tlist; itlist->has_ph_vars = false; + itlist->has_grp_vars = false; itlist->has_non_vars = false; itlist->has_conv_whole_rows = false; @@ -2016,6 +2107,8 @@ build_tlist_index(List *tlist) } else if (tle->expr && IsA(tle->expr, PlaceHolderVar)) itlist->has_ph_vars = true; + else if (tle->expr && IsA(tle->expr, GroupedVar)) + itlist->has_grp_vars = true; else if (is_converted_whole_row_reference((Node *) tle->expr)) itlist->has_conv_whole_rows = true; else @@ -2299,6 +2392,31 @@ fix_join_expr_mutator(Node *node, fix_join_expr_context *context) /* No referent found for Var */ elog(ERROR, "variable not found in subplan target lists"); } + if (IsA(node, GroupedVar)) + { + GroupedVar *gvar = (GroupedVar *) node; + + /* See if the GroupedVar has bubbled up from a lower plan node */ + if (context->outer_itlist && context->outer_itlist->has_grp_vars) + { + newvar = search_indexed_tlist_for_non_var((Expr *) gvar, + context->outer_itlist, + OUTER_VAR); + if (newvar) + return (Node *) newvar; + } + if (context->inner_itlist && context->inner_itlist->has_grp_vars) + { + newvar = search_indexed_tlist_for_non_var((Expr *) gvar, + context->inner_itlist, + INNER_VAR); + if (newvar) + return (Node *) newvar; + } + + /* No referent found for GroupedVar */ + elog(ERROR, "grouped variable not found in subplan target lists"); + } if (IsA(node, PlaceHolderVar)) { PlaceHolderVar *phv = (PlaceHolderVar *) node; @@ -2461,7 +2579,8 @@ fix_upper_expr_mutator(Node *node, fix_upper_expr_context *context) /* If no match, just fall through to process it normally */ } /* Try matching more complex expressions too, if tlist has any */ - if (context->subplan_itlist->has_non_vars || + if (context->subplan_itlist->has_grp_vars || + context->subplan_itlist->has_non_vars || (context->subplan_itlist->has_conv_whole_rows && is_converted_whole_row_reference(node))) { diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index 1d7e499..710f028 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -911,6 +911,7 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte, memset(subroot->upper_rels, 0, sizeof(subroot->upper_rels)); memset(subroot->upper_targets, 0, sizeof(subroot->upper_targets)); subroot->processed_tlist = NIL; + subroot->max_sortgroupref = 0; subroot->grouping_map = NULL; subroot->minmax_aggs = NIL; subroot->qual_security_level = 0; diff --git a/src/backend/optimizer/prep/prepunion.c b/src/backend/optimizer/prep/prepunion.c index f620243..8e4fd0e 100644 --- a/src/backend/optimizer/prep/prepunion.c +++ b/src/backend/optimizer/prep/prepunion.c @@ -221,7 +221,7 @@ plan_set_operations(PlannerInfo *root) root->processed_tlist = top_tlist; /* Add only the final path to the SETOP upperrel. */ - add_path(setop_rel, path); + add_path(setop_rel, path, false); /* Let extensions possibly add some more paths */ if (create_upper_paths_hook) diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index bc0841b..d9d6bd9 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -26,6 +26,7 @@ #include "optimizer/planmain.h" #include "optimizer/prep.h" #include "optimizer/restrictinfo.h" +/* TODO Remove this if get_grouping_expressions ends up in another module. */ #include "optimizer/tlist.h" #include "optimizer/var.h" #include "parser/parsetree.h" @@ -54,7 +55,12 @@ static List *translate_sub_tlist(List *tlist, int relid); static List *reparameterize_pathlist_by_child(PlannerInfo *root, List *pathlist, RelOptInfo *child_rel); - +static Path *add_grouping_expressions_to_subpath(PlannerInfo *root, + Path *subpath, + PathTarget *group_exprs); +static void make_uniquekeys_for_unique_index(PathTarget *reltarget, + IndexOptInfo *index, Path *path); +static void make_uniquekeys_for_append_path(PlannerInfo *root, Path *path); /***************************************************************************** * MISC. PATH UTILITIES @@ -417,8 +423,9 @@ set_cheapest(RelOptInfo *parent_rel) * Returns nothing, but modifies parent_rel->pathlist. */ void -add_path(RelOptInfo *parent_rel, Path *new_path) +add_path(RelOptInfo *parent_rel, Path *new_path, bool grouped) { + List *pathlist; bool accept_new = true; /* unless we find a superior old path */ ListCell *insert_after = NULL; /* where to insert new item */ List *new_path_pathkeys; @@ -435,6 +442,14 @@ add_path(RelOptInfo *parent_rel, Path *new_path) /* Pretend parameterized paths have no pathkeys, per comment above */ new_path_pathkeys = new_path->param_info ? NIL : new_path->pathkeys; + if (!grouped) + pathlist = parent_rel->pathlist; + else + { + Assert(parent_rel->gpi != NULL); + pathlist = parent_rel->gpi->pathlist; + } + /* * Loop to check proposed new path against old paths. Note it is possible * for more than one old path to be tossed out because new_path dominates @@ -444,7 +459,7 @@ add_path(RelOptInfo *parent_rel, Path *new_path) * list cell. */ p1_prev = NULL; - for (p1 = list_head(parent_rel->pathlist); p1 != NULL; p1 = p1_next) + for (p1 = list_head(pathlist); p1 != NULL; p1 = p1_next) { Path *old_path = (Path *) lfirst(p1); bool remove_old = false; /* unless new proves superior */ @@ -590,8 +605,7 @@ add_path(RelOptInfo *parent_rel, Path *new_path) */ if (remove_old) { - parent_rel->pathlist = list_delete_cell(parent_rel->pathlist, - p1, p1_prev); + pathlist = list_delete_cell(pathlist, p1, p1_prev); /* * Delete the data pointed-to by the deleted cell, if possible @@ -622,9 +636,14 @@ add_path(RelOptInfo *parent_rel, Path *new_path) { /* Accept the new path: insert it at proper place in pathlist */ if (insert_after) - lappend_cell(parent_rel->pathlist, insert_after, new_path); + lappend_cell(pathlist, insert_after, new_path); + else + pathlist = lcons(new_path, pathlist); + + if (!grouped) + parent_rel->pathlist = pathlist; else - parent_rel->pathlist = lcons(new_path, parent_rel->pathlist); + parent_rel->gpi->pathlist = pathlist; } else { @@ -654,8 +673,9 @@ add_path(RelOptInfo *parent_rel, Path *new_path) bool add_path_precheck(RelOptInfo *parent_rel, Cost startup_cost, Cost total_cost, - List *pathkeys, Relids required_outer) + List *pathkeys, Relids required_outer, bool grouped) { + List *pathlist; List *new_path_pathkeys; bool consider_startup; ListCell *p1; @@ -664,9 +684,18 @@ add_path_precheck(RelOptInfo *parent_rel, new_path_pathkeys = required_outer ? NIL : pathkeys; /* Decide whether new path's startup cost is interesting */ - consider_startup = required_outer ? parent_rel->consider_param_startup : parent_rel->consider_startup; + consider_startup = required_outer ? parent_rel->consider_param_startup : + parent_rel->consider_startup; + + if (!grouped) + pathlist = parent_rel->pathlist; + else + { + Assert(parent_rel->gpi != NULL); + pathlist = parent_rel->gpi->pathlist; + } - foreach(p1, parent_rel->pathlist) + foreach(p1, pathlist) { Path *old_path = (Path *) lfirst(p1); PathKeysComparison keyscmp; @@ -757,23 +786,32 @@ add_path_precheck(RelOptInfo *parent_rel, * referenced by partial BitmapHeapPaths. */ void -add_partial_path(RelOptInfo *parent_rel, Path *new_path) +add_partial_path(RelOptInfo *parent_rel, Path *new_path, bool grouped) { bool accept_new = true; /* unless we find a superior old path */ ListCell *insert_after = NULL; /* where to insert new item */ ListCell *p1; ListCell *p1_prev; ListCell *p1_next; + List *pathlist; /* Check for query cancel. */ CHECK_FOR_INTERRUPTS(); + if (!grouped) + pathlist = parent_rel->partial_pathlist; + else + { + Assert(parent_rel->gpi != NULL); + pathlist = parent_rel->gpi->partial_pathlist; + } + /* * As in add_path, throw out any paths which are dominated by the new * path, but throw out the new path if some existing path dominates it. */ p1_prev = NULL; - for (p1 = list_head(parent_rel->partial_pathlist); p1 != NULL; + for (p1 = list_head(pathlist); p1 != NULL; p1 = p1_next) { Path *old_path = (Path *) lfirst(p1); @@ -827,12 +865,11 @@ add_partial_path(RelOptInfo *parent_rel, Path *new_path) } /* - * Remove current element from partial_pathlist if dominated by new. + * Remove current element from pathlist if dominated by new. */ if (remove_old) { - parent_rel->partial_pathlist = - list_delete_cell(parent_rel->partial_pathlist, p1, p1_prev); + pathlist = list_delete_cell(pathlist, p1, p1_prev); pfree(old_path); /* p1_prev does not advance */ } @@ -847,8 +884,8 @@ add_partial_path(RelOptInfo *parent_rel, Path *new_path) /* * If we found an old path that dominates new_path, we can quit - * scanning the partial_pathlist; we will not add new_path, and we - * assume new_path cannot dominate any later path. + * scanning the pathlist; we will not add new_path, and we assume + * new_path cannot dominate any later path. */ if (!accept_new) break; @@ -858,10 +895,14 @@ add_partial_path(RelOptInfo *parent_rel, Path *new_path) { /* Accept the new path: insert it at proper place */ if (insert_after) - lappend_cell(parent_rel->partial_pathlist, insert_after, new_path); + lappend_cell(pathlist, insert_after, new_path); + else + pathlist = lcons(new_path, pathlist); + + if (!grouped) + parent_rel->partial_pathlist = pathlist; else - parent_rel->partial_pathlist = - lcons(new_path, parent_rel->partial_pathlist); + parent_rel->gpi->partial_pathlist = pathlist; } else { @@ -882,9 +923,18 @@ add_partial_path(RelOptInfo *parent_rel, Path *new_path) */ bool add_partial_path_precheck(RelOptInfo *parent_rel, Cost total_cost, - List *pathkeys) + List *pathkeys, bool grouped) { ListCell *p1; + List *pathlist; + + if (!grouped) + pathlist = parent_rel->partial_pathlist; + else + { + Assert(parent_rel->gpi != NULL); + pathlist = parent_rel->gpi->partial_pathlist; + } /* * Our goal here is twofold. First, we want to find out whether this path @@ -894,10 +944,11 @@ add_partial_path_precheck(RelOptInfo *parent_rel, Cost total_cost, * final cost computations. If so, we definitely want to consider it. * * Unlike add_path(), we always compare pathkeys here. This is because we - * expect partial_pathlist to be very short, and getting a definitive - * answer at this stage avoids the need to call add_path_precheck. + * expect partial_pathlist / grouped_pathlist to be very short, and + * getting a definitive answer at this stage avoids the need to call + * add_path_precheck. */ - foreach(p1, parent_rel->partial_pathlist) + foreach(p1, pathlist) { Path *old_path = (Path *) lfirst(p1); PathKeysComparison keyscmp; @@ -926,7 +977,7 @@ add_partial_path_precheck(RelOptInfo *parent_rel, Cost total_cost, * completion. */ if (!add_path_precheck(parent_rel, total_cost, total_cost, pathkeys, - NULL)) + NULL, grouped)) return false; return true; @@ -1259,11 +1310,13 @@ create_append_path(RelOptInfo *rel, List *subpaths, Relids required_outer, /* * create_merge_append_path * Creates a path corresponding to a MergeAppend plan, returning the - * pathnode. + * pathnode. target can be supplied by caller. If NULL is passed, the field + * is set to rel->reltarget. */ MergeAppendPath * create_merge_append_path(PlannerInfo *root, RelOptInfo *rel, + PathTarget *target, List *subpaths, List *pathkeys, Relids required_outer, @@ -1276,7 +1329,7 @@ create_merge_append_path(PlannerInfo *root, pathnode->path.pathtype = T_MergeAppend; pathnode->path.parent = rel; - pathnode->path.pathtarget = rel->reltarget; + pathnode->path.pathtarget = target ? target : rel->reltarget; pathnode->path.param_info = get_appendrel_parampathinfo(rel, required_outer); pathnode->path.parallel_aware = false; @@ -1413,6 +1466,7 @@ create_material_path(RelOptInfo *rel, Path *subpath) subpath->parallel_safe; pathnode->path.parallel_workers = subpath->parallel_workers; pathnode->path.pathkeys = subpath->pathkeys; + pathnode->path.uniquekeys = subpath->uniquekeys; pathnode->subpath = subpath; @@ -1446,8 +1500,12 @@ create_unique_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, MemoryContext oldcontext; int numCols; - /* Caller made a mistake if subpath isn't cheapest_total ... */ - Assert(subpath == rel->cheapest_total_path); + /* + * Caller made a mistake if subpath isn't cheapest_total (or the cheapest + * grouped). + */ + Assert(subpath == rel->cheapest_total_path || + (rel->gpi != NULL && subpath == linitial(rel->gpi->pathlist))); Assert(subpath->parent == rel); /* ... or if SpecialJoinInfo is the wrong one */ Assert(sjinfo->jointype == JOIN_SEMI); @@ -2075,6 +2133,7 @@ calc_non_nestloop_required_outer(Path *outer_path, Path *inner_path) * 'restrict_clauses' are the RestrictInfo nodes to apply at the join * 'pathkeys' are the path keys of the new join path * 'required_outer' is the set of required outer rels + * 'target' can be passed to override that of joinrel. * * Returns the resulting path node. */ @@ -2088,7 +2147,8 @@ create_nestloop_path(PlannerInfo *root, Path *inner_path, List *restrict_clauses, List *pathkeys, - Relids required_outer) + Relids required_outer, + PathTarget *target) { NestPath *pathnode = makeNode(NestPath); Relids inner_req_outer = PATH_REQ_OUTER(inner_path); @@ -2121,7 +2181,7 @@ create_nestloop_path(PlannerInfo *root, pathnode->path.pathtype = T_NestLoop; pathnode->path.parent = joinrel; - pathnode->path.pathtarget = joinrel->reltarget; + pathnode->path.pathtarget = target == NULL ? joinrel->reltarget : target; pathnode->path.param_info = get_joinrel_parampathinfo(root, joinrel, @@ -2179,13 +2239,15 @@ create_mergejoin_path(PlannerInfo *root, Relids required_outer, List *mergeclauses, List *outersortkeys, - List *innersortkeys) + List *innersortkeys, + PathTarget *target) { MergePath *pathnode = makeNode(MergePath); pathnode->jpath.path.pathtype = T_MergeJoin; pathnode->jpath.path.parent = joinrel; - pathnode->jpath.path.pathtarget = joinrel->reltarget; + pathnode->jpath.path.pathtarget = target == NULL ? joinrel->reltarget : + target; pathnode->jpath.path.param_info = get_joinrel_parampathinfo(root, joinrel, @@ -2230,6 +2292,7 @@ create_mergejoin_path(PlannerInfo *root, * 'required_outer' is the set of required outer rels * 'hashclauses' are the RestrictInfo nodes to use as hash clauses * (this should be a subset of the restrict_clauses list) + * 'target' can be passed to override that of joinrel. */ HashPath * create_hashjoin_path(PlannerInfo *root, @@ -2241,13 +2304,15 @@ create_hashjoin_path(PlannerInfo *root, Path *inner_path, List *restrict_clauses, Relids required_outer, - List *hashclauses) + List *hashclauses, + PathTarget *target) { HashPath *pathnode = makeNode(HashPath); pathnode->jpath.path.pathtype = T_HashJoin; pathnode->jpath.path.parent = joinrel; - pathnode->jpath.path.pathtarget = joinrel->reltarget; + pathnode->jpath.path.pathtarget = target == NULL ? joinrel->reltarget : + target; pathnode->jpath.path.param_info = get_joinrel_parampathinfo(root, joinrel, @@ -2294,12 +2359,14 @@ create_hashjoin_path(PlannerInfo *root, * 'rel' is the parent relation associated with the result * 'subpath' is the path representing the source of data * 'target' is the PathTarget to be computed + * 'force_result' enforces implementation using Result plan. */ ProjectionPath * create_projection_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, - PathTarget *target) + PathTarget *target, + bool force_result) { ProjectionPath *pathnode = makeNode(ProjectionPath); PathTarget *oldtarget = subpath->pathtarget; @@ -2316,8 +2383,10 @@ create_projection_path(PlannerInfo *root, pathnode->path.parallel_workers = subpath->parallel_workers; /* Projection does not change the sort order */ pathnode->path.pathkeys = subpath->pathkeys; + pathnode->path.uniquekeys = subpath->uniquekeys; pathnode->subpath = subpath; + pathnode->force_result = force_result; /* * We might not need a separate Result node. If the input plan node type @@ -2328,8 +2397,9 @@ create_projection_path(PlannerInfo *root, * Note: in the latter case, create_projection_plan has to recheck our * conclusion; see comments therein. */ - if (is_projection_capable_path(subpath) || - equal(oldtarget->exprs, target->exprs)) + if (!force_result && + (is_projection_capable_path(subpath) || + equal(oldtarget->exprs, target->exprs))) { /* No separate Result node needed */ pathnode->dummypp = true; @@ -2399,7 +2469,8 @@ apply_projection_to_path(PlannerInfo *root, * separate ProjectionPath. */ if (!is_projection_capable_path(path)) - return (Path *) create_projection_path(root, rel, path, target); + return (Path *) create_projection_path(root, rel, path, target, + false); /* * We can just jam the desired tlist into the existing path, being sure to @@ -2439,7 +2510,8 @@ apply_projection_to_path(PlannerInfo *root, create_projection_path(root, gpath->subpath->parent, gpath->subpath, - target); + target, + false); } else { @@ -2449,7 +2521,8 @@ apply_projection_to_path(PlannerInfo *root, create_projection_path(root, gmpath->subpath->parent, gmpath->subpath, - target); + target, + false); } } else if (path->parallel_safe && @@ -2562,6 +2635,7 @@ create_sort_path(PlannerInfo *root, subpath->parallel_safe; pathnode->path.parallel_workers = subpath->parallel_workers; pathnode->path.pathkeys = pathkeys; + pathnode->path.uniquekeys = subpath->uniquekeys; pathnode->subpath = subpath; @@ -2748,6 +2822,207 @@ create_agg_path(PlannerInfo *root, } /* + * Apply partial AGG_SORTED aggregation path to subpath if it's suitably + * sorted. ("partial" in the function name refers to AGGSPLIT_INITIAL_SERIAL + * strategy, as opposed to parallel processing.) + * + * first_call indicates whether the function is being called first time for + * given index --- since the target should not change, we can skip the check + * of sorting during subsequent calls. + * + * group_clauses, group_exprs and agg_exprs are pointers to lists we populate + * when called first time for particular index, and that user passes for + * subsequent calls. + * + * NULL is returned if sorting of subpath output is not suitable. + */ +AggPath * +create_partial_agg_sorted_path(PlannerInfo *root, Path *subpath, + bool first_call, + List **group_clauses, List **group_exprs, + List **agg_exprs, double input_rows, + bool parallel) +{ + RelOptInfo *rel; + AggClauseCosts agg_costs; + double dNumGroups; + AggPath *result = NULL; + + rel = subpath->parent; + Assert(rel->gpi != NULL); + Assert(rel->gpi->target != NULL); + + if (subpath->pathkeys == NIL) + return NULL; + + if (!grouping_is_sortable(root->parse->groupClause)) + return NULL; + + /* Add generic grouping expressions to the subpath if there are some. */ + if (rel->gpi->group_exprs != NULL) + subpath = add_grouping_expressions_to_subpath(root, subpath, + rel->gpi->group_exprs); + + if (first_call) + { + ListCell *lc1; + List *key_subset = NIL; + + /* + * Find all query pathkeys that our relation does affect. + */ + foreach(lc1, root->group_pathkeys) + { + PathKey *gkey = castNode(PathKey, lfirst(lc1)); + ListCell *lc2; + + foreach(lc2, subpath->pathkeys) + { + PathKey *skey = castNode(PathKey, lfirst(lc2)); + + if (skey == gkey) + { + key_subset = lappend(key_subset, gkey); + break; + } + } + } + + if (key_subset == NIL) + return NULL; + + /* Check if AGG_SORTED is useful for the whole query. */ + if (!pathkeys_contained_in(key_subset, subpath->pathkeys)) + return NULL; + } + + if (first_call) + get_grouping_expressions(root, rel->gpi->target, + rel->gpi->sortgroupclauses, group_clauses, + group_exprs, agg_exprs); + + MemSet(&agg_costs, 0, sizeof(AggClauseCosts)); + Assert(*agg_exprs != NIL); + get_agg_clause_costs(root, (Node *) *agg_exprs, AGGSPLIT_INITIAL_SERIAL, + &agg_costs); + + Assert(*group_exprs != NIL); + dNumGroups = estimate_num_groups(root, *group_exprs, input_rows, NULL); + + /* TODO HAVING qual. */ + Assert(*group_clauses != NIL); + result = create_agg_path(root, rel, subpath, rel->gpi->target, + AGG_SORTED, AGGSPLIT_INITIAL_SERIAL, + *group_clauses, NIL, &agg_costs, dNumGroups); + + /* + * Check if there's a chance to avoid final aggregation. + */ + if (!parallel) + make_uniquekeys(root, (Path *) result); + + return result; +} + +/* + * Appy partial AGG_HASHED aggregation to subpath. ("partial" in the function + * name refers to AGGSPLIT_INITIAL_SERIAL strategy, as opposed to parallel + * processing.) + * + * Arguments have the same meaning as those of create_agg_sorted_path. + */ +AggPath * +create_partial_agg_hashed_path(PlannerInfo *root, Path *subpath, + bool first_call, + List **group_clauses, List **group_exprs, + List **agg_exprs, double input_rows, + bool parallel) +{ + RelOptInfo *rel; + bool can_hash; + AggClauseCosts agg_costs; + double dNumGroups; + Size hashaggtablesize; + Query *parse = root->parse; + AggPath *result = NULL; + + rel = subpath->parent; + Assert(rel->gpi != NULL); + Assert(rel->gpi->target != NULL); + + /* Add generic grouping expressions to the subpath if there are some. */ + if (rel->gpi->group_exprs != NULL) + subpath = add_grouping_expressions_to_subpath(root, subpath, + rel->gpi->group_exprs); + + if (first_call) + { + /* + * Find one grouping clause per grouping column. + * + * All that create_agg_plan eventually needs of the clause is + * tleSortGroupRef, so we don't have to care that the clause + * expression might differ from texpr, in case texpr was derived from + * EC. + */ + get_grouping_expressions(root, rel->gpi->target, + rel->gpi->sortgroupclauses, + group_clauses, + group_exprs, agg_exprs); + } + + MemSet(&agg_costs, 0, sizeof(AggClauseCosts)); + Assert(*agg_exprs != NIL); + get_agg_clause_costs(root, (Node *) *agg_exprs, AGGSPLIT_INITIAL_SERIAL, + &agg_costs); + + can_hash = (parse->groupClause != NIL && + parse->groupingSets == NIL && + agg_costs.numOrderedAggs == 0 && + grouping_is_hashable(parse->groupClause)); + + if (can_hash) + { + Assert(*group_exprs != NIL); + dNumGroups = estimate_num_groups(root, *group_exprs, input_rows, + NULL); + + hashaggtablesize = estimate_hashagg_tablesize(subpath, &agg_costs, + dNumGroups); + + if (hashaggtablesize < work_mem * 1024L) + { + /* + * Create the partial aggregation path. + */ + Assert(*group_clauses != NIL); + + result = create_agg_path(root, rel, subpath, + rel->gpi->target, + AGG_HASHED, + AGGSPLIT_INITIAL_SERIAL, + *group_clauses, NIL, + &agg_costs, + dNumGroups); + + /* + * The agg path should require no fewer parameters than the plain + * one. + */ + result->path.param_info = subpath->param_info; + } + } + + /* + * Check if there's a chance to avoid final aggregation. + */ + if (result != NULL && !parallel) + make_uniquekeys(root, (Path *) result); + + return result; +} + +/* * create_groupingsets_path * Creates a pathnode that represents performing GROUPING SETS aggregation * @@ -3836,3 +4111,612 @@ reparameterize_pathlist_by_child(PlannerInfo *root, return result; } + +/* + * Add (non-Var) grouping expressions contained in group_exprs target to a + * subpath. The passed in subpath should not be touched so that it can still + * be used as a subpath for non-grouped paths. Therefore we wrap it into + * ResultPath, to which we actually add those expressions. + */ +static Path * +add_grouping_expressions_to_subpath(PlannerInfo *root, Path *subpath, + PathTarget *group_exprs) +{ + PathTarget *target; + ListCell *lc; + int i; + + /* + * The subpath must stay intact because, besides producing input data for + * aggregation, it can participate in a plan that does not use aggregation + * push-down. For the same reason we pass force_result=true to + * create_projection_path below. + */ + target = copy_pathtarget(subpath->pathtarget); + + Assert(group_exprs->sortgrouprefs != NULL); + i = 0; + foreach(lc, group_exprs->exprs) + { + Index sortgroupref; + GroupedVar *gvar; + + sortgroupref = group_exprs->sortgrouprefs[i++]; + gvar = lfirst_node(GroupedVar, lc); + add_column_to_pathtarget(target, (Expr *) gvar, sortgroupref); + } + + return (Path *) create_projection_path(root, + subpath->parent, + subpath, + target, + true); +} + +/* + * Find out if path produces an unique set of expressions and set uniquekeys + * accordingly. + * + * TODO Check if any expression of any unique key isn't nullable, whether in + * the table / index or by an outer join. + */ +void +make_uniquekeys(PlannerInfo *root, Path *path) +{ + RelOptInfo *rel; + + /* + * The unique keys are not interesting if there's no chance to push + * aggregation down to base relations / joins. + */ + if (root->grouped_var_list == NIL) + return; + + /* + * Do not accept repeated calls of the function on the same path. + */ + if (path->uniquekeys != NIL) + return; + + rel = path->parent; + + /* + * Base relations. + */ + if (IsA(path, IndexPath) || + (IsA(path, Path) &&path->pathtype == T_SeqScan) || + IsA(path, AppendPath) ||IsA(path, MergeAppendPath)) + { + ListCell *lc; + + /* + * Derive grouping keys from unique indexes. + */ + if (IsA(path, IndexPath) ||IsA(path, Path)) + { + foreach(lc, rel->indexlist) + { + IndexOptInfo *index = lfirst_node(IndexOptInfo, lc); + + make_uniquekeys_for_unique_index(rel->reltarget, index, path); + } + } + else if (IsA(path, AppendPath) ||IsA(path, MergeAppendPath)) + make_uniquekeys_for_append_path(root, path); +#ifdef USE_ASSERT_CHECKING + else + Assert(false); +#endif + return; + } + + if (IsA(path, AggPath)) + { + /* + * The immediate output of aggregation essentially produces an unique + * set of grouping keys. + */ + make_uniquekeys_for_agg_path(path); + } + else if (IS_JOIN_REL(path->parent)) + { + JoinPath *jpath = (JoinPath *) path; + Path *outerpath = jpath->outerjoinpath; + Path *innerpath = jpath->innerjoinpath; + ListCell *l1; + + /* + * Find out if the join produces unique keys for various combinations + * of input sets of unique keys. + * + * TODO Implement heuristic that picks a few most useful sets on each + * side, to avoid exponential growth of the uniquekeys list as we + * proceed from lower to higher joins. Maybe also discard the + * resulting sets containing unique expressions which are not grouping + * expressions (and of course which are not aggregates) of this join's + * target. + */ + foreach(l1, outerpath->uniquekeys) + { + Bitmapset *outerset = (Bitmapset *) lfirst(l1); + ListCell *l2; + + foreach(l2, innerpath->uniquekeys) + { + Bitmapset *innerset = (Bitmapset *) lfirst(l2); + Bitmapset *joinset; + + /* + * Given that unique keys of each input relation are contained + * in that relation's target, the union of the key sets should + * be contained in the join target. + */ + joinset = bms_union(outerset, innerset); + + /* Add the set to the path. */ + add_uniquekeys_to_path((Path *) jpath, joinset); + } + } + } +#ifdef USE_ASSERT_CHECKING + + /* + * TODO Consider other ones, e.g. UniquePath. + */ + else + Assert(false); +#endif +} + +/* + * Create a set of positions of expressions in reltarget if the index is + * unique and if reltarget contains all the index columns. Add the set to + * uniquekeys if identical one is not already there. + */ +static void +make_uniquekeys_for_unique_index(PathTarget *reltarget, IndexOptInfo *index, + Path *path) +{ + int i; + Bitmapset *new_set = NULL; + + /* + * Give up if the index does not guarantee uniqueness. + */ + if (!index->unique || !index->immediate || + (index->indpred != NIL && !index->predOK)) + return; + + /* + * For the index path to be acceptable, reltarget must contain all the + * index columns. + * + * reltarget is not supposed to contain non-var expressions, so the index + * should neither. + */ + if (index->indexprs != NULL) + return; + + for (i = 0; i < index->ncolumns; i++) + { + int indkey = index->indexkeys[i]; + ListCell *lc; + bool found = false; + int j = 0; + + foreach(lc, reltarget->exprs) + { + Var *var = lfirst_node(Var, lc); + + if (var->varno == index->rel->relid && var->varattno == indkey) + { + new_set = bms_add_member(new_set, j); + found = true; + break; + } + + j++; + } + + /* + * If rel needs less than the whole index key then the values of the + * columns matched so far can be duplicate. + */ + if (!found) + { + bms_free(new_set); + return; + } + } + + /* + * Add the set to the path, unless it's already there. + */ + add_uniquekeys_to_path(path, new_set); +} + +/* + * Create uniquekeys for a path that has Aggrefs in its target. + * set. + * + * Besides AggPath, ForeignPath is a known use case for this function. + */ +void +make_uniquekeys_for_agg_path(Path *path) +{ + PathTarget *target; + ListCell *lc; + Bitmapset *keyset = NULL; + int i = 0; + + target = path->pathtarget; + Assert(target->sortgrouprefs != NULL); + + foreach(lc, target->exprs) + { + Expr *expr = (Expr *) lfirst(lc); + + if (IsA(expr, GroupedVar)) + { + GroupedVar *gvar = castNode(GroupedVar, expr); + + if (!IsA(gvar->gvexpr, Aggref)) + { + /* + * Generic grouping expression. + */ + keyset = bms_add_member(keyset, i); + } + } + else + { + Assert(IsA(expr, Var)); + + if (target->sortgrouprefs[i] > 0) + { + /* + * Plain Var grouping expression. + */ + keyset = bms_add_member(keyset, i); + } + else + { + /* + * A column functionally dependent on the GROUP BY clause? + */ + } + } + + i++; + } + + add_uniquekeys_to_path((Path *) path, keyset); +} + +/* + * Create uniquekeys for a AppendPath or MergeAppendPath. + * + * Besides AggPath, ForeignPath is a known use case for this function. + */ +static void +make_uniquekeys_for_append_path(PlannerInfo *root, Path *path) +{ + RelOptInfo *rel = path->parent; + List *subpaths; + Path *subpath; + ListCell *l1; + bool first = true; + List *uniquekeys_common = NIL; + + /* + * In addition to the requirement that all subpaths must have uniquekeys + * set, the table needs to be partitioned in a specific way. Reject + * non-partitioned tables immediately, the other check to follow. + */ + if (!IS_PARTITIONED_REL(rel)) + return; + + if (IsA(path, AppendPath)) + subpaths = ((AppendPath *) path)->subpaths; + else if (IsA(path, MergeAppendPath)) + subpaths = ((MergeAppendPath *) path)->subpaths; +#ifdef USE_ASSERT_CHECKING + else + Assert(false); +#endif + + /* + * Check if each subpath has uniquekeys. + */ + foreach(l1, subpaths) + { + subpath = (Path *) lfirst(l1); + + /* + * If any subpath does not have uniquekeys, the whole append path can + * have them neither. + */ + if (subpath->uniquekeys == NIL) + return; + } + + /* + * Choose groupkey sets that each subpath has. + */ + foreach(l1, subpaths) + { + List *common_new = NIL; + ListCell *l2; + + subpath = (Path *) lfirst(l1); + + if (uniquekeys_common == NIL) + { + /* Get the initial list from the first subpath. */ + uniquekeys_common = subpath->uniquekeys; + continue; + } + + /* + * Remove items missing in the current subpath from uniquekeys_common. + */ + foreach(l2, uniquekeys_common) + { + Bitmapset *set1 = (Bitmapset *) lfirst(l2); + ListCell *l3; + + /* + * Does the current subpath contain this set? + */ + foreach(l3, subpath->uniquekeys) + { + Bitmapset *set2 = (Bitmapset *) lfirst(l3); + + if (bms_equal(set1, set2)) + { + common_new = lappend(common_new, set1); + break; + } + } + } + + /* + * If there are no sets in common, the next subpaths cannot help us. + */ + if (common_new == NIL) + return; + + /* + * Adopt the new, possibly reduced list. + */ + uniquekeys_common = common_new; + } + + foreach(l1, subpaths) + { + subpath = (Path *) lfirst(l1); + + /* + * The following tests should ideally be performed either on + * rel->reltarget or rel->gpi->target outside the iteration of + * subpaths. However there's no easy way to find out if path is + * grouped or not. So we check the target of the first subpath and + * assume that the other ones would pass or fail in the same way: + * another subpath target should be the same path target whose + * attributes are translated to another child relations. + */ + if (first) + { + List *partexprs = NIL; + List *partexprs_all = NIL; + ListCell *l2, + *l3; + AppendRelInfo **appinfos; + int i, + nappinfos; + bool found; + List *texprs = NIL; + + /* + * Put all partition expressions into two lists --- one for + * non-nullable expressions, one for nullable. + */ + for (i = 0; i < rel->part_scheme->partnatts; i++) + { + List *sublist; + + sublist = rel->partexprs[i]; + if (sublist != NIL) + { + partexprs = list_union(partexprs, sublist); + partexprs_all = list_union(partexprs_all, + sublist); + } + + /* + * The nullable expressions should only appear in + * partexprs_all. + */ + sublist = rel->nullable_partexprs[i]; + if (sublist != NIL) + partexprs_all = list_union(partexprs_all, + sublist); + } + Assert(partexprs != NIL); + + /* + * Translate the partitioning expressions so that they can match + * the subpath target. + */ + appinfos = + find_appinfos_by_relids(root, subpath->parent->relids, + &nappinfos); + partexprs = (List *) + adjust_appendrel_attrs(root, (Node *) partexprs, + nappinfos, appinfos); + partexprs_all = (List *) + adjust_appendrel_attrs(root, (Node *) partexprs_all, + nappinfos, appinfos); + pfree(appinfos); + + /* + * Since equivalence classes can be used to derive target + * expressions, the following checks have to consider ECs too. The + * targetlist we construct here contains the original expressions + * plus those generated from the appropriate ECs. + */ + foreach(l2, subpath->pathtarget->exprs) + { + Expr *texpr = (Expr *) lfirst(l2); + + if (IsA(texpr, GroupedVar)) + { + GroupedVar *gvar = castNode(GroupedVar, texpr); + + if (IsA(gvar->gvexpr, Aggref)) + { + /* + * Aggregate should not appear in any EC. + */ + texprs = lappend(texprs, texpr); + continue; + } + + /* + * The contained expression (i.e. generic grouping + * expression) is what we'll search for in the ECs. + */ + texpr = gvar->gvexpr; + } + + /* + * Search for matching EC. + */ + foreach(l3, root->group_pathkeys) + { + PathKey *pk = lfirst_node(PathKey, l3); + EquivalenceClass *ec = pk->pk_eclass; + ListCell *l4; + EquivalenceMember *em; + + if (ec->ec_below_outer_join) + continue; + + if (ec->ec_has_volatile) + continue; + + foreach(l4, ec->ec_members) + { + em = lfirst_node(EquivalenceMember, l4); + + if (em->em_nullable_relids) + continue; + + if (!bms_is_subset(em->em_relids, + subpath->parent->relids)) + continue; + + if (equal(em->em_expr, texpr)) + break; + } + + /* + * texpr not found in the current EC, try the next one. + */ + if (l4 == NULL) + continue; + + /* + * The EC does match, so add all its members to texprs. + */ + foreach(l4, ec->ec_members) + { + em = lfirst_node(EquivalenceMember, l4); + texprs = lappend(texprs, em->em_expr); + } + + /* + * No expression is supposed to appear in multiple EC. + */ + continue; + } + } + + /* + * To ensure that no output row can be produced by multiple + * partitions, check that: + * + * (a) at least one output column is a partitioning expression. + * Thus no output row of this path can appear in multiple + * partitions. + * + * Only search in the non-nullable partitioning expressions + * because NULL value can be generated by any partition. + */ + found = false; + foreach(l2, texprs) + { + Expr *texpr = (Expr *) lfirst(l2); + + /* + * Try to find the matching partitioning expression. + */ + foreach(l3, partexprs) + { + Expr *pexpr = lfirst(l3); + + if (equal(texpr, pexpr)) + { + found = true; + break; + } + } + + if (found) + break; + } + if (!found) + { + list_free(uniquekeys_common); + return; + } + + /* + * (b) there's no partitioning expression not contained in the + * target. If there was some, then the partitioning expression + * found above could appear in multiple partitions. + * + * Even nullable partition key makes harm here, so consider all + * partition keys this time. + */ + foreach(l2, partexprs_all) + { + Expr *pexpr = lfirst(l2); + + found = false; + foreach(l3, texprs) + { + Expr *texpr = (Expr *) lfirst(l3); + + if (equal(pexpr, texpr)) + { + found = true; + break; + } + } + if (!found) + { + list_free(uniquekeys_common); + return; + } + } + list_free(texprs); + + /* Do not repeat the checks for the other subpaths. */ + first = false; + } + } + + /* All the checks passed. */ + path->uniquekeys = uniquekeys_common; +} diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index f743871..f763a97 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -1466,6 +1466,51 @@ relation_excluded_by_constraints(PlannerInfo *root, return false; } +/* + * Remove from restrictions list items implied by table constraints + */ +void remove_restrictions_implied_by_constraints(PlannerInfo *root, + RelOptInfo *rel, RangeTblEntry *rte) +{ + List *constraint_pred; + List *safe_constraints = NIL; + List *safe_restrictions = NIL; + ListCell *lc; + + if (rte->rtekind != RTE_RELATION || rte->inh) + return; + + /* + * OK to fetch the constraint expressions. Include "col IS NOT NULL" + * expressions for attnotnull columns, in case we can refute those. + */ + constraint_pred = get_relation_constraints(root, rte->relid, rel, true); + + /* + * We do not currently enforce that CHECK constraints contain only + * immutable functions, so it's necessary to check here. We daren't draw + * conclusions from plan-time evaluation of non-immutable functions. Since + * they're ANDed, we can just ignore any mutable constraints in the list, + * and reason about the rest. + */ + foreach(lc, constraint_pred) + { + Node *pred = (Node*) lfirst(lc); + + if (!contain_mutable_functions(pred)) + safe_constraints = lappend(safe_constraints, pred); + } + + foreach(lc, rel->baserestrictinfo) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + if (!predicate_implied_by(list_make1(rinfo->clause), safe_constraints, false)) { + safe_restrictions = lappend(safe_restrictions, rinfo); + } + } + rel->baserestrictinfo = safe_restrictions; +} + /* * build_physical_tlist diff --git a/src/backend/optimizer/util/predtest.c b/src/backend/optimizer/util/predtest.c index 134460c..47cd4c9 100644 --- a/src/backend/optimizer/util/predtest.c +++ b/src/backend/optimizer/util/predtest.c @@ -17,6 +17,7 @@ #include "catalog/pg_proc.h" #include "catalog/pg_type.h" +#include "catalog/pg_operator.h" #include "executor/executor.h" #include "miscadmin.h" #include "nodes/nodeFuncs.h" @@ -1407,6 +1408,13 @@ static const StrategyNumber BT_refute_table[6][6] = { {none, none, BTEQ, none, none, none} /* NE */ }; +#define Int2LessOperator 95 +#define Int2LessOrEqualOperator 522 +#define Int4LessOrEqualOperator 523 +#define Int8LessOrEqualOperator 414 +#define DateLessOrEqualOperator 1096 +#define DateLessOperator 1095 + /* * operator_predicate_proof @@ -1600,6 +1608,17 @@ operator_predicate_proof(Expr *predicate, Node *clause, bool refute_it) if (clause_const->constisnull) return false; + if (!refute_it + && ((pred_op == Int4LessOrEqualOperator && clause_op == Int4LessOperator) + || (pred_op == Int8LessOrEqualOperator && clause_op == Int8LessOperator) + || (pred_op == Int2LessOrEqualOperator && clause_op == Int2LessOperator) + || (pred_op == DateLessOrEqualOperator && clause_op == DateLessOperator)) + && pred_const->constbyval && clause_const->constbyval + && pred_const->constvalue + 1 == clause_const->constvalue) + { + return true; + } + /* * Lookup the constant-comparison operator using the system catalogs and * the operator implication tables. diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index 674cfc6..065591b 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -27,6 +27,8 @@ #include "optimizer/prep.h" #include "optimizer/restrictinfo.h" #include "optimizer/tlist.h" +#include "optimizer/var.h" +#include "parser/parse_oper.h" #include "utils/hsearch.h" @@ -125,6 +127,7 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptInfo *parent) rel->cheapest_parameterized_paths = NIL; rel->direct_lateral_relids = NULL; rel->lateral_relids = NULL; + rel->gpi = NULL; rel->relid = relid; rel->rtekind = rte->rtekind; /* min_attr, max_attr, attr_needed, attr_widths are set below */ @@ -534,6 +537,7 @@ build_join_rel(PlannerInfo *root, inner_rel->direct_lateral_relids); joinrel->lateral_relids = min_join_parameterization(root, joinrel->relids, outer_rel, inner_rel); + joinrel->gpi = NULL; joinrel->relid = 0; /* indicates not a baserel */ joinrel->rtekind = RTE_JOIN; joinrel->min_attr = 0; @@ -587,6 +591,36 @@ build_join_rel(PlannerInfo *root, add_placeholders_to_joinrel(root, joinrel, outer_rel, inner_rel); /* + * If grouping is not applicable at relation level, try to build grouped + * targets --- both for aggregation and for joining grouped relation to + * non-grouped one. + */ + if (root->grouped_var_list != NIL) + { + /* + * TODO Consider if placeholders make sense here. If not, also make + * the related code below conditional. + */ + prepare_rel_for_grouping(root, joinrel); + + /* + * If the relation appears to be eligible for grouping, joinrel->gpi + * has been initialized. Compute cost of the grouping target. + */ + + /* + * TODO Store the costs of GroupedVars separate in GroupedPathInfo and + * only add it to the join cost if its result is actually aggregated. + * In contrast, if the grouped join is formed by joining a group + * relation to non-grouped one, the cost of GroupedVar should not + * included (but width should) because the grouped input relation was + * in charge of evaluation. + */ + if (joinrel->gpi != NULL) + set_pathtarget_cost_width(root, joinrel->gpi->target); + } + + /* * add_placeholders_to_joinrel also took care of adding the ph_lateral * sets of any PlaceHolderVars computed here to direct_lateral_relids, so * now we can finish computing that. This is much like the computation of @@ -708,6 +742,7 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, joinrel->cheapest_parameterized_paths = NIL; joinrel->direct_lateral_relids = NULL; joinrel->lateral_relids = NULL; + joinrel->gpi = NULL; joinrel->relid = 0; /* indicates not a baserel */ joinrel->rtekind = RTE_JOIN; joinrel->min_attr = 0; @@ -756,7 +791,6 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, (Node *) parent_joinrel->joininfo, nappinfos, appinfos); - pfree(appinfos); /* * Lateral relids referred in child join will be same as that referred in @@ -792,10 +826,65 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, /* Add the relation to the PlannerInfo. */ add_join_rel(root, joinrel); + pfree(appinfos); + return joinrel; } /* + * Initialize GroupedPathInfo of a child relation according to that of the + * parent. + */ +void +build_chiid_rel_gpi(PlannerInfo *root, RelOptInfo *child, RelOptInfo *parent, + int nappinfos, AppendRelInfo **appinfos) +{ + GroupedPathInfo *gpi, + *gpi_parent; + + Assert(child->gpi == NULL); + + gpi_parent = parent->gpi; + child->gpi = gpi = makeNode(GroupedPathInfo); + + /* + * Create grouping target translated to the child varnos. + * + * We need a copy of the target so that the translation does not affect + * the parent. + */ + gpi->target = copy_pathtarget(gpi_parent->target); + gpi->target->exprs = (List *) + adjust_appendrel_attrs(root, + (Node *) gpi->target->exprs, + nappinfos, appinfos); + + /* + * Non-var grouping expressions need to be translated as well. + */ + if (gpi_parent->group_exprs != NULL) + { + gpi->group_exprs = copy_pathtarget(gpi_parent->group_exprs); + gpi->group_exprs->exprs = (List *) + adjust_appendrel_attrs(root, + (Node *) gpi->group_exprs->exprs, + nappinfos, appinfos); + } + + /* + * sortgroupclauses need no kind of translation, so just copy the pointer. + */ + gpi->sortgroupclauses = gpi_parent->sortgroupclauses; + + /* + * If the output of the child rel's paths should be aggregated, + * sortgrouprefs are also needed. (initialize_grouped_targets should have + * initialized sortgrouprefs of the parent rel.) + */ + child->reltarget->sortgrouprefs = parent->reltarget->sortgrouprefs; +} + +/* * min_join_parameterization * * Determine the minimum possible parameterization of a joinrel, that is, the @@ -1748,3 +1837,303 @@ build_joinrel_partition_info(RelOptInfo *joinrel, RelOptInfo *outer_rel, joinrel->nullable_partexprs[cnt] = nullable_partexpr; } } + +/* + * If the relation can produce grouped paths, create GroupedPathInfo for it + * and create target for aggregation of the relation output. + */ +void +prepare_rel_for_grouping(PlannerInfo *root, RelOptInfo *rel) +{ + List *gvis; + List *aggregates = NIL; + List *grp_exprs = NIL; + bool found_higher_agg; + ListCell *lc; + GroupedPathInfo *gpi; + PathTarget *target; + List *grp_exprs_extra = NIL; + + /* + * The current implementation of aggregation push-down cannot handle + * PlaceHolderVar (PHV). + * + * If we knew that the PHV should be evaluated in this target (and of + * course, if its expression matched some grouping expression or Aggref + * argument), we'd just let initialize_grouped_targets create GroupedVar + * for the corresponding expression (phexpr). On the other hand, if we + * knew that the PHV is evaluated below the current rel, we'd ignore it + * because the referencing GroupedVar would take care of propagation of + * the value to upper joins. (PHV whose ph_eval_at is above the current + * rel make the aggregation push-down impossible in any case because the + * partial aggregation would receive wrong input if we ignored the + * ph_eval_at.) + * + * The problem is that the same PHV can be evaluated in the target of the + * current rel or in that of lower rel --- depending on the input paths. + * For example, consider rel->relids = {A, B, C} and if ph_eval_at = {B, + * C}. Path "A JOIN (B JOIN C)" implies that the PHV is evaluated by the + * "(B JOIN C)", while path "(A JOIN B) JOIN C" evaluates the PHV itself. + */ + foreach(lc, rel->reltarget->exprs) + { + Expr *expr = lfirst(lc); + + if (IsA(expr, PlaceHolderVar)) + return; + } + + /* + * target_agg will be used to aggregate the output of the current + * relation's paths. + */ + target = create_empty_pathtarget(); + + if (IS_SIMPLE_REL(rel)) + { + RangeTblEntry *rte = root->simple_rte_array[rel->relid];; + + /* + * rtekind != RTE_RELATION case is not supported yet. + */ + if (rte->rtekind != RTE_RELATION) + return; + } + + /* Caller should only pass base relations or joins. */ + Assert(rel->reloptkind == RELOPT_BASEREL || + rel->reloptkind == RELOPT_JOINREL); + + /* + * If any outer join can set the attribute value to NULL, the Agg plan + * would receive different input at the base rel level. + * + * XXX For RELOPT_JOINREL, do not return if all the joins that can set any + * entry of the grouped target (do we need to postpone this check until + * the grouped target is available, and initialize_grouped_targets_target + * take care?) of this rel to NULL are provably below rel. (It's ok if rel + * is one of these joins.) + */ + if (bms_overlap(rel->relids, root->nullable_baserels)) + return; + + /* + * Use equivalence classes to generate additional grouping expressions for + * the current rel. Without these we might not be able to apply + * aggregation to the relation result set. + * + * While create_grouping_expr_grouped_var_infos generates equivalence + * class implied plain-Var grouping expressions at once, doing so for + * generic expressions would be complex and might result in very long list + * to search in. So generate the EC-related grouping expressions for each + * particular set of relids. + */ + gvis = list_copy(root->grouped_var_list); + foreach(lc, root->grouped_var_list) + { + GroupedVarInfo *gvi = lfirst_node(GroupedVarInfo, lc); + + /* Only interested in grouping expressions. */ + if (IsA(gvi->gvexpr, Aggref)) + continue; + + /* + * Should have been processed by + * create_grouping_expr_grouped_var_infos. + */ + if (IsA(gvi->gvexpr, Var)) + continue; + + gvi = translate_expression_to_rels(root, gvi, rel->relids); + if (gvi != NULL) + gvis = lappend(gvis, gvi); + } + + /* + * Check if some aggregates or grouping expressions can be evaluated in + * this relation's target, and collect all vars referenced by these + * aggregates / grouping expressions; + */ + found_higher_agg = false; + foreach(lc, gvis) + { + GroupedVarInfo *gvi = lfirst_node(GroupedVarInfo, lc); + + /* + * The subset includes gv_eval_at uninitialized, which includes + * Aggref.aggstar. + */ + if (bms_is_subset(gvi->gv_eval_at, rel->relids)) + { + /* + * initialize_grouped_targets will handle plain Var grouping + * expressions because it needs to look them up in + * grouped_var_list anyway. + * + * XXX A plain Var could actually be handled w/o GroupedVar, but + * thus initialize_grouped_targets would have to spend extra + * effort looking for the EC-related vars, instead of relying on + * create_grouping_expr_grouped_var_infos. (Processing of + * particular expression would look different, so we could hardly + * reuse the same piece of code.) + */ + if (IsA(gvi->gvexpr, Var)) + continue; + + /* + * Accept the aggregate / grouping expression. + * + * (GroupedVarInfo is more convenient for the next processing than + * Aggref, see add_aggregates_to_grouped_target.) + */ + if (IsA(gvi->gvexpr, Aggref)) + aggregates = lappend(aggregates, gvi); + else + grp_exprs = lappend(grp_exprs, gvi); + } + else if (bms_overlap(gvi->gv_eval_at, rel->relids) && + IsA(gvi->gvexpr, Aggref)) + { + /* + * Remember that there is at least one aggregate / grouping + * expression that needs more than this rel. + */ + found_higher_agg = true; + } + } + list_free(gvis); + + /* + * Give up if some other aggregate(s) need multiple relations including + * the current one. The problem is that grouping of the current relation + * could make some input variables unavailable for the "higher aggregate", + * and it'd also decrease the number of input rows the "higher aggregate" + * receives. + * + * In contrast, grp_exprs is only supposed to contain generic grouping + * expression, so it can be NIL so far. If all the grouping keys are just + * plain Vars, initialize_grouped_targets will take care of them. + */ + if (found_higher_agg) + return; + + /* + * Add plain-var grouping expressions to the target. + */ + initialize_grouped_target(root, rel, target, &grp_exprs_extra); + + /* + * Grouping makes little sense w/o aggregate function and w/o grouping + * expressions. + * + * This test was postponed because initialize_grouped_targets initializes + * sortgrouprefs of rel->reltarget. This information is useful to avoid + * final aggregation. In particular, even non-grouped relation (e.g. + * unique index scan) can generate unique values of grouping keys. See + * check_group_key_uniqueness for details. + */ + if (aggregates == NIL) + return; + + /* + * Add (non-Var) grouping expressions (in the form of GroupedVar) to + * target_agg. + * + * Follow the convention that the grouping expressions should precede the + * aggregates. + */ + add_grouped_vars_to_target(root, target, grp_exprs); + + /* + * Initialize GroupedPathInfo. + */ + Assert(rel->gpi == NULL); + gpi = makeNode(GroupedPathInfo); + gpi->pathlist = NIL; + gpi->partial_pathlist = NIL; + rel->gpi = gpi; + + /* + * Partial aggregation makes no sense w/o grouping expressions. + */ + if (list_length(target->exprs) == 0) + { + pfree(rel->gpi); + rel->gpi = NULL; + return; + } + + /* + * If the aggregation target should have extra grouping expressions, add + * them now. This step includes assignment of tleSortGroupRef's which we + * can generate now (the "ordinary" grouping expression are present in the + * target by now). + */ + if (list_length(grp_exprs_extra) > 0) + { + Index sortgroupref; + + /* + * Always start at root->max_sortgroupref. The extra grouping + * expressions aren't used during the final aggregation, so the + * sortgroupref values don't need to be unique across the query. Thus + * we don't have to increase root->max_sortgroupref, which makes + * recognition of the extra grouping expressions pretty easy. + */ + sortgroupref = root->max_sortgroupref; + + /* + * Generate the SortGroupClause's and add the expressions to the + * target. + */ + foreach(lc, grp_exprs_extra) + { + Var *var = lfirst_node(Var, lc); + SortGroupClause *cl = makeNode(SortGroupClause); + + /* + * TODO Verify that these fields are sufficient for this special + * SortGroupClause. + */ + cl->tleSortGroupRef = ++sortgroupref; + get_sort_group_operators(var->vartype, + false, true, false, + NULL, &cl->eqop, NULL, + &cl->hashable); + gpi->sortgroupclauses = lappend(gpi->sortgroupclauses, cl); + add_column_to_pathtarget(target, (Expr *) var, + cl->tleSortGroupRef); + + /* + * The aggregation input target must emit this var too. + * + * TODO If the target already contains this var for another reason + * (e.g. a an input for generic grouping expression), it'll + * probably not have sortgrouprefs set. Consider removing that + * (and adjust target width accordingly) so that the var is not + * duplicated. + */ + add_column_to_pathtarget(rel->reltarget, (Expr *) var, + cl->tleSortGroupRef); + } + } + + /* + * Add aggregates (in the form of GroupedVar) to the target(s). + */ + add_grouped_vars_to_target(root, target, aggregates); + + /* TODO Check if copy is needed here. */ + gpi->target = copy_pathtarget(target); + + /* + * Add the non-Var group expressions to a separate target. If + * rel->reltarget should be used to generate input for partial + * aggregation, the corresponding path should include these expressions. + */ + if (list_length(grp_exprs) > 0) + { + rel->gpi->group_exprs = create_empty_pathtarget(); + add_grouped_vars_to_target(root, rel->gpi->group_exprs, grp_exprs); + } +} diff --git a/src/backend/optimizer/util/tlist.c b/src/backend/optimizer/util/tlist.c index 9345891..557d934 100644 --- a/src/backend/optimizer/util/tlist.c +++ b/src/backend/optimizer/util/tlist.c @@ -408,6 +408,94 @@ get_sortgrouplist_exprs(List *sgClauses, List *targetList) return result; } +/* + * get_sortgrouplist_clauses + * + * Given a "grouped target" (i.e. target where each non-GroupedVar + * element must have sortgroupref set), build a list of the referencing + * SortGroupClauses, a list of the corresponding grouping expressions and + * a list of aggregate expressions. + * + * target_group_clauses can contain a list of target-specific + * SortGroupClause's, which correspond to grouping expressions possibly + * added to the target (i.e. w/o being present in + * root->query->groupClause, see initialize_grouped_targets for + * details.). + */ +/* Refine the function name. */ +void +get_grouping_expressions(PlannerInfo *root, PathTarget *target, + List *target_group_clauses, + List **grouping_clauses, List **grouping_exprs, + List **agg_exprs) +{ + ListCell *l; + int i = 0; + + /* The target should contain at least one grouping column. */ + Assert(target->sortgrouprefs != NULL); + + foreach(l, target->exprs) + { + Index sortgroupref = 0; + SortGroupClause *cl; + Expr *texpr; + + texpr = (Expr *) lfirst(l); + + if (IsA(texpr, GroupedVar) && + IsA(((GroupedVar *) texpr)->gvexpr, Aggref)) + { + /* + * texpr should represent the first aggregate in the targetlist. + */ + break; + } + + /* + * Find the clause by sortgroupref. + */ + sortgroupref = target->sortgrouprefs[i++]; + + /* + * Besides being an aggregate, the target expression should have no + * other reason then being a column of a relation functionally + * dependent on the GROUP BY clause. So it's not actually a grouping + * column. + */ + if (sortgroupref == 0) + continue; + + cl = get_sortgroupref_clause_noerr(sortgroupref, root->parse->groupClause); + + /* + * If query does not have this clause, it must be target-specific. + */ + if (cl == NULL) + cl = get_sortgroupref_clause(sortgroupref, target_group_clauses); + + *grouping_clauses = list_append_unique(*grouping_clauses, cl); + + /* + * Add only unique clauses because of joins (both sides of a join can + * point at the same grouping clause). XXX Is it worth adding a bool + * argument indicating that we're dealing with join right now? + */ + *grouping_exprs = list_append_unique(*grouping_exprs, texpr); + } + + /* Now collect the aggregates. */ + while (l != NULL) + { + GroupedVar *gvar = castNode(GroupedVar, lfirst(l)); + + /* Currently, GroupedVarInfo can only represent aggregate. */ + Assert(gvar->agg_partial != NULL); + *agg_exprs = lappend(*agg_exprs, gvar->agg_partial); + l = lnext(l); + } +} + /***************************************************************************** * Functions to extract data from a list of SortGroupClauses @@ -783,6 +871,147 @@ apply_pathtarget_labeling_to_tlist(List *tlist, PathTarget *target) } /* + * Replace each "grouped var" in the source targetlist with the original + * expression. If agg_partial is true, restore the partial aggregate as + * opposed to the original "simple" one. + * + * TODO Think of more suitable name undo_grouped_var_substitutions? Also note + * that the partial aggregate is retrieved, not the original one. + */ +List * +restore_grouping_expressions(PlannerInfo *root, List *src, bool agg_partial) +{ + List *result = NIL; + ListCell *l; + + foreach(l, src) + { + TargetEntry *te, + *te_new; + Expr *expr_new = NULL; + + te = lfirst_node(TargetEntry, l); + + if (IsA(te->expr, GroupedVar)) + { + GroupedVar *gvar; + + gvar = castNode(GroupedVar, te->expr); + if (IsA(gvar->gvexpr, Aggref)) + { + if (agg_partial) + { + /* + * Partial aggregate should appear in the targetlist so + * that it looks as if convert_combining_aggrefs arranged + * it. + */ + expr_new = (Expr *) gvar->agg_partial; + } + else + expr_new = gvar->gvexpr; + } + else + expr_new = gvar->gvexpr; + } + + /* + * Alternatively --- if the query generates an unique set of grouping + * keys --- the targetlist may contain aggfinalfn referencing the + * partial aggregate. We replace this with the original + * (AGGSPLIT_SIMPLE) aggregate so that set_upper_references find a + * match. + */ + else if (IS_AGGFINALFN_STANDALONE(te->expr)) + { + FuncExpr *fexpr = castNode(FuncExpr, te->expr); + GroupedVar *gvar = linitial_node(GroupedVar, fexpr->args); + Aggref *aggref = castNode(Aggref, gvar->gvexpr); + + Assert(fexpr->funcid == aggref->aggfinalfn); + + expr_new = (Expr *) aggref; + } + + if (expr_new != NULL) + { + te_new = flatCopyTargetEntry(te); + te_new->expr = (Expr *) expr_new; + } + else + te_new = te; + result = lappend(result, te_new); + } + + return result; +} + +/* + * For each aggregate add GroupedVar to the grouped target. + * + * Caller passes the aggregates in the form of GroupedVarInfos so that we + * don't have to look for gvid. + */ +void +add_grouped_vars_to_target(PlannerInfo *root, PathTarget *target, + List *expressions) +{ + ListCell *lc; + + /* Create the vars and add them to the target. */ + foreach(lc, expressions) + { + GroupedVarInfo *gvi; + GroupedVar *gvar; + + gvi = lfirst_node(GroupedVarInfo, lc); + gvar = makeNode(GroupedVar); + gvar->gvid = gvi->gvid; + gvar->gvexpr = gvi->gvexpr; + gvar->agg_partial = gvi->agg_partial; + add_column_to_pathtarget(target, (Expr *) gvar, gvi->sortgroupref); + } +} + +/* + * Return GroupedVar containing the passed-in expression if one exists, or + * NULL if the expression cannot be used as grouping key. + * + * create_grouping_expr_grouped_var_infos should have been called before this + * function gets called the first time. + */ +GroupedVar * +get_grouping_expression(PlannerInfo *root, Expr *expr) +{ + ListCell *lc; + + /* Caller should have noticed the empty list much earlier. */ + Assert(root->grouped_var_list != NIL); + + foreach(lc, root->grouped_var_list) + { + GroupedVarInfo *gvi = lfirst_node(GroupedVarInfo, lc); + + if (IsA(gvi->gvexpr, Aggref)) + continue; + + if (equal(gvi->gvexpr, expr)) + { + GroupedVar *result = makeNode(GroupedVar); + + Assert(gvi->sortgroupref > 0); + result->gvexpr = gvi->gvexpr; + result->gvid = gvi->gvid; + result->sortgroupref = gvi->sortgroupref; + return result; + } + } + + /* The expression cannot be used as grouping key. */ + return NULL; +} + +/* * split_pathtarget_at_srfs * Split given PathTarget into multiple levels to position SRFs safely * diff --git a/src/backend/optimizer/util/var.c b/src/backend/optimizer/util/var.c index 81c60dc..8ecb21d 100644 --- a/src/backend/optimizer/util/var.c +++ b/src/backend/optimizer/util/var.c @@ -840,3 +840,25 @@ alias_relid_set(PlannerInfo *root, Relids relids) } return result; } + +/* + * Return GroupedVarInfo for given GroupedVar. + * + * XXX Consider better location of this routine. + */ +GroupedVarInfo * +find_grouped_var_info(PlannerInfo *root, GroupedVar *gvar) +{ + ListCell *l; + + foreach(l, root->grouped_var_list) + { + GroupedVarInfo *gvi = lfirst_node(GroupedVarInfo, l); + + if (gvi->gvid == gvar->gvid) + return gvi; + } + + elog(ERROR, "GroupedVarInfo not found"); + return NULL; /* keep compiler quiet */ +} diff --git a/src/backend/parser/parse_func.c b/src/backend/parser/parse_func.c index 2f20516..5be42e6 100644 --- a/src/backend/parser/parse_func.c +++ b/src/backend/parser/parse_func.c @@ -98,6 +98,8 @@ ParseFuncOrColumn(ParseState *pstate, List *funcname, List *fargs, Oid vatype; FuncDetailCode fdresult; char aggkind = 0; + Oid aggcombinefn = InvalidOid; + Oid aggfinalfn = InvalidOid; ParseCallbackState pcbstate; /* @@ -341,6 +343,8 @@ ParseFuncOrColumn(ParseState *pstate, List *funcname, List *fargs, elog(ERROR, "cache lookup failed for aggregate %u", funcid); classForm = (Form_pg_aggregate) GETSTRUCT(tup); aggkind = classForm->aggkind; + aggcombinefn = classForm->aggcombinefn; + aggfinalfn = classForm->aggfinalfn; catDirectArgs = classForm->aggnumdirectargs; ReleaseSysCache(tup); @@ -686,6 +690,8 @@ ParseFuncOrColumn(ParseState *pstate, List *funcname, List *fargs, aggref->aggstar = agg_star; aggref->aggvariadic = func_variadic; aggref->aggkind = aggkind; + aggref->aggcombinefn = aggcombinefn; + aggref->aggfinalfn = aggfinalfn; /* agglevelsup will be set by transformAggregateCall */ aggref->aggsplit = AGGSPLIT_SIMPLE; /* planner might change this */ aggref->location = location; diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 8514c21..ad8a0b9 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -7647,6 +7647,23 @@ get_rule_expr(Node *node, deparse_context *context, get_agg_expr((Aggref *) node, context, (Aggref *) node); break; + case T_GroupedVar: + { + GroupedVar *gvar = castNode(GroupedVar, node); + Expr *expr = gvar->gvexpr; + + if (IsA(expr, Aggref)) + get_agg_expr(gvar->agg_partial, context, (Aggref *) gvar->gvexpr); + else if (IsA(expr, Var)) + (void) get_variable((Var *) expr, 0, false, context); + else + { + Assert(IsA(gvar->gvexpr, OpExpr)); + get_oper_expr((OpExpr *) expr, context); + } + break; + } + case T_GroupingFunc: { GroupingFunc *gexpr = (GroupingFunc *) node; @@ -9132,10 +9149,18 @@ get_agg_combine_expr(Node *node, deparse_context *context, void *private) Aggref *aggref; Aggref *original_aggref = private; - if (!IsA(node, Aggref)) + if (IsA(node, Aggref)) + aggref = (Aggref *) node; + else if (IsA(node, GroupedVar)) + { + GroupedVar *gvar = castNode(GroupedVar, node); + + aggref = gvar->agg_partial; + original_aggref = castNode(Aggref, gvar->gvexpr); + } + else elog(ERROR, "combining Aggref does not point to an Aggref"); - aggref = (Aggref *) node; get_agg_expr(aggref, context, original_aggref); } diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index ea95b80..87dec17 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -114,6 +114,7 @@ #include "catalog/pg_statistic_ext.h" #include "catalog/pg_type.h" #include "executor/executor.h" +#include "executor/nodeAgg.h" #include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/makefuncs.h" @@ -3863,6 +3864,39 @@ estimate_hash_bucket_stats(PlannerInfo *root, Node *hashkey, double nbuckets, ReleaseVariableStats(vardata); } +/* + * estimate_hashagg_tablesize + * estimate the number of bytes that a hash aggregate hashtable will + * require based on the agg_costs, path width and dNumGroups. + * + * XXX this may be over-estimating the size now that hashagg knows to omit + * unneeded columns from the hashtable. Also for mixed-mode grouping sets, + * grouping columns not in the hashed set are counted here even though hashagg + * won't store them. Is this a problem? + */ +Size +estimate_hashagg_tablesize(Path *path, const AggClauseCosts *agg_costs, + double dNumGroups) +{ + Size hashentrysize; + + /* Estimate per-hash-entry space at tuple width... */ + hashentrysize = MAXALIGN(path->pathtarget->width) + + MAXALIGN(SizeofMinimalTupleHeader); + + /* plus space for pass-by-ref transition values... */ + hashentrysize += agg_costs->transitionSpace; + /* plus the per-hash-entry overhead */ + hashentrysize += hash_agg_entry_size(agg_costs->numAggs); + + /* + * Note that this disregards the effect of fill-factor and growth policy + * of the hash-table. That's probably ok, given default the default + * fill-factor is relatively high. It'd be hard to meaningfully factor in + * "double-in-size" growth policies here. + */ + return hashentrysize * dNumGroups; +} /*------------------------------------------------------------------------- * diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 6dcd738..c6da037 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -920,6 +920,15 @@ static struct config_bool ConfigureNamesBool[] = false, NULL, NULL, NULL }, + { + {"enable_agg_pushdown", PGC_USERSET, QUERY_TUNING_METHOD, + gettext_noop("Enables aggregation push-down."), + NULL + }, + &enable_agg_pushdown, + false, + NULL, NULL, NULL + }, { {"geqo", PGC_USERSET, QUERY_TUNING_GEQO, diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h index 04e43cc..16d6082 100644 --- a/src/include/foreign/fdwapi.h +++ b/src/include/foreign/fdwapi.h @@ -26,11 +26,13 @@ struct ExplainState; typedef void (*GetForeignRelSize_function) (PlannerInfo *root, RelOptInfo *baserel, - Oid foreigntableid); + Oid foreigntableid, + bool grouped); typedef void (*GetForeignPaths_function) (PlannerInfo *root, RelOptInfo *baserel, - Oid foreigntableid); + Oid foreigntableid, + bool grouped); typedef ForeignScan *(*GetForeignPlan_function) (PlannerInfo *root, RelOptInfo *baserel, @@ -57,7 +59,8 @@ typedef void (*GetForeignJoinPaths_function) (PlannerInfo *root, RelOptInfo *outerrel, RelOptInfo *innerrel, JoinType jointype, - JoinPathExtraData *extra); + JoinPathExtraData *extra, + bool grouped); typedef void (*GetForeignUpperPaths_function) (PlannerInfo *root, UpperRelationKind stage, diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index c5b5115..86693d1 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -218,6 +218,7 @@ typedef enum NodeTag T_IndexOptInfo, T_ForeignKeyOptInfo, T_ParamPathInfo, + T_GroupedPathInfo, T_Path, T_IndexPath, T_BitmapHeapPath, @@ -258,10 +259,12 @@ typedef enum NodeTag T_PathTarget, T_RestrictInfo, T_PlaceHolderVar, + T_GroupedVar, T_SpecialJoinInfo, T_AppendRelInfo, T_PartitionedChildRelInfo, T_PlaceHolderInfo, + T_GroupedVarInfo, T_MinMaxAggInfo, T_PlannerParamItem, T_RollupData, diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h index 074ae0a..9959dee 100644 --- a/src/include/nodes/primnodes.h +++ b/src/include/nodes/primnodes.h @@ -296,6 +296,8 @@ typedef struct Aggref Oid aggcollid; /* OID of collation of result */ Oid inputcollid; /* OID of collation that function should use */ Oid aggtranstype; /* type Oid of aggregate's transition value */ + Oid aggcombinefn; /* combine function (see pg_aggregate.h) */ + Oid aggfinalfn; /* final function (see pg_aggregate.h) */ List *aggargtypes; /* type Oids of direct and aggregated args */ List *aggdirectargs; /* direct arguments, if an ordered-set agg */ List *args; /* aggregated arguments and sort expressions */ @@ -306,6 +308,7 @@ typedef struct Aggref bool aggvariadic; /* true if variadic arguments have been * combined into an array last argument */ char aggkind; /* aggregate kind (see pg_aggregate.h) */ + Index agglevelsup; /* > 0 if agg belongs to outer query */ AggSplit aggsplit; /* expected agg-splitting mode of parent Agg */ int location; /* token location, or -1 if unknown */ @@ -459,6 +462,14 @@ typedef struct FuncExpr } FuncExpr; /* + * Is the expression an aggfinalfn function that can replace the final + * aggregation in some cases? + */ +#define IS_AGGFINALFN_STANDALONE(e) \ + (IsA((e), FuncExpr) && list_length(((FuncExpr *) (e))->args) == 1 &&\ + IsA(linitial(((FuncExpr *) (e))->args), GroupedVar)) + +/* * NamedArgExpr - a named argument of a function * * This node type can only appear in the args list of a FuncCall or FuncExpr diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index 51df8e9..f78f2d9 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -257,6 +257,8 @@ typedef struct PlannerInfo List *placeholder_list; /* list of PlaceHolderInfos */ + List *grouped_var_list; /* List of GroupedVarInfos. */ + List *fkey_list; /* list of ForeignKeyOptInfos */ List *query_pathkeys; /* desired pathkeys for query_planner() */ @@ -283,6 +285,12 @@ typedef struct PlannerInfo */ List *processed_tlist; + /* + * The maximum ressortgroupref among target entries in processed_list. + * Useful when adding extra grouping expressions for partial aggregation. + */ + int max_sortgroupref; + /* Fields filled during create_plan() for use in setrefs.c */ AttrNumber *grouping_map; /* for GroupingFunc fixup */ List *minmax_aggs; /* List of MinMaxAggInfos */ @@ -438,6 +446,8 @@ typedef struct PartitionSchemeData *PartitionScheme; * direct_lateral_relids - rels this rel has direct LATERAL references to * lateral_relids - required outer rels for LATERAL, as a Relids set * (includes both direct and indirect lateral references) + * gpi - GroupedPathInfo if the relation can produce grouped paths, NULL + * otherwise. * * If the relation is a base relation it will have these fields set: * @@ -609,6 +619,9 @@ typedef struct RelOptInfo Relids direct_lateral_relids; /* rels directly laterally referenced */ Relids lateral_relids; /* minimum parameterization of rel */ + /* Information needed to produce grouped paths. */ + struct GroupedPathInfo *gpi; + /* information about a base rel (not set for join rels!) */ Index relid; Oid reltablespace; /* containing tablespace */ @@ -1003,6 +1016,50 @@ typedef struct ParamPathInfo List *ppi_clauses; /* join clauses available from outer rels */ } ParamPathInfo; +/* + * GroupedPathInfo + * + * If RelOptInfo points to this structure, grouped paths can be created for + * it. + * + * "target" will be used as pathtarget of grouped paths produced either by + * "explicit aggregation" of the relation that owns this structure, or --- if + * the relation is a join --- by joining grouped path to a non-grouped + * one. + * + * The target contains plain-Var grouping expressions, generic grouping + * expressions wrapped in GroupedVar structure, or Aggrefs which are also + * wrapped in GroupedVar. Once GroupedVar is evaluated, its value is passed to + * the upper paths w/o being evaluated again. If final aggregation appears to + * be necessary above the final join, the contained Aggrefs are supposed to + * provide the final aggregation plan with input values, i.e. the aggregate + * transient state. + * + * Note: There's a convention that GroupedVars that contain Aggref expressions + * are supposed to follow the other expressions of the target. Iterations of + * target->exprs may rely on this arrangement. + * + * "group_exprs" contains grouping expressions which are not plain vars. These + * are only added to path target of a path which should generate input for + * partial aggregation. + * + * "sortgroupclauses" is a list of grouping clauses that the relation does + * have in its targetlist but the query does not. + * + * (Two grouped paths cannot be joined in general because grouping of one side + * of the join essentially reduces occurrence of groups of the other side in + * the input of the final aggregation.) + */ +typedef struct GroupedPathInfo +{ + NodeTag type; + + PathTarget *target; /* target of grouped paths aggregation. */ + PathTarget *group_exprs; /* non-Var grouping expressions. */ + List *pathlist; /* List of grouped paths. */ + List *partial_pathlist; /* List of partial grouped paths. */ + List *sortgroupclauses; /* Relation-specific grouping clauses. */ +} GroupedPathInfo; /* * Type "Path" is used as-is for sequential-scan paths, as well as some other @@ -1032,6 +1089,10 @@ typedef struct ParamPathInfo * * "pathkeys" is a List of PathKey nodes (see above), describing the sort * ordering of the path's output rows. + * + * "groupkeys" is a List of Bitmapset objects, each pointing at a set of + * expressions of "pathtarget" whose values within the path output are + * distinct. */ typedef struct Path { @@ -1055,6 +1116,10 @@ typedef struct Path List *pathkeys; /* sort ordering of path's output */ /* pathkeys is a List of PathKey nodes; see above */ + + List *uniquekeys; /* list of bitmapsets where each set contains + * positions of unique expressions within + * pathtarget. */ } Path; /* Macro for extracting a path's parameterization relids; beware double eval */ @@ -1473,12 +1538,16 @@ typedef struct HashPath * ProjectionPath node, which is marked dummy to indicate that we intend to * assign the work to the input plan node. The estimated cost for the * ProjectionPath node will account for whether a Result will be used or not. + * + * force_result field tells that the Result node must be used for some reason + * even though the subpath could normally handle the projection. */ typedef struct ProjectionPath { Path path; Path *subpath; /* path representing input source */ bool dummypp; /* true if no separate Result is needed */ + bool force_result; /* Is Result node required? */ } ProjectionPath; /* @@ -1943,6 +2012,41 @@ typedef struct PlaceHolderVar Index phlevelsup; /* > 0 if PHV belongs to outer query */ } PlaceHolderVar; + +/* + * Similar to the concept of PlaceHolderVar, we treat aggregates and grouping + * columns as special variables if grouping is possible below the top-level + * join. The reason is that aggregates having start as the argument can be + * evaluated at various places in the join tree (i.e. cannot be assigned to + * target list of exactly one relation). Also this concept seems to be less + * invasive than adding the grouped vars to reltarget (in which case + * attr_needed and attr_widths arrays of RelOptInfo) would also need + * additional changes. + * + * gvexpr is a pointer to gvexpr field of the corresponding instance + * GroupedVarInfo. It's there for the sake of exprType(), exprCollation(), + * etc. + * + * agg_partial also points to the corresponding field of GroupedVarInfo if the + * GroupedVar is in the target of a parent relation (RELOPT_BASEREL). However + * within a child relation's (RELOPT_OTHER_MEMBER_REL) target it points to a + * copy which has argument expressions translated, so they no longer reference + * the parent. + * + * XXX Currently we only create GroupedVar for aggregates, but sometime we can + * do it for grouping keys as well. That would allow grouping below the + * top-level join by keys other than plain Var. + */ +typedef struct GroupedVar +{ + Expr xpr; + Expr *gvexpr; /* the represented expression */ + Aggref *agg_partial; /* partial aggregate if gvexpr is aggregate */ + Index sortgroupref; /* SortGroupClause.tleSortGroupRef if gvexpr + * is grouping expression. */ + Index gvid; /* GroupedVarInfo */ +} GroupedVar; + /* * "Special join" info. * @@ -2158,6 +2262,25 @@ typedef struct PlaceHolderInfo } PlaceHolderInfo; /* + * Likewise, GroupedVarInfo exists for each distinct GroupedVar. + */ +typedef struct GroupedVarInfo +{ + NodeTag type; + + Index gvid; /* GroupedVar.gvid */ + Expr *gvexpr; /* the represented expression. */ + Aggref *agg_partial; /* if gvexpr is aggregate, agg_partial is the + * corresponding partial aggregate */ + Index sortgroupref; /* If gvexpr is a grouping expression, this is + * the tleSortGroupRef of the corresponding + * SortGroupClause. */ + Relids gv_eval_at; /* lowest level we can evaluate the expression + * at or NULL if it can happen anywhere. */ + int32 gv_width; /* estimated width of the expression */ +} GroupedVarInfo; + +/* * This struct describes one potentially index-optimizable MIN/MAX aggregate * function. MinMaxAggPath contains a list of these, and if we accept that * path, the list is stored into root->minmax_aggs for use during setrefs.c. diff --git a/src/include/optimizer/clauses.h b/src/include/optimizer/clauses.h index e367221..7090d02 100644 --- a/src/include/optimizer/clauses.h +++ b/src/include/optimizer/clauses.h @@ -84,5 +84,7 @@ extern Node *estimate_expression_value(PlannerInfo *root, Node *node); extern Query *inline_set_returning_function(PlannerInfo *root, RangeTblEntry *rte); - +extern GroupedVarInfo *translate_expression_to_rels(PlannerInfo *root, + GroupedVarInfo *gvi, + Relids relids); #endif /* CLAUSES_H */ diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h index 6c2317d..b7a8c60 100644 --- a/src/include/optimizer/cost.h +++ b/src/include/optimizer/cost.h @@ -68,6 +68,7 @@ extern bool enable_mergejoin; extern bool enable_hashjoin; extern bool enable_gathermerge; extern bool enable_partition_wise_join; +extern bool enable_agg_pushdown; extern int constraint_exclusion; extern double clamp_row_est(double nrows); @@ -189,7 +190,8 @@ extern void set_cte_size_estimates(PlannerInfo *root, RelOptInfo *rel, double cte_rows); extern void set_tablefunc_size_estimates(PlannerInfo *root, RelOptInfo *rel); extern void set_namedtuplestore_size_estimates(PlannerInfo *root, RelOptInfo *rel); -extern void set_foreign_size_estimates(PlannerInfo *root, RelOptInfo *rel); +extern void set_foreign_size_estimates(PlannerInfo *root, RelOptInfo *rel, + bool groupe); extern PathTarget *set_pathtarget_cost_width(PlannerInfo *root, PathTarget *target); extern double compute_bitmap_pages(PlannerInfo *root, RelOptInfo *baserel, Path *bitmapqual, int loop_count, Cost *cost, double *tuple); diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index e9ed16a..6c2c0f5 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -25,13 +25,15 @@ extern int compare_path_costs(Path *path1, Path *path2, extern int compare_fractional_path_costs(Path *path1, Path *path2, double fraction); extern void set_cheapest(RelOptInfo *parent_rel); -extern void add_path(RelOptInfo *parent_rel, Path *new_path); +extern void add_path(RelOptInfo *parent_rel, Path *new_path, bool grouped); extern bool add_path_precheck(RelOptInfo *parent_rel, Cost startup_cost, Cost total_cost, - List *pathkeys, Relids required_outer); -extern void add_partial_path(RelOptInfo *parent_rel, Path *new_path); + List *pathkeys, Relids required_outer, bool grouped); +extern void add_partial_path(RelOptInfo *parent_rel, Path *new_path, + bool grouped); extern bool add_partial_path_precheck(RelOptInfo *parent_rel, - Cost total_cost, List *pathkeys); + Cost total_cost, List *pathkeys, + bool grouped); extern Path *create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer, int parallel_workers); @@ -68,6 +70,7 @@ extern AppendPath *create_append_path(RelOptInfo *rel, List *subpaths, List *partitioned_rels); extern MergeAppendPath *create_merge_append_path(PlannerInfo *root, RelOptInfo *rel, + PathTarget *target, List *subpaths, List *pathkeys, Relids required_outer, @@ -127,7 +130,8 @@ extern NestPath *create_nestloop_path(PlannerInfo *root, Path *inner_path, List *restrict_clauses, List *pathkeys, - Relids required_outer); + Relids required_outer, + PathTarget *target); extern MergePath *create_mergejoin_path(PlannerInfo *root, RelOptInfo *joinrel, @@ -141,7 +145,8 @@ extern MergePath *create_mergejoin_path(PlannerInfo *root, Relids required_outer, List *mergeclauses, List *outersortkeys, - List *innersortkeys); + List *innersortkeys, + PathTarget *target); extern HashPath *create_hashjoin_path(PlannerInfo *root, RelOptInfo *joinrel, @@ -152,12 +157,14 @@ extern HashPath *create_hashjoin_path(PlannerInfo *root, Path *inner_path, List *restrict_clauses, Relids required_outer, - List *hashclauses); + List *hashclauses, + PathTarget *target); extern ProjectionPath *create_projection_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, - PathTarget *target); + PathTarget *target, + bool force_result); extern Path *apply_projection_to_path(PlannerInfo *root, RelOptInfo *rel, Path *path, @@ -193,6 +200,22 @@ extern AggPath *create_agg_path(PlannerInfo *root, List *qual, const AggClauseCosts *aggcosts, double numGroups); +extern AggPath *create_partial_agg_sorted_path(PlannerInfo *root, + Path *subpath, + bool first_call, + List **group_clauses, + List **group_exprs, + List **agg_exprs, + double input_rows, + bool parallel); +extern AggPath *create_partial_agg_hashed_path(PlannerInfo *root, + Path *subpath, + bool first_call, + List **group_clauses, + List **group_exprs, + List **agg_exprs, + double input_rows, + bool parallel); extern GroupingSetsPath *create_groupingsets_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, @@ -253,6 +276,8 @@ extern Path *reparameterize_path(PlannerInfo *root, Path *path, double loop_count); extern Path *reparameterize_path_by_child(PlannerInfo *root, Path *path, RelOptInfo *child_rel); +extern void make_uniquekeys(PlannerInfo *root, Path *path); +extern void make_uniquekeys_for_agg_path(Path *path); /* * prototypes for relnode.c @@ -296,5 +321,8 @@ extern RelOptInfo *build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, RelOptInfo *inner_rel, RelOptInfo *parent_joinrel, List *restrictlist, SpecialJoinInfo *sjinfo, JoinType jointype); - +extern void build_chiid_rel_gpi(PlannerInfo *root, RelOptInfo *child, + RelOptInfo *parent, int nappinfos, + AppendRelInfo **appinfos); +extern void prepare_rel_for_grouping(PlannerInfo *root, RelOptInfo *rel); #endif /* PATHNODE_H */ diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index ea886b6..15e94f9 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -53,7 +53,12 @@ extern void set_dummy_rel_pathlist(RelOptInfo *rel); extern RelOptInfo *standard_join_search(PlannerInfo *root, int levels_needed, List *initial_rels); -extern void generate_gather_paths(PlannerInfo *root, RelOptInfo *rel); +extern void generate_gather_paths(PlannerInfo *root, RelOptInfo *rel, + bool grouped); + +extern void create_grouped_path(PlannerInfo *root, RelOptInfo *rel, + Path *subpath, bool precheck, bool partial, + AggStrategy aggstrategy); extern int compute_parallel_worker(RelOptInfo *rel, double heap_pages, double index_pages); extern void create_partial_bitmap_paths(PlannerInfo *root, RelOptInfo *rel, @@ -69,7 +74,8 @@ extern void debug_print_rel(PlannerInfo *root, RelOptInfo *rel); * indxpath.c * routines to generate index paths */ -extern void create_index_paths(PlannerInfo *root, RelOptInfo *rel); +extern void create_index_paths(PlannerInfo *root, RelOptInfo *rel, + bool grouped); extern bool relation_has_unique_index_for(PlannerInfo *root, RelOptInfo *rel, List *restrictlist, List *exprlist, List *oprlist); @@ -233,5 +239,6 @@ extern bool has_useful_pathkeys(PlannerInfo *root, RelOptInfo *rel); extern PathKey *make_canonical_pathkey(PlannerInfo *root, EquivalenceClass *eclass, Oid opfamily, int strategy, bool nulls_first); - +extern void add_uniquekeys_to_path(Path *path, Bitmapset *new_set); +extern bool match_path_to_group_pathkeys(PlannerInfo *root, Path *path); #endif /* PATHS_H */ diff --git a/src/include/optimizer/plancat.h b/src/include/optimizer/plancat.h index 71f0faf..09e8927 100644 --- a/src/include/optimizer/plancat.h +++ b/src/include/optimizer/plancat.h @@ -38,6 +38,9 @@ extern int32 get_relation_data_width(Oid relid, int32 *attr_widths); extern bool relation_excluded_by_constraints(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); +extern void remove_restrictions_implied_by_constraints(PlannerInfo *root, + RelOptInfo *rel, RangeTblEntry *rte); + extern List *build_physical_tlist(PlannerInfo *root, RelOptInfo *rel); extern bool has_unique_index(RelOptInfo *rel, AttrNumber attno); diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h index d613322..a4c6f5d 100644 --- a/src/include/optimizer/planmain.h +++ b/src/include/optimizer/planmain.h @@ -76,6 +76,8 @@ extern void add_base_rels_to_query(PlannerInfo *root, Node *jtnode); extern void build_base_rel_tlists(PlannerInfo *root, List *final_tlist); extern void add_vars_to_targetlist(PlannerInfo *root, List *vars, Relids where_needed, bool create_new_ph); +extern void add_grouping_info_to_base_rels(PlannerInfo *root); +extern void add_grouped_vars_to_rels(PlannerInfo *root); extern void find_lateral_references(PlannerInfo *root); extern void create_lateral_join_info(PlannerInfo *root); extern List *deconstruct_jointree(PlannerInfo *root); diff --git a/src/include/optimizer/tlist.h b/src/include/optimizer/tlist.h index 0d3ec92..6b7667b 100644 --- a/src/include/optimizer/tlist.h +++ b/src/include/optimizer/tlist.h @@ -41,6 +41,10 @@ extern Node *get_sortgroupclause_expr(SortGroupClause *sgClause, List *targetList); extern List *get_sortgrouplist_exprs(List *sgClauses, List *targetList); +extern void get_grouping_expressions(PlannerInfo *root, PathTarget *target, + List *target_group_clauses, + List **grouping_clauses, + List **grouping_exprs, List **agg_exprs); extern SortGroupClause *get_sortgroupref_clause(Index sortref, List *clauses); @@ -65,6 +69,18 @@ extern void split_pathtarget_at_srfs(PlannerInfo *root, PathTarget *target, PathTarget *input_target, List **targets, List **targets_contain_srfs); +/* TODO Find the best location (position and in some cases even file) for the + * following ones. */ +extern List *restore_grouping_expressions(PlannerInfo *root, List *src, + bool agg_partial); +extern void add_grouped_vars_to_target(PlannerInfo *root, PathTarget *target, + List *expressions); +extern GroupedVar *get_grouping_expression(PlannerInfo *root, Expr *expr); +extern void initialize_grouped_target(PlannerInfo *root, + RelOptInfo *rel, + PathTarget *target_agg, + List **group_exprs_extra_p); + /* Convenience macro to get a PathTarget with valid cost/width fields */ #define create_pathtarget(root, tlist) \ set_pathtarget_cost_width(root, make_pathtarget_from_tlist(tlist)) diff --git a/src/include/optimizer/var.h b/src/include/optimizer/var.h index 6186152..22816db 100644 --- a/src/include/optimizer/var.h +++ b/src/include/optimizer/var.h @@ -36,5 +36,7 @@ extern bool contain_vars_of_level(Node *node, int levelsup); extern int locate_var_of_level(Node *node, int levelsup); extern List *pull_var_clause(Node *node, int flags); extern Node *flatten_join_alias_vars(PlannerInfo *root, Node *node); +extern GroupedVarInfo *find_grouped_var_info(PlannerInfo *root, + GroupedVar *gvar); #endif /* VAR_H */ diff --git a/src/include/utils/selfuncs.h b/src/include/utils/selfuncs.h index 199a631..e2435b6 100644 --- a/src/include/utils/selfuncs.h +++ b/src/include/utils/selfuncs.h @@ -210,6 +210,10 @@ extern void estimate_hash_bucket_stats(PlannerInfo *root, Node *hashkey, double nbuckets, Selectivity *mcv_freq, Selectivity *bucketsize_frac); +extern Size estimate_hashagg_tablesize(Path *path, + const AggClauseCosts *agg_costs, + double dNumGroups); + extern List *deconstruct_indexquals(IndexPath *path); extern void genericcostestimate(PlannerInfo *root, IndexPath *path, diff --git a/src/test/regress/expected/inherit.out b/src/test/regress/expected/inherit.out index fac7b62..f450209 100644 --- a/src/test/regress/expected/inherit.out +++ b/src/test/regress/expected/inherit.out @@ -1690,34 +1690,30 @@ explain (costs off) select * from list_parted where a is not null; --------------------------------- Append -> Seq Scan on part_ab_cd - Filter: (a IS NOT NULL) -> Seq Scan on part_ef_gh - Filter: (a IS NOT NULL) -> Seq Scan on part_null_xy Filter: (a IS NOT NULL) -(7 rows) +(5 rows) explain (costs off) select * from list_parted where a in ('ab', 'cd', 'ef'); QUERY PLAN ---------------------------------------------------------- Append -> Seq Scan on part_ab_cd - Filter: ((a)::text = ANY ('{ab,cd,ef}'::text[])) -> Seq Scan on part_ef_gh Filter: ((a)::text = ANY ('{ab,cd,ef}'::text[])) -(5 rows) +(4 rows) explain (costs off) select * from list_parted where a = 'ab' or a in (null, 'cd'); QUERY PLAN --------------------------------------------------------------------------------------- Append -> Seq Scan on part_ab_cd - Filter: (((a)::text = 'ab'::text) OR ((a)::text = ANY ('{NULL,cd}'::text[]))) -> Seq Scan on part_ef_gh Filter: (((a)::text = 'ab'::text) OR ((a)::text = ANY ('{NULL,cd}'::text[]))) -> Seq Scan on part_null_xy Filter: (((a)::text = 'ab'::text) OR ((a)::text = ANY ('{NULL,cd}'::text[]))) -(7 rows) +(6 rows) explain (costs off) select * from list_parted where a = 'ab'; QUERY PLAN @@ -1770,30 +1766,25 @@ explain (costs off) select * from range_list_parted where a = 5; (5 rows) explain (costs off) select * from range_list_parted where b = 'ab'; - QUERY PLAN ------------------------------------- + QUERY PLAN +---------------------------------- Append -> Seq Scan on part_1_10_ab - Filter: (b = 'ab'::bpchar) -> Seq Scan on part_10_20_ab - Filter: (b = 'ab'::bpchar) -> Seq Scan on part_21_30_ab - Filter: (b = 'ab'::bpchar) -> Seq Scan on part_40_inf_ab - Filter: (b = 'ab'::bpchar) -(9 rows) +(5 rows) explain (costs off) select * from range_list_parted where a between 3 and 23 and b in ('ab'); - QUERY PLAN ------------------------------------------------------------------ + QUERY PLAN +--------------------------------- Append -> Seq Scan on part_1_10_ab - Filter: ((a >= 3) AND (a <= 23) AND (b = 'ab'::bpchar)) + Filter: (a >= 3) -> Seq Scan on part_10_20_ab - Filter: ((a >= 3) AND (a <= 23) AND (b = 'ab'::bpchar)) -> Seq Scan on part_21_30_ab - Filter: ((a >= 3) AND (a <= 23) AND (b = 'ab'::bpchar)) -(7 rows) + Filter: (a <= 23) +(6 rows) /* Should select no rows because range partition key cannot be null */ explain (costs off) select * from range_list_parted where a is null; @@ -1809,44 +1800,34 @@ explain (costs off) select * from range_list_parted where b is null; ------------------------------------ Append -> Seq Scan on part_40_inf_null - Filter: (b IS NULL) -(3 rows) +(2 rows) explain (costs off) select * from range_list_parted where a is not null and a < 67; - QUERY PLAN ------------------------------------------------- + QUERY PLAN +------------------------------------ Append -> Seq Scan on part_1_10_ab - Filter: ((a IS NOT NULL) AND (a < 67)) -> Seq Scan on part_1_10_cd - Filter: ((a IS NOT NULL) AND (a < 67)) -> Seq Scan on part_10_20_ab - Filter: ((a IS NOT NULL) AND (a < 67)) -> Seq Scan on part_10_20_cd - Filter: ((a IS NOT NULL) AND (a < 67)) -> Seq Scan on part_21_30_ab - Filter: ((a IS NOT NULL) AND (a < 67)) -> Seq Scan on part_21_30_cd - Filter: ((a IS NOT NULL) AND (a < 67)) -> Seq Scan on part_40_inf_ab - Filter: ((a IS NOT NULL) AND (a < 67)) + Filter: (a < 67) -> Seq Scan on part_40_inf_cd - Filter: ((a IS NOT NULL) AND (a < 67)) + Filter: (a < 67) -> Seq Scan on part_40_inf_null - Filter: ((a IS NOT NULL) AND (a < 67)) -(19 rows) + Filter: (a < 67) +(13 rows) explain (costs off) select * from range_list_parted where a >= 30; QUERY PLAN ------------------------------------ Append -> Seq Scan on part_40_inf_ab - Filter: (a >= 30) -> Seq Scan on part_40_inf_cd - Filter: (a >= 30) -> Seq Scan on part_40_inf_null - Filter: (a >= 30) -(7 rows) +(4 rows) drop table list_parted; drop table range_list_parted; @@ -1887,7 +1868,7 @@ explain (costs off) select * from mcrparted where a = 10 and abs(b) = 5; -- scan -> Seq Scan on mcrparted1 Filter: ((a = 10) AND (abs(b) = 5)) -> Seq Scan on mcrparted2 - Filter: ((a = 10) AND (abs(b) = 5)) + Filter: (abs(b) = 5) -> Seq Scan on mcrparted_def Filter: ((a = 10) AND (abs(b) = 5)) (7 rows) @@ -1917,25 +1898,20 @@ explain (costs off) select * from mcrparted where a > -1; -- scans all partition -> Seq Scan on mcrparted0 Filter: (a > '-1'::integer) -> Seq Scan on mcrparted1 - Filter: (a > '-1'::integer) -> Seq Scan on mcrparted2 - Filter: (a > '-1'::integer) -> Seq Scan on mcrparted3 - Filter: (a > '-1'::integer) -> Seq Scan on mcrparted4 - Filter: (a > '-1'::integer) -> Seq Scan on mcrparted5 - Filter: (a > '-1'::integer) -> Seq Scan on mcrparted_def Filter: (a > '-1'::integer) -(15 rows) +(10 rows) explain (costs off) select * from mcrparted where a = 20 and abs(b) = 10 and c > 10; -- scans mcrparted4 - QUERY PLAN ------------------------------------------------------------ + QUERY PLAN +---------------------------------------------- Append -> Seq Scan on mcrparted4 - Filter: ((c > 10) AND (a = 20) AND (abs(b) = 10)) + Filter: ((c > 10) AND (abs(b) = 10)) (3 rows) explain (costs off) select * from mcrparted where a = 20 and c > 20; -- scans mcrparted3, mcrparte4, mcrparte5, mcrparted_def @@ -1945,7 +1921,7 @@ explain (costs off) select * from mcrparted where a = 20 and c > 20; -- scans mc -> Seq Scan on mcrparted3 Filter: ((c > 20) AND (a = 20)) -> Seq Scan on mcrparted4 - Filter: ((c > 20) AND (a = 20)) + Filter: (c > 20) -> Seq Scan on mcrparted5 Filter: ((c > 20) AND (a = 20)) -> Seq Scan on mcrparted_def @@ -1968,13 +1944,13 @@ explain (costs off) select min(a), max(a) from parted_minmax where b = '12345'; -> Merge Append Sort Key: parted_minmax1.a -> Index Only Scan using parted_minmax1i on parted_minmax1 - Index Cond: ((a IS NOT NULL) AND (b = '12345'::text)) + Index Cond: (b = '12345'::text) InitPlan 2 (returns $1) -> Limit -> Merge Append Sort Key: parted_minmax1_1.a DESC -> Index Only Scan Backward using parted_minmax1i on parted_minmax1 parted_minmax1_1 - Index Cond: ((a IS NOT NULL) AND (b = '12345'::text)) + Index Cond: (b = '12345'::text) (13 rows) select min(a), max(a) from parted_minmax where b = '12345'; diff --git a/src/test/regress/expected/partition_join.out b/src/test/regress/expected/partition_join.out index 27ab852..9ed3c5e 100644 --- a/src/test/regress/expected/partition_join.out +++ b/src/test/regress/expected/partition_join.out @@ -223,7 +223,7 @@ SELECT t1.a, t1.c, t2.b, t2.c FROM (SELECT * FROM prt1 WHERE a < 450) t1 LEFT JO -> Hash Left Join Hash Cond: (prt1_p1.a = b) -> Seq Scan on prt1_p1 - Filter: ((a < 450) AND (b = 0)) + Filter: (b = 0) -> Hash -> Result One-Time Filter: false @@ -261,7 +261,6 @@ SELECT t1.a, t1.c, t2.b, t2.c FROM (SELECT * FROM prt1 WHERE a < 450) t1 FULL JO Hash Cond: (prt1_p1.a = b) Filter: ((prt1_p1.b = 0) OR (a = 0)) -> Seq Scan on prt1_p1 - Filter: (a < 450) -> Hash -> Result One-Time Filter: false @@ -277,11 +276,10 @@ SELECT t1.a, t1.c, t2.b, t2.c FROM (SELECT * FROM prt1 WHERE a < 450) t1 FULL JO Hash Cond: (prt2_p3.b = a) Filter: ((b = 0) OR (prt2_p3.a = 0)) -> Seq Scan on prt2_p3 - Filter: (b > 250) -> Hash -> Result One-Time Filter: false -(27 rows) +(25 rows) SELECT t1.a, t1.c, t2.b, t2.c FROM (SELECT * FROM prt1 WHERE a < 450) t1 FULL JOIN (SELECT * FROM prt2 WHERE b > 250) t2 ON t1.a = t2.b WHERE t1.b = 0 OR t2.a = 0 ORDER BY t1.a, t2.b; a | c | b | c @@ -1019,7 +1017,7 @@ SELECT t1.a, t2.b FROM (SELECT * FROM prt1 WHERE a < 450) t1 LEFT JOIN (SELECT * -> Sort Sort Key: prt1_p1.a -> Seq Scan on prt1_p1 - Filter: ((a < 450) AND (b = 0)) + Filter: (b = 0) -> Sort Sort Key: b -> Result diff --git a/src/test/regress/expected/partition_prune.out b/src/test/regress/expected/partition_prune.out index aabb024..5a1c40a 100644 --- a/src/test/regress/expected/partition_prune.out +++ b/src/test/regress/expected/partition_prune.out @@ -25,22 +25,20 @@ explain (costs off) select * from lp where a > 'a' and a < 'd'; ----------------------------------------------------------- Append -> Seq Scan on lp_bc - Filter: ((a > 'a'::bpchar) AND (a < 'd'::bpchar)) -> Seq Scan on lp_default Filter: ((a > 'a'::bpchar) AND (a < 'd'::bpchar)) -(5 rows) +(4 rows) explain (costs off) select * from lp where a > 'a' and a <= 'd'; QUERY PLAN ------------------------------------------------------------ Append -> Seq Scan on lp_ad - Filter: ((a > 'a'::bpchar) AND (a <= 'd'::bpchar)) + Filter: (a > 'a'::bpchar) -> Seq Scan on lp_bc - Filter: ((a > 'a'::bpchar) AND (a <= 'd'::bpchar)) -> Seq Scan on lp_default Filter: ((a > 'a'::bpchar) AND (a <= 'd'::bpchar)) -(7 rows) +(6 rows) explain (costs off) select * from lp where a = 'a'; QUERY PLAN @@ -59,28 +57,22 @@ explain (costs off) select * from lp where 'a' = a; /* commuted */ (3 rows) explain (costs off) select * from lp where a is not null; - QUERY PLAN ---------------------------------- + QUERY PLAN +------------------------------ Append -> Seq Scan on lp_ad - Filter: (a IS NOT NULL) -> Seq Scan on lp_bc - Filter: (a IS NOT NULL) -> Seq Scan on lp_ef - Filter: (a IS NOT NULL) -> Seq Scan on lp_g - Filter: (a IS NOT NULL) -> Seq Scan on lp_default - Filter: (a IS NOT NULL) -(11 rows) +(6 rows) explain (costs off) select * from lp where a is null; - QUERY PLAN ------------------------------ + QUERY PLAN +--------------------------- Append -> Seq Scan on lp_null - Filter: (a IS NULL) -(3 rows) +(2 rows) explain (costs off) select * from lp where a = 'a' or a = 'c'; QUERY PLAN @@ -93,56 +85,44 @@ explain (costs off) select * from lp where a = 'a' or a = 'c'; (5 rows) explain (costs off) select * from lp where a is not null and (a = 'a' or a = 'c'); - QUERY PLAN --------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------- Append -> Seq Scan on lp_ad - Filter: ((a IS NOT NULL) AND ((a = 'a'::bpchar) OR (a = 'c'::bpchar))) + Filter: ((a = 'a'::bpchar) OR (a = 'c'::bpchar)) -> Seq Scan on lp_bc - Filter: ((a IS NOT NULL) AND ((a = 'a'::bpchar) OR (a = 'c'::bpchar))) + Filter: ((a = 'a'::bpchar) OR (a = 'c'::bpchar)) (5 rows) explain (costs off) select * from lp where a <> 'g'; - QUERY PLAN ------------------------------------- + QUERY PLAN +------------------------------ Append -> Seq Scan on lp_ad - Filter: (a <> 'g'::bpchar) -> Seq Scan on lp_bc - Filter: (a <> 'g'::bpchar) -> Seq Scan on lp_ef - Filter: (a <> 'g'::bpchar) -> Seq Scan on lp_default - Filter: (a <> 'g'::bpchar) -(9 rows) +(5 rows) explain (costs off) select * from lp where a <> 'a' and a <> 'd'; - QUERY PLAN -------------------------------------------------------------- + QUERY PLAN +------------------------------ Append -> Seq Scan on lp_bc - Filter: ((a <> 'a'::bpchar) AND (a <> 'd'::bpchar)) -> Seq Scan on lp_ef - Filter: ((a <> 'a'::bpchar) AND (a <> 'd'::bpchar)) -> Seq Scan on lp_g - Filter: ((a <> 'a'::bpchar) AND (a <> 'd'::bpchar)) -> Seq Scan on lp_default - Filter: ((a <> 'a'::bpchar) AND (a <> 'd'::bpchar)) -(9 rows) +(5 rows) explain (costs off) select * from lp where a not in ('a', 'd'); - QUERY PLAN ------------------------------------------------- + QUERY PLAN +------------------------------ Append -> Seq Scan on lp_bc - Filter: (a <> ALL ('{a,d}'::bpchar[])) -> Seq Scan on lp_ef - Filter: (a <> ALL ('{a,d}'::bpchar[])) -> Seq Scan on lp_g - Filter: (a <> ALL ('{a,d}'::bpchar[])) -> Seq Scan on lp_default - Filter: (a <> ALL ('{a,d}'::bpchar[])) -(9 rows) +(5 rows) -- collation matches the partitioning collation, pruning works create table coll_pruning (a text collate "C") partition by list (a); @@ -150,12 +130,11 @@ create table coll_pruning_a partition of coll_pruning for values in ('a'); create table coll_pruning_b partition of coll_pruning for values in ('b'); create table coll_pruning_def partition of coll_pruning default; explain (costs off) select * from coll_pruning where a collate "C" = 'a' collate "C"; - QUERY PLAN ---------------------------------------------- + QUERY PLAN +---------------------------------- Append -> Seq Scan on coll_pruning_a - Filter: (a = 'a'::text COLLATE "C") -(3 rows) +(2 rows) -- collation doesn't match the partitioning collation, no pruning occurs explain (costs off) select * from coll_pruning where a collate "POSIX" = 'a' collate "POSIX"; @@ -192,32 +171,29 @@ create table rlp5 partition of rlp for values from (31) to (maxvalue) partition create table rlp5_default partition of rlp5 default; create table rlp5_1 partition of rlp5 for values from (31) to (40); explain (costs off) select * from rlp where a < 1; - QUERY PLAN -------------------------- + QUERY PLAN +------------------------ Append -> Seq Scan on rlp1 - Filter: (a < 1) -(3 rows) +(2 rows) explain (costs off) select * from rlp where 1 > a; /* commuted */ - QUERY PLAN -------------------------- + QUERY PLAN +------------------------ Append -> Seq Scan on rlp1 - Filter: (1 > a) -(3 rows) +(2 rows) explain (costs off) select * from rlp where a <= 1; QUERY PLAN --------------------------------------- Append -> Seq Scan on rlp1 - Filter: (a <= 1) -> Seq Scan on rlp2 Filter: (a <= 1) -> Seq Scan on rlp_default_default Filter: (a <= 1) -(7 rows) +(6 rows) explain (costs off) select * from rlp where a = 1; QUERY PLAN @@ -276,65 +252,47 @@ explain (costs off) select * from rlp where a <= 10; --------------------------------------- Append -> Seq Scan on rlp1 - Filter: (a <= 10) -> Seq Scan on rlp2 - Filter: (a <= 10) -> Seq Scan on rlp_default_10 - Filter: (a <= 10) -> Seq Scan on rlp_default_default Filter: (a <= 10) -(9 rows) +(6 rows) explain (costs off) select * from rlp where a > 10; QUERY PLAN --------------------------------------- Append -> Seq Scan on rlp3abcd - Filter: (a > 10) -> Seq Scan on rlp3efgh - Filter: (a > 10) -> Seq Scan on rlp3nullxy - Filter: (a > 10) -> Seq Scan on rlp3_default - Filter: (a > 10) -> Seq Scan on rlp4_1 - Filter: (a > 10) -> Seq Scan on rlp4_2 - Filter: (a > 10) -> Seq Scan on rlp4_default - Filter: (a > 10) -> Seq Scan on rlp5_1 - Filter: (a > 10) -> Seq Scan on rlp5_default - Filter: (a > 10) -> Seq Scan on rlp_default_30 - Filter: (a > 10) -> Seq Scan on rlp_default_default Filter: (a > 10) -(23 rows) +(13 rows) explain (costs off) select * from rlp where a < 15; QUERY PLAN --------------------------------------- Append -> Seq Scan on rlp1 - Filter: (a < 15) -> Seq Scan on rlp2 - Filter: (a < 15) -> Seq Scan on rlp_default_10 - Filter: (a < 15) -> Seq Scan on rlp_default_default Filter: (a < 15) -(9 rows) +(6 rows) explain (costs off) select * from rlp where a <= 15; QUERY PLAN --------------------------------------- Append -> Seq Scan on rlp1 - Filter: (a <= 15) -> Seq Scan on rlp2 - Filter: (a <= 15) -> Seq Scan on rlp3abcd Filter: (a <= 15) -> Seq Scan on rlp3efgh @@ -344,10 +302,9 @@ explain (costs off) select * from rlp where a <= 15; -> Seq Scan on rlp3_default Filter: (a <= 15) -> Seq Scan on rlp_default_10 - Filter: (a <= 15) -> Seq Scan on rlp_default_default Filter: (a <= 15) -(17 rows) +(14 rows) explain (costs off) select * from rlp where a > 15 and b = 'ab'; QUERY PLAN @@ -356,17 +313,17 @@ explain (costs off) select * from rlp where a > 15 and b = 'ab'; -> Seq Scan on rlp3abcd Filter: ((a > 15) AND ((b)::text = 'ab'::text)) -> Seq Scan on rlp4_1 - Filter: ((a > 15) AND ((b)::text = 'ab'::text)) + Filter: ((b)::text = 'ab'::text) -> Seq Scan on rlp4_2 - Filter: ((a > 15) AND ((b)::text = 'ab'::text)) + Filter: ((b)::text = 'ab'::text) -> Seq Scan on rlp4_default - Filter: ((a > 15) AND ((b)::text = 'ab'::text)) + Filter: ((b)::text = 'ab'::text) -> Seq Scan on rlp5_1 - Filter: ((a > 15) AND ((b)::text = 'ab'::text)) + Filter: ((b)::text = 'ab'::text) -> Seq Scan on rlp5_default - Filter: ((a > 15) AND ((b)::text = 'ab'::text)) + Filter: ((b)::text = 'ab'::text) -> Seq Scan on rlp_default_30 - Filter: ((a > 15) AND ((b)::text = 'ab'::text)) + Filter: ((b)::text = 'ab'::text) -> Seq Scan on rlp_default_default Filter: ((a > 15) AND ((b)::text = 'ab'::text)) (17 rows) @@ -424,13 +381,13 @@ explain (costs off) select * from rlp where a = 16 and b is not null; ------------------------------------------------ Append -> Seq Scan on rlp3abcd - Filter: ((b IS NOT NULL) AND (a = 16)) + Filter: (a = 16) -> Seq Scan on rlp3efgh - Filter: ((b IS NOT NULL) AND (a = 16)) + Filter: (a = 16) -> Seq Scan on rlp3nullxy Filter: ((b IS NOT NULL) AND (a = 16)) -> Seq Scan on rlp3_default - Filter: ((b IS NOT NULL) AND (a = 16)) + Filter: (a = 16) (9 rows) explain (costs off) select * from rlp where a is null; @@ -438,96 +395,67 @@ explain (costs off) select * from rlp where a is null; ------------------------------------ Append -> Seq Scan on rlp_default_null - Filter: (a IS NULL) -(3 rows) +(2 rows) explain (costs off) select * from rlp where a is not null; QUERY PLAN --------------------------------------- Append -> Seq Scan on rlp1 - Filter: (a IS NOT NULL) -> Seq Scan on rlp2 - Filter: (a IS NOT NULL) -> Seq Scan on rlp3abcd - Filter: (a IS NOT NULL) -> Seq Scan on rlp3efgh - Filter: (a IS NOT NULL) -> Seq Scan on rlp3nullxy - Filter: (a IS NOT NULL) -> Seq Scan on rlp3_default - Filter: (a IS NOT NULL) -> Seq Scan on rlp4_1 - Filter: (a IS NOT NULL) -> Seq Scan on rlp4_2 - Filter: (a IS NOT NULL) -> Seq Scan on rlp4_default - Filter: (a IS NOT NULL) -> Seq Scan on rlp5_1 - Filter: (a IS NOT NULL) -> Seq Scan on rlp5_default - Filter: (a IS NOT NULL) -> Seq Scan on rlp_default_10 - Filter: (a IS NOT NULL) -> Seq Scan on rlp_default_30 - Filter: (a IS NOT NULL) -> Seq Scan on rlp_default_default - Filter: (a IS NOT NULL) -(29 rows) +(15 rows) explain (costs off) select * from rlp where a > 30; QUERY PLAN --------------------------------------- Append -> Seq Scan on rlp5_1 - Filter: (a > 30) -> Seq Scan on rlp5_default - Filter: (a > 30) -> Seq Scan on rlp_default_default Filter: (a > 30) -(7 rows) +(5 rows) explain (costs off) select * from rlp where a = 30; /* only default is scanned */ QUERY PLAN ---------------------------------- Append -> Seq Scan on rlp_default_30 - Filter: (a = 30) -(3 rows) +(2 rows) explain (costs off) select * from rlp where a <= 31; QUERY PLAN --------------------------------------- Append -> Seq Scan on rlp1 - Filter: (a <= 31) -> Seq Scan on rlp2 - Filter: (a <= 31) -> Seq Scan on rlp3abcd - Filter: (a <= 31) -> Seq Scan on rlp3efgh - Filter: (a <= 31) -> Seq Scan on rlp3nullxy - Filter: (a <= 31) -> Seq Scan on rlp3_default - Filter: (a <= 31) -> Seq Scan on rlp4_1 - Filter: (a <= 31) -> Seq Scan on rlp4_2 - Filter: (a <= 31) -> Seq Scan on rlp4_default - Filter: (a <= 31) -> Seq Scan on rlp5_1 Filter: (a <= 31) -> Seq Scan on rlp5_default Filter: (a <= 31) -> Seq Scan on rlp_default_10 - Filter: (a <= 31) -> Seq Scan on rlp_default_30 - Filter: (a <= 31) -> Seq Scan on rlp_default_default Filter: (a <= 31) -(29 rows) +(18 rows) explain (costs off) select * from rlp where a = 1 or a = 7; QUERY PLAN @@ -572,9 +500,9 @@ explain (costs off) select * from rlp where a > 20 and a < 27; ----------------------------------------- Append -> Seq Scan on rlp4_1 - Filter: ((a > 20) AND (a < 27)) + Filter: (a > 20) -> Seq Scan on rlp4_2 - Filter: ((a > 20) AND (a < 27)) + Filter: (a < 27) -> Seq Scan on rlp4_default Filter: ((a > 20) AND (a < 27)) (7 rows) @@ -594,51 +522,37 @@ explain (costs off) select * from rlp where a >= 29; -> Seq Scan on rlp4_default Filter: (a >= 29) -> Seq Scan on rlp5_1 - Filter: (a >= 29) -> Seq Scan on rlp5_default - Filter: (a >= 29) -> Seq Scan on rlp_default_30 - Filter: (a >= 29) -> Seq Scan on rlp_default_default Filter: (a >= 29) -(11 rows) +(8 rows) -- redundant clauses are eliminated explain (costs off) select * from rlp where a > 1 and a = 10; /* only default */ - QUERY PLAN ----------------------------------------- + QUERY PLAN +---------------------------------- Append -> Seq Scan on rlp_default_10 - Filter: ((a > 1) AND (a = 10)) -(3 rows) +(2 rows) explain (costs off) select * from rlp where a > 1 and a >=15; /* rlp3 onwards, including default */ QUERY PLAN ----------------------------------------- Append -> Seq Scan on rlp3abcd - Filter: ((a > 1) AND (a >= 15)) -> Seq Scan on rlp3efgh - Filter: ((a > 1) AND (a >= 15)) -> Seq Scan on rlp3nullxy - Filter: ((a > 1) AND (a >= 15)) -> Seq Scan on rlp3_default - Filter: ((a > 1) AND (a >= 15)) -> Seq Scan on rlp4_1 - Filter: ((a > 1) AND (a >= 15)) -> Seq Scan on rlp4_2 - Filter: ((a > 1) AND (a >= 15)) -> Seq Scan on rlp4_default - Filter: ((a > 1) AND (a >= 15)) -> Seq Scan on rlp5_1 - Filter: ((a > 1) AND (a >= 15)) -> Seq Scan on rlp5_default - Filter: ((a > 1) AND (a >= 15)) -> Seq Scan on rlp_default_30 - Filter: ((a > 1) AND (a >= 15)) -> Seq Scan on rlp_default_default Filter: ((a > 1) AND (a >= 15)) -(23 rows) +(13 rows) explain (costs off) select * from rlp where a = 1 and a = 3; /* empty */ QUERY PLAN @@ -727,28 +641,23 @@ explain (costs off) select * from mc3p where a = 10 and abs(b) between 5 and 35; -> Seq Scan on mc3p1 Filter: ((a = 10) AND (abs(b) >= 5) AND (abs(b) <= 35)) -> Seq Scan on mc3p2 - Filter: ((a = 10) AND (abs(b) >= 5) AND (abs(b) <= 35)) -> Seq Scan on mc3p3 - Filter: ((a = 10) AND (abs(b) >= 5) AND (abs(b) <= 35)) -> Seq Scan on mc3p4 - Filter: ((a = 10) AND (abs(b) >= 5) AND (abs(b) <= 35)) + Filter: (abs(b) <= 35) -> Seq Scan on mc3p_default Filter: ((a = 10) AND (abs(b) >= 5) AND (abs(b) <= 35)) -(11 rows) +(9 rows) explain (costs off) select * from mc3p where a > 10; QUERY PLAN -------------------------------- Append -> Seq Scan on mc3p5 - Filter: (a > 10) -> Seq Scan on mc3p6 - Filter: (a > 10) -> Seq Scan on mc3p7 - Filter: (a > 10) -> Seq Scan on mc3p_default Filter: (a > 10) -(9 rows) +(6 rows) explain (costs off) select * from mc3p where a >= 10; QUERY PLAN @@ -757,43 +666,36 @@ explain (costs off) select * from mc3p where a >= 10; -> Seq Scan on mc3p1 Filter: (a >= 10) -> Seq Scan on mc3p2 - Filter: (a >= 10) -> Seq Scan on mc3p3 - Filter: (a >= 10) -> Seq Scan on mc3p4 - Filter: (a >= 10) -> Seq Scan on mc3p5 - Filter: (a >= 10) -> Seq Scan on mc3p6 - Filter: (a >= 10) -> Seq Scan on mc3p7 - Filter: (a >= 10) -> Seq Scan on mc3p_default Filter: (a >= 10) -(17 rows) +(11 rows) explain (costs off) select * from mc3p where a < 10; QUERY PLAN -------------------------------- Append -> Seq Scan on mc3p0 - Filter: (a < 10) -> Seq Scan on mc3p1 Filter: (a < 10) -> Seq Scan on mc3p_default Filter: (a < 10) -(7 rows) +(6 rows) explain (costs off) select * from mc3p where a <= 10 and abs(b) < 10; QUERY PLAN ----------------------------------------------- Append -> Seq Scan on mc3p0 - Filter: ((a <= 10) AND (abs(b) < 10)) + Filter: (abs(b) < 10) -> Seq Scan on mc3p1 - Filter: ((a <= 10) AND (abs(b) < 10)) + Filter: (abs(b) < 10) -> Seq Scan on mc3p2 - Filter: ((a <= 10) AND (abs(b) < 10)) + Filter: (abs(b) < 10) -> Seq Scan on mc3p_default Filter: ((a <= 10) AND (abs(b) < 10)) (9 rows) @@ -807,11 +709,11 @@ explain (costs off) select * from mc3p where a = 11 and abs(b) = 0; (3 rows) explain (costs off) select * from mc3p where a = 20 and abs(b) = 10 and c = 100; - QUERY PLAN ------------------------------------------------------------- + QUERY PLAN +----------------------------------------------- Append -> Seq Scan on mc3p6 - Filter: ((a = 20) AND (c = 100) AND (abs(b) = 10)) + Filter: ((c = 100) AND (abs(b) = 10)) (3 rows) explain (costs off) select * from mc3p where a > 20; @@ -831,12 +733,10 @@ explain (costs off) select * from mc3p where a >= 20; -> Seq Scan on mc3p5 Filter: (a >= 20) -> Seq Scan on mc3p6 - Filter: (a >= 20) -> Seq Scan on mc3p7 - Filter: (a >= 20) -> Seq Scan on mc3p_default Filter: (a >= 20) -(9 rows) +(7 rows) explain (costs off) select * from mc3p where (a = 1 and abs(b) = 1 and c = 1) or (a = 10 and abs(b) = 5 and c = 10) or (a > 11 and a < 20); QUERY PLAN @@ -873,7 +773,6 @@ explain (costs off) select * from mc3p where (a = 1 and abs(b) = 1 and c = 1) or ------------------------------------------------------------------------------------------------------------------------------------------------------- Append -> Seq Scan on mc3p0 - Filter: (((a = 1) AND (abs(b) = 1) AND (c = 1)) OR ((a = 10) AND (abs(b) = 5) AND (c = 10)) OR ((a > 11) AND (a < 20)) OR (a < 1) OR (a = 1)) -> Seq Scan on mc3p1 Filter: (((a = 1) AND (abs(b) = 1) AND (c = 1)) OR ((a = 10) AND (abs(b) = 5) AND (c = 10)) OR ((a > 11) AND (a < 20)) OR (a < 1) OR (a = 1)) -> Seq Scan on mc3p2 @@ -882,7 +781,7 @@ explain (costs off) select * from mc3p where (a = 1 and abs(b) = 1 and c = 1) or Filter: (((a = 1) AND (abs(b) = 1) AND (c = 1)) OR ((a = 10) AND (abs(b) = 5) AND (c = 10)) OR ((a > 11) AND (a < 20)) OR (a < 1) OR (a = 1)) -> Seq Scan on mc3p_default Filter: (((a = 1) AND (abs(b) = 1) AND (c = 1)) OR ((a = 10) AND (abs(b) = 5) AND (c = 10)) OR ((a > 11) AND (a < 20)) OR (a < 1) OR (a = 1)) -(11 rows) +(10 rows) explain (costs off) select * from mc3p where a = 1 or abs(b) = 1 or c = 1; QUERY PLAN @@ -917,12 +816,11 @@ explain (costs off) select * from mc3p where (a = 1 and abs(b) = 1) or (a = 10 a -> Seq Scan on mc3p2 Filter: (((a = 1) AND (abs(b) = 1)) OR ((a = 10) AND (abs(b) = 10))) -> Seq Scan on mc3p3 - Filter: (((a = 1) AND (abs(b) = 1)) OR ((a = 10) AND (abs(b) = 10))) -> Seq Scan on mc3p4 Filter: (((a = 1) AND (abs(b) = 1)) OR ((a = 10) AND (abs(b) = 10))) -> Seq Scan on mc3p_default Filter: (((a = 1) AND (abs(b) = 1)) OR ((a = 10) AND (abs(b) = 10))) -(13 rows) +(12 rows) explain (costs off) select * from mc3p where (a = 1 and abs(b) = 1) or (a = 10 and abs(b) = 9); QUERY PLAN @@ -952,22 +850,18 @@ explain (costs off) select * from mc2p where a < 2; -------------------------------- Append -> Seq Scan on mc2p0 - Filter: (a < 2) -> Seq Scan on mc2p1 - Filter: (a < 2) -> Seq Scan on mc2p2 - Filter: (a < 2) -> Seq Scan on mc2p_default Filter: (a < 2) -(9 rows) +(6 rows) explain (costs off) select * from mc2p where a = 2 and b < 1; - QUERY PLAN ---------------------------------------- + QUERY PLAN +------------------------- Append -> Seq Scan on mc2p3 - Filter: ((b < 1) AND (a = 2)) -(3 rows) +(2 rows) explain (costs off) select * from mc2p where a > 1; QUERY PLAN @@ -976,14 +870,11 @@ explain (costs off) select * from mc2p where a > 1; -> Seq Scan on mc2p2 Filter: (a > 1) -> Seq Scan on mc2p3 - Filter: (a > 1) -> Seq Scan on mc2p4 - Filter: (a > 1) -> Seq Scan on mc2p5 - Filter: (a > 1) -> Seq Scan on mc2p_default Filter: (a > 1) -(11 rows) +(8 rows) explain (costs off) select * from mc2p where a = 1 and b > 1; QUERY PLAN @@ -999,14 +890,12 @@ create table boolpart_default partition of boolpart default; create table boolpart_t partition of boolpart for values in ('true'); create table boolpart_f partition of boolpart for values in ('false'); explain (costs off) select * from boolpart where a in (true, false); - QUERY PLAN ------------------------------------------------- + QUERY PLAN +------------------------------ Append -> Seq Scan on boolpart_f - Filter: (a = ANY ('{t,f}'::boolean[])) -> Seq Scan on boolpart_t - Filter: (a = ANY ('{t,f}'::boolean[])) -(5 rows) +(3 rows) explain (costs off) select * from boolpart where a = false; QUERY PLAN diff --git a/src/test/regress/expected/rowsecurity.out b/src/test/regress/expected/rowsecurity.out index b8dcf51..cf0ef42 100644 --- a/src/test/regress/expected/rowsecurity.out +++ b/src/test/regress/expected/rowsecurity.out @@ -1057,14 +1057,14 @@ NOTICE: f_leak => awesome science fiction (4 rows) EXPLAIN (COSTS OFF) SELECT * FROM part_document WHERE f_leak(dtitle); - QUERY PLAN --------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------- Append InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on part_document_fiction - Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) + Filter: ((dlevel <= $0) AND f_leak(dtitle)) (6 rows) -- pp1 ERROR @@ -1136,14 +1136,14 @@ NOTICE: f_leak => awesome science fiction (4 rows) EXPLAIN (COSTS OFF) SELECT * FROM part_document WHERE f_leak(dtitle); - QUERY PLAN --------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------- Append InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on part_document_fiction - Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) + Filter: ((dlevel <= $0) AND f_leak(dtitle)) (6 rows) -- viewpoint from regress_rls_carol diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out index cd1f7f3..3913847 100644 --- a/src/test/regress/expected/sysviews.out +++ b/src/test/regress/expected/sysviews.out @@ -72,6 +72,7 @@ select count(*) >= 0 as ok from pg_prepared_xacts; select name, setting from pg_settings where name like 'enable%'; name | setting ----------------------------+--------- + enable_agg_pushdown | off enable_bitmapscan | on enable_gathermerge | on enable_hashagg | on @@ -85,7 +86,7 @@ select name, setting from pg_settings where name like 'enable%'; enable_seqscan | on enable_sort | on enable_tidscan | on -(13 rows) +(14 rows) -- Test that the pg_timezone_names and pg_timezone_abbrevs views are -- more-or-less working. We can't test their contents in any great detail