From ea071133208844d5843a4803075c153ede07e409 Mon Sep 17 00:00:00 2001 From: Nikita Glukhov Date: Mon, 15 Aug 2022 15:04:17 +0300 Subject: [PATCH v7 3/4] Add EEOP_SUBTRANS executor step --- src/backend/executor/execExprInterp.c | 102 +++++++++++- src/backend/jit/llvm/llvmjit_expr.c | 218 ++++++++++++++++++++++++-- src/backend/jit/llvm/llvmjit_types.c | 2 + src/include/executor/execExpr.h | 15 ++ src/include/nodes/execnodes.h | 5 + 5 files changed, 330 insertions(+), 12 deletions(-) diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c index bf141a66066..66898198f33 100644 --- a/src/backend/executor/execExprInterp.c +++ b/src/backend/executor/execExprInterp.c @@ -158,6 +158,8 @@ static TupleDesc get_cached_rowtype(Oid type_id, int32 typmod, bool *changed); static void ExecEvalRowNullInt(ExprState *state, ExprEvalStep *op, ExprContext *econtext, bool checkisnull); +static void ExecEvalSubtransInterp(ExprState *state, ExprEvalStep *op, + ExprContext *econtext); /* fast-path evaluation functions */ static Datum ExecJustInnerVar(ExprState *state, ExprContext *econtext, bool *isnull); @@ -398,7 +400,8 @@ ExecReadyInterpretedExpr(ExprState *state) * (Only applies when EEO_USE_COMPUTED_GOTO is defined.) */ static Datum -ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull) +ExecInterpExprStep(ExprState *state, int stepno, + ExprContext *econtext, bool *isnull) { ExprEvalStep *op; TupleTableSlot *resultslot; @@ -488,6 +491,7 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull) &&CASE_EEOP_GROUPING_FUNC, &&CASE_EEOP_WINDOW_FUNC, &&CASE_EEOP_SUBPLAN, + &&CASE_EEOP_SUBTRANS, &&CASE_EEOP_JSON_CONSTRUCTOR, &&CASE_EEOP_IS_JSON, &&CASE_EEOP_JSONEXPR, @@ -519,7 +523,7 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull) #endif /* EEO_USE_COMPUTED_GOTO */ /* setup state */ - op = state->steps; + op = &state->steps[stepno]; resultslot = state->resultslot; innerslot = econtext->ecxt_innertuple; outerslot = econtext->ecxt_outertuple; @@ -1827,6 +1831,16 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull) EEO_NEXT(); } + EEO_CASE(EEOP_SUBTRANS) + { + /* too complex for an inline implementation */ + if (ExecEvalSubtrans(state, op, econtext, + ExecEvalSubtransInterp)) + EEO_JUMP(op->d.subtrans.jump_done); + else + EEO_JUMP(op->d.subtrans.jump_error); + } + EEO_CASE(EEOP_JSON_CONSTRUCTOR) { /* too complex for an inline implementation */ @@ -1861,6 +1875,12 @@ out: return state->resvalue; } +static Datum +ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull) +{ + return ExecInterpExprStep(state, 0, econtext, isnull); +} + /* * Expression evaluation callback that performs extra checks before executing * the expression. Declared extern so other methods of execution can use it @@ -4620,6 +4640,84 @@ ExecAggPlainTransByRef(AggState *aggstate, AggStatePerTrans pertrans, MemoryContextSwitchTo(oldContext); } +/* + * Execute subexpression steps using recursive interpreter call. + * "op" points EEOP_SUBTRANS step. + */ +static void +ExecEvalSubtransInterp(ExprState *state, ExprEvalStep *op, + ExprContext *econtext) +{ + /* Recursively call interpreter for next step ignoring result */ + bool isnull; + + (void) ExecInterpExprStep(state, op - state->steps + 1, econtext, + &isnull); +} + +/* + * Execute EEOP_SUBTRANS, calling "eval" function inside a + * subtransaction. This function executes sequence of subexpression + * steps that begins right after current step and ends with EEOP_DONE. + */ +bool +ExecEvalSubtrans(ExprState *state, ExprEvalStep *op, + ExprContext *econtext, ExecEvalSubroutine eval) +{ + MemoryContext oldcontext = CurrentMemoryContext; + ResourceOwner oldowner = CurrentResourceOwner; + bool error; + + /* + * We should catch exceptions of category ERRCODE_DATA_EXCEPTION and + * execute the corresponding ON ERROR behavior then. + */ + BeginInternalSubTransaction(NULL); + /* Want to execute expressions inside function's memory context */ + MemoryContextSwitchTo(oldcontext); + + PG_TRY(); + { + eval(state, op, econtext); + + /* Commit the inner transaction, return to outer xact context */ + ReleaseCurrentSubTransaction(); + MemoryContextSwitchTo(oldcontext); + CurrentResourceOwner = oldowner; + + error = false; + } + PG_CATCH(); + { + ErrorData *edata; + int ecategory; + + /* Save error info in oldcontext */ + MemoryContextSwitchTo(oldcontext); + edata = CopyErrorData(); + FlushErrorState(); + + /* Abort the inner transaction */ + RollbackAndReleaseCurrentSubTransaction(); + MemoryContextSwitchTo(oldcontext); + CurrentResourceOwner = oldowner; + + ecategory = ERRCODE_TO_CATEGORY(edata->sqlerrcode); + + if (ecategory != ERRCODE_DATA_EXCEPTION && /* jsonpath and other data errors */ + ecategory != ERRCODE_INTEGRITY_CONSTRAINT_VIOLATION) /* domain errors */ + ReThrowError(edata); + + *op->resvalue = (Datum) 0; + *op->resnull = true; + + error = true; + } + PG_END_TRY(); + + return !error; +} + /* * Evaluate a JSON constructor expression. */ diff --git a/src/backend/jit/llvm/llvmjit_expr.c b/src/backend/jit/llvm/llvmjit_expr.c index fd72630f5e6..6568228b996 100644 --- a/src/backend/jit/llvm/llvmjit_expr.c +++ b/src/backend/jit/llvm/llvmjit_expr.c @@ -45,14 +45,20 @@ #include "utils/typcache.h" #include "utils/xml.h" +#define LLVMJIT_SUBTRANS_RECURSIVE + typedef struct CompiledExprState { LLVMJitContext *context; const char *funcname; } CompiledExprState; - static Datum ExecRunCompiledExpr(ExprState *state, ExprContext *econtext, bool *isNull); +static void ExecEvalSubtransCompiled(ExprState *state, ExprEvalStep *op, + ExprContext *econtext); +#ifdef LLVMJIT_SUBTRANS_RECURSIVE +static Datum ExecRunCompileExprRecursive(ExprState *state, ExprContext *econtext, bool *isNull); +#endif static LLVMValueRef BuildV1Call(LLVMJitContext *context, LLVMBuilderRef b, LLVMModuleRef mod, FunctionCallInfo fcinfo, @@ -70,15 +76,15 @@ static LLVMValueRef create_LifetimeEnd(LLVMModuleRef mod); lengthof(((LLVMValueRef[]){__VA_ARGS__})), \ ((LLVMValueRef[]){__VA_ARGS__})) - /* - * JIT compile expression. + * JIT compile expression staring from the specified step. */ -bool -llvm_compile_expr(ExprState *state) +static bool +llvm_compile_expr_step(ExprState *state, int start_opno) { PlanState *parent = state->parent; char *funcname; + const char *functypename = "TypeExprStateEvalFunc"; LLVMJitContext *context = NULL; @@ -123,6 +129,12 @@ llvm_compile_expr(ExprState *state) instr_time starttime; instr_time endtime; +#ifdef LLVMJIT_SUBTRANS_RECURSIVE + LLVMValueRef v_opno; + int num_recursive_opnos = 1; + bool recursive = false; +#endif + llvm_enter_fatal_on_oom(); /* @@ -148,9 +160,17 @@ llvm_compile_expr(ExprState *state) funcname = llvm_expand_funcname(context, "evalexpr"); +#ifdef LLVMJIT_SUBTRANS_RECURSIVE + if (state->flags & EEO_FLAG_HAVE_SUBTRANS) + { + recursive = true; + functypename = "TypeExprStateEvalStepFunc"; + } +#endif + /* create function */ eval_fn = LLVMAddFunction(mod, funcname, - llvm_pg_var_func_type("TypeExprStateEvalFunc")); + llvm_pg_var_func_type(functypename)); LLVMSetLinkage(eval_fn, LLVMExternalLinkage); LLVMSetVisibility(eval_fn, LLVMDefaultVisibility); llvm_copy_attributes(AttributeTemplate, eval_fn); @@ -161,6 +181,10 @@ llvm_compile_expr(ExprState *state) v_state = LLVMGetParam(eval_fn, 0); v_econtext = LLVMGetParam(eval_fn, 1); v_isnullp = LLVMGetParam(eval_fn, 2); +#ifdef LLVMJIT_SUBTRANS_RECURSIVE + if (recursive) + v_opno = LLVMGetParam(eval_fn, 3); +#endif LLVMPositionBuilderAtEnd(b, entry); @@ -224,13 +248,58 @@ llvm_compile_expr(ExprState *state) /* allocate blocks for each op upfront, so we can do jumps easily */ opblocks = palloc(sizeof(LLVMBasicBlockRef) * state->steps_len); - for (int opno = 0; opno < state->steps_len; opno++) + for (int opno = start_opno; opno < state->steps_len; opno++) + { opblocks[opno] = l_bb_append_v(eval_fn, "b.op.%d.start", opno); +#ifdef LLVMJIT_SUBTRANS_RECURSIVE + if (recursive) + { + /* count recursive opnos */ + ExprEvalStep *op = &state->steps[opno]; + + if (ExecEvalStepOp(state, op) == EEOP_SUBTRANS) + num_recursive_opnos++; + + } +#endif + } + +#ifdef LLVMJIT_SUBTRANS_RECURSIVE + if (recursive) + { + /* + * Emit switch with jump to each recursive opno. + * + * switch (opno) { + * case N: + * goto b.op.N.start; + * ... + * } + */ + LLVMValueRef v_switch = + LLVMBuildSwitch(b, v_opno, opblocks[start_opno], + num_recursive_opnos); + + LLVMAddCase(v_switch, l_int32_const(start_opno), + opblocks[start_opno]); + + for (int opno = start_opno; opno < state->steps_len; opno++) + { + ExprEvalStep *op = &state->steps[opno]; + + if (ExecEvalStepOp(state, op) == EEOP_SUBTRANS) + /* SUBTRANS expression starts in the next step */ + LLVMAddCase(v_switch, l_int32_const(opno + 1), + opblocks[opno + 1]); + } + } + else +#endif /* jump from entry to first block */ - LLVMBuildBr(b, opblocks[0]); + LLVMBuildBr(b, opblocks[start_opno]); - for (int opno = 0; opno < state->steps_len; opno++) + for (int opno = start_opno; opno < state->steps_len; opno++) { ExprEvalStep *op; ExprEvalOp opcode; @@ -2395,6 +2464,72 @@ llvm_compile_expr(ExprState *state) LLVMBuildBr(b, opblocks[opno + 1]); break; + case EEOP_SUBTRANS: + { + LLVMValueRef v_ret; + LLVMTypeRef v_functype; + +#ifndef LLVMJIT_SUBTRANS_RECURSIVE + ExprState *substate; + + /* Create subexpression ExprState */ + substate = makeNode(ExprState); + substate->parent = state->parent; + substate->steps = state->steps; + substate->steps_len = op->d.subtrans.jump_next; + + /* + * Compile it starting from the first subexpression + * step, which is the next step. + */ + llvm_compile_expr_step(substate, opno + 1); + + /* + * Save subexpression state in the private area of + * the step. + */ + op->d.subtrans.private_data = substate; +#endif + + Assert(state->flags & EEO_FLAG_HAVE_SUBTRANS); + + /* + * Call ExecEvalJsonExprSubtrans() passing + * ExecEvalSubtransCompiled(), which will execute + * subexpression getting ExprState from the private + * area. + */ + v_functype = llvm_pg_var_func_type("TypeExecEvalSubroutine"); + + v_ret = build_EvalXFunc(b, mod, "ExecEvalSubtrans", + v_state, op, v_econtext, + l_ptr_const(ExecEvalSubtransCompiled, + LLVMPointerType(v_functype, 0))); + v_ret = LLVMBuildZExt(b, v_ret, TypeStorageBool, ""); + + /* + * Jump to "error" or "done" steps depending of + * returned boolean value (TRUE means "done"). + */ + LLVMBuildCondBr(b, + LLVMBuildICmp(b, + LLVMIntEQ, + v_ret, + l_sbool_const(0), + ""), + opblocks[op->d.subtrans.jump_error], + opblocks[op->d.subtrans.jump_done]); + +#ifndef LLVMJIT_SUBTRANS_RECURSIVE + /* + * Skip subexpression steps, which are compiled + * separately. + */ + opno = op->d.subtrans.jump_next - 1; +#endif + break; + } + case EEOP_JSON_CONSTRUCTOR: build_EvalXFunc(b, mod, "ExecEvalJsonConstructor", v_state, op, v_econtext); @@ -2428,12 +2563,16 @@ llvm_compile_expr(ExprState *state) * remapping overhead. */ { - CompiledExprState *cstate = palloc0(sizeof(CompiledExprState)); cstate->context = context; cstate->funcname = funcname; +#ifdef LLVMJIT_SUBTRANS_RECURSIVE + if (recursive) + state->evalfunc = ExecRunCompileExprRecursive; + else +#endif state->evalfunc = ExecRunCompiledExpr; state->evalfunc_private = cstate; } @@ -2447,6 +2586,15 @@ llvm_compile_expr(ExprState *state) return true; } +/* + * JIT compile expression staring from the first step. + */ +bool +llvm_compile_expr(ExprState *state) +{ + return llvm_compile_expr_step(state, 0); +} + /* * Run compiled expression. * @@ -2475,6 +2623,56 @@ ExecRunCompiledExpr(ExprState *state, ExprContext *econtext, bool *isNull) return func(state, econtext, isNull); } +#ifdef LLVMJIT_SUBTRANS_RECURSIVE +static Datum +ExecRunCompiledExprRecursive(ExprState *state, ExprContext *econtext, bool *isNull) +{ + ExprStateEvalStepFunc func = state->evalfunc_private; + + return func(state, econtext, isNull, 0); +} + +static Datum +ExecRunCompileExprRecursive(ExprState *state, ExprContext *econtext, bool *isNull) +{ + CompiledExprState *cstate = state->evalfunc_private; + ExprStateEvalStepFunc func; + + CheckExprStillValid(state, econtext); + + llvm_enter_fatal_on_oom(); + func = (ExprStateEvalStepFunc) llvm_get_function(cstate->context, + cstate->funcname); + llvm_leave_fatal_on_oom(); + Assert(func); + + /* remove indirection via this function for future calls */ + state->evalfunc = ExecRunCompiledExprRecursive; + state->evalfunc_private = func; + + return func(state, econtext, isNull, 0); +} +#endif + +static void +ExecEvalSubtransCompiled(ExprState *state, ExprEvalStep *op, + ExprContext *econtext) +{ + bool isnull; +#ifdef LLVMJIT_SUBTRANS_RECURSIVE + /* Recursively execute next opno */ + ExprStateEvalStepFunc func = state->evalfunc_private; + int opno = op - state->steps + 1; + + (void) func(state, econtext, &isnull, opno); +#else + /* Execute subexpression steps using compiled ExprState */ + ExprState *substate = op->d.subtrans.private_data; + + (void) ExecEvalExpr(substate, econtext, &isnull); +#endif +} + static LLVMValueRef BuildV1Call(LLVMJitContext *context, LLVMBuilderRef b, LLVMModuleRef mod, FunctionCallInfo fcinfo, diff --git a/src/backend/jit/llvm/llvmjit_types.c b/src/backend/jit/llvm/llvmjit_types.c index 37fe64654b6..bb96d46048f 100644 --- a/src/backend/jit/llvm/llvmjit_types.c +++ b/src/backend/jit/llvm/llvmjit_types.c @@ -49,6 +49,7 @@ PGFunction TypePGFunction; size_t TypeSizeT; bool TypeStorageBool; ExprStateEvalFunc TypeExprStateEvalFunc; +ExprStateEvalStepFunc TypeExprStateEvalStepFunc; ExecEvalSubroutine TypeExecEvalSubroutine; ExecEvalBoolSubroutine TypeExecEvalBoolSubroutine; @@ -133,6 +134,7 @@ void *referenced_functions[] = ExecEvalSysVar, ExecEvalWholeRowVar, ExecEvalXmlExpr, + ExecEvalSubtrans, ExecEvalJsonConstructor, ExecEvalJsonIsPredicate, ExecEvalJsonExpr, diff --git a/src/include/executor/execExpr.h b/src/include/executor/execExpr.h index 51e8d93178e..a51232b84de 100644 --- a/src/include/executor/execExpr.h +++ b/src/include/executor/execExpr.h @@ -30,6 +30,8 @@ struct JsonConstructorExprState; #define EEO_FLAG_INTERPRETER_INITIALIZED (1 << 1) /* jump-threading is in use */ #define EEO_FLAG_DIRECT_THREADED (1 << 2) +/* expression contains EEOP_SUBTRANS steps */ +#define EEO_FLAG_HAVE_SUBTRANS (1 << 3) /* Typical API for out-of-line evaluation subroutines */ typedef void (*ExecEvalSubroutine) (ExprState *state, @@ -242,6 +244,7 @@ typedef enum ExprEvalOp EEOP_GROUPING_FUNC, EEOP_WINDOW_FUNC, EEOP_SUBPLAN, + EEOP_SUBTRANS, EEOP_JSON_CONSTRUCTOR, EEOP_IS_JSON, EEOP_JSONEXPR, @@ -680,6 +683,15 @@ typedef struct ExprEvalStep int setoff; } agg_trans; + /* for EEOP_SUBTRANS */ + struct + { + void *private_data; + int jump_done; + int jump_error; + int jump_next; + } subtrans; + /* for EEOP_JSON_CONSTRUCTOR */ struct { @@ -878,6 +890,9 @@ extern void ExecEvalWholeRowVar(ExprState *state, ExprEvalStep *op, ExprContext *econtext); extern void ExecEvalSysVar(ExprState *state, ExprEvalStep *op, ExprContext *econtext, TupleTableSlot *slot); +extern bool ExecEvalSubtrans(ExprState *state, ExprEvalStep *op, + ExprContext *econtext, + ExecEvalSubroutine func); extern void ExecEvalJsonConstructor(ExprState *state, ExprEvalStep *op, ExprContext *econtext); extern void ExecEvalJsonExpr(ExprState *state, ExprEvalStep *op, diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 01b1727fc09..d6976b92828 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -70,6 +70,11 @@ typedef Datum (*ExprStateEvalFunc) (struct ExprState *expression, struct ExprContext *econtext, bool *isNull); +typedef Datum (*ExprStateEvalStepFunc) (struct ExprState *expression, + struct ExprContext *econtext, + bool *isNull, + int stepno); + /* Bits in ExprState->flags (see also execExpr.h for private flag bits): */ /* expression is for use with ExecQual() */ #define EEO_FLAG_IS_QUAL (1 << 0) -- 2.25.1