From 97b671aff4aafa834befaa3479af0cb2afade4d8 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Thu, 31 Aug 2017 23:06:04 -0700 Subject: [PATCH 16/16] Hacky/Preliminary inlining implementation. --- src/Makefile.global.in | 8 ++ src/backend/Makefile | 6 + src/backend/executor/execExprCompile.c | 111 +++++++++++++++ src/backend/lib/llvmjit.c | 239 ++++++++++++++++++++++++++++++++- src/backend/utils/misc/guc.c | 24 ++++ src/include/lib/llvmjit.h | 7 + 6 files changed, 391 insertions(+), 4 deletions(-) diff --git a/src/Makefile.global.in b/src/Makefile.global.in index ab5862b472..0d47734a6a 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -812,6 +812,10 @@ ifndef COMPILE.c COMPILE.c = $(CC) $(CFLAGS) $(CPPFLAGS) -c endif +ifndef COMPILE.bc +COMPILE.bc = $(CC) $(filter-out -g3 -g -ggdb3 -fno-strict-aliasing, $(CFLAGS)) -fstrict-aliasing $(CPPFLAGS) -emit-llvm -c +endif + DEPDIR = .deps ifeq ($(GCC), yes) @@ -821,6 +825,10 @@ ifeq ($(GCC), yes) @if test ! -d $(DEPDIR); then mkdir -p $(DEPDIR); fi $(COMPILE.c) -o $@ $< -MMD -MP -MF $(DEPDIR)/$(*F).Po +%.bc : %.c %.o + @if test ! -d $(DEPDIR); then mkdir -p $(DEPDIR); fi + $(COMPILE.bc) -o $@ $< + endif # GCC # Include all the dependency files generated for the current diff --git a/src/backend/Makefile b/src/backend/Makefile index c82ad75bda..1f1945b5f3 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -64,6 +64,12 @@ ifneq ($(PORTNAME), aix) postgres: $(OBJS) $(CC) $(CFLAGS) $(LDFLAGS) $(LDFLAGS_EX) $(export_dynamic) $(call expand_subsys,$^) $(LIBS) -o $@ + +ifeq ($(findstring clang,$(CC)), clang) +parsed: $(OBJS) + $(MAKE $(patsubst %.o,%.bc,$(call expand_subsys,$(OBJS)))) +endif + endif endif endif diff --git a/src/backend/executor/execExprCompile.c b/src/backend/executor/execExprCompile.c index d0b943530c..73412d0e1b 100644 --- a/src/backend/executor/execExprCompile.c +++ b/src/backend/executor/execExprCompile.c @@ -213,9 +213,21 @@ BuildFunctionCall(LLVMJitContext *context, LLVMBuilderRef builder, } else if (builtin) { + LLVMModuleRef inline_mod; + LLVMAddFunction(mod, builtin->funcName, TypePGFunction); v_fn_addr = LLVMGetNamedFunction(mod, builtin->funcName); Assert(v_fn_addr); + + inline_mod = llvm_module_for_function(builtin->funcName); + if (inline_mod) + { + context->inline_modules = + list_append_unique_ptr(context->inline_modules, + inline_mod); + } + + forceinline = true; } else { @@ -263,6 +275,20 @@ BuildFunctionCall(LLVMJitContext *context, LLVMBuilderRef builder, LLVMValueRef v_lifetime = get_LifetimeEnd(mod); LLVMValueRef params[2]; + params[0] = LLVMConstInt(LLVMInt64Type(), sizeof(fcinfo->arg), false); + params[1] = LLVMBuildBitCast( + builder, LLVMConstInt(TypeSizeT, (intptr_t) fcinfo->arg, false), + LLVMPointerType(LLVMInt8Type(), 0), + ""); + LLVMBuildCall(builder, v_lifetime, params, lengthof(params), ""); + + params[0] = LLVMConstInt(LLVMInt64Type(), sizeof(fcinfo->argnull), false); + params[1] = LLVMBuildBitCast( + builder, LLVMConstInt(TypeSizeT, (intptr_t) fcinfo->argnull, false), + LLVMPointerType(LLVMInt8Type(), 0), + ""); + LLVMBuildCall(builder, v_lifetime, params, lengthof(params), ""); + params[0] = LLVMConstInt(LLVMInt64Type(), sizeof(FunctionCallInfoData), false); params[1] = LLVMBuildBitCast( builder, v_fcinfo, @@ -314,6 +340,9 @@ ExecRunCompiledExpr(ExprState *state, ExprContext *econtext, bool *isNull) return func(state, econtext, isNull); } +static void +emit_lifetime_end(ExprState *state, LLVMModuleRef mod, LLVMBuilderRef builder); + bool ExecReadyCompiledExpr(ExprState *state, PlanState *parent) { @@ -519,6 +548,7 @@ ExecReadyCompiledExpr(ExprState *state, PlanState *parent) LLVMBuildCall(builder, v_lifetime, params, lengthof(params), ""); } + emit_lifetime_end(state, mod, builder); LLVMBuildRet(builder, v_tmpvalue); break; @@ -2793,4 +2823,85 @@ ExecReadyCompiledExpr(ExprState *state, PlanState *parent) return true; } + +static void +emit_lifetime_end(ExprState *state, LLVMModuleRef mod, LLVMBuilderRef builder) +{ + ExprEvalStep *op; + int i = 0; + int argno = 0; + LLVMValueRef v_lifetime = get_LifetimeEnd(mod); + + + /* + * Add lifetime-end annotation, signalling that writes to memory don't + * have to be retained (important for inlining potential). + */ + + for (i = 0; i < state->steps_len; i++) + { + FunctionCallInfo fcinfo = NULL; + LLVMValueRef v_ptr; + LLVMValueRef params[2]; + + op = &state->steps[i]; + + switch ((ExprEvalOp) op->opcode) + { + case EEOP_FUNCEXPR: + case EEOP_FUNCEXPR_STRICT: + case EEOP_NULLIF: + case EEOP_DISTINCT: + fcinfo = op->d.func.fcinfo_data; + + for (argno = 0; argno < op->d.func.nargs; argno++) + { + params[0] = LLVMConstInt(LLVMInt64Type(), sizeof(Datum), false); + params[1] = LLVMBuildIntToPtr( + builder, LLVMConstInt(TypeSizeT, (intptr_t) &fcinfo->arg[argno], false), + LLVMPointerType(LLVMInt8Type(), 0), + ""); + LLVMBuildCall(builder, v_lifetime, params, lengthof(params), ""); + + params[0] = LLVMConstInt(LLVMInt64Type(), sizeof(bool), false); + params[1] = LLVMBuildIntToPtr( + builder, LLVMConstInt(TypeSizeT, (intptr_t) &fcinfo->argnull[argno], false), + LLVMPointerType(LLVMInt8Type(), 0), + ""); + LLVMBuildCall(builder, v_lifetime, params, lengthof(params), ""); + } + params[0] = LLVMConstInt(LLVMInt64Type(), sizeof(FunctionCallInfoData), false); + params[1] = LLVMBuildIntToPtr( + builder, LLVMConstInt(TypeSizeT, (intptr_t) fcinfo, false), + LLVMPointerType(LLVMInt8Type(), 0), + ""); + LLVMBuildCall(builder, v_lifetime, params, lengthof(params), ""); + + break; + case EEOP_ROWCOMPARE_STEP: + fcinfo = op->d.rowcompare_step.fcinfo_data;; + break; + case EEOP_BOOLTEST_IS_TRUE: + case EEOP_BOOLTEST_IS_NOT_FALSE: + case EEOP_BOOLTEST_IS_FALSE: + case EEOP_BOOLTEST_IS_NOT_TRUE: + if (op->d.boolexpr.anynull) + { + v_ptr = LLVMBuildIntToPtr( + builder, + LLVMConstInt(LLVMInt64Type(), (intptr_t) op->d.boolexpr.anynull, false), + LLVMPointerType(LLVMInt8Type(), 0), + "anynull"); + + params[0] = LLVMConstInt(LLVMInt64Type(), sizeof(bool), false); + params[1] = v_ptr; + LLVMBuildCall(builder, v_lifetime, params, lengthof(params), ""); + } + break; + default: + break; + } + + } +} #endif diff --git a/src/backend/lib/llvmjit.c b/src/backend/lib/llvmjit.c index 57d0663410..86d43c3c07 100644 --- a/src/backend/lib/llvmjit.c +++ b/src/backend/lib/llvmjit.c @@ -9,6 +9,7 @@ #include "utils/memutils.h" #include "utils/resowner_private.h" +#include "utils/varlena.h" #ifdef USE_LLVM @@ -32,6 +33,8 @@ /* GUCs */ bool jit_log_ir = 0; bool jit_dump_bitcode = 0; +bool jit_perform_inlining = 0; +char *jit_inline_directories = NULL; static bool llvm_initialized = false; static LLVMPassManagerBuilderRef llvm_pmb; @@ -72,6 +75,8 @@ static LLVMOrcJITStackRef llvm_orc; static void llvm_shutdown(void); static void llvm_create_types(void); +static void llvm_search_inline_directories(void); + static void llvm_shutdown(void) @@ -103,6 +108,8 @@ llvm_initialize(void) LLVMLoadLibraryPermanently(""); llvm_triple = LLVMGetDefaultTargetTriple(); + /* FIXME: overwrite with clang compatible one? */ + llvm_triple = "x86_64-pc-linux-gnu"; if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0) { @@ -117,6 +124,7 @@ llvm_initialize(void) llvm_pmb = LLVMPassManagerBuilderCreate(); LLVMPassManagerBuilderSetOptLevel(llvm_pmb, 3); + LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 0); llvm_orc = LLVMOrcCreateInstance(llvm_targetmachine); @@ -128,9 +136,188 @@ llvm_initialize(void) llvm_create_types(); llvm_initialized = true; + + llvm_search_inline_directories(); + MemoryContextSwitchTo(oldcontext); } +#include "common/string.h" +#include "storage/fd.h" +#include "miscadmin.h" + +static HTAB *InlineModuleHash = NULL; + +typedef struct InlineableFunction +{ + NameData fname; + const char *path; + LLVMModuleRef mod; +} InlineableFunction; + +static void +llvm_preload_bitcode(const char *filename) +{ + LLVMMemoryBufferRef buf; + char *msg; + LLVMValueRef func; + LLVMModuleRef mod = NULL; + + mod = LLVMModuleCreateWithName("tmp"); + + if (LLVMCreateMemoryBufferWithContentsOfFile( + filename, &buf, &msg)) + { + elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s", + filename, msg); + } + +#if 1 + if (LLVMParseBitcode2(buf, &mod)) + { + elog(ERROR, "LLVMParseBitcode2 failed: %s", msg); + } +#else + if (LLVMGetBitcodeModule2(buf, &mod)) + { + elog(ERROR, "LLVMGetBitcodeModule2 failed: %s", msg); + } +#endif + + func = LLVMGetFirstFunction(mod); + while (func) + { + const char *funcname = LLVMGetValueName(func); + + if (!LLVMIsDeclaration(func)) + { + if (LLVMGetLinkage(func) == LLVMExternalLinkage) + { + InlineableFunction *fentry; + bool found; + + fentry = (InlineableFunction *) + hash_search(InlineModuleHash, + (void *) funcname, + HASH_ENTER, &found); + + if (found) + { + elog(LOG, "skiping loading func %s, already exists at %s, loading %s", + funcname, fentry->path, filename); + } + else + { + fentry->path = pstrdup(filename); + fentry->mod = mod; + } + + LLVMSetLinkage(func, LLVMAvailableExternallyLinkage); + } + } + + func = LLVMGetNextFunction(func); + } +} + +static void +llvm_search_inline_directory(const char *path) +{ + DIR *dir; + struct dirent *de; + + dir = AllocateDir(path); + if (dir == NULL) + { + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open directory \"%s\": %m", path))); + return; + } + + while ((de = ReadDir(dir, path)) != NULL) + { + char subpath[MAXPGPATH * 2]; + struct stat fst; + int sret; + + CHECK_FOR_INTERRUPTS(); + + if (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0) + continue; + + snprintf(subpath, sizeof(subpath), "%s/%s", path, de->d_name); + + sret = lstat(subpath, &fst); + + if (sret < 0) + { + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", subpath))); + continue; + } + + if (S_ISREG(fst.st_mode)) + { + if (pg_str_endswith(subpath, ".bc")) + { + llvm_preload_bitcode(subpath); + } + } + else if (S_ISDIR(fst.st_mode)) + { + llvm_search_inline_directory(subpath); + } + } + + FreeDir(dir); /* we ignore any error here */ +} + +static void +llvm_search_inline_directories(void) +{ + List *elemlist; + ListCell *lc; + HASHCTL ctl; + + Assert(InlineModuleHash == NULL); + /* First time through: initialize the hash table */ + + MemSet(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(NameData); + ctl.entrysize = sizeof(InlineableFunction); + InlineModuleHash = hash_create("inlineable function cache", 64, + &ctl, HASH_ELEM); + + SplitDirectoriesString(pstrdup(jit_inline_directories), ';', &elemlist); + + foreach(lc, elemlist) + { + char *curdir = (char *) lfirst(lc); + + llvm_search_inline_directory(curdir); + } +} + +LLVMModuleRef +llvm_module_for_function(const char *funcname) +{ + InlineableFunction *fentry; + bool found; + + fentry = (InlineableFunction *) + hash_search(InlineModuleHash, + (void *) funcname, + HASH_FIND, &found); + + if (fentry) + return fentry->mod; + return NULL; +} + + static void llvm_create_types(void) { @@ -399,7 +586,11 @@ llvm_create_types(void) static uint64_t llvm_resolve_symbol(const char *name, void *ctx) { - return (uint64_t) LLVMSearchForAddressOfSymbol(name); + uint64_t addr = (uint64_t) LLVMSearchForAddressOfSymbol(name); + + if (!addr) + elog(ERROR, "failed to resolve name %s", name); + return addr; } void * @@ -412,9 +603,22 @@ llvm_get_function(LLVMJitContext *context, const char *funcname) if (!context->compiled) { int handle; - LLVMSharedModuleRef smod = LLVMOrcMakeSharedModule(context->module); + LLVMSharedModuleRef smod; MemoryContext oldcontext; + if (jit_perform_inlining) + { + ListCell *lc; + + foreach(lc, context->inline_modules) + { + LLVMModuleRef inline_mod = lfirst(lc); + + inline_mod = LLVMCloneModule(inline_mod); + LLVMLinkModules2Needed(context->module, inline_mod); + } + } + if (jit_log_ir) { LLVMDumpModule(context->module); @@ -439,13 +643,26 @@ llvm_get_function(LLVMJitContext *context, const char *funcname) llvm_mpm = LLVMCreatePassManager(); LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm); - LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb, llvm_mpm); + //LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb, llvm_mpm); LLVMPassManagerBuilderPopulateLTOPassManager(llvm_pmb, llvm_mpm, true, true); + LLVMAddCFGSimplificationPass(llvm_fpm); + LLVMAddJumpThreadingPass(llvm_fpm); + LLVMAddTypeBasedAliasAnalysisPass(llvm_fpm); + LLVMAddDeadStoreEliminationPass(llvm_fpm); + LLVMAddConstantPropagationPass(llvm_fpm); + LLVMAddSCCPPass(llvm_fpm); + LLVMAddAnalysisPasses(llvm_targetmachine, llvm_mpm); LLVMAddAnalysisPasses(llvm_targetmachine, llvm_fpm); - LLVMAddDeadStoreEliminationPass(llvm_fpm); + /* do function level optimization */ + LLVMInitializeFunctionPassManager(llvm_fpm); + for (func = LLVMGetFirstFunction(context->module); + func != NULL; + func = LLVMGetNextFunction(func)) + LLVMRunFunctionPassManager(llvm_fpm, func); + LLVMFinalizeFunctionPassManager(llvm_fpm); /* do function level optimization */ LLVMInitializeFunctionPassManager(llvm_fpm); @@ -457,11 +674,24 @@ llvm_get_function(LLVMJitContext *context, const char *funcname) /* do module level optimization */ LLVMRunPassManager(llvm_mpm, context->module); + LLVMRunPassManager(llvm_mpm, context->module); + LLVMRunPassManager(llvm_mpm, context->module); + LLVMRunPassManager(llvm_mpm, context->module); LLVMDisposePassManager(llvm_fpm); LLVMDisposePassManager(llvm_mpm); } + if (jit_dump_bitcode) + { + /* FIXME: invent module rather than function specific name */ + char *filename = psprintf("%s.optimized.bc", funcname); + LLVMWriteBitcodeToFile(context->module, filename); + pfree(filename); + } + + smod = LLVMOrcMakeSharedModule(context->module); + /* and emit the code */ { handle = @@ -480,6 +710,7 @@ llvm_get_function(LLVMJitContext *context, const char *funcname) context->module = NULL; context->compiled = true; + context->inline_modules = NIL; } /* search all emitted modules for function we're asked for */ diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 4cc9f305a2..82359b1616 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -1043,6 +1043,17 @@ static struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, + { + {"jit_perform_inlining", PGC_USERSET, DEVELOPER_OPTIONS, + gettext_noop("inline functions for JIT"), + NULL, + GUC_NOT_IN_SAMPLE + }, + &jit_perform_inlining, + false, + NULL, NULL, NULL + }, + #endif { @@ -3700,6 +3711,19 @@ static struct config_string ConfigureNamesString[] = check_wal_consistency_checking, assign_wal_consistency_checking, NULL }, +#ifdef USE_LLVM + { + {"jit_inline_directories", PGC_BACKEND, DEVELOPER_OPTIONS, + gettext_noop("Sets the directories where inline contents for JIT are located."), + NULL, + GUC_SUPERUSER_ONLY + }, + &jit_inline_directories, + "", + NULL, NULL, NULL + }, +#endif + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, NULL, NULL, NULL, NULL diff --git a/src/include/lib/llvmjit.h b/src/include/lib/llvmjit.h index 47f9b6d64c..e2a8cccafc 100644 --- a/src/include/lib/llvmjit.h +++ b/src/include/lib/llvmjit.h @@ -27,12 +27,15 @@ typedef struct LLVMJitContext { int counter; LLVMModuleRef module; + List *inline_modules; bool compiled; List *handles; } LLVMJitContext; extern bool jit_log_ir; extern bool jit_dump_bitcode; +extern bool jit_perform_inlining; +extern char *jit_inline_directories; extern LLVMTargetMachineRef llvm_targetmachine; extern const char *llvm_triple; @@ -74,6 +77,10 @@ extern void llvm_shutdown_orc_perf_support(LLVMOrcJITStackRef llvm_orc); extern LLVMValueRef slot_compile_deform(struct LLVMJitContext *context, TupleDesc desc, int natts); + +extern LLVMModuleRef llvm_module_for_function(const char *funcname); + + #else struct LLVMJitContext; -- 2.14.1.2.g4274c698f4.dirty