From 146a3dc852f04216b8aa805e4e7739cbcf58b117 Mon Sep 17 00:00:00 2001 From: Nazir Bilal Yavuz Date: Fri, 7 Mar 2025 12:10:58 +0300 Subject: [PATCH v7 4/6] meson: Add architecture for LLVM bitcode emission This commit adds suport for bitcode emission for both normal and generated source files (processed by bison, flex, etc). These bitcode files are installed into $pkglibdir/bitcode/ directory if the LLVM is found. New variable `bitcode_modules` is introduced to generate bitcode files. All required information is gathered in this variable. Then, this variable is processed by the main meson LLVM bitcode emission scripts: src/backend/jit/llvm/bitcode/meson.build -> src/tools/irlink. An example of a possible structure of bitcode_modules is: ``` bitcode_modules = [ { 'name': '...', 'target': ..., 'srcfiles': [ '...', '...', ], 'additional_flags': [ '-I...', '-I...', ], 'gen_srcfiles': [ { 'srcfiles': [ , , ], 'additional_flags': [ '-I...', '-I...', ] } ] } ] ``` Author: Andres Freund Author: Nazir Bilal Yavuz Author: Diego Fronza Reviewed-by: Diego Fronza Discussion: https://postgr.es/m/206b001d-1884-4081-bd02-bed5c92f02ba%40eisentraut.org --- src/backend/jit/llvm/bitcode/meson.build | 69 ++++++++++++++++++++++++ src/backend/jit/llvm/meson.build | 31 +++++++---- src/backend/meson.build | 6 +++ meson.build | 21 ++++++++ src/tools/irlink | 25 +++++++++ 5 files changed, 141 insertions(+), 11 deletions(-) create mode 100644 src/backend/jit/llvm/bitcode/meson.build create mode 100644 src/tools/irlink diff --git a/src/backend/jit/llvm/bitcode/meson.build b/src/backend/jit/llvm/bitcode/meson.build new file mode 100644 index 00000000000..546e4d8b898 --- /dev/null +++ b/src/backend/jit/llvm/bitcode/meson.build @@ -0,0 +1,69 @@ +# Copyright (c) 2022-2024, PostgreSQL Global Development Group +# +# emit LLVM bitcode for JIT inlining + +assert(llvm.found()) + +foreach bitcode_module : bitcode_modules + bitcode_targets = [] + bitcode_obj = bitcode_module['target'] + bitcode_cflags_local = bitcode_cflags + bitcode_module.get('additional_flags', []) + bitcode_name = bitcode_module.get('name', bitcode_obj.name()) + + foreach srcfile : bitcode_module['srcfiles'] + if meson.version().version_compare('>=0.59') + srcfilename = fs.parent(srcfile) / fs.name(srcfile) + else + srcfilename = '@0@'.format(srcfile) + endif + + targetname = '@0@_@1@.bc'.format( + bitcode_name, + srcfilename.underscorify(), + ) + bitcode_targets += custom_target( + targetname, + depends: [bitcode_obj], + input: [srcfile], + output: targetname, + command: [llvm_irgen_command, bitcode_cflags_local], + install: true, + install_dir: dir_bitcode, + ) + endforeach + + # Process generated sources, which may include custom compilation flags. + foreach gen_sources: bitcode_module.get('gen_sources', []) + bitcode_cflags_gen_local = bitcode_cflags_local + gen_sources.get('additional_flags', []) + + foreach srcfile: gen_sources['srcfiles'] + # Generated sources are stored in some folder under meson.build_root()/**, + # remove the build prefix from the string. + srcfilename = srcfile.full_path().split(meson.build_root() + '/')[1] + + targetname = '@0@_@1@.bc'.format( + bitcode_name, + srcfilename.underscorify(), + ) + bitcode_targets += custom_target( + targetname, + depends: [bitcode_obj], + input: [srcfile], + output: targetname, + command: [llvm_irgen_command, bitcode_cflags_gen_local], + install: true, + install_dir: dir_bitcode, + ) + endforeach + endforeach + + index_name = '@0@.index.bc'.format(bitcode_name) + bitcode_index = custom_target('@0@'.format(bitcode_name), + output: index_name, + input: bitcode_targets, + command: [irlink, '--lto', llvm_lto, '--outdir', '@OUTDIR@', '--index', index_name, '@INPUT@'], + install: true, + install_dir: dir_bitcode, + ) + backend_targets += bitcode_index +endforeach diff --git a/src/backend/jit/llvm/meson.build b/src/backend/jit/llvm/meson.build index 805fbd69006..2dd922e573b 100644 --- a/src/backend/jit/llvm/meson.build +++ b/src/backend/jit/llvm/meson.build @@ -42,21 +42,22 @@ backend_targets += llvmjit # Define a few bits and pieces used here and elsewhere to generate bitcode -llvm_irgen_args = [ - '-c', '-o', '@OUTPUT@', '@INPUT@', +llvm_irgen_command = [] +if ccache.found() + llvm_irgen_command += ccache +endif + +llvm_irgen_command += [ + clang, + '-c', '-o', '@OUTPUT0@', '@INPUT0@', '-flto=thin', '-emit-llvm', - '-MD', '-MQ', '@OUTPUT@', '-MF', '@DEPFILE@', '-O2', '-Wno-ignored-attributes', '-Wno-empty-body', + '-Wno-unknown-warning-option', + '-Wno-compound-token-split-by-macro', ] - -if ccache.found() - llvm_irgen_command = ccache - llvm_irgen_args = [clang.full_path()] + llvm_irgen_args -else - llvm_irgen_command = clang -endif +llvm_irgen_dep_args = ['-MD', '-MQ', '@OUTPUT0@', '-MF', '@DEPFILE@'] # XXX: Need to determine proper version of the function cflags for clang @@ -73,7 +74,7 @@ bitcode_cflags += '-I@SOURCE_ROOT@/src/include' # Note this is intentionally not installed to bitcodedir, as it's not for # inlining llvmjit_types = custom_target('llvmjit_types.bc', - command: [llvm_irgen_command] + llvm_irgen_args + bitcode_cflags, + command: llvm_irgen_command + llvm_irgen_dep_args + bitcode_cflags, input: 'llvmjit_types.c', output: 'llvmjit_types.bc', depends: [postgres], @@ -82,3 +83,11 @@ llvmjit_types = custom_target('llvmjit_types.bc', depfile: '@BASENAME@.c.bc.d', ) backend_targets += llvmjit_types + +# Figure out -I's needed to build all postgres code, including all its +# dependencies +pkg_config = find_program(['pkg-config', 'pkgconf'], required: true) +r = run_command(pkg_config, + ['--cflags-only-I', meson.build_root() / 'meson-uninstalled/postgresql-extension-uninstalled.pc'], + check: true) +bitcode_cflags += r.stdout().split() diff --git a/src/backend/meson.build b/src/backend/meson.build index dd903591e34..276ecbd6d74 100644 --- a/src/backend/meson.build +++ b/src/backend/meson.build @@ -140,6 +140,12 @@ postgres = executable('postgres', backend_targets += postgres +bitcode_modules += { + 'name': 'postgres', + 'target': postgres_lib, + 'srcfiles': backend_sources, +} + pg_mod_c_args = cflags_mod pg_mod_cpp_args = cxxflags_mod pg_mod_link_args = ldflags_sl + ldflags_mod diff --git a/meson.build b/meson.build index a1142db800c..646148e8373 100644 --- a/meson.build +++ b/meson.build @@ -820,6 +820,8 @@ if add_languages('cpp', required: llvmopt, native: false) # Some distros put LLVM and clang in different paths, so fallback to # find via PATH, too. clang = find_program(llvm_binpath / 'clang', 'clang', required: true) + llvm_lto = find_program(llvm_binpath / 'llvm-lto', required: true) + irlink = find_program('src/tools/irlink', native: true) endif elif llvmopt.auto() message('llvm requires a C++ compiler') @@ -3053,6 +3055,11 @@ test_deps = [] tests = [] meson_extension_tests = [] +# List of object files + source files to generated LLVM IR for inlining. +# Each element is a hash of: +# {'target': target, 'srcfiles': ..., 'additional_flags': ...}. +bitcode_modules = [] + # Default options for targets @@ -3375,6 +3382,11 @@ subdir('src/interfaces/ecpg/test') subdir('doc/src/sgml') +# generate bitcode for JIT inlining after giving contrib modules etc a chance +# to add themselves to bitcode_modules[] +subdir('src/backend/jit/llvm/bitcode', if_found: llvm) + + generated_sources_ac += {'': ['GNUmakefile']} # After processing src/test, add test_install_libs to the testprep_targets @@ -4004,6 +4016,15 @@ summary( section: 'Programs', ) +if llvm.found() + summary( + { + 'clang': clang, + }, + section: 'Programs', + ) +endif + summary( { 'bonjour': bonjour, diff --git a/src/tools/irlink b/src/tools/irlink new file mode 100644 index 00000000000..793c0abf91a --- /dev/null +++ b/src/tools/irlink @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +import argparse +import os +import shutil +import subprocess +import sys + +parser = argparse.ArgumentParser( + description='generate PostgreSQL JIT IR module') + +parser.add_argument('--index', type=str, required=True) +parser.add_argument('--lto', type=str, required=True) +parser.add_argument('--outdir', type=str, required=True) +parser.add_argument('INPUT', type=str, nargs='+') + +args = parser.parse_args() + +file_names = [os.path.basename(f) for f in args.INPUT] +command = [args.lto, + '-thinlto', '-thinlto-action=thinlink', + '-o', args.index] + file_names +res = subprocess.run(command, cwd=args.outdir) + +exit(res.returncode) -- 2.50.1