From 0ecc76e22d9e4d14d317fef7238fed8882b7df2a Mon Sep 17 00:00:00 2001 From: Nazir Bilal Yavuz Date: Fri, 7 Mar 2025 12:10:58 +0300 Subject: [PATCH v12 4/7] meson: Add architecture for LLVM bitcode emission This commit adds suport for bitcode emission for both normal and generated source files (processed by bison, flex, etc). These bitcode files are installed into $pkglibdir/bitcode/ directory if the LLVM is found. New variable `bitcode_modules` is introduced to generate bitcode files. All required information is gathered in this variable. Then, this variable is processed by the main meson LLVM bitcode emission scripts: src/backend/jit/llvm/bitcode/meson.build -> src/tools/irlink. An example of a possible structure of bitcode_modules is: ``` bitcode_modules = [ { 'name': '...', 'target': ..., 'srcfiles': [ '...', '...', ], 'additional_flags': [ '-I...', '-I...', ], 'gen_srcfiles': [ { 'srcfiles': [ , , ], 'extra_depends' : [ ..., ..., ], 'additional_flags': [ '-I...', '-I...', ] } ] } ] ``` Author: Andres Freund Author: Nazir Bilal Yavuz Author: Diego Fronza Reviewed-by: Peter Eisentraut Reviewed-by: Diego Fronza Reviewed-by: Zsolt Parragi Discussion: https://postgr.es/m/206b001d-1884-4081-bd02-bed5c92f02ba%40eisentraut.org --- meson.build | 21 +++++++ src/backend/jit/llvm/bitcode/meson.build | 72 ++++++++++++++++++++++++ src/backend/jit/llvm/meson.build | 40 ++++++++----- src/backend/meson.build | 9 +++ src/makefiles/meson.build | 5 +- src/tools/irlink | 25 ++++++++ 6 files changed, 155 insertions(+), 17 deletions(-) create mode 100644 src/backend/jit/llvm/bitcode/meson.build create mode 100644 src/tools/irlink diff --git a/meson.build b/meson.build index f7b59eda65c..d25fbdec509 100644 --- a/meson.build +++ b/meson.build @@ -942,6 +942,8 @@ if have_cxx # Some distros put LLVM and clang in different paths, so fallback to # find via PATH, too. clang = find_program(llvm_binpath / 'clang', 'clang', required: true) + llvm_lto = find_program(llvm_binpath / 'llvm-lto', required: true) + irlink = find_program('src/tools/irlink', native: true) endif else msg = 'llvm requires a C++ compiler' @@ -3275,6 +3277,11 @@ test_deps = [] tests = [] meson_extension_tests = [] +# List of object files + source files to generated LLVM IR for inlining. +# Each element is a hash of: +# {'target': target, 'srcfiles': ..., 'additional_flags': ...}. +bitcode_modules = [] + # Default options for targets @@ -3598,6 +3605,11 @@ subdir('src/interfaces/ecpg/test') subdir('doc/src/sgml') +# generate bitcode for JIT inlining after giving contrib modules etc a chance +# to add themselves to bitcode_modules[] +subdir('src/backend/jit/llvm/bitcode', if_found: llvm) + + generated_sources_ac += {'': ['GNUmakefile']} # After processing src/test, add test_install_libs to the testprep_targets @@ -4243,6 +4255,15 @@ summary( section: 'Programs', ) +if llvm.found() + summary( + { + 'clang': clang, + }, + section: 'Programs', + ) +endif + summary( { 'bonjour': bonjour, diff --git a/src/backend/jit/llvm/bitcode/meson.build b/src/backend/jit/llvm/bitcode/meson.build new file mode 100644 index 00000000000..daf6a55e31d --- /dev/null +++ b/src/backend/jit/llvm/bitcode/meson.build @@ -0,0 +1,72 @@ +# Copyright (c) 2022-2024, PostgreSQL Global Development Group +# +# emit LLVM bitcode for JIT inlining + +assert(llvm.found()) + +foreach bitcode_module : bitcode_modules + bitcode_targets = [] + bitcode_obj = bitcode_module['target'] + bitcode_cflags_local = bitcode_cflags + bitcode_module.get('additional_flags', []) + bitcode_name = bitcode_module.get('name', bitcode_obj.name()) + + foreach srcfile : bitcode_module['srcfiles'] + if meson.version().version_compare('>=0.59') + srcfilename = fs.parent(srcfile) / fs.name(srcfile) + else + srcfilename = '@0@'.format(srcfile) + endif + + targetname = '@0@_@1@.bc'.format( + bitcode_name, + srcfilename.underscorify(), + ) + bitcode_targets += custom_target( + targetname, + depends: [generated_backend_headers_stamp], + input: [srcfile], + output: targetname, + command: [llvm_irgen_command, llvm_irgen_dep_args, bitcode_cflags_local], + depfile: targetname + '.d', + install: true, + install_dir: dir_bitcode, + ) + endforeach + + # Process generated sources, which may include custom compilation flags. + foreach gen_sources: bitcode_module.get('gen_sources', []) + bitcode_cflags_gen_local = bitcode_cflags_local + gen_sources.get('additional_flags', []) + + foreach srcfile: gen_sources['srcfiles'] + # Generated sources are stored in some folder under meson.build_root()/**, + # remove the build prefix from the string. + srcfilename = srcfile.full_path().split(meson.project_build_root() + '/')[1] + + targetname = '@0@_@1@.bc'.format( + bitcode_name, + srcfilename.underscorify(), + ) + bitcode_targets += custom_target( + targetname, + depends: [generated_backend_headers_stamp] + gen_sources.get('extra_depends', []), + input: [srcfile], + output: targetname, + command: [llvm_irgen_command, llvm_irgen_dep_args, + bitcode_cflags_gen_local], + depfile: targetname + '.d', + install: true, + install_dir: dir_bitcode, + ) + endforeach + endforeach + + index_name = '@0@.index.bc'.format(bitcode_name) + bitcode_index = custom_target('@0@'.format(bitcode_name), + output: index_name, + input: bitcode_targets, + command: [irlink, '--lto', llvm_lto, '--outdir', '@OUTDIR@', '--index', index_name, '@INPUT@'], + install: true, + install_dir: dir_bitcode, + ) + backend_targets += bitcode_index +endforeach diff --git a/src/backend/jit/llvm/meson.build b/src/backend/jit/llvm/meson.build index 7df8453ad6f..ceb5bc7bd88 100644 --- a/src/backend/jit/llvm/meson.build +++ b/src/backend/jit/llvm/meson.build @@ -42,28 +42,32 @@ backend_targets += llvmjit # Define a few bits and pieces used here and elsewhere to generate bitcode -llvm_irgen_args = [ - '-c', '-o', '@OUTPUT@', '@INPUT@', +llvm_irgen_command = [] +if ccache.found() + llvm_irgen_command += ccache +endif + +llvm_irgen_command += [ + clang, + '-c', '-o', '@OUTPUT0@', '@INPUT0@', '-flto=thin', '-emit-llvm', - '-MD', '-MQ', '@OUTPUT@', '-MF', '@DEPFILE@', - '-O2', '-Wno-ignored-attributes', '-Wno-empty-body', + '-Wno-unknown-warning-option', + '-Wno-compound-token-split-by-macro', ] - -if ccache.found() - llvm_irgen_command = ccache - llvm_irgen_args = [clang.full_path()] + llvm_irgen_args -else - llvm_irgen_command = clang -endif +llvm_irgen_dep_args = ['-MD', '-MQ', '@OUTPUT0@', '-MF', '@DEPFILE@'] # XXX: Need to determine proper version of the function cflags for clang -bitcode_cflags = ['-fno-strict-aliasing', '-fwrapv'] -bitcode_cflags += get_option('c_args') +bitcode_cflags = ['-fno-strict-aliasing', '-fwrapv', '-O2'] bitcode_cflags += cppflags +var_bitcode_cxxflags = bitcode_cflags +var_bitcode_cxxflags += get_option('cpp_args') +bitcode_cflags += get_option('c_args') +var_bitcode_cflags = bitcode_cflags + # XXX: Worth improving on the logic to find directories here bitcode_cflags += '-I@BUILD_ROOT@/src/include' bitcode_cflags += '-I@BUILD_ROOT@/src/backend/utils/misc' @@ -73,7 +77,7 @@ bitcode_cflags += '-I@SOURCE_ROOT@/src/include' # Note this is intentionally not installed to bitcodedir, as it's not for # inlining llvmjit_types = custom_target('llvmjit_types.bc', - command: [llvm_irgen_command] + llvm_irgen_args + bitcode_cflags, + command: llvm_irgen_command + llvm_irgen_dep_args + bitcode_cflags, input: 'llvmjit_types.c', output: 'llvmjit_types.bc', depends: [postgres], @@ -82,3 +86,11 @@ llvmjit_types = custom_target('llvmjit_types.bc', depfile: '@BASENAME@.c.bc.d', ) backend_targets += llvmjit_types + +# Figure out -I's needed to build all postgres code, including all its +# dependencies +pkg_config = find_program(['pkg-config', 'pkgconf'], required: true) +r = run_command(pkg_config, + ['--cflags-only-I', meson.project_build_root() / 'meson-uninstalled/postgresql-extension-uninstalled.pc'], + check: true) +bitcode_cflags += r.stdout().split() diff --git a/src/backend/meson.build b/src/backend/meson.build index 30a77f45fe9..2877127954b 100644 --- a/src/backend/meson.build +++ b/src/backend/meson.build @@ -7,6 +7,9 @@ backend_link_with = [pgport_srv, common_srv] generated_backend_sources = [] post_export_backend_sources = [] +var_bitcode_cflags = [] +var_bitcode_cxxflags = [] + subdir('access') subdir('archive') subdir('backup') @@ -165,6 +168,12 @@ postgres = executable('postgres', backend_targets += postgres +bitcode_modules += { + 'name': 'postgres', + 'target': postgres_lib, + 'srcfiles': backend_sources, +} + pg_mod_c_args = cflags_mod pg_mod_cxx_args = cxxflags_mod pg_mod_link_args = ldflags_sl + ldflags_mod diff --git a/src/makefiles/meson.build b/src/makefiles/meson.build index 2401025d1cd..1d09aed1d86 100644 --- a/src/makefiles/meson.build +++ b/src/makefiles/meson.build @@ -113,9 +113,8 @@ pgxs_kv = { ' '.join(cc.get_supported_link_arguments('-Wl,--export-dynamic')), 'LDFLAGS_SL': var_ldflags_sl, - # TODO: requires bitcode generation to be implemented for meson - 'BITCODE_CFLAGS': '', - 'BITCODE_CXXFLAGS': '', + 'BITCODE_CFLAGS': ' '.join(var_bitcode_cflags), + 'BITCODE_CXXFLAGS': ' '.join(var_bitcode_cxxflags), 'BISONFLAGS': ' '.join(bison_flags), 'FLEXFLAGS': ' '.join(flex_flags), diff --git a/src/tools/irlink b/src/tools/irlink new file mode 100644 index 00000000000..793c0abf91a --- /dev/null +++ b/src/tools/irlink @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +import argparse +import os +import shutil +import subprocess +import sys + +parser = argparse.ArgumentParser( + description='generate PostgreSQL JIT IR module') + +parser.add_argument('--index', type=str, required=True) +parser.add_argument('--lto', type=str, required=True) +parser.add_argument('--outdir', type=str, required=True) +parser.add_argument('INPUT', type=str, nargs='+') + +args = parser.parse_args() + +file_names = [os.path.basename(f) for f in args.INPUT] +command = [args.lto, + '-thinlto', '-thinlto-action=thinlink', + '-o', args.index] + file_names +res = subprocess.run(command, cwd=args.outdir) + +exit(res.returncode) -- 2.47.3