From fbc86097e438e64f04a4e3eebc8ecfcc4b7736eb Mon Sep 17 00:00:00 2001 From: Nazir Bilal Yavuz Date: Fri, 7 Mar 2025 12:10:58 +0300 Subject: [PATCH v10 4/7] meson: Add architecture for LLVM bitcode emission This commit adds suport for bitcode emission for both normal and generated source files (processed by bison, flex, etc). These bitcode files are installed into $pkglibdir/bitcode/ directory if the LLVM is found. New variable `bitcode_modules` is introduced to generate bitcode files. All required information is gathered in this variable. Then, this variable is processed by the main meson LLVM bitcode emission scripts: src/backend/jit/llvm/bitcode/meson.build -> src/tools/irlink. An example of a possible structure of bitcode_modules is: ``` bitcode_modules = [ { 'name': '...', 'target': ..., 'srcfiles': [ '...', '...', ], 'additional_flags': [ '-I...', '-I...', ], 'gen_srcfiles': [ { 'srcfiles': [ , , ], 'additional_flags': [ '-I...', '-I...', ] } ] } ] ``` Author: Andres Freund Author: Nazir Bilal Yavuz Author: Diego Fronza Reviewed-by: Diego Fronza Discussion: https://postgr.es/m/206b001d-1884-4081-bd02-bed5c92f02ba%40eisentraut.org --- meson.build | 21 ++++++++ src/backend/jit/llvm/bitcode/meson.build | 69 ++++++++++++++++++++++++ src/backend/jit/llvm/meson.build | 31 +++++++---- src/backend/meson.build | 6 +++ src/tools/irlink | 25 +++++++++ 5 files changed, 141 insertions(+), 11 deletions(-) create mode 100644 src/backend/jit/llvm/bitcode/meson.build create mode 100644 src/tools/irlink diff --git a/meson.build b/meson.build index 902ec60a74e..3fb538a583d 100644 --- a/meson.build +++ b/meson.build @@ -942,6 +942,8 @@ if have_cxx # Some distros put LLVM and clang in different paths, so fallback to # find via PATH, too. clang = find_program(llvm_binpath / 'clang', 'clang', required: true) + llvm_lto = find_program(llvm_binpath / 'llvm-lto', required: true) + irlink = find_program('src/tools/irlink', native: true) endif else msg = 'llvm requires a C++ compiler' @@ -3270,6 +3272,11 @@ test_deps = [] tests = [] meson_extension_tests = [] +# List of object files + source files to generated LLVM IR for inlining. +# Each element is a hash of: +# {'target': target, 'srcfiles': ..., 'additional_flags': ...}. +bitcode_modules = [] + # Default options for targets @@ -3593,6 +3600,11 @@ subdir('src/interfaces/ecpg/test') subdir('doc/src/sgml') +# generate bitcode for JIT inlining after giving contrib modules etc a chance +# to add themselves to bitcode_modules[] +subdir('src/backend/jit/llvm/bitcode', if_found: llvm) + + generated_sources_ac += {'': ['GNUmakefile']} # After processing src/test, add test_install_libs to the testprep_targets @@ -4234,6 +4246,15 @@ summary( section: 'Programs', ) +if llvm.found() + summary( + { + 'clang': clang, + }, + section: 'Programs', + ) +endif + summary( { 'bonjour': bonjour, diff --git a/src/backend/jit/llvm/bitcode/meson.build b/src/backend/jit/llvm/bitcode/meson.build new file mode 100644 index 00000000000..570f8abefa4 --- /dev/null +++ b/src/backend/jit/llvm/bitcode/meson.build @@ -0,0 +1,69 @@ +# Copyright (c) 2022-2024, PostgreSQL Global Development Group +# +# emit LLVM bitcode for JIT inlining + +assert(llvm.found()) + +foreach bitcode_module : bitcode_modules + bitcode_targets = [] + bitcode_obj = bitcode_module['target'] + bitcode_cflags_local = bitcode_cflags + bitcode_module.get('additional_flags', []) + bitcode_name = bitcode_module.get('name', bitcode_obj.name()) + + foreach srcfile : bitcode_module['srcfiles'] + if meson.version().version_compare('>=0.59') + srcfilename = fs.parent(srcfile) / fs.name(srcfile) + else + srcfilename = '@0@'.format(srcfile) + endif + + targetname = '@0@_@1@.bc'.format( + bitcode_name, + srcfilename.underscorify(), + ) + bitcode_targets += custom_target( + targetname, + depends: [bitcode_obj], + input: [srcfile], + output: targetname, + command: [llvm_irgen_command, bitcode_cflags_local], + install: true, + install_dir: dir_bitcode, + ) + endforeach + + # Process generated sources, which may include custom compilation flags. + foreach gen_sources: bitcode_module.get('gen_sources', []) + bitcode_cflags_gen_local = bitcode_cflags_local + gen_sources.get('additional_flags', []) + + foreach srcfile: gen_sources['srcfiles'] + # Generated sources are stored in some folder under meson.build_root()/**, + # remove the build prefix from the string. + srcfilename = srcfile.full_path().split(meson.project_build_root() + '/')[1] + + targetname = '@0@_@1@.bc'.format( + bitcode_name, + srcfilename.underscorify(), + ) + bitcode_targets += custom_target( + targetname, + depends: [bitcode_obj], + input: [srcfile], + output: targetname, + command: [llvm_irgen_command, bitcode_cflags_gen_local], + install: true, + install_dir: dir_bitcode, + ) + endforeach + endforeach + + index_name = '@0@.index.bc'.format(bitcode_name) + bitcode_index = custom_target('@0@'.format(bitcode_name), + output: index_name, + input: bitcode_targets, + command: [irlink, '--lto', llvm_lto, '--outdir', '@OUTDIR@', '--index', index_name, '@INPUT@'], + install: true, + install_dir: dir_bitcode, + ) + backend_targets += bitcode_index +endforeach diff --git a/src/backend/jit/llvm/meson.build b/src/backend/jit/llvm/meson.build index 7df8453ad6f..fca61a82779 100644 --- a/src/backend/jit/llvm/meson.build +++ b/src/backend/jit/llvm/meson.build @@ -42,21 +42,22 @@ backend_targets += llvmjit # Define a few bits and pieces used here and elsewhere to generate bitcode -llvm_irgen_args = [ - '-c', '-o', '@OUTPUT@', '@INPUT@', +llvm_irgen_command = [] +if ccache.found() + llvm_irgen_command += ccache +endif + +llvm_irgen_command += [ + clang, + '-c', '-o', '@OUTPUT0@', '@INPUT0@', '-flto=thin', '-emit-llvm', - '-MD', '-MQ', '@OUTPUT@', '-MF', '@DEPFILE@', '-O2', '-Wno-ignored-attributes', '-Wno-empty-body', + '-Wno-unknown-warning-option', + '-Wno-compound-token-split-by-macro', ] - -if ccache.found() - llvm_irgen_command = ccache - llvm_irgen_args = [clang.full_path()] + llvm_irgen_args -else - llvm_irgen_command = clang -endif +llvm_irgen_dep_args = ['-MD', '-MQ', '@OUTPUT0@', '-MF', '@DEPFILE@'] # XXX: Need to determine proper version of the function cflags for clang @@ -73,7 +74,7 @@ bitcode_cflags += '-I@SOURCE_ROOT@/src/include' # Note this is intentionally not installed to bitcodedir, as it's not for # inlining llvmjit_types = custom_target('llvmjit_types.bc', - command: [llvm_irgen_command] + llvm_irgen_args + bitcode_cflags, + command: llvm_irgen_command + llvm_irgen_dep_args + bitcode_cflags, input: 'llvmjit_types.c', output: 'llvmjit_types.bc', depends: [postgres], @@ -82,3 +83,11 @@ llvmjit_types = custom_target('llvmjit_types.bc', depfile: '@BASENAME@.c.bc.d', ) backend_targets += llvmjit_types + +# Figure out -I's needed to build all postgres code, including all its +# dependencies +pkg_config = find_program(['pkg-config', 'pkgconf'], required: true) +r = run_command(pkg_config, + ['--cflags-only-I', meson.project_build_root() / 'meson-uninstalled/postgresql-extension-uninstalled.pc'], + check: true) +bitcode_cflags += r.stdout().split() diff --git a/src/backend/meson.build b/src/backend/meson.build index 30a77f45fe9..6c2ff2a9f6e 100644 --- a/src/backend/meson.build +++ b/src/backend/meson.build @@ -165,6 +165,12 @@ postgres = executable('postgres', backend_targets += postgres +bitcode_modules += { + 'name': 'postgres', + 'target': postgres_lib, + 'srcfiles': backend_sources, +} + pg_mod_c_args = cflags_mod pg_mod_cxx_args = cxxflags_mod pg_mod_link_args = ldflags_sl + ldflags_mod diff --git a/src/tools/irlink b/src/tools/irlink new file mode 100644 index 00000000000..793c0abf91a --- /dev/null +++ b/src/tools/irlink @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +import argparse +import os +import shutil +import subprocess +import sys + +parser = argparse.ArgumentParser( + description='generate PostgreSQL JIT IR module') + +parser.add_argument('--index', type=str, required=True) +parser.add_argument('--lto', type=str, required=True) +parser.add_argument('--outdir', type=str, required=True) +parser.add_argument('INPUT', type=str, nargs='+') + +args = parser.parse_args() + +file_names = [os.path.basename(f) for f in args.INPUT] +command = [args.lto, + '-thinlto', '-thinlto-action=thinlink', + '-o', args.index] + file_names +res = subprocess.run(command, cwd=args.outdir) + +exit(res.returncode) -- 2.47.3