From a9fc008dfb3babb14dfd88aee2a281b776cdc784 Mon Sep 17 00:00:00 2001 From: Nazir Bilal Yavuz Date: Fri, 7 Mar 2025 12:10:58 +0300 Subject: [PATCH v5 5/7] meson: Add architecture for LLVM bitcode emission This commit adds suport for bitcode emission for both normal and generated source files (processed by bison, flex, etc). These bitcode files are installed into $pkglibdir/bitcode/ directory if the LLVM is found. New variable `bitcode_modules` is introduced to generate bitcode files. All required information is gathered in this variable. Then, this variable is processed by the main meson LLVM bitcode emission scripts: src/backend/jit/llvm/bitcode/meson.build -> src/tools/irlink. An example of a possible structure of bitcode_modules is: ``` bitcode_modules = [ { 'name': '...', 'target': ..., 'srcfiles': [ '...', '...', ], 'additional_flags': [ '-I...', '-I...', ], 'gen_srcfiles': [ { 'srcfiles': [ , , ], 'additional_flags': [ '-I...', '-I...', ] } ] } ] ``` Author: Andres Freund Author: Nazir Bilal Yavuz Author: Diego Fronza Reviewed-by: Diego Fronza Discussion: https://postgr.es/m/206b001d-1884-4081-bd02-bed5c92f02ba%40eisentraut.org --- src/backend/jit/llvm/bitcode/meson.build | 69 ++++++++++++++++++++++++ src/backend/jit/llvm/meson.build | 31 +++++++---- src/backend/meson.build | 6 +++ meson.build | 21 ++++++++ src/tools/irlink | 25 +++++++++ 5 files changed, 141 insertions(+), 11 deletions(-) create mode 100644 src/backend/jit/llvm/bitcode/meson.build create mode 100644 src/tools/irlink diff --git a/src/backend/jit/llvm/bitcode/meson.build b/src/backend/jit/llvm/bitcode/meson.build new file mode 100644 index 00000000000..546e4d8b898 --- /dev/null +++ b/src/backend/jit/llvm/bitcode/meson.build @@ -0,0 +1,69 @@ +# Copyright (c) 2022-2024, PostgreSQL Global Development Group +# +# emit LLVM bitcode for JIT inlining + +assert(llvm.found()) + +foreach bitcode_module : bitcode_modules + bitcode_targets = [] + bitcode_obj = bitcode_module['target'] + bitcode_cflags_local = bitcode_cflags + bitcode_module.get('additional_flags', []) + bitcode_name = bitcode_module.get('name', bitcode_obj.name()) + + foreach srcfile : bitcode_module['srcfiles'] + if meson.version().version_compare('>=0.59') + srcfilename = fs.parent(srcfile) / fs.name(srcfile) + else + srcfilename = '@0@'.format(srcfile) + endif + + targetname = '@0@_@1@.bc'.format( + bitcode_name, + srcfilename.underscorify(), + ) + bitcode_targets += custom_target( + targetname, + depends: [bitcode_obj], + input: [srcfile], + output: targetname, + command: [llvm_irgen_command, bitcode_cflags_local], + install: true, + install_dir: dir_bitcode, + ) + endforeach + + # Process generated sources, which may include custom compilation flags. + foreach gen_sources: bitcode_module.get('gen_sources', []) + bitcode_cflags_gen_local = bitcode_cflags_local + gen_sources.get('additional_flags', []) + + foreach srcfile: gen_sources['srcfiles'] + # Generated sources are stored in some folder under meson.build_root()/**, + # remove the build prefix from the string. + srcfilename = srcfile.full_path().split(meson.build_root() + '/')[1] + + targetname = '@0@_@1@.bc'.format( + bitcode_name, + srcfilename.underscorify(), + ) + bitcode_targets += custom_target( + targetname, + depends: [bitcode_obj], + input: [srcfile], + output: targetname, + command: [llvm_irgen_command, bitcode_cflags_gen_local], + install: true, + install_dir: dir_bitcode, + ) + endforeach + endforeach + + index_name = '@0@.index.bc'.format(bitcode_name) + bitcode_index = custom_target('@0@'.format(bitcode_name), + output: index_name, + input: bitcode_targets, + command: [irlink, '--lto', llvm_lto, '--outdir', '@OUTDIR@', '--index', index_name, '@INPUT@'], + install: true, + install_dir: dir_bitcode, + ) + backend_targets += bitcode_index +endforeach diff --git a/src/backend/jit/llvm/meson.build b/src/backend/jit/llvm/meson.build index c8e06dfbe35..2dd922e573b 100644 --- a/src/backend/jit/llvm/meson.build +++ b/src/backend/jit/llvm/meson.build @@ -42,21 +42,22 @@ backend_targets += llvmjit # Define a few bits and pieces used here and elsewhere to generate bitcode -llvm_irgen_args = [ - '-c', '-o', '@OUTPUT@', '@INPUT@', +llvm_irgen_command = [] +if ccache.found() + llvm_irgen_command += ccache +endif + +llvm_irgen_command += [ + clang, + '-c', '-o', '@OUTPUT0@', '@INPUT0@', '-flto=thin', '-emit-llvm', - '-MD', '-MQ', '@OUTPUT@', '-MF', '@DEPFILE@', '-O2', '-Wno-ignored-attributes', '-Wno-empty-body', + '-Wno-unknown-warning-option', + '-Wno-compound-token-split-by-macro', ] - -if ccache.found() - llvm_irgen_command = ccache - llvm_irgen_args = [clang.path()] + llvm_irgen_args -else - llvm_irgen_command = clang -endif +llvm_irgen_dep_args = ['-MD', '-MQ', '@OUTPUT0@', '-MF', '@DEPFILE@'] # XXX: Need to determine proper version of the function cflags for clang @@ -73,7 +74,7 @@ bitcode_cflags += '-I@SOURCE_ROOT@/src/include' # Note this is intentionally not installed to bitcodedir, as it's not for # inlining llvmjit_types = custom_target('llvmjit_types.bc', - command: [llvm_irgen_command] + llvm_irgen_args + bitcode_cflags, + command: llvm_irgen_command + llvm_irgen_dep_args + bitcode_cflags, input: 'llvmjit_types.c', output: 'llvmjit_types.bc', depends: [postgres], @@ -82,3 +83,11 @@ llvmjit_types = custom_target('llvmjit_types.bc', depfile: '@BASENAME@.c.bc.d', ) backend_targets += llvmjit_types + +# Figure out -I's needed to build all postgres code, including all its +# dependencies +pkg_config = find_program(['pkg-config', 'pkgconf'], required: true) +r = run_command(pkg_config, + ['--cflags-only-I', meson.build_root() / 'meson-uninstalled/postgresql-extension-uninstalled.pc'], + check: true) +bitcode_cflags += r.stdout().split() diff --git a/src/backend/meson.build b/src/backend/meson.build index 9d79d4d058c..5fb33660d3d 100644 --- a/src/backend/meson.build +++ b/src/backend/meson.build @@ -140,6 +140,12 @@ postgres = executable('postgres', backend_targets += postgres +bitcode_modules += { + 'name': 'postgres', + 'target': postgres_lib, + 'srcfiles': backend_sources, +} + pg_mod_c_args = cflags_mod pg_mod_cpp_args = cxxflags_mod pg_mod_link_args = ldflags_sl + ldflags_mod diff --git a/meson.build b/meson.build index 1b7388096a6..19e46df5d07 100644 --- a/meson.build +++ b/meson.build @@ -814,6 +814,8 @@ if add_languages('cpp', required: llvmopt, native: false) # Some distros put LLVM and clang in different paths, so fallback to # find via PATH, too. clang = find_program(llvm_binpath / 'clang', 'clang', required: true) + llvm_lto = find_program(llvm_binpath / 'llvm-lto', required: true) + irlink = find_program('src/tools/irlink', native: true) endif elif llvmopt.auto() message('llvm requires a C++ compiler') @@ -3025,6 +3027,11 @@ test_deps = [] tests = [] meson_extension_tests = [] +# List of object files + source files to generated LLVM IR for inlining. +# Each element is a hash of: +# {'target': target, 'srcfiles': ..., 'additional_flags': ...}. +bitcode_modules = [] + # Default options for targets @@ -3346,6 +3353,11 @@ subdir('src/interfaces/ecpg/test') subdir('doc/src/sgml') +# generate bitcode for JIT inlining after giving contrib modules etc a chance +# to add themselves to bitcode_modules[] +subdir('src/backend/jit/llvm/bitcode', if_found: llvm) + + generated_sources_ac += {'': ['GNUmakefile']} # After processing src/test, add test_install_libs to the testprep_targets @@ -3981,6 +3993,15 @@ if meson.version().version_compare('>=0.57') section: 'Programs', ) + if llvm.found() + summary( + { + 'clang': clang, + }, + section: 'Programs', + ) + endif + summary( { 'bonjour': bonjour, diff --git a/src/tools/irlink b/src/tools/irlink new file mode 100644 index 00000000000..793c0abf91a --- /dev/null +++ b/src/tools/irlink @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +import argparse +import os +import shutil +import subprocess +import sys + +parser = argparse.ArgumentParser( + description='generate PostgreSQL JIT IR module') + +parser.add_argument('--index', type=str, required=True) +parser.add_argument('--lto', type=str, required=True) +parser.add_argument('--outdir', type=str, required=True) +parser.add_argument('INPUT', type=str, nargs='+') + +args = parser.parse_args() + +file_names = [os.path.basename(f) for f in args.INPUT] +command = [args.lto, + '-thinlto', '-thinlto-action=thinlink', + '-o', args.index] + file_names +res = subprocess.run(command, cwd=args.outdir) + +exit(res.returncode) -- 2.49.0