From a0aadba866912e1328e114447dcde91052a816f2 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Wed, 21 Sep 2022 22:08:00 -0700 Subject: [PATCH v17 02/23] meson: ci: dontmerge: Add additional CI coverage This is part of the series to be able to test meson on more platforms than normally part of CI. Author: Andres Freund Author: Nazir Bilal Yavuz Author: Justin Pryzby --- .cirrus.yml | 322 +++++++++++++++++- src/tools/ci/docker/linux_centos7 | 57 ++++ src/tools/ci/docker/linux_centos8 | 54 +++ src/tools/ci/docker/linux_fedora_rawhide | 49 +++ src/tools/ci/docker/linux_opensuse_tumbleweed | 56 +++ 5 files changed, 522 insertions(+), 16 deletions(-) create mode 100644 src/tools/ci/docker/linux_centos7 create mode 100644 src/tools/ci/docker/linux_centos8 create mode 100644 src/tools/ci/docker/linux_fedora_rawhide create mode 100644 src/tools/ci/docker/linux_opensuse_tumbleweed diff --git a/.cirrus.yml b/.cirrus.yml index eb33fdc4855..efb2d2c4832 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -126,6 +126,90 @@ task: cores_script: src/tools/ci/cores_backtrace.sh freebsd /tmp/cores +task: + env: + CPUS: 2 + BUILD_JOBS: 3 + TEST_JOBS: 3 + + CIRRUS_WORKING_DIR: /home/postgres/postgres + CCACHE_DIR: /tmp/ccache_dir + + PATH: /usr/sbin:$PATH + + # Postgres interprets LANG as a 'en_US.UTF-8' but it is 'C', then + # Postgres tries to set 'LC_COLLATE' to 'en_US.UTF-8' but it is not + # changeable. Initdb fails because of that. So, LANG is forced to be 'C'. + LANG: "C" + LC_ALL: "C" + + matrix: + - name: NetBSD - 9 - Meson + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*netbsd.*' + env: + IMAGE_NAME: pg-ci-netbsd-9-postgres + PLATFORM: netbsd + INCLUDE_DIRS: -Dextra_lib_dirs=/usr/pkg/lib -Dextra_include_dirs=/usr/pkg/include + + - name: OpenBSD - 7 - Meson + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*openbsd.*' + env: + IMAGE_NAME: pg-ci-openbsd-7-postgres + PLATFORM: openbsd + INCLUDE_DIRS: -Dextra_include_dirs=/usr/local/include -Dextra_lib_dirs=/usr/local/lib + UUID: -Duuid=e2fs + + compute_engine_instance: + image_project: $IMAGE_PROJECT + image: family/${IMAGE_NAME} + platform: ${PLATFORM} + cpu: $CPUS + memory: 4G + disk: 25 + + sysinfo_script: | + locale + id + uname -a + ulimit -a -H && ulimit -a -S + export + + ccache_cache: + folder: $CCACHE_DIR + + create_user_script: | + useradd postgres + chown -R postgres:users /home/postgres + mkdir -p ${CCACHE_DIR} + chown -R postgres:users ${CCACHE_DIR} + + # -Duuid=bsd is not set since 'bsd' uuid option + # is not working on netBSD & openBSD. See + # https://www.postgresql.org/message-id/17358-89806e7420797025@postgresql.org + # And other uuid options are not available on netBSD + configure_script: | + su postgres <<-EOF + meson setup \ + --buildtype debug \ + -Dcassert=true -Dssl=openssl ${UUID} \ + -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \ + ${INCLUDE_DIRS} \ + build + EOF + + build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS}' + upload_caches: ccache + + test_world_script: | + su postgres <<-EOF + ulimit -c unlimited + meson test $MTEST_ARGS --num-processes ${TEST_JOBS} + EOF + + on_failure: + <<: *on_failure_meson + + # configure feature flags, shared between the task running the linux tests and # the CompilerWarnings task LINUX_CONFIGURE_FEATURES: &LINUX_CONFIGURE_FEATURES >- @@ -152,6 +236,22 @@ LINUX_MESON_FEATURES: &LINUX_MESON_FEATURES >- -Duuid=e2fs +# configure preparation scripts, shared between tasks running the linux tests +linux_preparation_scripts_template: &linux_preparation_scripts_template + sysinfo_script: | + id + uname -a + cat /proc/cmdline + ulimit -a -H && ulimit -a -S + export + create_user_script: | + useradd -m -U postgres + chown -R postgres:postgres . + mkdir -p ${CCACHE_DIR} + chown -R postgres:postgres ${CCACHE_DIR} + su postgres -c "ulimit -l -H && ulimit -l -S" + + task: env: CPUS: 4 @@ -181,19 +281,9 @@ task: ccache_cache: folder: ${CCACHE_DIR} - sysinfo_script: | - id - uname -a - cat /proc/cmdline - ulimit -a -H && ulimit -a -S - export - create_user_script: | - useradd -m postgres - chown -R postgres:postgres . - mkdir -p ${CCACHE_DIR} - chown -R postgres:postgres ${CCACHE_DIR} + set_limits_script: | echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf - su postgres -c "ulimit -l -H && ulimit -l -S" + <<: *linux_preparation_scripts_template setup_core_files_script: | mkdir -m 770 /tmp/cores chown root:postgres /tmp/cores @@ -227,7 +317,13 @@ task: on_failure: <<: *on_failure_ac - - name: Linux - Debian Bullseye - Meson + - matrix: + - name: Linux - Debian Bullseye - Meson + - name: Linux - Debian Sid - Meson + trigger_type: manual + + compute_engine_instance: + image: family/pg-ci-sid configure_script: | su postgres <<-EOF @@ -255,6 +351,81 @@ task: cores_script: src/tools/ci/cores_backtrace.sh linux /tmp/cores +task: + env: + CPUS: 4 + BUILD_JOBS: 4 + TEST_JOBS: 8 # experimentally derived to be a decent choice + + CCACHE_DIR: /tmp/ccache_dir + DEBUGINFOD_URLS: ${DEBUGINFO} + + TCL_DIR: /usr/lib64/ + UUID: e2fs + + CFLAGS: "-Og -ggdb" + CXXFLAGS: "-Og -ggdb" + + LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES + + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*' + + container: + dockerfile: ${DOCKERFILE_PATH} + cpu: $CPUS + memory: 4G + + ccache_cache: + folder: ${CCACHE_DIR} + + <<: *linux_preparation_scripts_template + + matrix: + - name: Linux - OpenSuse Tumbleweed (LLVM) - Meson + env: + DOCKERFILE_PATH: src/tools/ci/docker/linux_opensuse_tumbleweed + DEBUGINFO: "https://debuginfod.opensuse.org/" + LLVM: -Dllvm=enabled + + - trigger_type: manual + matrix: + - name: Linux - Fedora Rawhide - Meson + env: + DOCKERFILE_PATH: src/tools/ci/docker/linux_fedora_rawhide + DEBUGINFO: "https://debuginfod.fedoraproject.org/" + + - name: Linux - Centos 8 - Meson + env: + DOCKERFILE_PATH: src/tools/ci/docker/linux_centos7 + DEBUGINFO: "http://debuginfo.centos.org/" + + - name: Linux - Centos 7 - Meson + env: + DOCKERFILE_PATH: src/tools/ci/docker/linux_centos7 + DEBUGINFO: "http://debuginfo.centos.org/" + + configure_script: | + su postgres <<-EOF + meson setup \ + --buildtype debug \ + -Dcassert=true -Dssl=openssl -Duuid=e2fs ${LLVM} \ + -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \ + build + EOF + + build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS}' + upload_caches: ccache + + test_world_script: | + su postgres <<-EOF + ulimit -c unlimited + meson test $MTEST_ARGS --num-processes ${TEST_JOBS} + EOF + + on_failure: + <<: *on_failure_meson + + task: name: macOS - Monterey - Meson @@ -355,9 +526,7 @@ task: cores_script: src/tools/ci/cores_backtrace.sh macos "${HOME}/cores" -task: - name: Windows - Server 2019, VS 2019 - Meson & ninja - +windows_template: &windows_template env: # Half the allowed per-user CPU cores CPUS: 4 @@ -373,6 +542,20 @@ task: # Avoids port conflicts between concurrent tap test runs PG_TEST_USE_UNIX_SOCKETS: 1 PG_REGRESS_SOCK_DIR: "c:/cirrus/" + # -m enables parallelism + # verbosity:minimal + Summary reduce verbosity, while keeping a summary of + # errors/warnings + # ForceNoAlign prevents msbuild from introducing line-breaks for long lines + # disable file tracker, we're never going to rebuild, and it slows down the + # build + MSBFLAGS: -m -verbosity:minimal "-consoleLoggerParameters:Summary;ForceNoAlign" /p:TrackFileAccess=false -nologo + + # If tests hang forever, cirrus eventually times out. In that case log + # output etc is not uploaded, making the problem hard to debug. Of course + # tests internally should have shorter timeouts, but that's proven to not + # be sufficient. 15min currently is fast enough to finish individual test + # "suites". + T_C: "\"C:/Program Files/Git/usr/bin/timeout.exe\" -v -k60s 15m" # startcreate_script starts a postgres instance that we don't want to get # killed at the end of that script (it's stopped in stop_script). Can't @@ -406,6 +589,16 @@ task: setup_additional_packages_script: | REM choco install -y --no-progress ... + +task: + <<: *windows_template + name: Windows - Server 2019, VS 2019 - Meson & ninja + + meson_install_wraps_script: | + mkdir subprojects + meson wrap install lz4 + meson wrap install zlib + # Use /DEBUG:FASTLINK to avoid high memory usage during linking configure_script: | vcvarsall x64 @@ -426,6 +619,103 @@ task: type: text/plain +task: + <<: *windows_template + name: Windows - Server 2019, VS 2019 - Meson & msbuild + + meson_install_wraps_script: | + mkdir subprojects + meson wrap install lz4 + meson wrap install zlib + + configure_script: | + vcvarsall x64 + meson setup --buildtype debug --backend vs -Dcassert=true -Dssl=openssl -Dlz4=enabled -Dzlib=enabled -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=c:/windows/system32/tar.exe -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" build + + build_script: | + vcvarsall x64 + msbuild %MSBFLAGS% build\postgresql.sln + + check_world_script: | + vcvarsall x64 + meson test %MTEST_ARGS% --num-processes %TEST_JOBS% + + on_failure: + <<: *on_failure_meson + crashlog_artifacts: + path: "crashlog-*.txt" + type: text/plain + + +task: + <<: *windows_template + name: Windows - Server 2019, VS 2019 - Homegrown + + env: + # Our windows infrastructure doesn't have test concurrency above the level + # of a single vcregress test target. Due to that, it's useful to run prove + # with multiple jobs. For the other tasks it isn't, because two sources + # (make and prove) of concurrency can overload machines. + # + # The concrete choice of 10 is based on a small bit of experimentation and + # likely can be improved upon further. + PROVE_FLAGS: -j10 --timer + + configure_script: + # copy errors out when using forward slashes + - copy src\tools\ci\windows_build_config.pl src\tools\msvc\config.pl + - vcvarsall x64 + - perl src/tools/msvc/mkvcbuild.pl + build_script: + - vcvarsall x64 + - msbuild %MSBFLAGS% pgsql.sln + tempinstall_script: + # Installation on windows currently only completely works from src/tools/msvc + - cd src/tools/msvc && perl install.pl %CIRRUS_WORKING_DIR%/tmp_install + + test_regress_parallel_script: | + %T_C% perl src/tools/msvc/vcregress.pl check parallel + startcreate_script: | + rem paths to binaries need backslashes + tmp_install\bin\pg_ctl.exe initdb -D tmp_check/db -l tmp_check/initdb.log --options=--no-sync + echo include '%TEMP_CONFIG%' >> tmp_check/db/postgresql.conf + tmp_install\bin\pg_ctl.exe start -D tmp_check/db -l tmp_check/postmaster.log + + test_pl_script: | + %T_C% perl src/tools/msvc/vcregress.pl plcheck + test_isolation_script: | + %T_C% perl src/tools/msvc/vcregress.pl isolationcheck + test_modules_script: | + %T_C% perl src/tools/msvc/vcregress.pl modulescheck + test_contrib_script: | + %T_C% perl src/tools/msvc/vcregress.pl contribcheck + stop_script: | + tmp_install\bin\pg_ctl.exe stop -D tmp_check/db -l tmp_check/postmaster.log + test_ssl_script: | + set with_ssl=openssl + %T_C% perl src/tools/msvc/vcregress.pl taptest ./src/test/ssl/ + test_subscription_script: | + %T_C% perl src/tools/msvc/vcregress.pl taptest ./src/test/subscription/ + test_authentication_script: | + %T_C% perl src/tools/msvc/vcregress.pl taptest ./src/test/authentication/ + test_recovery_script: | + %T_C% perl src/tools/msvc/vcregress.pl recoverycheck + test_bin_script: | + %T_C% perl src/tools/msvc/vcregress.pl bincheck + test_ecpg_script: | + rem tries to build additional stuff + vcvarsall x64 + rem References ecpg_regression.proj in the current dir + cd src/tools/msvc + %T_C% perl vcregress.pl ecpgcheck + + on_failure: + <<: *on_failure_ac + crashlog_artifacts: + path: "crashlog-*.txt" + type: text/plain + + task: name: CompilerWarnings diff --git a/src/tools/ci/docker/linux_centos7 b/src/tools/ci/docker/linux_centos7 new file mode 100644 index 00000000000..3742890d961 --- /dev/null +++ b/src/tools/ci/docker/linux_centos7 @@ -0,0 +1,57 @@ +FROM centos:centos7 +# Enable powertools, EPEL and devtoolset repository +RUN \ + yum -y install dnf-plugins-core \ + epel-release \ + centos-release-scl-rh && \ + \ + yum -y update && \ + yum -y install \ + \ + git \ + perl \ + perl-ExtUtils-Embed \ + perl-IPC-Run \ + perl-Test-Harness \ + perl-Test-Simple \ + \ + bison \ + ccache \ + clang \ + devtoolset-7-make \ + flex \ + gcc \ + gettext \ + \ + krb5-devel \ + libicu-devel \ + libuuid-devel \ + libxml2-devel \ + libxslt-devel \ + libzstd-devel \ + lz4-devel \ + openldap-devel \ + openssl-devel \ + pam-devel \ + python3-devel \ + readline-devel \ + systemd-devel \ + tcl-devel \ + \ + krb5-server \ + krb5-server-ldap \ + krb5-workstation \ + lz4 \ + openldap-clients \ + openldap-servers \ + openssl \ + zstd \ + \ + && \ + pip3 install meson && \ + pip3 install ninja && \ + yum clean all + +# Make version is 3.82 but required is > 4, adding devtoolset-7-make to the PATH +ENV PATH="/opt/rh/devtoolset-7/root/usr/bin/:${PATH}" +ENV LANG=en_US.UTF-8 diff --git a/src/tools/ci/docker/linux_centos8 b/src/tools/ci/docker/linux_centos8 new file mode 100644 index 00000000000..58f95847ef1 --- /dev/null +++ b/src/tools/ci/docker/linux_centos8 @@ -0,0 +1,54 @@ +FROM quay.io/centos/centos:stream8 +# Enable powertools and EPEL repository +RUN \ + dnf -y install dnf-plugins-core && \ + dnf config-manager --set-enabled powertools && \ + dnf -y install epel-release && \ + dnf -y update && \ + dnf -y install \ + \ + git \ + meson \ + perl \ + perl-IPC-Run \ + \ + bison \ + ccache \ + clang \ + diffutils \ + flex \ + gcc \ + gettext \ + \ + krb5-devel \ + libicu-devel \ + libuuid-devel \ + libxml2-devel \ + libxslt-devel \ + libzstd-devel \ + llvm-devel \ + lz4-devel \ + openldap-devel \ + openssl-devel \ + pam-devel \ + python3-devel \ + readline-devel \ + systemd-devel \ + tcl-devel \ + \ + krb5-server \ + krb5-server-ldap \ + krb5-workstation \ + lz4 \ + openldap-clients \ + openldap-servers \ + openssl \ + zstd \ + \ + glibc-langpack-en \ + \ + && \ + pip3 install ninja && \ + yum clean all + +ENV LANG=en_US.UTF-8 diff --git a/src/tools/ci/docker/linux_fedora_rawhide b/src/tools/ci/docker/linux_fedora_rawhide new file mode 100644 index 00000000000..c95027cd6a7 --- /dev/null +++ b/src/tools/ci/docker/linux_fedora_rawhide @@ -0,0 +1,49 @@ +FROM fedora:rawhide +RUN \ + dnf -y update && \ + dnf -y install \ + git \ + meson \ + perl \ + perl-IPC-Run \ + \ + bison \ + ccache \ + clang \ + flex \ + gcc \ + gettext \ + ninja-build \ + \ + krb5-devel \ + libicu-devel \ + libuuid-devel \ + libxml2-devel \ + libxslt-devel \ + libzstd-devel \ + llvm-devel \ + lz4-devel \ + openldap-devel \ + openssl-devel \ + pam-devel \ + python3-devel \ + readline-devel \ + systemd-devel \ + tcl-devel \ + \ + krb5-server \ + krb5-server-ldap \ + krb5-workstation \ + lz4 \ + openldap-clients \ + openldap-servers \ + openssl \ + zstd \ + \ + langpacks-en \ + glibc-langpack-en \ + \ + && \ + yum clean all + +ENV LANG=en_US.UTF-8 diff --git a/src/tools/ci/docker/linux_opensuse_tumbleweed b/src/tools/ci/docker/linux_opensuse_tumbleweed new file mode 100644 index 00000000000..07364ee85ff --- /dev/null +++ b/src/tools/ci/docker/linux_opensuse_tumbleweed @@ -0,0 +1,56 @@ +FROM opensuse/tumbleweed +RUN \ + zypper -n clean -a && \ + zypper -n ref && \ + zypper -n dup && \ + zypper -n update && \ + zypper -n install \ + diffutils \ + git \ + icu \ + make \ + meson \ + perl-IPC-Run \ + shadow \ + systemd-devel \ + util-linux \ + \ + bison \ + ccache \ + clang \ + flex \ + gcc \ + gcc-c++ \ + gettext-runtime \ + ninja \ + \ + krb5-devel \ + libicu-devel \ + libldapcpp-devel \ + liblz4-devel \ + libopenssl-devel \ + libselinux-devel \ + libuuid-devel \ + libxml2-devel \ + libxslt-devel \ + libzstd-devel \ + llvm-devel \ + pam-devel \ + python38-devel \ + readline-devel \ + tcl-devel \ + \ + openldap2 \ + openldap2-client \ + \ + krb5-client \ + krb5-plugin-kdb-ldap \ + krb5-server \ + lz4 \ + zstd \ + \ + && \ + zypper -n clean -a + +# Fixing ICU errors caused by locale being set to 'POSIX' +ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 -- 2.37.3.542.gdd3f6c4cae