From 651fce87b7bda15029c5b55c4fca2a1e63dfaf29 Mon Sep 17 00:00:00 2001 From: Justin Pryzby Date: Mon, 17 Jan 2022 00:54:28 -0600 Subject: [PATCH 6/8] cirrus: show coverage report of new code for every patch Coverage is shown only for changed files. This is useful to see coverage of newly-added code, but won't show added/lost coverage in files which this patch doesn't modify. Some alternatives: - could build with "--coverage -fprofile-filter-files=", but that means that ccache will never work (both because it doesn't support that option and also because the arguments will be different for every patch). - could use ninja coverage-html, but that can't filter only changed files, and would take a long time and a lot of space to upload a lot of useless files. https://www.postgresql.org/message-id/202202111821.w3gqblvfp4pr%40alvherre.pgsql https://www.postgresql.org/message-id/flat/20220409021853.GP24419@telsasoft.com ci-os-only: freebsd --- .cirrus.yml | 20 ++++++++++++- src/tools/ci/code-coverage-report | 48 +++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100755 src/tools/ci/code-coverage-report diff --git a/.cirrus.yml b/.cirrus.yml index 82ae18145b1..0f48bd2ab9d 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -20,22 +20,30 @@ env: # target to test, for all but windows CHECK: check-world PROVE_FLAGS=$PROVE_FLAGS CHECKFLAGS: -Otarget PROVE_FLAGS: --timer MTEST_ARGS: --print-errorlogs --no-rebuild -C build PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf PG_TEST_EXTRA: kerberos ldap ssl load_balance + # The commit that this branch is rebased on. There's no easy way to find this. + # This does the right thing for cfbot, which always squishes all patches into a single commit. + # And does the right thing for any 1-patch commits. + # Patch series manually submitted to cirrus would benefit from setting this to the + # number of patches in the series (or directly to the commit the series was rebased on). + #BASE_COMMIT: HEAD~1 + # For demo purposes: + BASE_COMMIT: HEAD~11 # What files to preserve in case tests fail on_failure_ac: &on_failure_ac log_artifacts: paths: - "**/*.log" - "**/*.diffs" - "**/regress_log_*" type: text/plain on_failure_meson: &on_failure_meson @@ -149,57 +157,67 @@ task: image: family/pg-ci-freebsd-13 platform: freebsd cpu: $CPUS memory: 4G disk: 50 sysinfo_script: | id uname -a ulimit -a -H && ulimit -a -S export + git diff --name-only "$BASE_COMMIT" ccache_cache: folder: $CCACHE_DIR create_user_script: | pw useradd postgres chown -R postgres:postgres . mkdir -p ${CCACHE_DIR} chown -R postgres:postgres ${CCACHE_DIR} setup_core_files_script: | mkdir -m 770 /tmp/cores chown root:postgres /tmp/cores sysctl kern.corefile='/tmp/cores/%N.%P.core' setup_additional_packages_script: | - #pkg install -y ... + pkg install -y lcov # NB: Intentionally build without -Dllvm. The freebsd image size is already # large enough to make VM startup slow configure_script: | su postgres <<-EOF meson setup \ --buildtype=debug \ + -Db_coverage=true \ -Dcassert=true -Duuid=bsd -Dtcl_version=tcl86 -Ddtrace=auto \ -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \ -Dextra_lib_dirs=/usr/local/lib -Dextra_include_dirs=/usr/local/include/ \ build EOF build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS}' upload_caches: ccache test_world_script: | su postgres <<-EOF + set -e ulimit -c unlimited + # Write initial coverage files before running tests: + time ./src/tools/ci/code-coverage-report "$BASE_COMMIT" ./build ./coverage "--initial" meson test $MTEST_ARGS --num-processes ${TEST_JOBS} + # Create coverage report for files changed since the base commit. + time ./src/tools/ci/code-coverage-report "$BASE_COMMIT" ./build ./coverage EOF + coverage_artifacts: + path: 'coverage/**' + # test runningcheck, freebsd chosen because it's currently fast enough test_running_script: | su postgres <<-EOF set -e ulimit -c unlimited meson test $MTEST_ARGS --quiet --suite setup export LD_LIBRARY_PATH="$(pwd)/build/tmp_install/usr/local/pgsql/lib/:$LD_LIBRARY_PATH" mkdir -p build/testrun ${PG_FAILED_TESTDIR} build/tmp_install/usr/local/pgsql/bin/initdb -N build/runningcheck --no-instructions -A trust echo "include '$(pwd)/src/tools/ci/pg_ci_base.conf'" >> build/runningcheck/postgresql.conf build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l ${PG_FAILED_TESTDIR}/runningcheck.log start diff --git a/src/tools/ci/code-coverage-report b/src/tools/ci/code-coverage-report new file mode 100755 index 00000000000..db448e802ba --- /dev/null +++ b/src/tools/ci/code-coverage-report @@ -0,0 +1,48 @@ +#! /bin/sh +# Called during CI to generate a code coverage report of changed files. +set -e + +base_branch=$1 +build_dir=$2 +outdir=$3 +args=$4 + +changed=`git diff --name-only "$base_branch" '*.c'` +[ -z "$changed" ] && exit 0 # Nothing changed + +[ -d "$outdir" ] || + mkdir "$outdir" + +# This could be used to map from object file back to source file: +# readelf --debug-dump=info src/backend/postgres_lib.a.p/utils_adt_array_userfuncs.c.o |awk '/DW_AT_name/{print $NF;exit}' +# gcov ./src/port/libpgport_shlib.a.p/inet_net_ntop.c.gcno --stdout + +gcov=$outdir/coverage.gcov +lcov --quiet --capture --directory "$build_dir" $args >"$gcov.new" + +# Filter to include only changed files +echo "$changed" |sed 's,^,*/,' | + xargs -rt lcov --extract "$gcov.new" >"$gcov.filtered" +rm "$gcov.new" + +echo "$args" |grep initial >/dev/null && { + mv "$gcov.filtered" "$gcov.init" + exit 0 +} + +# Exit successfully if no relevant files were changed +[ -s "$gcov.filtered" ] || { + rm "$gcov.init" + exit 0 +} + +# Otherwise combine with the init file: +lcov -a "$gcov.init" -a "$gcov.filtered" >"$gcov" + +genhtml "$gcov" --show-details --legend --quiet --num-spaces=4 --output-directory "$outdir" \ + --title="Coverage report of files changed since: $base_branch" +cp "$outdir"/index.html "$outdir"/00-index.html + +gzip "$gcov" "$gcov.init" "$gcov.filtered" +ls -l "$outdir" +du -sh "$outdir" -- 2.34.1