Re: CI and test improvements

From 64fb5139309c62efad3342cc4ad9d9387978db21 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Tue, 19 Jul 2022 12:38:45 -0500
Subject: [PATCH 21/23] cirrus/warnings: use ./configure cache in
 headerscheck..

This is desirable since since configure is slow.  It's necessary for the
environment variables to match, so disable ccache a different way.

ci-os-only: warnings
---
 .cirrus.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 4f0695e455d..6f743ee03c0 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -633,11 +633,13 @@ task:
   ###
   always:
     headers_headerscheck_script: |
+      export CCACHE_DISABLE=1
       time ./configure \
+        --cache gcc.cache \
         ${LINUX_CONFIGURE_FEATURES} \
         --without-icu \
         --quiet \
-        CC="gcc" CXX"=g++" CLANG="clang"
+        CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
       make -s -j${BUILD_JOBS} clean
       time make -s headerscheck EXTRAFLAGS='-fmax-errors=10'
     headers_cpluspluscheck_script: |
-- 
2.17.1

From 196e61c3b885e4d7de7f861c8a99e5d9dce1ffff Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Tue, 19 Jul 2022 12:38:45 -0500
Subject: [PATCH 22/23] cirrus/warnings: move use a single/common 'always'
 block

ci-os-only: warnings
---
 .cirrus.yml | 48 +++++++++++++++++++++---------------------------
 1 file changed, 21 insertions(+), 27 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 6f743ee03c0..f8cd7e030e9 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -536,11 +536,6 @@ task:
     clang -v
     export
 
-  ccache_cache:
-    folder: $CCACHE_DIR
-    fingerprint_key: ccache/warnings
-    reupload_on_changes: true
-
   setup_additional_packages_script: |
     #apt-get update
     #DEBIAN_FRONTEND=noninteractive apt-get -y install ...
@@ -555,8 +550,13 @@ task:
   # different compilers to build with different combinations of dtrace on/off
   # and cassert on/off.
 
-  # gcc, cassert off, dtrace on
   always:
+    ccache_cache:
+      folder: $CCACHE_DIR
+      fingerprint_key: ccache/warnings
+      reupload_on_changes: true
+
+    # gcc, cassert off, dtrace on
     gcc_warning_script: |
       time ./configure \
         --cache gcc.cache \
@@ -568,8 +568,7 @@ task:
       time make -s -j${BUILD_JOBS} world-bin
       ccache --show-stats
 
-  # gcc, cassert on, dtrace off
-  always:
+    # gcc, cassert on, dtrace off
     gcc_a_warning_script: |
       time ./configure \
         --cache gcc.cache \
@@ -581,8 +580,7 @@ task:
       time make -s -j${BUILD_JOBS} world-bin
       ccache --show-stats
 
-  # clang, cassert off, dtrace off
-  always:
+    # clang, cassert off, dtrace off
     clang_warning_script: |
       time ./configure \
         --cache clang.cache \
@@ -593,8 +591,7 @@ task:
       time make -s -j${BUILD_JOBS} world-bin
       ccache --show-stats
 
-  # clang, cassert on, dtrace on
-  always:
+    # clang, cassert on, dtrace on
     clang_a_warning_script: |
       time ./configure \
         --cache clang.cache \
@@ -607,8 +604,7 @@ task:
       time make -s -j${BUILD_JOBS} world-bin
       ccache --show-stats
 
-  # cross-compile to windows
-  always:
+    # cross-compile to windows
     mingw_cross_warning_script: |
       time ./configure \
         --host=x86_64-w64-mingw32 \
@@ -620,18 +616,17 @@ task:
       time make -s -j${BUILD_JOBS} world-bin
       ccache --show-stats
 
-  ###
-  # Verify headerscheck / cpluspluscheck succeed
-  #
-  # - Don't use ccache, the files are uncacheable, polluting ccache's
-  #   cache
-  # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
-  # - XXX have to disable ICU to avoid errors:
-  #   https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
-  # - XXX: the -Wno-register avoids verbose warnings:
-  #   https://postgr.es/m/20220308181837.aun3tdtdvao4vb7o%40alap3.anarazel.de
-  ###
-  always:
+    ###
+    # Verify headerscheck / cpluspluscheck succeed
+    #
+    # - Don't use ccache, the files are uncacheable, polluting ccache's
+    #   cache
+    # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
+    # - XXX have to disable ICU to avoid errors:
+    #   https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
+    # - XXX: the -Wno-register avoids verbose warnings:
+    #   https://postgr.es/m/20220308181837.aun3tdtdvao4vb7o%40alap3.anarazel.de
+    ###
     headers_headerscheck_script: |
       export CCACHE_DISABLE=1
       time ./configure \
@@ -645,7 +640,6 @@ task:
     headers_cpluspluscheck_script: |
       time make -s cpluspluscheck EXTRAFLAGS='-Wno-register -fmax-errors=10'
 
-  always:
     upload_caches: ccache
 
 
-- 
2.17.1

From 0566ad149df1fc2ee5ba96e523933fc073165194 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 25 May 2022 21:53:22 -0500
Subject: [PATCH 01/23] cirrus/windows: add compiler_warnings_script

I'm not sure how to write this test in windows shell; it's also not easy to
write it in posix sh, since windows shell is somehow interpretting && and ||...

https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de

See also: 8a1ce5e54f6d144e4f8e19af7c767b026ee0c956

ci-os-only: windows

https://cirrus-ci.com/task/6183879907213312
https://cirrus-ci.com/task/4876271443247104
---
 .cirrus.yml                            | 14 +++++++++++++-
 src/tools/ci/windows-compiler-warnings | 16 ++++++++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100755 src/tools/ci/windows-compiler-warnings

diff --git a/.cirrus.yml b/.cirrus.yml
index 81eb8a9996d..2be62791448 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -370,7 +370,14 @@ task:
     # ForceNoAlign prevents msbuild from introducing line-breaks for long lines
     # disable file tracker, we're never going to rebuild, and it slows down the
     #   build
-    MSBFLAGS: -m -verbosity:minimal "-consoleLoggerParameters:Summary;ForceNoAlign" /p:TrackFileAccess=false -nologo
+    # -fileLoggerParameters1: write warnings to msbuild.warn.log.
+    MSBFLAGS: >-
+      "-consoleLoggerParameters:Summary;ForceNoAlign"
+      -fileLoggerParameters1:warningsonly;logfile=msbuild.warn.log
+      -m
+      -nologo
+      -p:TrackFileAccess=false
+      -verbosity:minimal
 
     # If tests hang forever, cirrus eventually times out. In that case log
     # output etc is not uploaded, making the problem hard to debug. Of course
@@ -450,6 +457,11 @@ task:
     cd src/tools/msvc
     %T_C% perl vcregress.pl ecpgcheck
 
+  # These should be last, so all the important checks are always run
+  always:
+    compiler_warnings_script:
+      - sh src\tools\ci\windows-compiler-warnings msbuild.warn.log
+
   on_failure:
     <<: *on_failure
     crashlog_artifacts:
diff --git a/src/tools/ci/windows-compiler-warnings b/src/tools/ci/windows-compiler-warnings
new file mode 100755
index 00000000000..d6f9a1fc569
--- /dev/null
+++ b/src/tools/ci/windows-compiler-warnings
@@ -0,0 +1,16 @@
+#! /bin/sh
+# Success if the given file doesn't exist or is empty, else fail
+# This is a separate file only to avoid dealing with windows shell quoting and escaping.
+set -e
+
+fn=$1
+
+if [ -s "$fn" ]
+then
+	# Display the file's content, then exit indicating failure
+	cat "$fn"
+	exit 1
+else
+	# Success
+	exit 0
+fi
-- 
2.17.1

From ac09946b9d5f1220a690d51086329fe0dbad0811 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sun, 9 Jan 2022 18:25:02 -0600
Subject: [PATCH 02/23] cirrus/vcregress: test modules/contrib with
 NO_INSTALLCHECK=1

--temp-config must be specified with an "=" because otherwise vcregress runs
pg_regress --temp-config test1 test2 [...],
..which means test1 gets eaten as the argument to --temp-config

https://www.postgresql.org/message-id/20220109191649.GL14051%40telsasoft.com
https://www.postgresql.org/message-id/CA%2BhUKGLneD%2Bq%2BE7upHGwn41KGvbxhsKbJ%2BM-y9nvv7_Xjv8Qog%40mail.gmail.com
---
 .cirrus.yml                                |  4 +-
 contrib/basic_archive/Makefile             |  2 +-
 contrib/pg_stat_statements/Makefile        |  2 +-
 contrib/test_decoding/Makefile             |  2 +-
 src/test/modules/snapshot_too_old/Makefile |  2 +-
 src/test/modules/worker_spi/Makefile       |  2 +-
 src/tools/msvc/vcregress.pl                | 46 +++++++++++++++++++---
 7 files changed, 48 insertions(+), 12 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 2be62791448..591e802bee2 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -434,9 +434,9 @@ task:
   test_isolation_script: |
     %T_C% perl src/tools/msvc/vcregress.pl isolationcheck
   test_modules_script: |
-    %T_C% perl src/tools/msvc/vcregress.pl modulescheck
+    %T_C% perl src/tools/msvc/vcregress.pl modulescheck install
   test_contrib_script: |
-    %T_C% perl src/tools/msvc/vcregress.pl contribcheck
+    %T_C% perl src/tools/msvc/vcregress.pl contribcheck install
   stop_script: |
     tmp_install\bin\pg_ctl.exe stop -D tmp_check/db -l tmp_check/postmaster.log
   test_ssl_script: |
diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile
index 14d036e1c42..246358973fe 100644
--- a/contrib/basic_archive/Makefile
+++ b/contrib/basic_archive/Makefile
@@ -4,7 +4,7 @@ MODULES = basic_archive
 PGFILEDESC = "basic_archive - basic archive module"
 
 REGRESS = basic_archive
-REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.conf
+REGRESS_OPTS = --temp-config=$(top_srcdir)/contrib/basic_archive/basic_archive.conf
 
 NO_INSTALLCHECK = 1
 
diff --git a/contrib/pg_stat_statements/Makefile b/contrib/pg_stat_statements/Makefile
index edc40c8bbfb..8684ea0be9c 100644
--- a/contrib/pg_stat_statements/Makefile
+++ b/contrib/pg_stat_statements/Makefile
@@ -16,7 +16,7 @@ PGFILEDESC = "pg_stat_statements - execution statistics of SQL statements"
 
 LDFLAGS_SL += $(filter -lm, $(LIBS))
 
-REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/pg_stat_statements/pg_stat_statements.conf
+REGRESS_OPTS = --temp-config=$(top_srcdir)/contrib/pg_stat_statements/pg_stat_statements.conf
 REGRESS = pg_stat_statements oldextversions
 # Disabled because these tests require "shared_preload_libraries=pg_stat_statements",
 # which typical installcheck users do not have (e.g. buildfarm clients).
diff --git a/contrib/test_decoding/Makefile b/contrib/test_decoding/Makefile
index c7ce6037064..a6289f606d3 100644
--- a/contrib/test_decoding/Makefile
+++ b/contrib/test_decoding/Makefile
@@ -10,7 +10,7 @@ ISOLATION = mxact delayed_startup ondisk_startup concurrent_ddl_dml \
 	oldest_xmin snapshot_transfer subxact_without_top concurrent_stream \
 	twophase_snapshot slot_creation_error catalog_change_snapshot
 
-REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/test_decoding/logical.conf
+REGRESS_OPTS = --temp-config=$(top_srcdir)/contrib/test_decoding/logical.conf
 ISOLATION_OPTS = --temp-config $(top_srcdir)/contrib/test_decoding/logical.conf
 
 # Disabled because these tests require "wal_level=logical", which
diff --git a/src/test/modules/snapshot_too_old/Makefile b/src/test/modules/snapshot_too_old/Makefile
index dfb4537f63c..752a0039fdc 100644
--- a/src/test/modules/snapshot_too_old/Makefile
+++ b/src/test/modules/snapshot_too_old/Makefile
@@ -5,7 +5,7 @@
 EXTRA_CLEAN = $(pg_regress_clean_files)
 
 ISOLATION = sto_using_cursor sto_using_select sto_using_hash_index
-ISOLATION_OPTS = --temp-config $(top_srcdir)/src/test/modules/snapshot_too_old/sto.conf
+ISOLATION_OPTS = --temp-config=$(top_srcdir)/src/test/modules/snapshot_too_old/sto.conf
 
 # Disabled because these tests require "old_snapshot_threshold" >= 0, which
 # typical installcheck users do not have (e.g. buildfarm clients).
diff --git a/src/test/modules/worker_spi/Makefile b/src/test/modules/worker_spi/Makefile
index cbf9b2e37fd..d9f7d9bab6d 100644
--- a/src/test/modules/worker_spi/Makefile
+++ b/src/test/modules/worker_spi/Makefile
@@ -9,7 +9,7 @@ PGFILEDESC = "worker_spi - background worker example"
 REGRESS = worker_spi
 
 # enable our module in shared_preload_libraries for dynamic bgworkers
-REGRESS_OPTS = --temp-config $(top_srcdir)/src/test/modules/worker_spi/dynamic.conf
+REGRESS_OPTS = --temp-config=$(top_srcdir)/src/test/modules/worker_spi/dynamic.conf
 
 # Disable installcheck to ensure we cover dynamic bgworkers.
 NO_INSTALLCHECK = 1
diff --git a/src/tools/msvc/vcregress.pl b/src/tools/msvc/vcregress.pl
index c3729f6be5e..2d6ccd45419 100644
--- a/src/tools/msvc/vcregress.pl
+++ b/src/tools/msvc/vcregress.pl
@@ -426,6 +426,7 @@ sub plcheck
 sub subdircheck
 {
 	my $module = shift;
+	my $installcheck = shift || 1;
 
 	if (   !-d "$module/sql"
 		|| !-d "$module/expected"
@@ -435,7 +436,7 @@ sub subdircheck
 	}
 
 	chdir $module;
-	my @tests = fetchTests();
+	my @tests = fetchTests($installcheck);
 
 	# Leave if no tests are listed in the module.
 	if (scalar @tests == 0)
@@ -445,12 +446,13 @@ sub subdircheck
 	}
 
 	my @opts = fetchRegressOpts();
+	push @opts, "--temp-instance=tmp_check" if $installcheck == -1;
 
 	print "============================================================\n";
 	print "Checking $module\n";
 	my @args = (
 		"$topdir/$Config/pg_regress/pg_regress",
-		"--bindir=${topdir}/${Config}/psql",
+		"--bindir=$tmp_installdir/bin",
 		"--dbname=contrib_regression", @opts, @tests);
 	print join(' ', @args), "\n";
 	system(@args);
@@ -460,6 +462,8 @@ sub subdircheck
 
 sub contribcheck
 {
+	my $mode = shift || '';
+
 	chdir "../../../contrib";
 	my $mstat = 0;
 	foreach my $module (glob("*"))
@@ -477,12 +481,25 @@ sub contribcheck
 		my $status = $? >> 8;
 		$mstat ||= $status;
 	}
+
+	# As above, but creates new DB instance for each module.  For CI.
+	if ($mode eq "install")
+	{
+		foreach my $module (glob("*"))
+		{
+			subdircheck("$module", -1);
+			$mstat ||= $? >> 8;
+		}
+	}
+
 	exit $mstat if $mstat;
 	return;
 }
 
 sub modulescheck
 {
+	my $mode = shift || '';
+
 	chdir "../../../src/test/modules";
 	my $mstat = 0;
 	foreach my $module (glob("*"))
@@ -491,6 +508,17 @@ sub modulescheck
 		my $status = $? >> 8;
 		$mstat ||= $status;
 	}
+
+	# As above, but creates new DB instance for each module.  For CI.
+	if ($mode eq "install")
+	{
+		foreach my $module (glob("*"))
+		{
+			subdircheck("$module", -1);
+			$mstat ||= $? >> 8;
+		}
+	}
+
 	exit $mstat if $mstat;
 	return;
 }
@@ -558,6 +586,7 @@ sub fetchRegressOpts
 		# option starting with "--".
 		@opts = grep { !/\$\(/ && /^--/ }
 		  map { (my $x = $_) =~ s/\Q$(top_builddir)\E/\"$topdir\"/; $x; }
+		  map { (my $x = $_) =~ s/\Q$(top_srcdir)\E/\"$topdir\"/; $x; }
 		  split(/\s+/, $1);
 	}
 	if ($m =~ /^\s*ENCODING\s*=\s*(\S+)/m)
@@ -583,14 +612,19 @@ sub fetchTests
 	my $m = <$handle>;
 	close($handle);
 	my $t = "";
+	my $installcheck = shift || 1;
 
 	$m =~ s{\\\r?\n}{}g;
 
-	# A module specifying NO_INSTALLCHECK does not support installcheck,
-	# so bypass its run by returning an empty set of tests.
 	if ($m =~ /^\s*NO_INSTALLCHECK\s*=\s*\S+/m)
 	{
-		return ();
+		# Skip modules marked installcheck unless running installcheck tests.
+		return () if $installcheck == 1;
+	}
+	else
+	{
+		# Skip modules not marked installcheck if running installcheck tests.
+		return () if $installcheck == -1;
 	}
 
 	if ($m =~ /^REGRESS\s*=\s*(.*)$/gm)
@@ -656,6 +690,8 @@ sub usage
 	  "\nOptions for <arg>: (used by check and installcheck)\n",
 	  "  serial         serial mode\n",
 	  "  parallel       parallel mode\n",
+	  "\nOptions for <arg>: (used by contribcheck and modulescheck)\n",
+	  "  install        also run tests which require a new instance\n",
 	  "\nOption for <arg>: for taptest\n",
 	  "  TEST_DIR       (required) directory where tests reside\n";
 	exit(1);
-- 
2.17.1

From 4ed5eb427de4508a4c3422e60891b45c8512814a Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sun, 3 Apr 2022 00:10:20 -0500
Subject: [PATCH 03/23] cirrus/ccache: disable compression and show stats

Since v4.0, ccache enables zstd compression by default, saving roughly
2x-3x.  But, cirrus caches are compressed as tar.gz, so we could disable
ccache compression, allowing cirrus to gzip the uncompressed data
(better than ccache's default of zstd-1).

With default compression enabled (https://cirrus-ci.com/task/6692342840164352):
linux/debian/bullseye has 4.2; 99MB cirrus cache; cache_size_kibibyte	109616
macos has 4.5.1: 47MB cirrus cache; cache_size_kibibyte	52500
freebsd has 3.7.12: 42MB cirrus cache; cache_size_kibibyte	134064
windows has 4.6.1; 180MB cirrus cache; cache_size_kibibyte	51179
todo: compiler warnings

With compression disabled (https://cirrus-ci.com/task/4614182514458624):
linux: 91MB cirrus cache; cache_size_kibibyte	316136
macos: 41MB cirrus cache; cache_size_kibibyte	118068
windows: 42MB cirrus cache; cache_size_kibibyte	134064
freebsd is the same

The stats should either be shown or logged (or maybe run with CCACHE_NOSTATS,
to avoid re-uploading cache tarball in a 100% cached build, due only to
updating ./**/stats).

Note that ccache 4.4 supports CCACHE_STATSLOG, which seems ideal.

freebsd, linux
ci-os-only: macos
---
 .cirrus.yml | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 591e802bee2..54256570030 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -16,7 +16,9 @@ env:
   # Useful to be able to analyse what in a script takes long
   CIRRUS_LOG_TIMESTAMP: true
 
-  CCACHE_MAXSIZE: "250M"
+  CCACHE_MAXSIZE: "500M"
+  CCACHE_NOCOMPRESS: 1
+  #CCACHE_STATSLOG: ccache-stats.log
 
   # target to test, for all but windows
   CHECK: check-world PROVE_FLAGS=$PROVE_FLAGS
@@ -110,7 +112,10 @@ task:
         CXX="ccache c++" \
         CFLAGS="-Og -ggdb"
     EOF
-  build_script: su postgres -c "gmake -s -j${BUILD_JOBS} world-bin"
+  build_script: |
+    su postgres -c "ccache --zero-stats"
+    su postgres -c "gmake -s -j${BUILD_JOBS} world-bin"
+    su postgres -c "ccache --show-stats"
   upload_caches: ccache
 
   # The use of script avoids make -Otarget complaints about fcntl() on
@@ -206,7 +211,11 @@ task:
         CFLAGS="-Og -ggdb" \
         CXXFLAGS="-Og -ggdb"
     EOF
-  build_script: su postgres -c "make -s -j${BUILD_JOBS} world-bin"
+
+  build_script: |
+    su postgres -c "ccache --zero-stats"
+    su postgres -c "make -s -j${BUILD_JOBS} world-bin"
+    su postgres -c "ccache --show-stats"
   upload_caches: ccache
 
   test_world_script: |
@@ -324,7 +333,11 @@ task:
       \
       LLVM_CONFIG=${brewpath}/llvm/bin/llvm-config \
       PYTHON=python3
-  build_script: gmake -s -j${BUILD_JOBS} world-bin
+
+  build_script: |
+    ccache --zero-stats
+    gmake -s -j${BUILD_JOBS} world-bin
+    ccache --show-stats -v
   upload_caches: ccache
 
   test_world_script: |
@@ -482,7 +495,7 @@ task:
 
     # Use larger ccache cache, as this task compiles with multiple compilers /
     # flag combinations
-    CCACHE_MAXSIZE: "1GB"
+    CCACHE_MAXSIZE: "1500MB"
     CCACHE_DIR: "/tmp/ccache_dir"
 
     LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
@@ -530,7 +543,9 @@ task:
         ${LINUX_CONFIGURE_FEATURES} \
         CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
       make -s -j${BUILD_JOBS} clean
+      ccache --zero-stats
       time make -s -j${BUILD_JOBS} world-bin
+      ccache --show-stats
 
   # gcc, cassert on, dtrace off
   always:
@@ -541,7 +556,9 @@ task:
         ${LINUX_CONFIGURE_FEATURES} \
         CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
       make -s -j${BUILD_JOBS} clean
+      ccache --zero-stats
       time make -s -j${BUILD_JOBS} world-bin
+      ccache --show-stats
 
   # clang, cassert off, dtrace off
   always:
@@ -551,7 +568,9 @@ task:
         ${LINUX_CONFIGURE_FEATURES} \
         CC="ccache clang" CXX="ccache clang++" CLANG="ccache clang"
       make -s -j${BUILD_JOBS} clean
+      ccache --zero-stats
       time make -s -j${BUILD_JOBS} world-bin
+      ccache --show-stats
 
   # clang, cassert on, dtrace on
   always:
@@ -563,7 +582,9 @@ task:
         ${LINUX_CONFIGURE_FEATURES} \
         CC="ccache clang" CXX="ccache clang++" CLANG="ccache clang"
       make -s -j${BUILD_JOBS} clean
+      ccache --zero-stats
       time make -s -j${BUILD_JOBS} world-bin
+      ccache --show-stats
 
   # cross-compile to windows
   always:
@@ -574,7 +595,9 @@ task:
         CC="ccache x86_64-w64-mingw32-gcc" \
         CXX="ccache x86_64-w64-mingw32-g++"
       make -s -j${BUILD_JOBS} clean
+      ccache --zero-stats
       time make -s -j${BUILD_JOBS} world-bin
+      ccache --show-stats
 
   ###
   # Verify docs can be built
-- 
2.17.1

From 634af7835902a70bd733fb96e5531a81f97869d5 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Tue, 26 Jul 2022 20:30:02 -0500
Subject: [PATCH 04/23] cirrus/ccache: add explicit cache keys..

Since otherwise, building with ci-os-only will probably fail to use the normal
cache, since the cache key is computed using both the task name and its *index*
in the list of caches (internal/executor/cache.go:184).
---
 .cirrus.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.cirrus.yml b/.cirrus.yml
index 54256570030..183e8746ce6 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -68,6 +68,9 @@ task:
 
   ccache_cache:
     folder: $CCACHE_DIR
+    fingerprint_key: ccache/freebsd
+    reupload_on_changes: true
+
   # Workaround around performance issues due to 32KB block size
   repartition_script: src/tools/ci/gcp_freebsd_repartition.sh
   create_user_script: |
@@ -175,6 +178,8 @@ task:
 
   ccache_cache:
     folder: ${CCACHE_DIR}
+    fingerprint_key: ccache/linux
+    reupload_on_changes: true
 
   sysinfo_script: |
     id
@@ -289,6 +294,9 @@ task:
 
   ccache_cache:
     folder: $CCACHE_DIR
+    fingerprint_key: ccache/macos
+    reupload_on_changes: true
+
   configure_script: |
     brewpath="/usr/local"
     INCLUDES="${brewpath}/include:${INCLUDES}"
@@ -519,6 +527,8 @@ task:
 
   ccache_cache:
     folder: $CCACHE_DIR
+    fingerprint_key: ccache/warnings
+    reupload_on_changes: true
 
   setup_additional_packages_script: |
     #apt-get update
-- 
2.17.1

From 0bd5f51b8c143ed87a867987309d66b8554b1fd6 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Thu, 14 Apr 2022 06:27:07 -0500
Subject: [PATCH 05/23] cirrus: enable various runtime checks on macos and
 freebsd

windows is slower than freebsd and mac, so it's okay to enable options which
will slow them down some.  Also, the cirrusci mac instances always have lot of
cores available.

See:
https://www.postgresql.org/message-id/20211217193159.pwrelhiyx7kevgsn@alap3.anarazel.de
https://www.postgresql.org/message-id/20211213211223.vkgg3wwiss2tragj%40alap3.anarazel.de
https://www.postgresql.org/message-id/CAH2-WzmevBhKNEtqX3N-Tkb0gVBHH62C0KfeTxXzqYES_PiFiA%40mail.gmail.com
https://www.postgresql.org/message-id/20220325000933.vgazz7pjk2ytj65d@alap3.anarazel.de

ci-os-only: freebsd, macos
---
 .cirrus.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 183e8746ce6..4ad20892eeb 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -113,7 +113,9 @@ task:
         \
         CC="ccache cc" \
         CXX="ccache c++" \
-        CFLAGS="-Og -ggdb"
+        CPPFLAGS="-DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST" \
+        CXXFLAGS="-Og -ggdb -march=native -mtune=native" \
+        CFLAGS="-Og -ggdb -march=native -mtune=native"
     EOF
   build_script: |
     su postgres -c "ccache --zero-stats"
@@ -336,8 +338,8 @@ task:
       CC="ccache cc" \
       CXX="ccache c++" \
       CLANG="ccache ${brewpath}/llvm/bin/ccache" \
-      CFLAGS="-Og -ggdb" \
-      CXXFLAGS="-Og -ggdb" \
+      CFLAGS="-Og -ggdb -DRANDOMIZE_ALLOCATED_MEMORY" \
+      CXXFLAGS="-Og -ggdb -DRANDOMIZE_ALLOCATED_MEMORY" \
       \
       LLVM_CONFIG=${brewpath}/llvm/bin/llvm-config \
       PYTHON=python3
-- 
2.17.1

From 52f97fc5445b739f1fa3577c8c6c563de24fc6cd Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 27 Jul 2022 16:54:47 -0500
Subject: [PATCH 06/23] cirrus/freebsd: run build+check in a make vpath

ci-os-only: freebsd
---
 .cirrus.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 4ad20892eeb..aa3746d0440 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -90,7 +90,7 @@ task:
   # freebsd already takes longer than other platforms except for windows.
   configure_script: |
     su postgres <<-EOF
-      ./configure \
+      mkdir build; cd build; ../configure \
         --enable-cassert --enable-debug --enable-tap-tests \
         --enable-nls \
         \
@@ -119,7 +119,7 @@ task:
     EOF
   build_script: |
     su postgres -c "ccache --zero-stats"
-    su postgres -c "gmake -s -j${BUILD_JOBS} world-bin"
+    su postgres -c "cd build && gmake -s -j${BUILD_JOBS} world-bin"
     su postgres -c "ccache --show-stats"
   upload_caches: ccache
 
@@ -128,7 +128,7 @@ task:
   # https://savannah.gnu.org/bugs/?60774
   # script uses pseudo-ttys, which do support locking.
   test_world_script:
-    - su postgres -c "time script test.log gmake -s -j${TEST_JOBS} ${CHECK} ${CHECKFLAGS}"
+    - su postgres -c "cd build && time script test.log gmake -s -j${TEST_JOBS} ${CHECK} ${CHECKFLAGS}"
 
   on_failure:
     <<: *on_failure
-- 
2.17.1

From aaefdb02e7d891fbb70b83af2bc94b338b439ca6 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 24 Jun 2022 00:09:12 -0500
Subject: [PATCH 07/23] cirrus/freebsd: run with more CPUs+RAM and do not
 repartitiion

There was some historic problem where tests under freebsd took 8+ minutes (and
before 4a288a37f took 15 minutes).

This reduces test time from 10min to 3min.
4 CPUs 4 tests https://cirrus-ci.com/task/4880240739614720
4 CPUs 6 tests https://cirrus-ci.com/task/4664440120410112 https://cirrus-ci.com/task/4586784884523008
4 CPUs 8 tests https://cirrus-ci.com/task/5001995491737600

6 CPUs https://cirrus-ci.com/task/6678321684545536
8 CPUs https://cirrus-ci.com/task/6264854121021440

See also:
https://www.postgresql.org/message-id/flat/20220310033347.hgxk4pyarzq4hxwp@alap3.anarazel.de#f36c0b17e33e31e7925e7e5812998686
8 jobs 7min https://cirrus-ci.com/task/6186376667332608

//-os-only: freebsd
---
 .cirrus.yml | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index aa3746d0440..c5a7f4d8cc4 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -42,11 +42,9 @@ task:
   name: FreeBSD - 13
 
   env:
-    # FreeBSD on GCP is slow when running with larger number of CPUS /
-    # jobs. Using one more job than cpus seems to work best.
-    CPUS: 2
-    BUILD_JOBS: 3
-    TEST_JOBS: 3
+    CPUS: 4
+    BUILD_JOBS: 4
+    TEST_JOBS: 6
 
     CCACHE_DIR: /tmp/ccache_dir
 
@@ -57,7 +55,7 @@ task:
     image: family/pg-ci-freebsd-13
     platform: freebsd
     cpu: $CPUS
-    memory: 2G
+    memory: 3G
     disk: 50
 
   sysinfo_script: |
@@ -71,8 +69,6 @@ task:
     fingerprint_key: ccache/freebsd
     reupload_on_changes: true
 
-  # Workaround around performance issues due to 32KB block size
-  repartition_script: src/tools/ci/gcp_freebsd_repartition.sh
   create_user_script: |
     pw useradd postgres
     chown -R postgres:postgres .
-- 
2.17.1

From 9544f4411ef1ed9379ad42ce0ca124196f20f541 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 2 Apr 2022 18:01:39 -0500
Subject: [PATCH 08/23] cirrus/linux: compile with -fsanitize

One concern about this is that the CompilerWarnings task currently
depends on it, and it seems important to avoid slowing down cfbot by
slowing down the Linux task so much that the Linux+Warnings takes longer
than windows.  Maybe Warnings should depend on Freebsd, which is now the
fastest OS.

ci-os-only: linux
---
 .cirrus.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index c5a7f4d8cc4..2cabe6b7ec1 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -211,8 +211,8 @@ task:
         CC="ccache gcc" \
         CXX="ccache g++" \
         CLANG="ccache clang" \
-        CFLAGS="-Og -ggdb" \
-        CXXFLAGS="-Og -ggdb"
+        CFLAGS="-Og -ggdb -fsanitize=undefined,alignment -fno-sanitize-recover=all" \
+        CXXFLAGS="-Og -ggdb -fsanitize=undefined,alignment -fno-sanitize-recover=all"
     EOF
 
   build_script: |
@@ -491,9 +491,13 @@ task:
 task:
   name: CompilerWarnings
 
-  # To limit unnecessary work only run this once the normal linux test succeeds
+  # To limit unnecessary work only run this after success of freebsd
   depends_on:
-    - Linux - Debian Bullseye
+    - FreeBSD - 13
+
+  # task that did not run, count as a success, so we need to recheck freebsd's
+  # condition here; cirus warns if the "only_if" condition doesn't match the task being depended on
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
 
   env:
     CPUS: 4
@@ -506,10 +510,6 @@ task:
 
     LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
 
-  # task that did not run, count as a success, so we need to recheck Linux'
-  # condition here ...
-  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
-
   container:
     image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
     cpu: $CPUS
-- 
2.17.1

From 3d04645aa9c0b87fe2d40503f8263579bb8cfbf6 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 25 Feb 2022 17:00:33 -0600
Subject: [PATCH 09/23] vcregress: add alltaptests

https://www.postgresql.org/message-id/20220326031353.GG28503@telsasoft.com
https://www.postgresql.org/message-id/flat/87a81b91-87bf-c0bc-7e4f-06dffadcf737@dunslane.net

In passing, document taptest PROVE_FLAGS

ci-os-only: windows

See also:
d835dd6685246f0737ca42ab68242210681bb220
13d856e177e69083f543d6383eeda9e12ce3c55c
fed6df486dca1b9e53d3f560031b9a236c99f4bb
---
 .cirrus.yml                 | 14 ++++--------
 src/tools/msvc/vcregress.pl | 43 +++++++++++++++++++++++++++++++++----
 2 files changed, 43 insertions(+), 14 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 2cabe6b7ec1..7b8ce6eadc3 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -401,9 +401,9 @@ task:
     # If tests hang forever, cirrus eventually times out. In that case log
     # output etc is not uploaded, making the problem hard to debug. Of course
     # tests internally should have shorter timeouts, but that's proven to not
-    # be sufficient. 15min currently is fast enough to finish individual test
+    # be sufficient. 25min currently is fast enough to finish individual test
     # "suites".
-    T_C: "\"C:/Program Files/Git/usr/bin/timeout.exe\" -v -k60s 15m"
+    T_C: "\"C:/Program Files/Git/usr/bin/timeout.exe\" -v -k60s 25m"
 
     # startcreate_script starts a postgres instance that we don't want to get
     # killed at the end of that script (it's stopped in stop_script). Can't
@@ -461,14 +461,8 @@ task:
   test_ssl_script: |
     set with_ssl=openssl
     %T_C% perl src/tools/msvc/vcregress.pl taptest ./src/test/ssl/
-  test_subscription_script: |
-    %T_C% perl src/tools/msvc/vcregress.pl taptest ./src/test/subscription/
-  test_authentication_script: |
-    %T_C% perl src/tools/msvc/vcregress.pl taptest ./src/test/authentication/
-  test_recovery_script: |
-    %T_C% perl src/tools/msvc/vcregress.pl recoverycheck
-  test_bin_script: |
-    %T_C% perl src/tools/msvc/vcregress.pl bincheck
+  test_tap_script: |
+    %T_C% perl src/tools/msvc/vcregress.pl alltaptests
   test_ecpg_script: |
     rem tries to build additional stuff
     vcvarsall x64
diff --git a/src/tools/msvc/vcregress.pl b/src/tools/msvc/vcregress.pl
index 2d6ccd45419..6495f33e593 100644
--- a/src/tools/msvc/vcregress.pl
+++ b/src/tools/msvc/vcregress.pl
@@ -51,7 +51,7 @@ if (-e "src/tools/msvc/buildenv.pl")
 
 my $what = shift || "";
 if ($what =~
-	/^(check|installcheck|plcheck|contribcheck|modulescheck|ecpgcheck|isolationcheck|upgradecheck|bincheck|recoverycheck|taptest)$/i
+	/^(check|installcheck|plcheck|contribcheck|modulescheck|ecpgcheck|isolationcheck|upgradecheck|bincheck|recoverycheck|taptest|alltaptests)$/i
   )
 {
 	$what = uc $what;
@@ -109,6 +109,7 @@ my %command = (
 	BINCHECK       => \&bincheck,
 	RECOVERYCHECK  => \&recoverycheck,
 	UPGRADECHECK   => \&upgradecheck,     # no-op
+	ALLTAPTESTS    => \&alltaptests,
 	TAPTEST        => \&taptest,);
 
 my $proc = $command{$what};
@@ -296,6 +297,9 @@ sub tap_check
 	# add the module build dir as the second element in the PATH
 	$ENV{PATH} =~ s!;!;$topdir/$Config/$module;!;
 
+	print "============================================================\n";
+	print "Checking $dir: @args\n";
+
 	rmtree('tmp_check');
 	system(@args);
 	my $status = $? >> 8;
@@ -310,8 +314,7 @@ sub bincheck
 
 	my $mstat = 0;
 
-	# Find out all the existing TAP tests by looking for t/ directories
-	# in the tree.
+	# Find the TAP tests by looking for t/ directories
 	my @bin_dirs = glob("$topdir/src/bin/*");
 
 	# Process each test
@@ -326,6 +329,36 @@ sub bincheck
 	return;
 }
 
+sub alltaptests
+{
+	InstallTemp();
+
+	my $mstat = 0;
+
+	# Find out all the existing TAP tests by looking for t/ directories
+	# in the tree.
+	my @tap_dirs = ();
+	my @top_dir = ($topdir);
+	File::Find::find(
+		{   wanted => sub {
+			/^t\z/s
+			  && $File::Find::name !~ /\/(kerberos|ldap|ssl|ssl_passphrase_callback)\// # opt-in: warn about these?
+			  && push(@tap_dirs, $File::Find::name);
+			}
+		},
+		@top_dir);
+
+	# Process each test
+	foreach my $test_path (@tap_dirs)
+	{
+		my $dir = dirname($test_path);
+		my $status = tap_check($dir);
+		$mstat ||= $status;
+	}
+	exit $mstat if $mstat;
+	return;
+}
+
 sub taptest
 {
 	my $dir = shift;
@@ -676,6 +709,7 @@ sub usage
 	print STDERR
 	  "Usage: vcregress.pl <mode> [<arg>]\n\n",
 	  "Options for <mode>:\n",
+	  "  alltaptests    run all tap tests (except kerberos, ldap, ssl, ssl_passphrase_callback)\n",
 	  "  bincheck       run tests of utilities in src/bin/\n",
 	  "  check          deploy instance and run regression tests on it\n",
 	  "  contribcheck   run tests of modules in contrib/\n",
@@ -693,6 +727,7 @@ sub usage
 	  "\nOptions for <arg>: (used by contribcheck and modulescheck)\n",
 	  "  install        also run tests which require a new instance\n",
 	  "\nOption for <arg>: for taptest\n",
-	  "  TEST_DIR       (required) directory where tests reside\n";
+	  "  TEST_DIR       (required) directory where tests reside\n",
+	  "  PROVE_FLAGS    flags to pass to prove\n";
 	exit(1);
 }
-- 
2.17.1

From 5a334d04ed60bb28c77ddce3fa0d4773a1e59be2 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Thu, 31 Mar 2022 23:52:10 -0500
Subject: [PATCH 10/23] tmp: run tap tests first

ci-os-only: windows
---
 .cirrus.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 7b8ce6eadc3..a1541db2bc8 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -440,6 +440,11 @@ task:
     # Installation on windows currently only completely works from src/tools/msvc
     - cd src/tools/msvc && perl install.pl %CIRRUS_WORKING_DIR%/tmp_install
 
+  this_tap_script: |
+    %T_C% perl src/tools/msvc/vcregress.pl taptest src/interfaces/libpq
+  test_tap_script: |
+    %T_C% perl src/tools/msvc/vcregress.pl alltaptests
+
   test_regress_parallel_script: |
     %T_C% perl src/tools/msvc/vcregress.pl check parallel
   startcreate_script: |
@@ -461,8 +466,6 @@ task:
   test_ssl_script: |
     set with_ssl=openssl
     %T_C% perl src/tools/msvc/vcregress.pl taptest ./src/test/ssl/
-  test_tap_script: |
-    %T_C% perl src/tools/msvc/vcregress.pl alltaptests
   test_ecpg_script: |
     rem tries to build additional stuff
     vcvarsall x64
-- 
2.17.1

From 81fa3fe92fce7b0e37e7f4950d25563ce4d30f1f Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 19 Feb 2022 13:06:52 -0600
Subject: [PATCH 11/23] set TESTDIR from src/test/perl rather than
 Makefile/vcregress

https://www.postgresql.org/message-id/flat/20220219234148.GC9008%40telsasoft.com

These seem most likely to break:

make check -C src/bin/psql
make check -C src/bin/pgbench
make check -C src/test/modules/test_misc
make check -C src/test/modules/libpq_pipeline
PROVE_TESTS=t/027_stream_regress.pl make check -C src/test/recovery
---
 src/Makefile.global.in                 |  9 ++++++---
 src/test/perl/PostgreSQL/Test/Utils.pm | 16 +++++++++-------
 src/tools/msvc/vcregress.pl            |  1 -
 3 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index 5664c645f82..e48b12edfac 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -453,8 +453,9 @@ echo "+++ tap install-check in $(subdir) +++" && \
 rm -rf '$(CURDIR)'/tmp_check && \
 $(MKDIR_P) '$(CURDIR)'/tmp_check && \
 cd $(srcdir) && \
-   TESTDIR='$(CURDIR)' PATH="$(bindir):$(CURDIR):$$PATH" \
+   PATH="$(bindir):$(CURDIR):$$PATH" \
    PGPORT='6$(DEF_PGPORT)' top_builddir='$(CURDIR)/$(top_builddir)' \
+   PG_SUBDIR='$(CURDIR)' \
    PG_REGRESS='$(CURDIR)/$(top_builddir)/src/test/regress/pg_regress' \
    $(PROVE) $(PG_PROVE_FLAGS) $(PROVE_FLAGS) $(if $(PROVE_TESTS),$(PROVE_TESTS),t/*.pl)
 endef
@@ -464,8 +465,9 @@ echo "+++ tap install-check in $(subdir) +++" && \
 rm -rf '$(CURDIR)'/tmp_check && \
 $(MKDIR_P) '$(CURDIR)'/tmp_check && \
 cd $(srcdir) && \
-   TESTDIR='$(CURDIR)' PATH="$(bindir):$(CURDIR):$$PATH" \
+   PATH="$(bindir):$(CURDIR):$$PATH" \
    PGPORT='6$(DEF_PGPORT)' top_builddir='$(top_builddir)' \
+   PG_SUBDIR='$(CURDIR)' \
    PG_REGRESS='$(top_builddir)/src/test/regress/pg_regress' \
    $(PROVE) $(PG_PROVE_FLAGS) $(PROVE_FLAGS) $(if $(PROVE_TESTS),$(PROVE_TESTS),t/*.pl)
 endef
@@ -476,7 +478,8 @@ echo "+++ tap check in $(subdir) +++" && \
 rm -rf '$(CURDIR)'/tmp_check && \
 $(MKDIR_P) '$(CURDIR)'/tmp_check && \
 cd $(srcdir) && \
-   TESTDIR='$(CURDIR)' $(with_temp_install) PGPORT='6$(DEF_PGPORT)' \
+   $(with_temp_install) PGPORT='6$(DEF_PGPORT)' \
+   PG_SUBDIR='$(CURDIR)' \
    PG_REGRESS='$(CURDIR)/$(top_builddir)/src/test/regress/pg_regress' \
    $(PROVE) $(PG_PROVE_FLAGS) $(PROVE_FLAGS) $(if $(PROVE_TESTS),$(PROVE_TESTS),t/*.pl)
 endef
diff --git a/src/test/perl/PostgreSQL/Test/Utils.pm b/src/test/perl/PostgreSQL/Test/Utils.pm
index 1ca2cc59170..063a19b1df5 100644
--- a/src/test/perl/PostgreSQL/Test/Utils.pm
+++ b/src/test/perl/PostgreSQL/Test/Utils.pm
@@ -189,19 +189,21 @@ INIT
 	# test may still fail, but it's more likely to report useful facts.
 	$SIG{PIPE} = 'IGNORE';
 
-	# Determine output directories, and create them.  The base path is the
-	# TESTDIR environment variable, which is normally set by the invoking
-	# Makefile.
-	$tmp_check = $ENV{TESTDIR} ? "$ENV{TESTDIR}/tmp_check" : "tmp_check";
+	my $test_dir = File::Spec->rel2abs($ENV{PG_SUBDIR} || dirname(dirname($0)));
+	my $test_name = basename($0);
+	$test_name =~ s/\.[^.]+$//;
+
+	$ENV{TESTDIR} = $test_dir;
+
+	# Determine output directories, and create them.
+	$tmp_check = "$test_dir/tmp_check";
 	$log_path = "$tmp_check/log";
 
 	mkdir $tmp_check;
 	mkdir $log_path;
 
 	# Open the test log file, whose name depends on the test name.
-	$test_logfile = basename($0);
-	$test_logfile =~ s/\.[^.]+$//;
-	$test_logfile = "$log_path/regress_log_$test_logfile";
+	$test_logfile = "$log_path/regress_log_$test_name";
 	open my $testlog, '>', $test_logfile
 	  or die "could not open STDOUT to logfile \"$test_logfile\": $!";
 
diff --git a/src/tools/msvc/vcregress.pl b/src/tools/msvc/vcregress.pl
index 6495f33e593..822cbcdef47 100644
--- a/src/tools/msvc/vcregress.pl
+++ b/src/tools/msvc/vcregress.pl
@@ -292,7 +292,6 @@ sub tap_check
 	$ENV{PG_REGRESS}    = "$topdir/$Config/pg_regress/pg_regress";
 	$ENV{REGRESS_SHLIB} = "$topdir/src/test/regress/regress.dll";
 
-	$ENV{TESTDIR} = "$dir";
 	my $module = basename $dir;
 	# add the module build dir as the second element in the PATH
 	$ENV{PATH} =~ s!;!;$topdir/$Config/$module;!;
-- 
2.17.1

From 412da43eede25aa59e461c10d6073d3c20d7aac5 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 25 Feb 2022 17:23:26 -0600
Subject: [PATCH 12/23] ..also set PATH.

This essentially reverts commit f4ce6c4d3a30ec3a12c7f64b90a6fc82887ddd7b
(See also 795862c280c5949bafcd8c44543d887fd32b590a).

NO And partially reverts 6b04abdfc5e0653542ac5d586e639185a8c61a39

XXX: also set PATH=bindir ?

 linux, windows
ci-os-only: freebsd,
---
 src/Makefile.global.in                 |  6 +++---
 src/test/perl/PostgreSQL/Test/Utils.pm | 10 ++++++++++
 src/tools/msvc/vcregress.pl            |  4 ----
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index e48b12edfac..ac970215495 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -441,7 +441,7 @@ ld_library_path_var = LD_LIBRARY_PATH
 # need something more here. If not defined then the expansion does
 # nothing.
 with_temp_install = \
-	PATH="$(abs_top_builddir)/tmp_install$(bindir):$(CURDIR):$$PATH" \
+	PATH="$(abs_top_builddir)/tmp_install$(bindir):$$PATH" \
 	$(call add_to_path,$(strip $(ld_library_path_var)),$(abs_top_builddir)/tmp_install$(libdir)) \
 	$(with_temp_install_extra)
 
@@ -453,7 +453,7 @@ echo "+++ tap install-check in $(subdir) +++" && \
 rm -rf '$(CURDIR)'/tmp_check && \
 $(MKDIR_P) '$(CURDIR)'/tmp_check && \
 cd $(srcdir) && \
-   PATH="$(bindir):$(CURDIR):$$PATH" \
+   PATH="$(bindir):$$PATH" \
    PGPORT='6$(DEF_PGPORT)' top_builddir='$(CURDIR)/$(top_builddir)' \
    PG_SUBDIR='$(CURDIR)' \
    PG_REGRESS='$(CURDIR)/$(top_builddir)/src/test/regress/pg_regress' \
@@ -465,7 +465,7 @@ echo "+++ tap install-check in $(subdir) +++" && \
 rm -rf '$(CURDIR)'/tmp_check && \
 $(MKDIR_P) '$(CURDIR)'/tmp_check && \
 cd $(srcdir) && \
-   PATH="$(bindir):$(CURDIR):$$PATH" \
+   PATH="$(bindir):$$PATH" \
    PGPORT='6$(DEF_PGPORT)' top_builddir='$(top_builddir)' \
    PG_SUBDIR='$(CURDIR)' \
    PG_REGRESS='$(top_builddir)/src/test/regress/pg_regress' \
diff --git a/src/test/perl/PostgreSQL/Test/Utils.pm b/src/test/perl/PostgreSQL/Test/Utils.pm
index 063a19b1df5..4a1629a8e16 100644
--- a/src/test/perl/PostgreSQL/Test/Utils.pm
+++ b/src/test/perl/PostgreSQL/Test/Utils.pm
@@ -195,6 +195,16 @@ INIT
 
 	$ENV{TESTDIR} = $test_dir;
 
+	if ($PostgreSQL::Test::Utils::windows_os &&
+		$Config{osname} eq 'MSWin32')
+	{
+		$ENV{PATH} =~ s!;!;$test_dir;$test_dir/test;!;
+	}
+	else
+	{
+		$ENV{PATH} =~ s!:!:$test_dir:$test_dir/test:!;
+	}
+
 	# Determine output directories, and create them.
 	$tmp_check = "$test_dir/tmp_check";
 	$log_path = "$tmp_check/log";
diff --git a/src/tools/msvc/vcregress.pl b/src/tools/msvc/vcregress.pl
index 822cbcdef47..5dcc3419ad7 100644
--- a/src/tools/msvc/vcregress.pl
+++ b/src/tools/msvc/vcregress.pl
@@ -292,10 +292,6 @@ sub tap_check
 	$ENV{PG_REGRESS}    = "$topdir/$Config/pg_regress/pg_regress";
 	$ENV{REGRESS_SHLIB} = "$topdir/src/test/regress/regress.dll";
 
-	my $module = basename $dir;
-	# add the module build dir as the second element in the PATH
-	$ENV{PATH} =~ s!;!;$topdir/$Config/$module;!;
-
 	print "============================================================\n";
 	print "Checking $dir: @args\n";
 
-- 
2.17.1

From 68cf18b094d9ccb6d72948440568ead019de12bd Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 26 Aug 2022 12:00:10 -0500
Subject: [PATCH 13/23] ..and chdir

compare: 1053f85 91ae677 557eb01

sending the srcdir means that libpq_pipeline fails; it needs to set path to the
build dir (testdir).  Why doesn't it break the other tests that set PATH, though ??

pipeline is working now, but failing in basebackup test??
https://github.com/justinpryzby/postgres/runs/8044385257

XX-os-only: windows

ci-os-only: freebsd
---
 src/Makefile.global.in                 | 15 ++++++---------
 src/test/perl/PostgreSQL/Test/Utils.pm | 12 ++++++++++--
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index ac970215495..b65a4630228 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -452,24 +452,22 @@ define prove_installcheck
 echo "+++ tap install-check in $(subdir) +++" && \
 rm -rf '$(CURDIR)'/tmp_check && \
 $(MKDIR_P) '$(CURDIR)'/tmp_check && \
-cd $(srcdir) && \
    PATH="$(bindir):$$PATH" \
    PGPORT='6$(DEF_PGPORT)' top_builddir='$(CURDIR)/$(top_builddir)' \
-   PG_SUBDIR='$(CURDIR)' \
+   PG_SRCDIR='$(srcdir)' \
    PG_REGRESS='$(CURDIR)/$(top_builddir)/src/test/regress/pg_regress' \
-   $(PROVE) $(PG_PROVE_FLAGS) $(PROVE_FLAGS) $(if $(PROVE_TESTS),$(PROVE_TESTS),t/*.pl)
+   $(PROVE) $(PG_PROVE_FLAGS) $(PROVE_FLAGS) $(if $(PROVE_TESTS),$(PROVE_TESTS),$(srcdir)/t/*.pl)
 endef
 else # PGXS case
 define prove_installcheck
 echo "+++ tap install-check in $(subdir) +++" && \
 rm -rf '$(CURDIR)'/tmp_check && \
 $(MKDIR_P) '$(CURDIR)'/tmp_check && \
-cd $(srcdir) && \
    PATH="$(bindir):$$PATH" \
    PGPORT='6$(DEF_PGPORT)' top_builddir='$(top_builddir)' \
-   PG_SUBDIR='$(CURDIR)' \
+   PG_SRCDIR='$(srcdir)' \
    PG_REGRESS='$(top_builddir)/src/test/regress/pg_regress' \
-   $(PROVE) $(PG_PROVE_FLAGS) $(PROVE_FLAGS) $(if $(PROVE_TESTS),$(PROVE_TESTS),t/*.pl)
+   $(PROVE) $(PG_PROVE_FLAGS) $(PROVE_FLAGS) $(if $(PROVE_TESTS),$(PROVE_TESTS),$(srcdir)/t/*.pl)
 endef
 endif # PGXS
 
@@ -477,11 +475,10 @@ define prove_check
 echo "+++ tap check in $(subdir) +++" && \
 rm -rf '$(CURDIR)'/tmp_check && \
 $(MKDIR_P) '$(CURDIR)'/tmp_check && \
-cd $(srcdir) && \
    $(with_temp_install) PGPORT='6$(DEF_PGPORT)' \
-   PG_SUBDIR='$(CURDIR)' \
+   PG_SRCDIR='$(srcdir)' \
    PG_REGRESS='$(CURDIR)/$(top_builddir)/src/test/regress/pg_regress' \
-   $(PROVE) $(PG_PROVE_FLAGS) $(PROVE_FLAGS) $(if $(PROVE_TESTS),$(PROVE_TESTS),t/*.pl)
+   $(PROVE) $(PG_PROVE_FLAGS) $(PROVE_FLAGS) $(if $(PROVE_TESTS),$(PROVE_TESTS),$(srcdir)/t/*.pl)
 endef
 
 else
diff --git a/src/test/perl/PostgreSQL/Test/Utils.pm b/src/test/perl/PostgreSQL/Test/Utils.pm
index 4a1629a8e16..0b52101e835 100644
--- a/src/test/perl/PostgreSQL/Test/Utils.pm
+++ b/src/test/perl/PostgreSQL/Test/Utils.pm
@@ -47,6 +47,7 @@ use warnings;
 use Carp;
 use Config;
 use Cwd;
+use Cwd qw(abs_path);
 use Exporter 'import';
 use Fcntl qw(:mode :seek);
 use File::Basename;
@@ -189,20 +190,27 @@ INIT
 	# test may still fail, but it's more likely to report useful facts.
 	$SIG{PIPE} = 'IGNORE';
 
-	my $test_dir = File::Spec->rel2abs($ENV{PG_SUBDIR} || dirname(dirname($0)));
+	my $test_dir = abs_path($ENV{PG_SRCDIR} || dirname(dirname($0)));
+	#my $test_dir = File::Spec->rel2abs($ENV{PG_SUBDIR} || dirname(dirname($0)));
 	my $test_name = basename($0);
 	$test_name =~ s/\.[^.]+$//;
 
 	$ENV{TESTDIR} = $test_dir;
 
+	my $cwd = getcwd();
+	chdir($test_dir);
+	#$ENV{BUILD_DIR} = $cwd;
 	if ($PostgreSQL::Test::Utils::windows_os &&
 		$Config{osname} eq 'MSWin32')
 	{
+		# $test_dir is what's needed for windows
+		#$ENV{PATH} =~ s!;!;$test_dir;$test_dir/test;$cwd;$cwd/test;!;
 		$ENV{PATH} =~ s!;!;$test_dir;$test_dir/test;!;
 	}
 	else
 	{
-		$ENV{PATH} =~ s!:!:$test_dir:$test_dir/test:!;
+		# $cwd is what's needed for a vpath build (and same as testdir for non-vpath build)
+		$ENV{PATH} =~ s!:!:$cwd:$cwd/test:!;
 	}
 
 	# Determine output directories, and create them.
-- 
2.17.1

From b92f3f702b03e94f47bba608afb0687c96d65a34 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 19 Feb 2022 13:41:36 -0600
Subject: [PATCH 14/23] vcregress: run alltaptests in parallel

XX-os-only: windows

The test changes are needed to avoid these failures:

https://github.com/justinpryzby/postgres/runs/5174636590
[15:59:59.408] Bailout called.  Further testing stopped:  could not create test directory "c:/cirrus/tmp_check/t_C:\cirrus\src\bin\pgbench\t\002_pgbench_no_server_stuff": Invalid argument
[15:59:59.408] FAILED--Further testing stopped: could not create test directory "c:/cirrus/tmp_check/t_C:\cirrus\src\bin\pgbench\t\002_pgbench_no_server_stuff": Invalid argument

https://github.com/justinpryzby/postgres/runs/5174788205
[16:37:09.891] #   Failed test 'reading traces/disallowed_in_pipeline.trace: could not open "traces/disallowed_in_pipeline.trace": The system cannot find the path specified at C:\cirrus\src\test\modules\libpq_pipeline\t\001_libpq_pipeline.pl line 70.

https://github.com/justinpryzby/postgres/runs/5174877506
pg_regress: could not open file "../regress/parallel_schedule" for reading: No such file or directory

XXX: avoid breaking src/bin/psql/t/010_tab_completion.pl:

See also:
f4ce6c4d3a30ec3a12c7f64b90a6fc82887ddd7b
795862c280c5949bafcd8c44543d887fd32b590a
db973ffb3ca43e65a0bf15175a35184a53bf977d
---
 src/bin/pgbench/t/002_pgbench_no_server.pl |  4 ++--
 src/tools/msvc/vcregress.pl                | 14 ++++----------
 2 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/src/bin/pgbench/t/002_pgbench_no_server.pl b/src/bin/pgbench/t/002_pgbench_no_server.pl
index 50bde7dd0fc..bbbb49bb23d 100644
--- a/src/bin/pgbench/t/002_pgbench_no_server.pl
+++ b/src/bin/pgbench/t/002_pgbench_no_server.pl
@@ -8,12 +8,12 @@
 use strict;
 use warnings;
 
+use File::Basename;
 use PostgreSQL::Test::Utils;
 use Test::More;
 
 # create a directory for scripts
-my $testname = $0;
-$testname =~ s,.*/,,;
+my $testname = basename($0);
 $testname =~ s/\.pl$//;
 
 my $testdir = "$PostgreSQL::Test::Utils::tmp_check/t_${testname}_stuff";
diff --git a/src/tools/msvc/vcregress.pl b/src/tools/msvc/vcregress.pl
index 5dcc3419ad7..49168e298ac 100644
--- a/src/tools/msvc/vcregress.pl
+++ b/src/tools/msvc/vcregress.pl
@@ -328,8 +328,6 @@ sub alltaptests
 {
 	InstallTemp();
 
-	my $mstat = 0;
-
 	# Find out all the existing TAP tests by looking for t/ directories
 	# in the tree.
 	my @tap_dirs = ();
@@ -343,14 +341,10 @@ sub alltaptests
 		},
 		@top_dir);
 
-	# Process each test
-	foreach my $test_path (@tap_dirs)
-	{
-		my $dir = dirname($test_path);
-		my $status = tap_check($dir);
-		$mstat ||= $status;
-	}
-	exit $mstat if $mstat;
+	# Run all the tap tests in a single prove instance for better performance
+	$ENV{PROVE_TESTS} = "@tap_dirs";
+	my $status = tap_check('PROVE_FLAGS=--ext=.pl', "$topdir");
+	exit $status if $status;
 	return;
 }
 
-- 
2.17.1

From 20737bb180402c615aa9a67cc4e06b29bdc531d9 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Thu, 26 May 2022 22:24:41 -0500
Subject: [PATCH 15/23] another way to install uri_regress/testclient

This reverts commit a17fd67d2f2861ae0ce00d1aeefdf2facc47cd5e.

TODO: partially revert 6b04abdfc5e0653542ac5d586e639185a8c61a39

See also:
a17fd67d2f2861ae0ce00d1aeefdf2facc47cd5e
https://www.postgresql.org/message-id/20220416144454.GX26620%40telsasoft.com
https://www.postgresql.org/message-id/20220223211606.ikcwoolsz2nohsw6%40alap3.anarazel.de

TODO: doc/src/sgml/extend.sgml

ci-os-only: linux, windows
---
 src/interfaces/libpq/test/Makefile |  8 +++----
 src/tools/msvc/Mkvcbuild.pm        | 36 +++++++++++++-----------------
 2 files changed, 19 insertions(+), 25 deletions(-)

diff --git a/src/interfaces/libpq/test/Makefile b/src/interfaces/libpq/test/Makefile
index 75ac08f943d..91c8d87d528 100644
--- a/src/interfaces/libpq/test/Makefile
+++ b/src/interfaces/libpq/test/Makefile
@@ -14,11 +14,11 @@ endif
 override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += $(libpq_pgport)
 
-PROGS = libpq_testclient libpq_uri_regress
+PROGRAMS = libpq_testclient libpq_uri_regress
 
-all: $(PROGS)
+all: $(PROGRAMS)
 
-$(PROGS): $(WIN32RES)
+$(PROGRAMS): $(WIN32RES)
 
 clean distclean maintainer-clean:
-	rm -f $(PROGS) *.o
+	rm -f $(PROGRAMS) *.o
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index 098bc3f1b09..5277ef068f1 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -36,11 +36,14 @@ my @unlink_on_exit;
 
 # Set of variables for modules in contrib/ and src/test/modules/
 my @contrib_uselibpq       = ();
-my @contrib_uselibpgport   = ();
+my @contrib_uselibpgport   = ('libpq_uri_regress', 'libpq_testclient');
 my @contrib_uselibpgcommon = ();
 my $contrib_extralibs      = { 'libpq_pipeline' => ['ws2_32.lib'] };
 my $contrib_extraincludes  = {};
-my $contrib_extrasource    = {};
+my $contrib_extrasource    = {
+	'libpq_uri_regress' => ['src/interfaces/libpq/test/libpq_uri_regress.c'],
+	'libpq_testclient'  => ['src/interfaces/libpq/test/libpq_testclient.c'],
+};
 my @contrib_excludes       = (
 	'bool_plperl',     'commit_ts',
 	'hstore_plperl',   'hstore_plpython',
@@ -289,24 +292,6 @@ sub mkvcbuild
 	$libpqwalreceiver->AddIncludeDir('src/interfaces/libpq');
 	$libpqwalreceiver->AddReference($postgres, $libpq);
 
-	my $libpq_testclient =
-	  $solution->AddProject('libpq_testclient', 'exe', 'misc',
-		'src/interfaces/libpq/test');
-	$libpq_testclient->AddFile(
-		'src/interfaces/libpq/test/libpq_testclient.c');
-	$libpq_testclient->AddIncludeDir('src/interfaces/libpq');
-	$libpq_testclient->AddReference($libpgport, $libpq);
-	$libpq_testclient->AddLibrary('ws2_32.lib');
-
-	my $libpq_uri_regress =
-	  $solution->AddProject('libpq_uri_regress', 'exe', 'misc',
-		'src/interfaces/libpq/test');
-	$libpq_uri_regress->AddFile(
-		'src/interfaces/libpq/test/libpq_uri_regress.c');
-	$libpq_uri_regress->AddIncludeDir('src/interfaces/libpq');
-	$libpq_uri_regress->AddReference($libpgport, $libpq);
-	$libpq_uri_regress->AddLibrary('ws2_32.lib');
-
 	my $pgoutput = $solution->AddProject('pgoutput', 'dll', '',
 		'src/backend/replication/pgoutput');
 	$pgoutput->AddReference($postgres);
@@ -478,7 +463,7 @@ sub mkvcbuild
 		push @contrib_excludes, 'uuid-ossp';
 	}
 
-	foreach my $subdir ('contrib', 'src/test/modules')
+	foreach my $subdir ('contrib', 'src/test/modules', 'src/interfaces/libpq')
 	{
 		opendir($D, $subdir) || croak "Could not opendir on $subdir!\n";
 		while (my $d = readdir($D))
@@ -989,6 +974,15 @@ sub AddContrib
 		AdjustContribProj($proj);
 		push @projects, $proj;
 	}
+	elsif ($mf =~ /^PROGRAMS\s*=\s*(.*)$/mg)
+	{
+		foreach my $proj (split /\s+/, $1)
+		{
+			my $p = $solution->AddProject($proj, 'exe', 'contrib', "$subdir/$n");
+			AdjustContribProj($p);
+			push @projects, $p;
+		}
+	}
 	else
 	{
 		croak "Could not determine contrib module type for $n\n";
-- 
2.17.1

From e6469e3ed09a78884da7da318306b6ab30e9046b Mon Sep 17 00:00:00 2001
From: Andres Freund <andres@anarazel.de>
Date: Thu, 24 Feb 2022 08:27:41 -0800
Subject: [PATCH 16/23] Move libpq_pipeline test into src/interfaces/libpq.

https://www.postgresql.org/message-id/20220227014626.vi7g7km554gmoape%40alap3.anarazel.de

windows,
ci-os-only: freebsd, linux
---
 .../libpq/t/002_pipeline.pl}                  |  2 +-
 src/interfaces/libpq/test/.gitignore          |  1 +
 src/interfaces/libpq/test/Makefile            |  2 +-
 .../libpq/test}/libpq_pipeline.c              |  0
 .../test}/traces/disallowed_in_pipeline.trace |  0
 .../libpq/test}/traces/multi_pipelines.trace  |  0
 .../libpq/test}/traces/nosync.trace           |  0
 .../libpq/test}/traces/pipeline_abort.trace   |  0
 .../libpq/test}/traces/pipeline_idle.trace    |  0
 .../libpq/test}/traces/prepared.trace         |  0
 .../libpq/test}/traces/simple_pipeline.trace  |  0
 .../libpq/test}/traces/singlerow.trace        |  0
 .../libpq/test}/traces/transaction.trace      |  0
 src/test/modules/Makefile                     |  1 -
 src/test/modules/libpq_pipeline/.gitignore    |  5 ----
 src/test/modules/libpq_pipeline/Makefile      | 25 -------------------
 src/test/modules/libpq_pipeline/README        |  1 -
 src/tools/msvc/Mkvcbuild.pm                   |  5 ++--
 18 files changed, 6 insertions(+), 36 deletions(-)
 rename src/{test/modules/libpq_pipeline/t/001_libpq_pipeline.pl => interfaces/libpq/t/002_pipeline.pl} (96%)
 rename src/{test/modules/libpq_pipeline => interfaces/libpq/test}/libpq_pipeline.c (100%)
 rename src/{test/modules/libpq_pipeline => interfaces/libpq/test}/traces/disallowed_in_pipeline.trace (100%)
 rename src/{test/modules/libpq_pipeline => interfaces/libpq/test}/traces/multi_pipelines.trace (100%)
 rename src/{test/modules/libpq_pipeline => interfaces/libpq/test}/traces/nosync.trace (100%)
 rename src/{test/modules/libpq_pipeline => interfaces/libpq/test}/traces/pipeline_abort.trace (100%)
 rename src/{test/modules/libpq_pipeline => interfaces/libpq/test}/traces/pipeline_idle.trace (100%)
 rename src/{test/modules/libpq_pipeline => interfaces/libpq/test}/traces/prepared.trace (100%)
 rename src/{test/modules/libpq_pipeline => interfaces/libpq/test}/traces/simple_pipeline.trace (100%)
 rename src/{test/modules/libpq_pipeline => interfaces/libpq/test}/traces/singlerow.trace (100%)
 rename src/{test/modules/libpq_pipeline => interfaces/libpq/test}/traces/transaction.trace (100%)
 delete mode 100644 src/test/modules/libpq_pipeline/.gitignore
 delete mode 100644 src/test/modules/libpq_pipeline/Makefile
 delete mode 100644 src/test/modules/libpq_pipeline/README

diff --git a/src/test/modules/libpq_pipeline/t/001_libpq_pipeline.pl b/src/interfaces/libpq/t/002_pipeline.pl
similarity index 96%
rename from src/test/modules/libpq_pipeline/t/001_libpq_pipeline.pl
rename to src/interfaces/libpq/t/002_pipeline.pl
index 0821329c8d3..5ddde3425cc 100644
--- a/src/test/modules/libpq_pipeline/t/001_libpq_pipeline.pl
+++ b/src/interfaces/libpq/t/002_pipeline.pl
@@ -50,7 +50,7 @@ for my $testname (@tests)
 		my $expected;
 		my $result;
 
-		$expected = slurp_file_eval("traces/$testname.trace");
+		$expected = slurp_file_eval("test/traces/$testname.trace");
 		next unless $expected ne "";
 		$result = slurp_file_eval($traceout);
 		next unless $result ne "";
diff --git a/src/interfaces/libpq/test/.gitignore b/src/interfaces/libpq/test/.gitignore
index 6ba78adb678..5c1a281d679 100644
--- a/src/interfaces/libpq/test/.gitignore
+++ b/src/interfaces/libpq/test/.gitignore
@@ -1,2 +1,3 @@
 /libpq_testclient
 /libpq_uri_regress
+/libpq_pipeline
diff --git a/src/interfaces/libpq/test/Makefile b/src/interfaces/libpq/test/Makefile
index 91c8d87d528..bd459062434 100644
--- a/src/interfaces/libpq/test/Makefile
+++ b/src/interfaces/libpq/test/Makefile
@@ -14,7 +14,7 @@ endif
 override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += $(libpq_pgport)
 
-PROGRAMS = libpq_testclient libpq_uri_regress
+PROGRAMS = libpq_testclient libpq_uri_regress libpq_pipeline
 
 all: $(PROGRAMS)
 
diff --git a/src/test/modules/libpq_pipeline/libpq_pipeline.c b/src/interfaces/libpq/test/libpq_pipeline.c
similarity index 100%
rename from src/test/modules/libpq_pipeline/libpq_pipeline.c
rename to src/interfaces/libpq/test/libpq_pipeline.c
diff --git a/src/test/modules/libpq_pipeline/traces/disallowed_in_pipeline.trace b/src/interfaces/libpq/test/traces/disallowed_in_pipeline.trace
similarity index 100%
rename from src/test/modules/libpq_pipeline/traces/disallowed_in_pipeline.trace
rename to src/interfaces/libpq/test/traces/disallowed_in_pipeline.trace
diff --git a/src/test/modules/libpq_pipeline/traces/multi_pipelines.trace b/src/interfaces/libpq/test/traces/multi_pipelines.trace
similarity index 100%
rename from src/test/modules/libpq_pipeline/traces/multi_pipelines.trace
rename to src/interfaces/libpq/test/traces/multi_pipelines.trace
diff --git a/src/test/modules/libpq_pipeline/traces/nosync.trace b/src/interfaces/libpq/test/traces/nosync.trace
similarity index 100%
rename from src/test/modules/libpq_pipeline/traces/nosync.trace
rename to src/interfaces/libpq/test/traces/nosync.trace
diff --git a/src/test/modules/libpq_pipeline/traces/pipeline_abort.trace b/src/interfaces/libpq/test/traces/pipeline_abort.trace
similarity index 100%
rename from src/test/modules/libpq_pipeline/traces/pipeline_abort.trace
rename to src/interfaces/libpq/test/traces/pipeline_abort.trace
diff --git a/src/test/modules/libpq_pipeline/traces/pipeline_idle.trace b/src/interfaces/libpq/test/traces/pipeline_idle.trace
similarity index 100%
rename from src/test/modules/libpq_pipeline/traces/pipeline_idle.trace
rename to src/interfaces/libpq/test/traces/pipeline_idle.trace
diff --git a/src/test/modules/libpq_pipeline/traces/prepared.trace b/src/interfaces/libpq/test/traces/prepared.trace
similarity index 100%
rename from src/test/modules/libpq_pipeline/traces/prepared.trace
rename to src/interfaces/libpq/test/traces/prepared.trace
diff --git a/src/test/modules/libpq_pipeline/traces/simple_pipeline.trace b/src/interfaces/libpq/test/traces/simple_pipeline.trace
similarity index 100%
rename from src/test/modules/libpq_pipeline/traces/simple_pipeline.trace
rename to src/interfaces/libpq/test/traces/simple_pipeline.trace
diff --git a/src/test/modules/libpq_pipeline/traces/singlerow.trace b/src/interfaces/libpq/test/traces/singlerow.trace
similarity index 100%
rename from src/test/modules/libpq_pipeline/traces/singlerow.trace
rename to src/interfaces/libpq/test/traces/singlerow.trace
diff --git a/src/test/modules/libpq_pipeline/traces/transaction.trace b/src/interfaces/libpq/test/traces/transaction.trace
similarity index 100%
rename from src/test/modules/libpq_pipeline/traces/transaction.trace
rename to src/interfaces/libpq/test/traces/transaction.trace
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 6c31c8707c2..bb7a80748fe 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -10,7 +10,6 @@ SUBDIRS = \
 		  delay_execution \
 		  dummy_index_am \
 		  dummy_seclabel \
-		  libpq_pipeline \
 		  plsample \
 		  snapshot_too_old \
 		  spgist_name_ops \
diff --git a/src/test/modules/libpq_pipeline/.gitignore b/src/test/modules/libpq_pipeline/.gitignore
deleted file mode 100644
index 3a11e786b83..00000000000
--- a/src/test/modules/libpq_pipeline/.gitignore
+++ /dev/null
@@ -1,5 +0,0 @@
-# Generated subdirectories
-/log/
-/results/
-/tmp_check/
-/libpq_pipeline
diff --git a/src/test/modules/libpq_pipeline/Makefile b/src/test/modules/libpq_pipeline/Makefile
deleted file mode 100644
index 65acc3e997e..00000000000
--- a/src/test/modules/libpq_pipeline/Makefile
+++ /dev/null
@@ -1,25 +0,0 @@
-# src/test/modules/libpq_pipeline/Makefile
-
-PGFILEDESC = "libpq_pipeline - test program for pipeline execution"
-PGAPPICON = win32
-
-PROGRAM = libpq_pipeline
-OBJS = $(WIN32RES) libpq_pipeline.o
-
-NO_INSTALL = 1
-
-PG_CPPFLAGS = -I$(libpq_srcdir)
-PG_LIBS_INTERNAL += $(libpq_pgport)
-
-TAP_TESTS = 1
-
-ifdef USE_PGXS
-PG_CONFIG = pg_config
-PGXS := $(shell $(PG_CONFIG) --pgxs)
-include $(PGXS)
-else
-subdir = src/test/modules/libpq_pipeline
-top_builddir = ../../../..
-include $(top_builddir)/src/Makefile.global
-include $(top_srcdir)/contrib/contrib-global.mk
-endif
diff --git a/src/test/modules/libpq_pipeline/README b/src/test/modules/libpq_pipeline/README
deleted file mode 100644
index d8174dd579a..00000000000
--- a/src/test/modules/libpq_pipeline/README
+++ /dev/null
@@ -1 +0,0 @@
-Test programs and libraries for libpq
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index 5277ef068f1..449587fe50b 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -36,13 +36,14 @@ my @unlink_on_exit;
 
 # Set of variables for modules in contrib/ and src/test/modules/
 my @contrib_uselibpq       = ();
-my @contrib_uselibpgport   = ('libpq_uri_regress', 'libpq_testclient');
-my @contrib_uselibpgcommon = ();
+my @contrib_uselibpgport   = ('libpq_uri_regress', 'libpq_testclient', 'libpq_pipeline');
+my @contrib_uselibpgcommon = ('libpq_pipeline');
 my $contrib_extralibs      = { 'libpq_pipeline' => ['ws2_32.lib'] };
 my $contrib_extraincludes  = {};
 my $contrib_extrasource    = {
 	'libpq_uri_regress' => ['src/interfaces/libpq/test/libpq_uri_regress.c'],
 	'libpq_testclient'  => ['src/interfaces/libpq/test/libpq_testclient.c'],
+	'libpq_pipeline'    => ['src/interfaces/libpq/test/libpq_pipeline.c'],
 };
 my @contrib_excludes       = (
 	'bool_plperl',     'commit_ts',
-- 
2.17.1

From 66939379946922c63f90f9380936d486f925dfcb Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Mon, 17 Jan 2022 00:54:28 -0600
Subject: [PATCH 17/23] cirrus: code coverage

https://www.postgresql.org/message-id/202202111821.w3gqblvfp4pr%40alvherre.pgsql
https://www.postgresql.org/message-id/flat/20220409021853.GP24419@telsasoft.com

ci-os-only: linux
---
 .cirrus.yml                       | 16 ++++++++++++++++
 src/tools/ci/code-coverage-report | 29 +++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+)
 create mode 100755 src/tools/ci/code-coverage-report

diff --git a/.cirrus.yml b/.cirrus.yml
index a1541db2bc8..e5e5d113c4c 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -28,6 +28,13 @@ env:
   TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
   PG_TEST_EXTRA: kerberos ldap ssl
 
+  # The commit that this branch is rebased on.  There's no easy way to find this.
+  # This does the right thing for cfbot, which always squishes all patches into a single commit.
+  # And does the right thing for any 1-patch commits.
+  # Patch series manually submitted to cirrus would benefit from setting this to the
+  # number of patches in the series (or directly to the commit the series was rebased on).
+  BASE_COMMIT: HEAD~1
+
 
 # What files to preserve in case tests fail
 on_failure: &on_failure
@@ -185,6 +192,7 @@ task:
     cat /proc/cmdline
     ulimit -a -H && ulimit -a -S
     export
+    git diff --name-only "$BASE_COMMIT"
   create_user_script: |
     useradd -m postgres
     chown -R postgres:postgres .
@@ -204,6 +212,7 @@ task:
     su postgres <<-EOF
       ./configure \
         --enable-cassert --enable-debug --enable-tap-tests \
+        --enable-coverage \
         --enable-nls \
         \
         ${LINUX_CONFIGURE_FEATURES} \
@@ -227,6 +236,13 @@ task:
       make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
     EOF
 
+  # Build coverage report for files changed since the base commit.
+  generate_coverage_report_script: |
+    src/tools/ci/code-coverage-report "$BASE_COMMIT"
+
+  coverage_artifacts:
+    paths: ['coverage/**/*.html', 'coverage/**/*.png', 'coverage/**/*.gcov', 'coverage/**/*.css' ]
+
   on_failure:
     <<: *on_failure
     cores_script: src/tools/ci/cores_backtrace.sh linux /tmp/cores
diff --git a/src/tools/ci/code-coverage-report b/src/tools/ci/code-coverage-report
new file mode 100755
index 00000000000..0dce149dcf8
--- /dev/null
+++ b/src/tools/ci/code-coverage-report
@@ -0,0 +1,29 @@
+#! /bin/sh
+# Called during the linux CI task to generate a code coverage report.
+set -e
+
+base_branch=$1
+changed=`git diff --name-only "$base_branch" '*.c'` ||
+	[ $? -eq 1 ]
+
+outdir=coverage
+mkdir "$outdir"
+
+# Coverage is shown only for changed files
+# This is useful to see coverage of newly-added code, but won't
+# show added/lost coverage in files which this patch doesn't modify.
+
+gcov=$outdir/coverage.gcov
+for f in $changed
+do
+	# Avoid removed files
+	[ -f "$f" ] || continue
+
+	lcov --quiet --capture --directory "$f"
+done >"$gcov"
+
+# Exit successfully if no relevant files were changed
+[ -s "$gcov" ] || exit 0
+
+genhtml "$gcov" --show-details --legend --quiet --num-spaces=4 --output-directory "$outdir" --title="Coverage report of files changed since: $base_branch"
+cp "$outdir"/index.html "$outdir"/00-index.html
-- 
2.17.1

From 9bcc7fb925a32eacd9881ecca7955c98cb8c6e8f Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 26 Feb 2022 19:34:35 -0600
Subject: [PATCH 18/23] cirrus: build docs as a separate task..

This will run the doc build if any docs have changed, even if Linux
fails, to allow catch doc build failures.

This'll automatically show up as a separate "column" on cfbot.

Also, in the future, this will hopefully upload each patch's changed HTML docs
as an artifact, for easy review.

XX-os-only: html
---
 .cirrus.yml | 49 +++++++++++++++++++++++++++++++++++--------------
 1 file changed, 35 insertions(+), 14 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index e5e5d113c4c..b3391e8aae6 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -620,20 +620,6 @@ task:
       time make -s -j${BUILD_JOBS} world-bin
       ccache --show-stats
 
-  ###
-  # Verify docs can be built
-  ###
-  # XXX: Only do this if there have been changes in doc/ since last build
-  always:
-    docs_build_script: |
-      time ./configure \
-        --cache gcc.cache \
-        CC="ccache gcc" \
-        CXX="ccache g++" \
-        CLANG="ccache clang"
-      make -s -j${BUILD_JOBS} clean
-      time make -s -j${BUILD_JOBS} -C doc
-
   ###
   # Verify headerscheck / cpluspluscheck succeed
   #
@@ -659,3 +645,38 @@ task:
 
   always:
     upload_caches: ccache
+
+
+###
+# Verify docs can be built
+# changesInclude() will skip this task if none of the commits since
+# CIRRUS_LAST_GREEN_CHANGE touched any relevant files. The comparison appears
+# to be like "git log a..b -- ./file", not "git diff a..b -- ./file"
+###
+
+task:
+  name: Documentation
+
+  env:
+    CPUS: 1
+    BUILD_JOBS: 1
+
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(docs|html).*'
+  skip: "!changesInclude('.cirrus.yml', 'doc/**')"
+
+  container:
+    image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
+    cpu: $CPUS
+    memory: 2G
+
+  sysinfo_script: |
+    id
+    uname -a
+    cat /proc/cmdline
+    ulimit -a -H && ulimit -a -S
+    export
+
+  # Exercise HTML and other docs:
+  docs_build_script: |
+    time ./configure
+    make -s -j${BUILD_JOBS} -C doc
-- 
2.17.1

From b64d942554d2598f502d48586a1876988a777b55 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 26 Feb 2022 19:39:10 -0600
Subject: [PATCH 19/23] cirrus: upload changed html docs as artifacts

This could be done on the client side (cfbot).  One advantage of doing it here
is that fewer docs are uploaded - many patches won't upload docs at all.

https://www.postgresql.org/message-id/flat/20220409021853.GP24419@telsasoft.com

https://cirrus-ci.com/task/5396696388599808

ci-os-only: html
---
 .cirrus.yml                    | 19 +++++++++++++++++--
 src/tools/ci/copy-changed-docs | 23 +++++++++++++++++++++++
 2 files changed, 40 insertions(+), 2 deletions(-)
 create mode 100755 src/tools/ci/copy-changed-docs

diff --git a/.cirrus.yml b/.cirrus.yml
index b3391e8aae6..8fca4de219f 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -648,7 +648,7 @@ task:
 
 
 ###
-# Verify docs can be built
+# Verify docs can be built, and upload changed docs as artifacts
 # changesInclude() will skip this task if none of the commits since
 # CIRRUS_LAST_GREEN_CHANGE touched any relevant files. The comparison appears
 # to be like "git log a..b -- ./file", not "git diff a..b -- ./file"
@@ -662,7 +662,7 @@ task:
     BUILD_JOBS: 1
 
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(docs|html).*'
-  skip: "!changesInclude('.cirrus.yml', 'doc/**')"
+  #skip: "!changesInclude('.cirrus.yml', 'doc/**', 'src/tools/ci/copy-changed-docs')"
 
   container:
     image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
@@ -675,8 +675,23 @@ task:
     cat /proc/cmdline
     ulimit -a -H && ulimit -a -S
     export
+    git diff --name-only "$BASE_COMMIT"
 
   # Exercise HTML and other docs:
   docs_build_script: |
     time ./configure
     make -s -j${BUILD_JOBS} -C doc
+    cp -r doc new-docs
+
+    # Build HTML docs from the base commit.
+    git checkout "$BASE_COMMIT" -- doc
+    #html-stamp
+    make -s -C doc clean
+    make -s -C doc html
+    cp -r doc old-docs
+
+  copy_changed_docs_script: |
+    src/tools/ci/copy-changed-docs "old-docs" "new-docs" "html_docs"
+
+  html_docs_artifacts:
+    paths: ['html_docs/*.html', 'html_docs/*.png', 'html_docs/*.css']
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
new file mode 100755
index 00000000000..1584f974d94
--- /dev/null
+++ b/src/tools/ci/copy-changed-docs
@@ -0,0 +1,23 @@
+#! /bin/sh
+# Copy HTML which differ between $old and $new into $outdir
+set -e
+
+old=$1
+new=$2
+outdir=$3
+
+mkdir "$outdir"
+cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
+
+changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
+	[ $? -eq 1 ]
+
+for f in $changed
+do
+	# Avoid removed files
+	[ -f "$f" ] || continue
+
+	cp -v "$f" "$outdir"
+done
+
+exit 0
-- 
2.17.1

From 24c579e3347e69b7e322cd008c14e5c96bdc6aee Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Mon, 28 Feb 2022 23:18:19 -0600
Subject: [PATCH 20/23] +html index file

This allows linking to the artifacts from the last successful build, which
itself allows *not* rebuilding when sources haven't changed.

//freebsd
ci-os-only: html,
---
 .cirrus.yml                    |  3 ++-
 src/tools/ci/copy-changed-docs | 45 ++++++++++++++++++++++++++++++++--
 2 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 8fca4de219f..4f0695e455d 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -662,7 +662,8 @@ task:
     BUILD_JOBS: 1
 
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(docs|html).*'
-  #skip: "!changesInclude('.cirrus.yml', 'doc/**', 'src/tools/ci/copy-changed-docs')"
+  skip: "!changesInclude('.cirrus.yml', 'doc/**', 'src/tools/ci/copy-changed-docs')"
+  #skip: "!changesInclude('doc/**')"
 
   container:
     image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
index 1584f974d94..0efad386cca 100755
--- a/src/tools/ci/copy-changed-docs
+++ b/src/tools/ci/copy-changed-docs
@@ -5,10 +5,27 @@ set -e
 old=$1
 new=$2
 outdir=$3
+branch=$CIRRUS_BRANCH
+
+# The index is large and changes often
+skippages="bookindex.html"
 
 mkdir "$outdir"
 cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
 
+# The index is useful to allow a static link (not specific to a cirrus run) to the artifacts for the most-recent, successful CI run for a branch
+branchurl=https://api.cirrus-ci.com/v1/artifact/github/$CIRRUS_REPO_FULL_NAME/$CIRRUS_TASK_NAME/html_docs/html_docs/00-index.html?branch=$branch
+
+index="$outdir/00-index.html"
+cat >"$index" <<EOF
+<html>
+<head><title>Index of docs changed since: $BASE_COMMIT</title></head>
+<body>
+<!-- A link to documentation for the most recent successful build: $branchurl -->
+<h1>Index of docs changed since: $BASE_COMMIT</h1>
+<ul>
+EOF
+
 changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
 	[ $? -eq 1 ]
 
@@ -17,7 +34,31 @@ do
 	# Avoid removed files
 	[ -f "$f" ] || continue
 
-	cp -v "$f" "$outdir"
-done
+	echo "$f" |grep -Ew "$skippages" >/dev/null &&
+		continue
+
+	cp -v "$f" "$outdir" >&2
+	fn=${f##*/}
+	# ?branch=... is needed when accessing the artifacts for the static link for the branch
+	# It's not needed and ignored if accessing artifacts for *this* CI run
+	echo "<li><a href='$fn?branch=$branch'>$fn</a>"
+done >>"$index"
+
+github=https://github.com/$CIRRUS_REPO_FULL_NAME/commit
+cirrus=https://cirrus-ci.com/build
+
+cat >>"$index" <<EOF
+</ul>
+<hr>
+<code>
+<br>This file was written on: `date --rfc-822 --utc`
+<br>CIRRUS_CHANGE_TITLE: $CIRRUS_CHANGE_TITLE
+<br>CIRRUS_CHANGE_IN_REPO: <a href="$github/$CIRRUS_CHANGE_IN_REPO">$CIRRUS_CHANGE_IN_REPO</a>
+<br>CIRRUS_BUILD_ID: <a href="$cirrus/$CIRRUS_BUILD_ID">$CIRRUS_BUILD_ID</a>
+<br>CIRRUS_LAST_GREEN_CHANGE: <a href="$github/$CIRRUS_LAST_GREEN_CHANGE">$CIRRUS_LAST_GREEN_CHANGE</a>
+<br>CIRRUS_LAST_GREEN_BUILD_ID: <a href="$cirrus/$CIRRUS_LAST_GREEN_BUILD_ID">$CIRRUS_LAST_GREEN_BUILD_ID</a>
+</code>
+</body></html>
+EOF
 
 exit 0
-- 
2.17.1

From c56a83a2f4f986c519ffd0302b8cb584c1b08412 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Mon, 5 Sep 2022 18:36:46 -0500
Subject: [PATCH 23/23] WIP: skip building if only docs have changed

changesOnlyInclude() will skip tasks if the commits since
CIRRUS_LAST_GREEN_CHANGE touched no files outside of doc/.  The
comparison appears to be like "git log a..b -- ./file", not
"git diff a..b -- ./file"

For the moment this is limited to windows, since it's 3x slower than
other OSs.
---
 .cirrus.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.cirrus.yml b/.cirrus.yml
index f8cd7e030e9..0bf7747abba 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -429,6 +429,7 @@ task:
     CIRRUS_ESCAPING_PROCESSES: 1
 
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
+  skip: "changesIncludeOnly('doc/*')"
 
   windows_container:
     image: $CONTAINER_REPO/windows_ci_vs_2019:latest
-- 
2.17.1

From fb7f8413a9efa136e73911e56f9970d8289ee178 Mon Sep 17 00:00:00 2001
From: Andres Freund <andres@anarazel.de>
Date: Sun, 2 Oct 2022 11:26:19 -0700
Subject: [PATCH v1] ci: Introduce a SanityCheck task that other tasks depend
 on

---
 .cirrus.yml | 94 +++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 87 insertions(+), 7 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 531cfe96f65..87dc7025eaf 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -55,6 +55,83 @@ on_failure_meson: &on_failure_meson
     type: text/plain
 
 
+# To avoid unnecessarily spinning up a lot of VMs / containers for entirely
+# broken commits, have a very minimal test that all others depend on.
+task:
+  name: SanityCheck
+
+  # If a specific OS is requested, don't run the sanity check. This shortens
+  # push-wait-for-ci cycles a bit when debugging operating system specific
+  # failures.
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*'
+
+  env:
+    CPUS: 4
+    BUILD_JOBS: 8
+    TEST_JOBS: 8
+    CCACHE_DIR: ${CIRRUS_WORKING_DIR}/ccache_dir
+    # no options enabled, should be small
+    CCACHE_MAXSIZE: "150M"
+
+  container:
+    image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
+    cpu: $CPUS
+
+  ccache_cache:
+    folder: $CCACHE_DIR
+
+  create_user_script: |
+    useradd -m postgres
+    chown -R postgres:postgres .
+    mkdir -p ${CCACHE_DIR}
+    chown -R postgres:postgres ${CCACHE_DIR}
+    echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf
+    su postgres -c "ulimit -l -H && ulimit -l -S"
+    # Can't change container's kernel.core_pattern. Postgres user can't write
+    # to / normally. Change that.
+    chown root:postgres /
+    chmod g+rwx /
+
+  configure_script: |
+    su postgres <<-EOF
+      meson setup \
+        --buildtype=debug \
+        --auto-features=disabled \
+        -Dtap_tests=enabled \
+        build
+    EOF
+  build_script: |
+    su postgres <<-EOF
+      set -e
+      ccache -s
+      ccache -z
+      ninja -C build -j${BUILD_JOBS}
+      ccache -s
+    EOF
+  upload_caches: ccache
+
+  # Run a minimal set of tests. The main regression tests take too long for
+  # this purpose. I chose a random pg_regress style test, and a tap test that
+  # exercises both a frontend binary and the backend.
+  test_minimal_script: |
+    su postgres <<-EOF
+      ulimit -c unlimited
+      meson test $MTEST_ARGS --num-processes ${TEST_JOBS} \
+        tmp_install cube/regress pg_ctl/001_start_stop
+    EOF
+
+  on_failure:
+    <<: *on_failure_meson
+    cores_script: |
+      shopt -s nullglob
+      mkdir -m 770 /tmp/cores
+      ls -l /
+      if [ -n "$(echo /core*)" ] ; then
+        mv /core* /tmp/cores/
+        src/tools/ci/cores_backtrace.sh linux /tmp/cores
+      fi
+
+
 task:
   name: FreeBSD - 13 - Meson
 
@@ -69,6 +146,7 @@ task:
     CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST
     CFLAGS: -Og -ggdb
 
+  depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
 
   compute_engine_instance:
@@ -170,6 +248,7 @@ task:
     LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
     LINUX_MESON_FEATURES: *LINUX_MESON_FEATURES
 
+  depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
 
   compute_engine_instance:
@@ -311,6 +390,7 @@ task:
     CFLAGS: -Og -ggdb
     CXXFLAGS: -Og -ggdb
 
+  depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
 
   osx_instance:
@@ -428,6 +508,7 @@ task:
     # 0x8001 is SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX
     CIRRUS_WINDOWS_ERROR_MODE: 0x8001
 
+  depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
 
   windows_container:
@@ -467,9 +548,12 @@ task:
 task:
   name: CompilerWarnings
 
-  # To limit unnecessary work only run this once the normal linux test succeeds
-  depends_on:
-    - Linux - Debian Bullseye - Meson
+  # To limit unnecessary work only run this once the SanityCheck
+  # succeeds. This is particularly important for this task as we intentionally
+  # use always: to continue after failures. Task that did not run count as a
+  # success, so we need to recheck SanityChecks's condition here ...
+  depends_on: SanityCheck
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*'
 
   env:
     CPUS: 4
@@ -483,10 +567,6 @@ task:
     LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
     LINUX_MESON_FEATURES: *LINUX_MESON_FEATURES
 
-  # task that did not run, count as a success, so we need to recheck Linux'
-  # condition here ...
-  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
-
   container:
     image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
     cpu: $CPUS
-- 
2.37.3.542.gdd3f6c4cae

From a994127b0361e2bc3ff59919176ec5f81faa426e Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 30 Sep 2022 13:39:43 -0500
Subject: [PATCH 2/8] meson: other fixes for cygwin

XXX: what about HAVE_BUGGY_STRTOF ?

See: 20221021034040.GT16921@telsasoft.com
---
 meson.build                  | 8 ++++++--
 src/port/meson.build         | 4 ++++
 src/test/regress/meson.build | 2 +-
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/meson.build b/meson.build
index ce2f223a409..ed24370672a 100644
--- a/meson.build
+++ b/meson.build
@@ -211,6 +211,10 @@ if host_system == 'aix'
 
 elif host_system == 'cygwin'
   cppflags += '-D_GNU_SOURCE'
+  dlsuffix = '.dll'
+  mod_link_args_fmt = ['@0@']
+  mod_link_with_name = 'lib@0@.exe.a'
+  mod_link_with_dir = 'libdir'
 
 elif host_system == 'darwin'
   dlsuffix = '.dylib'
@@ -2301,8 +2305,8 @@ gnugetopt_dep = cc.find_library('gnugetopt', required: false)
 #   (i.e., allow '-' as a flag character), so use our version on those platforms
 # - We want to use system's getopt_long() only if the system provides struct
 #   option
-always_replace_getopt = host_system in ['windows', 'openbsd', 'solaris']
-always_replace_getopt_long = host_system == 'windows' or not cdata.has('HAVE_STRUCT_OPTION')
+always_replace_getopt = host_system in ['windows', 'cygwin', 'openbsd', 'solaris']
+always_replace_getopt_long = host_system in ['windows', 'cygwin'] or not cdata.has('HAVE_STRUCT_OPTION')
 
 # Required on BSDs
 execinfo_dep = cc.find_library('execinfo', required: false)
diff --git a/src/port/meson.build b/src/port/meson.build
index c2222696f1b..0ba83cc7930 100644
--- a/src/port/meson.build
+++ b/src/port/meson.build
@@ -40,6 +40,10 @@ if host_system == 'windows'
     'win32setlocale.c',
     'win32stat.c',
   )
+elif host_system == 'cygwin'
+  pgport_sources += files(
+    'dirmod.c',
+  )
 endif
 
 if cc.get_id() == 'msvc'
diff --git a/src/test/regress/meson.build b/src/test/regress/meson.build
index 3dcfc11278f..6ec3c77af53 100644
--- a/src/test/regress/meson.build
+++ b/src/test/regress/meson.build
@@ -10,7 +10,7 @@ regress_sources = pg_regress_c + files(
 # patterns like ".*-.*-mingw.*". We probably can do better, but for now just
 # replace 'gcc' with 'mingw' on windows.
 host_tuple_cc = cc.get_id()
-if host_system == 'windows' and host_tuple_cc == 'gcc'
+if host_system in ['windows', 'cygwin'] and host_tuple_cc == 'gcc'
   host_tuple_cc = 'mingw'
 endif
 host_tuple = '@0@-@1@-@2@'.format(host_cpu, host_system, host_tuple_cc)
-- 
2.25.1

From 3825287f88ad03ed3335ab6c7ea1ca2b95d88356 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 30 Sep 2022 08:56:07 -0500
Subject: [PATCH 1/8] meson: PROVE is not required

It ought to be possible to build the application without running tests,
or without running TAP tests.  And it's may be needed to support
cygwin/buildfarm, where TAP tests consistently fail..

See: 20221021034040.GT16921@telsasoft.com
---
 meson.build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meson.build b/meson.build
index bfacbdc0af6..ce2f223a409 100644
--- a/meson.build
+++ b/meson.build
@@ -323,7 +323,7 @@ python = find_program(get_option('PYTHON'), required: true, native: true)
 flex = find_program(get_option('FLEX'), native: true, version: '>= 2.5.35')
 bison = find_program(get_option('BISON'), native: true, version: '>= 2.3')
 sed = find_program(get_option('SED'), 'sed', native: true)
-prove = find_program(get_option('PROVE'), native: true)
+prove = find_program(get_option('PROVE'), native: true, required: false)
 tar = find_program(get_option('TAR'), native: true)
 gzip = find_program(get_option('GZIP'), native: true)
 program_lz4 = find_program(get_option('LZ4'), native: true, required: false)
-- 
2.25.1

From 8eddfa78e4ec82e14bff1c8af36efe0b622336f0 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 4 Nov 2022 11:53:20 -0500
Subject: [PATCH 3/8] meson: rename 'main' tasks to 'regress' and 'isolation'

See also:
20221001221514.2yy257v4zdfhwiy2@awork3.anarazel.de
20221021123435.GU16921@telsasoft.com
---
 src/test/isolation/meson.build | 2 +-
 src/test/regress/meson.build   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/isolation/meson.build b/src/test/isolation/meson.build
index ba27b8c1d44..e2ed400a80a 100644
--- a/src/test/isolation/meson.build
+++ b/src/test/isolation/meson.build
@@ -58,7 +58,7 @@ isolationtester = executable('isolationtester',
 bin_targets += isolationtester
 
 tests += {
-  'name': 'main',
+  'name': 'isolation',
   'sd': meson.current_source_dir(),
   'bd': meson.current_build_dir(),
   'isolation': {
diff --git a/src/test/regress/meson.build b/src/test/regress/meson.build
index 6ec3c77af53..4f7122f99df 100644
--- a/src/test/regress/meson.build
+++ b/src/test/regress/meson.build
@@ -64,7 +64,7 @@ testprep_targets += refint_regress
 
 
 tests += {
-  'name': 'main',
+  'name': 'regress',
   'sd': meson.current_source_dir(),
   'bd': meson.current_build_dir(),
   'regress': {
-- 
2.25.1

From 0acbbd2fdd97bbafc5c4552e26f92d52c597eea9 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 25 May 2022 21:53:22 -0500
Subject: [PATCH 4/8] cirrus/windows: add compiler_warnings_script

I'm not sure how to write this test in windows shell; it's also not easy to
write it in posix sh, since windows shell is somehow interpretting && and ||...

https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de

See also:
8a1ce5e54f6d144e4f8e19af7c767b026ee0c956
https://cirrus-ci.com/task/6241060062494720
https://cirrus-ci.com/task/6496366607204352

ci-os-only: windows
---
 .cirrus.yml                            | 10 +++++++++-
 src/tools/ci/windows-compiler-warnings | 24 ++++++++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100755 src/tools/ci/windows-compiler-warnings

diff --git a/.cirrus.yml b/.cirrus.yml
index 9f2282471a9..99ac09dc679 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -451,12 +451,20 @@ task:
 
   build_script: |
     vcvarsall x64
-    ninja -C build
+    ninja -C build |tee build/meson-logs/build.txt
+    REM Since pipes lose exit status of the preceding command, rerun compilation,
+    REM without the pipe exiting now if it fails, rather than trying to run checks
+    ninja -C build > nul
 
   check_world_script: |
     vcvarsall x64
     meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
 
+  # This should be last, so check_world is always run
+  always:
+    compiler_warnings_script:
+      - sh src\tools\ci\windows-compiler-warnings build/meson-logs/build.txt build/meson-logs/build-warnings.txt
+
   on_failure:
     <<: *on_failure_meson
     crashlog_artifacts:
diff --git a/src/tools/ci/windows-compiler-warnings b/src/tools/ci/windows-compiler-warnings
new file mode 100755
index 00000000000..685aa995d5c
--- /dev/null
+++ b/src/tools/ci/windows-compiler-warnings
@@ -0,0 +1,24 @@
+#! /bin/sh
+# Success if the given exists and is empty, else fail
+# This is a separate file to avoid dealing with windows shell quoting and escaping,
+# or giving the impression that one-liner scripts will work trivially.
+set -e
+
+infile=$1
+outfile=$2
+
+# Looks like:
+# [19:39:27.130] c:\cirrus\src\backend\backup\basebackup_zstd.c(80) : warning C4715: 'bbsink_zstd_new': not all control paths return a value
+# should include linker warnings?
+grep ": warning " "$infile" > $outfile ||
+	[ $? -eq 1 ]
+
+if [ -s "$outfile" ]
+then
+	# Display the file's content, then exit indicating failure
+	cat "$outfile"
+	exit 1
+else
+	# Success
+	exit 0
+fi
-- 
2.25.1

From e85fe83fd8a4b4c79a96d2bf66cd6a5e1bdfcd1e Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 26 Feb 2022 19:34:35 -0600
Subject: [PATCH 5/8] cirrus: build docs as a separate task..

This will run the doc build if any docs have changed, even if Linux
fails, to allow catch doc build failures.

This'll automatically show up as a separate "column" on cfbot.

Also, in the future, this will hopefully upload each patch's changed HTML docs
as an artifact, for easy review.

Note that this is currently building docs with both autoconf and meson.

ci-os-only: html
---
 .cirrus.yml | 62 +++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 44 insertions(+), 18 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 99ac09dc679..37fd79e5b77 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -472,6 +472,9 @@ task:
       type: text/plain
 
 
+###
+# Test that code can be built with gcc/clang without warnings
+###
 task:
   name: CompilerWarnings
 
@@ -515,10 +518,6 @@ task:
     #apt-get update
     #DEBIAN_FRONTEND=noninteractive apt-get -y install ...
 
-  ###
-  # Test that code can be built with gcc/clang without warnings
-  ###
-
   setup_script: echo "COPT=-Werror" > src/Makefile.custom
 
   # Trace probes have a history of getting accidentally broken. Use the
@@ -580,20 +579,6 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  ###
-  # Verify docs can be built
-  ###
-  # XXX: Only do this if there have been changes in doc/ since last build
-  always:
-    docs_build_script: |
-      time ./configure \
-        --cache gcc.cache \
-        CC="ccache gcc" \
-        CXX="ccache g++" \
-        CLANG="ccache clang"
-      make -s -j${BUILD_JOBS} clean
-      time make -s -j${BUILD_JOBS} -C doc
-
   ###
   # Verify headerscheck / cpluspluscheck succeed
   #
@@ -617,3 +602,44 @@ task:
 
   always:
     upload_caches: ccache
+
+
+###
+# Verify docs can be built
+# changesInclude() will skip this task if none of the commits since
+# CIRRUS_LAST_GREEN_CHANGE touched any relevant files. The comparison appears
+# to be like "git log a..b -- ./file", not "git diff a..b -- ./file"
+###
+
+task:
+  name: Documentation
+
+  env:
+    CPUS: 1
+    BUILD_JOBS: 1
+
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(docs|html).*'
+  skip: "!changesInclude('.cirrus.yml', 'doc/**')"
+
+  container:
+    image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
+    cpu: $CPUS
+    memory: 2G
+
+  sysinfo_script: |
+    id
+    uname -a
+    cat /proc/cmdline
+    ulimit -a -H && ulimit -a -S
+    export
+
+  # Exercise HTML and other docs:
+  ninja_docs_build_script: |
+    mkdir build.ninja
+    cd build.ninja
+    time meson setup
+    time ninja docs
+
+  docs_build_script: |
+    time ./configure
+    make -s -j${BUILD_JOBS} -C doc
-- 
2.25.1

From adebe93a4409990e929f2775d45c6613134a4243 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Tue, 26 Jul 2022 20:30:02 -0500
Subject: [PATCH 6/8] cirrus/ccache: add explicit cache keys..

Since otherwise, building with ci-os-only will probably fail to use the
normal cache, since the cache key is computed using both the task name
and its *index* in the list of caches (internal/executor/cache.go:184).
---
 .cirrus.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.cirrus.yml b/.cirrus.yml
index 37fd79e5b77..221431e70c4 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -87,6 +87,9 @@ task:
 
   ccache_cache:
     folder: $CCACHE_DIR
+    fingerprint_key: ccache/freebsd
+    reupload_on_changes: true
+
   # Workaround around performance issues due to 32KB block size
   repartition_script: src/tools/ci/gcp_freebsd_repartition.sh
   create_user_script: |
@@ -181,6 +184,8 @@ task:
 
   ccache_cache:
     folder: ${CCACHE_DIR}
+    fingerprint_key: ccache/linux
+    reupload_on_changes: true
 
   sysinfo_script: |
     id
@@ -359,6 +364,9 @@ task:
 
   ccache_cache:
     folder: $CCACHE_DIR
+    fingerprint_key: ccache/macos
+    reupload_on_changes: true
+
   configure_script: |
     brewpath="/usr/local"
     PKG_CONFIG_PATH="${brewpath}/lib/pkgconfig:${PKG_CONFIG_PATH}"
@@ -513,6 +521,8 @@ task:
 
   ccache_cache:
     folder: $CCACHE_DIR
+    fingerprint_key: ccache/warnings
+    reupload_on_changes: true
 
   setup_additional_packages_script: |
     #apt-get update
-- 
2.25.1

From f16739bc5d2087847129baf663aa25fa9edb8449 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sun, 3 Apr 2022 00:10:20 -0500
Subject: [PATCH 7/8] cirrus/ccache: disable compression and show stats

Since v4.0, ccache enables zstd compression by default, saving roughly
2x-3x.  But, cirrus caches are compressed as tar.gz, so we could disable
ccache compression, allowing cirrus to gzip the uncompressed data
(better than ccache's default of zstd-1).

With default compression enabled (https://cirrus-ci.com/task/6692342840164352):
linux/debian/bullseye has 4.2; 99MB cirrus cache; cache_size_kibibyte	109616
macos has 4.5.1: 47MB cirrus cache; cache_size_kibibyte	52500
freebsd has 3.7.12: 42MB cirrus cache; cache_size_kibibyte	134064
XXX windows has 4.7.2; 180MB cirrus cache; cache_size_kibibyte	51179
todo: compiler warnings

With compression disabled (https://cirrus-ci.com/task/4614182514458624):
linux: 91MB cirrus cache; cache_size_kibibyte	316136
macos: 41MB cirrus cache; cache_size_kibibyte	118068
windows: 42MB cirrus cache; cache_size_kibibyte	134064
freebsd is the same

The stats should be shown and/or logged.
ccache --show-stats shows the *cumulative* stats (including prior
compilations)
ccache --zero-stats clears out not only the global stats, but the
per-file cache stats (from which the global stats are derived) - which
obviously makes the cache work poorly.

Note that ccache 4.4 supports CCACHE_STATSLOG, which seems ideal.
The log should be written *outside* the ccache folder - it shouldn't be
preserved across cirrusci task invocations.

freebsd, linux, macos
ci-os-only: linux
---
 .cirrus.yml | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 221431e70c4..d1c3119e4ff 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -16,7 +16,9 @@ env:
   # Useful to be able to analyse what in a script takes long
   CIRRUS_LOG_TIMESTAMP: true
 
-  CCACHE_MAXSIZE: "250M"
+  CCACHE_MAXSIZE: "750M"
+  CCACHE_NOCOMPRESS: 1
+  #CCACHE_STATSLOG: ccache-stats.log
 
   # target to test, for all but windows
   CHECK: check-world PROVE_FLAGS=$PROVE_FLAGS
@@ -236,7 +238,7 @@ task:
     - name: Linux - Debian Bullseye - Meson
 
       env:
-        CCACHE_MAXSIZE: "400M" # tests two different builds
+        CCACHE_MAXSIZE: "1200M" # tests two different builds
 
       configure_script: |
         su postgres <<-EOF
@@ -307,6 +309,7 @@ task:
 
     CIRRUS_WORKING_DIR: ${HOME}/pgsql/
     CCACHE_DIR: ${HOME}/ccache
+    CCACHE_STATSLOG: ${CCACHE_DIR}.stats.log
     HOMEBREW_CACHE: ${HOME}/homebrew-cache
     PERL5LIB: ${HOME}/perl5/lib/perl5
 
@@ -388,7 +391,7 @@ task:
       -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
       build
 
-  build_script: ninja -C build -j${BUILD_JOBS}
+  build_script: ninja -C build -j${BUILD_JOBS} && ccache --show-log-stats
   upload_caches: ccache
 
   test_world_script: |
@@ -496,14 +499,14 @@ task:
 
     # Use larger ccache cache, as this task compiles with multiple compilers /
     # flag combinations
-    CCACHE_MAXSIZE: "1GB"
+    CCACHE_MAXSIZE: "3G"
     CCACHE_DIR: "/tmp/ccache_dir"
 
     LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
     LINUX_MESON_FEATURES: *LINUX_MESON_FEATURES
 
   # task that did not run, count as a success, so we need to recheck Linux'
-  # condition here ...
+  # condition here; cirus warns if the "only_if" condition doesn't match the task being depended on
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
 
   container:
-- 
2.25.1

From de86a044d663ed0a6d5c292e44a4d56bd9b0cd4b Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Tue, 19 Jul 2022 12:38:45 -0500
Subject: [PATCH 8/8] cirrus/warnings: use a single/common 'always' block

ci-os-only: warnings
---
 .cirrus.yml | 44 +++++++++++++++++++-------------------------
 1 file changed, 19 insertions(+), 25 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index d1c3119e4ff..9ab8ce30e09 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -522,11 +522,6 @@ task:
     clang -v
     export
 
-  ccache_cache:
-    folder: $CCACHE_DIR
-    fingerprint_key: ccache/warnings
-    reupload_on_changes: true
-
   setup_additional_packages_script: |
     #apt-get update
     #DEBIAN_FRONTEND=noninteractive apt-get -y install ...
@@ -537,8 +532,13 @@ task:
   # different compilers to build with different combinations of dtrace on/off
   # and cassert on/off.
 
-  # gcc, cassert off, dtrace on
   always:
+    ccache_cache:
+      folder: $CCACHE_DIR
+      fingerprint_key: ccache/warnings
+      reupload_on_changes: true
+
+    # gcc, cassert off, dtrace on
     gcc_warning_script: |
       time ./configure \
         --cache gcc.cache \
@@ -548,8 +548,7 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  # gcc, cassert on, dtrace off
-  always:
+    # gcc, cassert on, dtrace off
     gcc_a_warning_script: |
       time ./configure \
         --cache gcc.cache \
@@ -559,8 +558,7 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  # clang, cassert off, dtrace off
-  always:
+    # clang, cassert off, dtrace off
     clang_warning_script: |
       time ./configure \
         --cache clang.cache \
@@ -569,8 +567,7 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  # clang, cassert on, dtrace on
-  always:
+    # clang, cassert on, dtrace on
     clang_a_warning_script: |
       time ./configure \
         --cache clang.cache \
@@ -581,8 +578,7 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  # cross-compile to windows
-  always:
+    # cross-compile to windows
     mingw_cross_warning_script: |
       time ./configure \
         --host=x86_64-w64-mingw32 \
@@ -592,16 +588,15 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  ###
-  # Verify headerscheck / cpluspluscheck succeed
-  #
-  # - Don't use ccache, the files are uncacheable, polluting ccache's
-  #   cache
-  # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
-  # - XXX have to disable ICU to avoid errors:
-  #   https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
-  ###
-  always:
+    ###
+    # Verify headerscheck / cpluspluscheck succeed
+    #
+    # - Don't use ccache, the files are uncacheable, polluting ccache's
+    #   cache
+    # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
+    # - XXX have to disable ICU to avoid errors:
+    #   https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
+    ###
     headers_headerscheck_script: |
       time ./configure \
         ${LINUX_CONFIGURE_FEATURES} \
@@ -613,7 +608,6 @@ task:
     headers_cpluspluscheck_script: |
       time make -s cpluspluscheck EXTRAFLAGS='-fmax-errors=10'
 
-  always:
     upload_caches: ccache
 
 
-- 
2.25.1

From 8f3dd74efa522b0db0ca27456d1b9717b2bd8447 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Tue, 26 Jul 2022 20:30:02 -0500
Subject: [PATCH 03/11] cirrus/ccache: add explicit cache keys..

Since otherwise, building with ci-os-only will probably fail to use the
normal cache, since the cache key is computed using both the task name
and its *index* in the list of caches (internal/executor/cache.go:184).
---
 .cirrus.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.cirrus.yml b/.cirrus.yml
index a3b84d3b1c0..061c8e1c7df 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -87,6 +87,9 @@ task:
 
   ccache_cache:
     folder: $CCACHE_DIR
+    fingerprint_key: ccache/freebsd
+    reupload_on_changes: true
+
   # Workaround around performance issues due to 32KB block size
   repartition_script: src/tools/ci/gcp_freebsd_repartition.sh
   create_user_script: |
@@ -181,6 +184,8 @@ task:
 
   ccache_cache:
     folder: ${CCACHE_DIR}
+    fingerprint_key: ccache/linux
+    reupload_on_changes: true
 
   sysinfo_script: |
     id
@@ -359,6 +364,9 @@ task:
 
   ccache_cache:
     folder: $CCACHE_DIR
+    fingerprint_key: ccache/macos
+    reupload_on_changes: true
+
   configure_script: |
     brewpath="/usr/local"
     PKG_CONFIG_PATH="${brewpath}/lib/pkgconfig:${PKG_CONFIG_PATH}"
@@ -510,6 +518,8 @@ task:
 
   ccache_cache:
     folder: $CCACHE_DIR
+    fingerprint_key: ccache/warnings
+    reupload_on_changes: true
 
   setup_additional_packages_script: |
     #apt-get update
-- 
2.25.1

From 12d90810ad86afbad4d692cf27ac0b363433a382 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 25 May 2022 21:53:22 -0500
Subject: [PATCH 01/11] cirrus/windows: add compiler_warnings_script

I'm not sure how to write this test in windows shell; it's also not easy to
write it in posix sh, since windows shell is somehow interpretting && and ||...

https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de

See also:
8a1ce5e54f6d144e4f8e19af7c767b026ee0c956
https://cirrus-ci.com/task/6241060062494720
https://cirrus-ci.com/task/6496366607204352

ci-os-only: windows
---
 .cirrus.yml                            | 10 +++++++++-
 src/tools/ci/windows-compiler-warnings | 24 ++++++++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100755 src/tools/ci/windows-compiler-warnings

diff --git a/.cirrus.yml b/.cirrus.yml
index 9f2282471a9..7596b2058c6 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -451,12 +451,20 @@ task:
 
   build_script: |
     vcvarsall x64
-    ninja -C build
+    ninja -C build |tee build/meson-logs/build.txt
+    REM Since pipes lose the exit status of the preceding command, rerun the compilation
+    REM without the pipe, exiting now if it fails, to avoid trying to run checks
+    ninja -C build > nul
 
   check_world_script: |
     vcvarsall x64
     meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
 
+  # This should be last, so check_world is always run
+  always:
+    compiler_warnings_script:
+      - sh src\tools\ci\windows-compiler-warnings build/meson-logs/build.txt build/meson-logs/build-warnings.txt
+
   on_failure:
     <<: *on_failure_meson
     crashlog_artifacts:
diff --git a/src/tools/ci/windows-compiler-warnings b/src/tools/ci/windows-compiler-warnings
new file mode 100755
index 00000000000..685aa995d5c
--- /dev/null
+++ b/src/tools/ci/windows-compiler-warnings
@@ -0,0 +1,24 @@
+#! /bin/sh
+# Success if the given exists and is empty, else fail
+# This is a separate file to avoid dealing with windows shell quoting and escaping,
+# or giving the impression that one-liner scripts will work trivially.
+set -e
+
+infile=$1
+outfile=$2
+
+# Looks like:
+# [19:39:27.130] c:\cirrus\src\backend\backup\basebackup_zstd.c(80) : warning C4715: 'bbsink_zstd_new': not all control paths return a value
+# should include linker warnings?
+grep ": warning " "$infile" > $outfile ||
+	[ $? -eq 1 ]
+
+if [ -s "$outfile" ]
+then
+	# Display the file's content, then exit indicating failure
+	cat "$outfile"
+	exit 1
+else
+	# Success
+	exit 0
+fi
-- 
2.25.1

From 1df20b9bd9883a2f77618ac9e7611752128ddad1 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 26 Feb 2022 19:34:35 -0600
Subject: [PATCH 02/11] cirrus: build docs as a separate task..

This will run the doc build if any docs have changed, even if Linux
fails, to allow catch doc build failures.

XXX: it'd be nice to have cfbot inject this cirrus.yml for a couple days
to check that changesInclude() is working as intended.

This automatically shows up as a separate "column" on cfbot.

Also, in the future, this will hopefully upload each patch's changed HTML docs
as an artifact, for easy review.

ci-os-only: html
---
 .cirrus.yml | 49 +++++++++++++++++++++++++++++++++++--------------
 1 file changed, 35 insertions(+), 14 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 7596b2058c6..a3b84d3b1c0 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -580,20 +580,6 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  ###
-  # Verify docs can be built
-  ###
-  # XXX: Only do this if there have been changes in doc/ since last build
-  always:
-    docs_build_script: |
-      time ./configure \
-        --cache gcc.cache \
-        CC="ccache gcc" \
-        CXX="ccache g++" \
-        CLANG="ccache clang"
-      make -s -j${BUILD_JOBS} clean
-      time make -s -j${BUILD_JOBS} -C doc
-
   ###
   # Verify headerscheck / cpluspluscheck succeed
   #
@@ -617,3 +603,38 @@ task:
 
   always:
     upload_caches: ccache
+
+
+###
+# Verify docs can be built
+# changesInclude() will skip this task if none of the commits since
+# CIRRUS_LAST_GREEN_CHANGE touched any relevant files. The comparison appears
+# to be like "git log a..b -- ./file", not "git diff a..b -- ./file"
+###
+
+task:
+  name: Documentation
+
+  env:
+    CPUS: 1
+    BUILD_JOBS: 1
+
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(docs|html).*'
+  skip: "!changesInclude('.cirrus.yml', 'doc/**')"
+
+  container:
+    image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
+    cpu: $CPUS
+    memory: 2G
+
+  # Exercise HTML and other docs:
+  ninja_docs_build_script: |
+    mkdir build-ninja
+    cd build-ninja
+    time meson setup
+    time ninja docs
+
+  # Again, with autoconf
+  docs_build_script: |
+    time ./configure
+    make -s -j${BUILD_JOBS} -C doc
-- 
2.25.1

From 54338b5a26c7eb8dcd0586bf93d67a1cb5c06803 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 2 Nov 2022 14:14:38 -0500
Subject: [PATCH 04/11] cirrus/ccache: avoid clearing cache until after all
 compilations

Local storage use is of no concern, but it's bad if we have lower cache
hit ratio due to pruning which occurs during earlier compilations.
---
 .cirrus.yml | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 061c8e1c7df..47c4caf51ac 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -236,7 +236,7 @@ task:
     - name: Linux - Debian Bullseye - Meson
 
       env:
-        CCACHE_MAXSIZE: "400M" # tests two different builds
+        CCACHE_MAXSIZE: "800M" # tests two different builds
 
       configure_script: |
         su postgres <<-EOF
@@ -287,6 +287,10 @@ task:
           PYTHONCOERCECLOCALE=0 LANG=C meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
         EOF
 
+      ccache_stats_end_script:
+        ccache -s
+        CCACHE_MAXSIZE=400M ccache --cleanup
+
       on_failure:
         <<: *on_failure_meson
 
@@ -492,8 +496,10 @@ task:
     BUILD_JOBS: 4
 
     # Use larger ccache cache, as this task compiles with multiple compilers /
-    # flag combinations
-    CCACHE_MAXSIZE: "1GB"
+    # flag combinations.  The cache size is set high now, and manually pruned later,
+    # to avoid cache cleaning cycles during the early compilations pruning
+    # compilation results which could've been used during later compilations.
+    CCACHE_MAXSIZE: "2G"
     CCACHE_DIR: "/tmp/ccache_dir"
 
     LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
@@ -612,6 +618,10 @@ task:
       time make -s cpluspluscheck EXTRAFLAGS='-fmax-errors=10'
 
   always:
+    ccache_stats_end_script:
+      ccache -s
+      CCACHE_MAXSIZE=1G ccache --cleanup
+
     upload_caches: ccache
 
 
-- 
2.25.1

From 1914ac033f7fe416c691961c27f2b08cefb701b0 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sun, 3 Apr 2022 00:10:20 -0500
Subject: [PATCH 05/11] cirrus/ccache: disable compression and show stats

Since v4.0, ccache enables zstd compression by default, saving roughly
2x-3x.  But, cirrus caches are compressed as tar.gz, so we could disable
ccache compression, allowing cirrus to gzip the uncompressed data
(better than ccache's default of zstd-1).

With default compression enabled (https://cirrus-ci.com/task/6692342840164352):
linux/debian/bullseye has 4.2; 99MB cirrus cache; cache_size_kibibyte	109616
macos has 4.5.1: 47MB cirrus cache; cache_size_kibibyte	52500
freebsd has 3.7.12: 42MB cirrus cache; cache_size_kibibyte	134064
XXX windows has 4.7.2; 180MB cirrus cache; cache_size_kibibyte	51179
todo: compiler warnings

With compression disabled (https://cirrus-ci.com/task/4614182514458624):
linux: 91MB cirrus cache; cache_size_kibibyte	316136
macos: 41MB cirrus cache; cache_size_kibibyte	118068
windows: 42MB cirrus cache; cache_size_kibibyte	134064
freebsd is the same

The stats should be shown and/or logged.
ccache --show-stats shows the *cumulative* stats (including prior
compilations)
ccache --zero-stats clears out not only the global stats, but the
per-file cache stats (from which the global stats are derived) - which
obviously makes the cache work poorly.

Note that ccache 4.4 supports CCACHE_STATSLOG, which seems ideal.
The log should be written *outside* the ccache folder - it shouldn't be
preserved across cirrusci task invocations.

freebsd, linux, macos
ci-os-only: linux
---
 .cirrus.yml | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 47c4caf51ac..d9660149a73 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -16,7 +16,11 @@ env:
   # Useful to be able to analyse what in a script takes long
   CIRRUS_LOG_TIMESTAMP: true
 
-  CCACHE_MAXSIZE: "250M"
+  CCACHE_MAXSIZE: "750M"
+  CCACHE_HARDLINK: 1
+  #CCACHE_NOCOMPRESS: 1
+  # The statslog should be stored somewhere *outside* the persistent ccache dir
+  CCACHE_STATSLOG: ${CIRRUS_WORKING_DIR}/ccache-stats.log
 
   # target to test, for all but windows
   CHECK: check-world PROVE_FLAGS=$PROVE_FLAGS
@@ -236,7 +240,7 @@ task:
     - name: Linux - Debian Bullseye - Meson
 
       env:
-        CCACHE_MAXSIZE: "800M" # tests two different builds
+        CCACHE_MAXSIZE: "2400M" # tests two different builds
 
       configure_script: |
         su postgres <<-EOF
@@ -289,7 +293,7 @@ task:
 
       ccache_stats_end_script:
         ccache -s
-        CCACHE_MAXSIZE=400M ccache --cleanup
+        CCACHE_MAXSIZE=1200M ccache --cleanup
 
       on_failure:
         <<: *on_failure_meson
@@ -392,7 +396,7 @@ task:
       -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
       build
 
-  build_script: ninja -C build -j${BUILD_JOBS}
+  build_script: ninja -C build -j${BUILD_JOBS} && ccache --show-log-stats
   upload_caches: ccache
 
   test_world_script: |
@@ -499,7 +503,7 @@ task:
     # flag combinations.  The cache size is set high now, and manually pruned later,
     # to avoid cache cleaning cycles during the early compilations pruning
     # compilation results which could've been used during later compilations.
-    CCACHE_MAXSIZE: "2G"
+    CCACHE_MAXSIZE: "6G"
     CCACHE_DIR: "/tmp/ccache_dir"
 
     LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
@@ -620,7 +624,7 @@ task:
   always:
     ccache_stats_end_script:
       ccache -s
-      CCACHE_MAXSIZE=1G ccache --cleanup
+      CCACHE_MAXSIZE=3G ccache --cleanup
 
     upload_caches: ccache
 
-- 
2.25.1

From 61e27060dd249f38ce2fd96a6031101e95c492e5 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Tue, 19 Jul 2022 12:38:45 -0500
Subject: [PATCH 06/11] cirrus/warnings: use a single/common 'always' block

ci-os-only: warnings
---
 .cirrus.yml | 44 +++++++++++++++++++-------------------------
 1 file changed, 19 insertions(+), 25 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index d9660149a73..6fb8ed0cded 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -526,11 +526,6 @@ task:
     clang -v
     export
 
-  ccache_cache:
-    folder: $CCACHE_DIR
-    fingerprint_key: ccache/warnings
-    reupload_on_changes: true
-
   setup_additional_packages_script: |
     #apt-get update
     #DEBIAN_FRONTEND=noninteractive apt-get -y install ...
@@ -545,8 +540,13 @@ task:
   # different compilers to build with different combinations of dtrace on/off
   # and cassert on/off.
 
-  # gcc, cassert off, dtrace on
   always:
+    ccache_cache:
+      folder: $CCACHE_DIR
+      fingerprint_key: ccache/warnings
+      reupload_on_changes: true
+
+    # gcc, cassert off, dtrace on
     gcc_warning_script: |
       time ./configure \
         --cache gcc.cache \
@@ -556,8 +556,7 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  # gcc, cassert on, dtrace off
-  always:
+    # gcc, cassert on, dtrace off
     gcc_a_warning_script: |
       time ./configure \
         --cache gcc.cache \
@@ -567,8 +566,7 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  # clang, cassert off, dtrace off
-  always:
+    # clang, cassert off, dtrace off
     clang_warning_script: |
       time ./configure \
         --cache clang.cache \
@@ -577,8 +575,7 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  # clang, cassert on, dtrace on
-  always:
+    # clang, cassert on, dtrace on
     clang_a_warning_script: |
       time ./configure \
         --cache clang.cache \
@@ -589,8 +586,7 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  # cross-compile to windows
-  always:
+    # cross-compile to windows
     mingw_cross_warning_script: |
       time ./configure \
         --host=x86_64-w64-mingw32 \
@@ -600,16 +596,15 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  ###
-  # Verify headerscheck / cpluspluscheck succeed
-  #
-  # - Don't use ccache, the files are uncacheable, polluting ccache's
-  #   cache
-  # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
-  # - XXX have to disable ICU to avoid errors:
-  #   https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
-  ###
-  always:
+    ###
+    # Verify headerscheck / cpluspluscheck succeed
+    #
+    # - Don't use ccache, the files are uncacheable, polluting ccache's
+    #   cache
+    # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
+    # - XXX have to disable ICU to avoid errors:
+    #   https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
+    ###
     headers_headerscheck_script: |
       time ./configure \
         ${LINUX_CONFIGURE_FEATURES} \
@@ -621,7 +616,6 @@ task:
     headers_cpluspluscheck_script: |
       time make -s cpluspluscheck EXTRAFLAGS='-fmax-errors=10'
 
-  always:
     ccache_stats_end_script:
       ccache -s
       CCACHE_MAXSIZE=3G ccache --cleanup
-- 
2.25.1

From b76dd329bcf6af63de96ed26d8bb1945149467c7 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sun, 13 Nov 2022 14:03:27 -0600
Subject: [PATCH 07/11] cirrus: clean up windows task

 - remove CIRRUS_ESCAPING_PROCESSES, which wasn't needed since the task
   was converted to meson in e6b6ea025;
 - name the task the same as the other tasks using meson;
 - use the otherwise-unused TAR variable;
---
 .cirrus.yml | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 6fb8ed0cded..3045278f4b1 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -410,7 +410,7 @@ task:
 
 
 task:
-  name: Windows - Server 2019, VS 2019 - Meson & ninja
+  name: Windows - Server 2019, VS 2019 - Meson
 
   env:
     # Half the allowed per-user CPU cores
@@ -428,13 +428,6 @@ task:
     PG_TEST_USE_UNIX_SOCKETS: 1
     PG_REGRESS_SOCK_DIR: "c:/cirrus/"
 
-    # startcreate_script starts a postgres instance that we don't want to get
-    # killed at the end of that script (it's stopped in stop_script). Can't
-    # trivially use background_scripts because a) need pg_ctl's dropping of
-    # permissions b) need to wait for startup to have finished, and we don't
-    # currently have a tool for that...
-    CIRRUS_ESCAPING_PROCESSES: 1
-
     # Cirrus defaults to SetErrorMode(SEM_NOGPFAULTERRORBOX | ...). That
     # prevents crash reporting from working unless binaries do SetErrorMode()
     # themselves. Furthermore, it appears that either python or, more likely,
@@ -463,7 +456,7 @@ task:
   # Use /DEBUG:FASTLINK to avoid high memory usage during linking
   configure_script: |
     vcvarsall x64
-    meson setup --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Db_pch=true -Dssl=openssl -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=c:/windows/system32/tar.exe -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" build
+    meson setup --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Db_pch=true -Dssl=openssl -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" build
 
   build_script: |
     vcvarsall x64
-- 
2.25.1

From 1ce768f023526313b86df24a32c8584a2fb578e1 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 11 Nov 2022 21:14:39 -0600
Subject: [PATCH 08/11] cirrus: switch to "macos_instance"..

This uses an M1 ARM processor rather than intel.

See:

https://cirrus-ci.org/blog/2022/11/08/sunsetting-intel-macos-instances/
https://github.com/justinpryzby/postgres/runs/9446478561
| warning We are sunsetting support for Intel-based macOS instances! Please migrate before January 1st 2023.

ci-os-only: macos
---
 .cirrus.yml                     | 10 ++++++----
 src/test/kerberos/t/001_auth.pl | 14 ++++++++++++--
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 3045278f4b1..ca49b2973ac 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -306,7 +306,7 @@ task:
   name: macOS - Monterey - Meson
 
   env:
-    CPUS: 12 # always get that much for cirrusci macOS instances
+    CPUS: 4 # always get that much for cirrusci macOS instances
     BUILD_JOBS: $CPUS
     # Test performance regresses noticably when using all cores. 8 seems to
     # work OK. See
@@ -326,8 +326,10 @@ task:
 
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
 
-  osx_instance:
-    image: monterey-base
+  macos_instance:
+    image: ghcr.io/cirruslabs/macos-monterey-base:latest
+    #cpus: 4
+    #memory: 8G
 
   sysinfo_script: |
     id
@@ -376,7 +378,7 @@ task:
     reupload_on_changes: true
 
   configure_script: |
-    brewpath="/usr/local"
+    brewpath="/opt/homebrew"
     PKG_CONFIG_PATH="${brewpath}/lib/pkgconfig:${PKG_CONFIG_PATH}"
 
     for pkg in icu4c krb5 openldap openssl zstd ; do
diff --git a/src/test/kerberos/t/001_auth.pl b/src/test/kerberos/t/001_auth.pl
index a2bc8a5351e..1f6e45b372e 100644
--- a/src/test/kerberos/t/001_auth.pl
+++ b/src/test/kerberos/t/001_auth.pl
@@ -34,8 +34,18 @@ my ($krb5_bin_dir, $krb5_sbin_dir);
 
 if ($^O eq 'darwin')
 {
-	$krb5_bin_dir  = '/usr/local/opt/krb5/bin';
-	$krb5_sbin_dir = '/usr/local/opt/krb5/sbin';
+	if (-d "/usr/local/opt/krb5")
+	{
+		# old
+		$krb5_bin_dir  = '/usr/local/opt/krb5/bin';
+		$krb5_sbin_dir = '/usr/local/opt/krb5/sbin';
+	}
+	elsif (-d "/opt/homebrew/opt/krb5")
+	{
+		# new
+		$krb5_bin_dir  = '/opt/homebrew/opt/krb5/bin';
+		$krb5_sbin_dir = '/opt/homebrew/opt/krb5/sbin';
+	}
 }
 elsif ($^O eq 'freebsd')
 {
-- 
2.25.1

From 0f4e47d0f6157ab8ee6ecd17d7933155d0bdc91a Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 4 Nov 2022 15:43:56 -0500
Subject: [PATCH 09/11] cirrus: update to macos ventura

ci-os-only: macos
---
 .cirrus.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index ca49b2973ac..aa381432050 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -303,7 +303,7 @@ task:
 
 
 task:
-  name: macOS - Monterey - Meson
+  name: macOS - Ventura - Meson
 
   env:
     CPUS: 4 # always get that much for cirrusci macOS instances
@@ -327,7 +327,7 @@ task:
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
 
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-monterey-base:latest
+    image: ghcr.io/cirruslabs/macos-ventura-base:latest
     #cpus: 4
     #memory: 8G
 
-- 
2.25.1

From 045de16bf365302805f33557c0824011300545e8 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 24 Jun 2022 00:09:12 -0500
Subject: [PATCH 10/11] cirrus/freebsd: run with more CPUs+RAM and do not
 repartition

There was some historic problem where tests under freebsd took 8+ minutes (and
before 4a288a37f took 15 minutes).

This reduces test time from 10min to 3min.
4 CPUs 4 tests https://cirrus-ci.com/task/4880240739614720
4 CPUs 6 tests https://cirrus-ci.com/task/4664440120410112 https://cirrus-ci.com/task/4586784884523008
4 CPUs 8 tests https://cirrus-ci.com/task/5001995491737600

6 CPUs https://cirrus-ci.com/task/6678321684545536
8 CPUs https://cirrus-ci.com/task/6264854121021440

See also:
https://www.postgresql.org/message-id/flat/20220310033347.hgxk4pyarzq4hxwp@alap3.anarazel.de#f36c0b17e33e31e7925e7e5812998686
8 jobs 7min https://cirrus-ci.com/task/6186376667332608

//-os-only: freebsd
---
 .cirrus.yml | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index aa381432050..3473d792561 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -63,11 +63,9 @@ task:
   name: FreeBSD - 13 - Meson
 
   env:
-    # FreeBSD on GCP is slow when running with larger number of CPUS /
-    # jobs. Using one more job than cpus seems to work best.
-    CPUS: 2
-    BUILD_JOBS: 3
-    TEST_JOBS: 3
+    CPUS: 4
+    BUILD_JOBS: 4
+    TEST_JOBS: 6
 
     CCACHE_DIR: /tmp/ccache_dir
     CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST
@@ -94,8 +92,6 @@ task:
     fingerprint_key: ccache/freebsd
     reupload_on_changes: true
 
-  # Workaround around performance issues due to 32KB block size
-  repartition_script: src/tools/ci/gcp_freebsd_repartition.sh
   create_user_script: |
     pw useradd postgres
     chown -R postgres:postgres .
-- 
2.25.1

From 81f5e76049bc831f651e8e398eee17f0a9b9e26a Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sun, 13 Nov 2022 16:32:41 -0600
Subject: [PATCH 11/11] cirrus: split linux and move the 'only_if:' line ..

..above the big 'env' blocks

This allows (for example) only running only tests under autoconf.
---
 .cirrus.yml | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 3473d792561..0a280f2cd71 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -61,6 +61,7 @@ on_failure_meson: &on_failure_meson
 
 task:
   name: FreeBSD - 13 - Meson
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
 
   env:
     CPUS: 4
@@ -71,8 +72,6 @@ task:
     CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST
     CFLAGS: -Og -ggdb
 
-  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
-
   compute_engine_instance:
     image_project: $IMAGE_PROJECT
     image: family/pg-ci-freebsd-13
@@ -173,8 +172,6 @@ task:
     LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
     LINUX_MESON_FEATURES: *LINUX_MESON_FEATURES
 
-  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
-
   compute_engine_instance:
     image_project: $IMAGE_PROJECT
     image: family/pg-ci-bullseye
@@ -210,6 +207,7 @@ task:
 
   matrix:
     - name: Linux - Debian Bullseye - Autoconf
+      only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux-autoconf.*'
 
       configure_script: |
         su postgres <<-EOF
@@ -234,6 +232,7 @@ task:
         <<: *on_failure_ac
 
     - name: Linux - Debian Bullseye - Meson
+      only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux-meson.*'
 
       env:
         CCACHE_MAXSIZE: "2400M" # tests two different builds
@@ -300,6 +299,7 @@ task:
 
 task:
   name: macOS - Ventura - Meson
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
 
   env:
     CPUS: 4 # always get that much for cirrusci macOS instances
@@ -320,8 +320,6 @@ task:
     CFLAGS: -Og -ggdb
     CXXFLAGS: -Og -ggdb
 
-  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
-
   macos_instance:
     image: ghcr.io/cirruslabs/macos-ventura-base:latest
     #cpus: 4
@@ -409,6 +407,7 @@ task:
 
 task:
   name: Windows - Server 2019, VS 2019 - Meson
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
 
   env:
     # Half the allowed per-user CPU cores
@@ -435,8 +434,6 @@ task:
     # 0x8001 is SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX
     CIRRUS_WINDOWS_ERROR_MODE: 0x8001
 
-  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
-
   windows_container:
     image: $CONTAINER_REPO/windows_ci_vs_2019:latest
     cpu: $CPUS
@@ -481,6 +478,9 @@ task:
 
 task:
   name: CompilerWarnings
+  # task that did not run, count as a success, so we need to recheck Linux'
+  # condition here ...
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
 
   # To limit unnecessary work only run this once the normal linux test succeeds
   depends_on:
@@ -500,10 +500,6 @@ task:
     LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
     LINUX_MESON_FEATURES: *LINUX_MESON_FEATURES
 
-  # task that did not run, count as a success, so we need to recheck Linux'
-  # condition here ...
-  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
-
   container:
     image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
     cpu: $CPUS
@@ -623,14 +619,13 @@ task:
 
 task:
   name: Documentation
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(docs|html).*'
+  skip: "!changesInclude('.cirrus.yml', 'doc/**')"
 
   env:
     CPUS: 1
     BUILD_JOBS: 1
 
-  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(docs|html).*'
-  skip: "!changesInclude('.cirrus.yml', 'doc/**')"
-
   container:
     image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
     cpu: $CPUS
-- 
2.25.1

From be7c1c7683da02b7d0f65c77eb8dc9e5ad852864 Mon Sep 17 00:00:00 2001
From: Andres Freund <andres@anarazel.de>
Date: Sun, 2 Oct 2022 11:26:19 -0700
Subject: [PATCH v4 1/2] ci: Introduce SanityCheck task that other tasks depend
 on

Reviewed-by: Justin Pryzby <pryzby@telsasoft.com>
Discussion: https://postgr.es/m/20221002205201.injtofbx4ax4erww@awork3.anarazel.de
---
 .cirrus.yml | 91 ++++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 84 insertions(+), 7 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 6f665a207f8..a5679fdfb9b 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -55,6 +55,78 @@ on_failure_meson: &on_failure_meson
     type: text/plain
 
 
+# To avoid unnecessarily spinning up a lot of VMs / containers for entirely
+# broken commits, have a minimal task that all others depend on.
+task:
+  name: SanityCheck
+
+  # If a specific OS is requested, don't run the sanity check. This shortens
+  # push-wait-for-ci cycle time a bit when debugging operating system specific
+  # failures. Uses skip instead of only_if, as cirrus otherwise warns about
+  # only_if conditions not matching.
+  skip: $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:.*'
+
+  env:
+    CPUS: 4
+    BUILD_JOBS: 8
+    TEST_JOBS: 8
+    CCACHE_DIR: ${CIRRUS_WORKING_DIR}/ccache_dir
+    # no options enabled, should be small
+    CCACHE_MAXSIZE: "150M"
+
+  # Container starts up quickly, but is slower at runtime, particularly for
+  # tests. Good for the briefly running sanity check.
+  container:
+    image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
+    cpu: $CPUS
+
+  ccache_cache:
+    folder: $CCACHE_DIR
+
+  create_user_script: |
+    useradd -m postgres
+    chown -R postgres:postgres .
+    mkdir -p ${CCACHE_DIR}
+    chown -R postgres:postgres ${CCACHE_DIR}
+    echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf
+    su postgres -c "ulimit -l -H && ulimit -l -S"
+    # Can't change container's kernel.core_pattern. Postgres user can't write
+    # to / normally. Change that.
+    chown root:postgres /
+    chmod g+rwx /
+
+  configure_script: |
+    su postgres <<-EOF
+      meson setup \
+        --buildtype=debug \
+        --auto-features=disabled \
+        -Dtap_tests=enabled \
+        build
+    EOF
+  build_script: |
+    su postgres <<-EOF
+      ninja -C build -j${BUILD_JOBS}
+    EOF
+  upload_caches: ccache
+
+  # Run a minimal set of tests. The main regression tests take too long for
+  # this purpose. For now this is a random quick pg_regress style test, and a
+  # tap test that exercises both a frontend binary and the backend.
+  test_minimal_script: |
+    su postgres <<-EOF
+      ulimit -c unlimited
+      meson test $MTEST_ARGS --num-processes ${TEST_JOBS} \
+        tmp_install cube/regress pg_ctl/001_start_stop
+    EOF
+
+  on_failure:
+    <<: *on_failure_meson
+    cores_script: |
+      mkdir -m 770 /tmp/cores
+      find / -maxdepth 1 -type f -name 'core*' -exec mv '{}' /tmp/cores/ \;
+      src/tools/ci/cores_backtrace.sh linux /tmp/cores
+
+
 task:
   name: FreeBSD - 13 - Meson
 
@@ -69,6 +141,7 @@ task:
     CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST
     CFLAGS: -Og -ggdb
 
+  depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
 
   compute_engine_instance:
@@ -170,6 +243,7 @@ task:
     LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
     LINUX_MESON_FEATURES: *LINUX_MESON_FEATURES
 
+  depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
 
   compute_engine_instance:
@@ -311,6 +385,7 @@ task:
     CFLAGS: -Og -ggdb
     CXXFLAGS: -Og -ggdb
 
+  depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
 
   osx_instance:
@@ -439,6 +514,7 @@ task:
     # 0x8001 is SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX
     CIRRUS_WINDOWS_ERROR_MODE: 0x8001
 
+  depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
 
   windows_container:
@@ -478,6 +554,8 @@ task:
   # worth using only_if despite being manual, otherwise this task will show up
   # when e.g. ci-os-only: linux is used.
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*mingw.*'
+  # otherwise it'll be sorted before other tasks
+  depends_on: SanityCheck
 
   windows_container:
     image: $CONTAINER_REPO/windows_ci_mingw64:latest
@@ -532,9 +610,12 @@ task:
 task:
   name: CompilerWarnings
 
-  # To limit unnecessary work only run this once the normal linux test succeeds
-  depends_on:
-    - Linux - Debian Bullseye - Meson
+  # To limit unnecessary work only run this once the SanityCheck
+  # succeeds. This is particularly important for this task as we intentionally
+  # use always: to continue after failures. Task that did not run count as a
+  # success, so we need to recheck SanityChecks's condition here ...
+  depends_on: SanityCheck
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*'
 
   env:
     CPUS: 4
@@ -548,10 +629,6 @@ task:
     LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
     LINUX_MESON_FEATURES: *LINUX_MESON_FEATURES
 
-  # task that did not run, count as a success, so we need to recheck Linux'
-  # condition here ...
-  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
-
   container:
     image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
     cpu: $CPUS
-- 
2.37.1.188.g71a8fab31b

From ef127f2b13f82f9da683a65e082cebcf9c7be15d Mon Sep 17 00:00:00 2001
From: Andres Freund <andres@anarazel.de>
Date: Thu, 29 Sep 2022 17:44:45 -0700
Subject: [PATCH v4 2/2] ci: use -fsanitize=undefined,alignment in linux tasks

---
 .cirrus.yml | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index a5679fdfb9b..93ea6524a7a 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -235,8 +235,28 @@ task:
     CCACHE_DIR: /tmp/ccache_dir
     DEBUGINFOD_URLS: "https://debuginfod.debian.net"
 
-    CFLAGS: -Og -ggdb
-    CXXFLAGS: -Og -ggdb
+    # Enable a reasonable set of sanitizers. Use the linux task for that, as
+    # it currently is the fastest task. Also several of the sanitizers work
+    # best on linux.
+    #
+    # The overhead of alignment sanitizer is low, undefined behaviour has
+    # moderate overhead. Test alignment sanitizer in the meson task, as it
+    # does both 32 and 64 bit builds and is thus more likely to expose
+    # alignment bugs.
+    #
+    # Address sanitizer in contrast somewhat expensive. Enable it in the
+    # autoconf task, as the meson task tests both 32 and 64bit.
+    #
+    # disable_coredump=0, abort_on_error=1: for useful backtraces in case of crashes
+    # print_stacktraces=1,verbosity=2, duh
+    # detect_leaks=0: too many uninteresting leak errors in short-lived binaries
+    UBSAN_OPTIONS: print_stacktrace=1:disable_coredump=0:abort_on_error=1:verbosity=2
+    ASAN_OPTIONS: print_stacktrace=1:disable_coredump=0:abort_on_error=1:detect_leaks=0
+
+    # SANITIZER_FLAGS is set in the tasks below
+    CFLAGS: -Og -ggdb -fno-sanitize-recover=all $SANITIZER_FLAGS
+    CXXFLAGS: $CFLAGS
+    LDFLAGS: $SANITIZER_FLAGS
     CC: ccache gcc
     CXX: ccache g++
 
@@ -280,6 +300,9 @@ task:
   matrix:
     - name: Linux - Debian Bullseye - Autoconf
 
+      env:
+        SANITIZER_FLAGS: -fsanitize=address
+
       configure_script: |
         su postgres <<-EOF
           ./configure \
@@ -306,6 +329,7 @@ task:
 
       env:
         CCACHE_MAXSIZE: "400M" # tests two different builds
+        SANITIZER_FLAGS: -fsanitize=alignment,undefined
 
       configure_script: |
         su postgres <<-EOF
-- 
2.37.1.188.g71a8fab31b

From e7eb22b85dcfe503810ea0f89f48833f04f80d3f Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 25 May 2022 21:53:22 -0500
Subject: [PATCH 01/10] cirrus/windows: add compiler_warnings_script

I'm not sure how to write this test in windows shell; it's also easy to
write something that doesn't work in posix sh, since windows shell is
interpretting && and ||...

https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de

See also:
8a1ce5e54f6d144e4f8e19af7c767b026ee0c956
https://cirrus-ci.com/task/6241060062494720
https://cirrus-ci.com/task/6496366607204352

ci-os-only: windows
---
 .cirrus.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index f31923333ef..6ce4f393e29 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -547,12 +547,21 @@ task:
 
   build_script: |
     vcvarsall x64
-    ninja -C build
+    ninja -C build |tee build.txt
+    REM Since pipes lose the exit status of the preceding command, rerun the compilation
+    REM without the pipe, exiting now if it fails, to avoid trying to run checks
+    ninja -C build > nul
 
   check_world_script: |
     vcvarsall x64
     meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
 
+  # This should be last, so check_world is run even if there are warnings
+  always:
+    compiler_warnings_script:
+      # this avoids using metachars which would be interpretted by the windows shell
+      - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0'
+
   on_failure:
     <<: *on_failure_meson
     crashlog_artifacts:
-- 
2.25.1

From 79586f7ac46ce48648b150b6ad08987578c730f5 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 11 Nov 2022 21:14:39 -0600
Subject: [PATCH 02/10] cirrus/macos: switch to "macos_instance" / M1..

This uses an M1 ARM processor rather than intel.

See:

https://cirrus-ci.org/blog/2022/11/08/sunsetting-intel-macos-instances/
https://github.com/justinpryzby/postgres/runs/9446478561
| warning We are sunsetting support for Intel-based macOS instances! Please migrate before January 1st 2023.

ci-os-only: macos
---
 .cirrus.yml                     | 10 ++++++----
 src/test/kerberos/t/001_auth.pl | 14 ++++++++++++--
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 6ce4f393e29..ed05c4adf87 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -391,7 +391,7 @@ task:
   name: macOS - Monterey - Meson
 
   env:
-    CPUS: 12 # always get that much for cirrusci macOS instances
+    CPUS: 4 # always get that much for cirrusci macOS instances
     BUILD_JOBS: $CPUS
     # Test performance regresses noticably when using all cores. 8 seems to
     # work OK. See
@@ -412,8 +412,10 @@ task:
   depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
 
-  osx_instance:
-    image: monterey-base
+  macos_instance:
+    image: ghcr.io/cirruslabs/macos-monterey-base:latest
+    #cpus: 4
+    #memory: 8G
 
   sysinfo_script: |
     id
@@ -459,7 +461,7 @@ task:
   ccache_cache:
     folder: $CCACHE_DIR
   configure_script: |
-    brewpath="/usr/local"
+    brewpath="/opt/homebrew"
     PKG_CONFIG_PATH="${brewpath}/lib/pkgconfig:${PKG_CONFIG_PATH}"
 
     for pkg in icu4c krb5 openldap openssl zstd ; do
diff --git a/src/test/kerberos/t/001_auth.pl b/src/test/kerberos/t/001_auth.pl
index a2bc8a5351e..1f6e45b372e 100644
--- a/src/test/kerberos/t/001_auth.pl
+++ b/src/test/kerberos/t/001_auth.pl
@@ -34,8 +34,18 @@ my ($krb5_bin_dir, $krb5_sbin_dir);
 
 if ($^O eq 'darwin')
 {
-	$krb5_bin_dir  = '/usr/local/opt/krb5/bin';
-	$krb5_sbin_dir = '/usr/local/opt/krb5/sbin';
+	if (-d "/usr/local/opt/krb5")
+	{
+		# old
+		$krb5_bin_dir  = '/usr/local/opt/krb5/bin';
+		$krb5_sbin_dir = '/usr/local/opt/krb5/sbin';
+	}
+	elsif (-d "/opt/homebrew/opt/krb5")
+	{
+		# new
+		$krb5_bin_dir  = '/opt/homebrew/opt/krb5/bin';
+		$krb5_sbin_dir = '/opt/homebrew/opt/krb5/sbin';
+	}
 }
 elsif ($^O eq 'freebsd')
 {
-- 
2.25.1

From 9d340b707897985d18c4e6f10fb9da41b14e59e9 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 4 Nov 2022 15:43:56 -0500
Subject: [PATCH 03/10] cirrus/macos: update to macos ventura

ci-os-only: macos
---
 .cirrus.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index ed05c4adf87..7c6e3691093 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -388,7 +388,7 @@ task:
 
 
 task:
-  name: macOS - Monterey - Meson
+  name: macOS - Ventura - Meson
 
   env:
     CPUS: 4 # always get that much for cirrusci macOS instances
@@ -413,7 +413,7 @@ task:
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
 
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-monterey-base:latest
+    image: ghcr.io/cirruslabs/macos-ventura-base:latest
     #cpus: 4
     #memory: 8G
 
-- 
2.25.1

From 766f9c2b8114ace0431877cbb1fc8214aeeb8e5b Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 24 Jun 2022 00:09:12 -0500
Subject: [PATCH 04/10] cirrus/freebsd: run with more CPUs+RAM and do not
 repartition

There was some historic problem where tests under freebsd took 8+ minutes (and
before 4a288a37f took 15 minutes).

This reduces test time from 10min to 3min.
4 CPUs 4 tests https://cirrus-ci.com/task/4880240739614720
4 CPUs 6 tests https://cirrus-ci.com/task/4664440120410112 https://cirrus-ci.com/task/4586784884523008
4 CPUs 8 tests https://cirrus-ci.com/task/5001995491737600

6 CPUs https://cirrus-ci.com/task/6678321684545536
8 CPUs https://cirrus-ci.com/task/6264854121021440

See also:
https://www.postgresql.org/message-id/flat/20220310033347.hgxk4pyarzq4hxwp@alap3.anarazel.de#f36c0b17e33e31e7925e7e5812998686
8 jobs 7min https://cirrus-ci.com/task/6186376667332608

//-os-only: freebsd
---
 .cirrus.yml | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 7c6e3691093..6aaaf62d6a9 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -131,11 +131,9 @@ task:
   name: FreeBSD - 13 - Meson
 
   env:
-    # FreeBSD on GCP is slow when running with larger number of CPUS /
-    # jobs. Using one more job than cpus seems to work best.
-    CPUS: 2
-    BUILD_JOBS: 3
-    TEST_JOBS: 3
+    CPUS: 4
+    BUILD_JOBS: 4
+    TEST_JOBS: 6
 
     CCACHE_DIR: /tmp/ccache_dir
     CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST
@@ -160,8 +158,6 @@ task:
 
   ccache_cache:
     folder: $CCACHE_DIR
-  # Workaround around performance issues due to 32KB block size
-  repartition_script: src/tools/ci/gcp_freebsd_repartition.sh
   create_user_script: |
     pw useradd postgres
     chown -R postgres:postgres .
-- 
2.25.1

From b36213a7801d354e8f47189d19b7c9166b1ad22b Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sun, 13 Nov 2022 14:03:27 -0600
Subject: [PATCH 05/10] cirrus: clean up typos

---
 .cirrus.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 6aaaf62d6a9..3eda0268161 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -47,7 +47,7 @@ on_failure_meson: &on_failure_meson
 
   # In theory it'd be nice to upload the junit files meson generates, so that
   # cirrus will nicely annotate the commit. Unfortunately the files don't
-  # contain identifieable file + line numbers right now, so the annotations
+  # contain identifiable file + line numbers right now, so the annotations
   # don't end up useful. We could probably improve on that with a some custom
   # conversion script, but ...
   meson_log_artifacts:
@@ -74,7 +74,7 @@ task:
     # no options enabled, should be small
     CCACHE_MAXSIZE: "150M"
 
-  # Container starts up quickly, but is slower at runtime, particularly for
+  # Containers start up quickly, but is slower at runtime, particularly for
   # tests. Good for the briefly running sanity check.
   container:
     image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
@@ -590,7 +590,7 @@ task:
     CCACHE_MAXSIZE: "500M"
     CCACHE_SLOPPINESS: pch_defines,time_macros
     CCACHE_DEPEND: 1
-    # for some reason mingw plpython cannot find it's installation without this
+    # for some reason mingw plpython cannot find its installation without this
     PYTHONHOME: C:/msys64/ucrt64
     # prevents MSYS bash from resetting error mode
     MSYS: winjitdebug
@@ -645,7 +645,7 @@ task:
 
     # Use larger ccache cache, as this task compiles with multiple compilers /
     # flag combinations
-    CCACHE_MAXSIZE: "1GB"
+    CCACHE_MAXSIZE: "1G"
     CCACHE_DIR: "/tmp/ccache_dir"
 
     LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
-- 
2.25.1

From 7133474e6280538439f43f799726275ce69ee2cd Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 26 Feb 2022 19:34:35 -0600
Subject: [PATCH 06/10] cirrus: build docs as a separate task..

This will run the doc build if any docs have changed, even if Linux
fails, to allow catch doc build failures.

This automatically shows up as a separate "column" on cfbot.

Also, in the future, this will hopefully upload each patch's changed HTML docs
as an artifact, for easy review.

XXX: it ought to be possible to skip (re)building docs if they haven't
changed since the last build.  But that doesn't appear to be working.
https://cirrus-ci.com/task/4523764229603328
skip: "!changesInclude('.cirrus.yml', 'doc/**')"

ci-os-only: html
---
 .cirrus.yml | 48 ++++++++++++++++++++++++++++++++++--------------
 1 file changed, 34 insertions(+), 14 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 3eda0268161..5cb18032779 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -736,20 +736,6 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  ###
-  # Verify docs can be built
-  ###
-  # XXX: Only do this if there have been changes in doc/ since last build
-  always:
-    docs_build_script: |
-      time ./configure \
-        --cache gcc.cache \
-        CC="ccache gcc" \
-        CXX="ccache g++" \
-        CLANG="ccache clang"
-      make -s -j${BUILD_JOBS} clean
-      time make -s -j${BUILD_JOBS} -C doc
-
   ###
   # Verify headerscheck / cpluspluscheck succeed
   #
@@ -773,3 +759,37 @@ task:
 
   always:
     upload_caches: ccache
+
+
+###
+# Verify docs can be built
+# changesInclude() will skip this task if none of the commits since
+# CIRRUS_LAST_GREEN_CHANGE touched any relevant files. The comparison appears
+# to be like "git log a..b -- ./file", not "git diff a..b -- ./file"
+###
+
+task:
+  name: Documentation
+
+  env:
+    CPUS: 1
+    BUILD_JOBS: 1
+
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(docs|html).*'
+
+  container:
+    image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
+    cpu: $CPUS
+    memory: 2G
+
+  # Exercise HTML and other docs:
+  ninja_docs_build_script: |
+    mkdir build-ninja
+    cd build-ninja
+    time meson setup
+    time ninja docs
+
+  # Again, with autoconf
+  docs_build_script: |
+    time ./configure
+    make -s -j${BUILD_JOBS} -C doc
-- 
2.25.1

From 0686c222fdf36bd2b844cbe9c86d522dfc030fc8 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Tue, 19 Jul 2022 12:38:45 -0500
Subject: [PATCH 07/10] cirrus/warnings: use a single/common 'always' block

ci-os-only: warnings
---
 .cirrus.yml | 40 +++++++++++++++++-----------------------
 1 file changed, 17 insertions(+), 23 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 5cb18032779..0bc7c5e3796 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -664,9 +664,6 @@ task:
     clang -v
     export
 
-  ccache_cache:
-    folder: $CCACHE_DIR
-
   setup_additional_packages_script: |
     #apt-get update
     #DEBIAN_FRONTEND=noninteractive apt-get -y install ...
@@ -681,8 +678,11 @@ task:
   # different compilers to build with different combinations of dtrace on/off
   # and cassert on/off.
 
-  # gcc, cassert off, dtrace on
   always:
+    ccache_cache:
+      folder: $CCACHE_DIR
+
+    # gcc, cassert off, dtrace on
     gcc_warning_script: |
       time ./configure \
         --cache gcc.cache \
@@ -692,8 +692,7 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  # gcc, cassert on, dtrace off
-  always:
+    # gcc, cassert on, dtrace off
     gcc_a_warning_script: |
       time ./configure \
         --cache gcc.cache \
@@ -703,8 +702,7 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  # clang, cassert off, dtrace off
-  always:
+    # clang, cassert off, dtrace off
     clang_warning_script: |
       time ./configure \
         --cache clang.cache \
@@ -713,8 +711,7 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  # clang, cassert on, dtrace on
-  always:
+    # clang, cassert on, dtrace on
     clang_a_warning_script: |
       time ./configure \
         --cache clang.cache \
@@ -725,8 +722,7 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  # cross-compile to windows
-  always:
+    # cross-compile to windows
     mingw_cross_warning_script: |
       time ./configure \
         --host=x86_64-w64-mingw32 \
@@ -736,16 +732,15 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  ###
-  # Verify headerscheck / cpluspluscheck succeed
-  #
-  # - Don't use ccache, the files are uncacheable, polluting ccache's
-  #   cache
-  # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
-  # - XXX have to disable ICU to avoid errors:
-  #   https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
-  ###
-  always:
+    ###
+    # Verify headerscheck / cpluspluscheck succeed
+    #
+    # - Don't use ccache, the files are uncacheable, polluting ccache's
+    #   cache
+    # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
+    # - XXX have to disable ICU to avoid errors:
+    #   https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
+    ###
     headers_headerscheck_script: |
       time ./configure \
         ${LINUX_CONFIGURE_FEATURES} \
@@ -757,7 +752,6 @@ task:
     headers_cpluspluscheck_script: |
       time make -s cpluspluscheck EXTRAFLAGS='-fmax-errors=10'
 
-  always:
     upload_caches: ccache
 
 
-- 
2.25.1

From 994c0df0b98fab4f10ac987f13302e0eb3aa07a9 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 26 Feb 2022 19:39:10 -0600
Subject: [PATCH 08/10] cirrus: upload changed html docs as artifacts

This could be done on the client side (cfbot).  One advantage of doing it here
is that fewer docs are uploaded - many patches won't upload docs at all.

https://www.postgresql.org/message-id/flat/20220409021853.GP24419@telsasoft.com
https://www.postgresql.org/message-id/CAB8KJ=i4qmEuopQ+PCSMBzGd4O-Xv0FCnC+q1x7hN9hsdvkBug@mail.gmail.com

https://cirrus-ci.com/task/5396696388599808

ci-os-only: html
---
 .cirrus.yml                    | 31 ++++++++++++++++++++++++++++---
 src/tools/ci/copy-changed-docs | 29 +++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+), 3 deletions(-)
 create mode 100755 src/tools/ci/copy-changed-docs

diff --git a/.cirrus.yml b/.cirrus.yml
index 0bc7c5e3796..2f2c09027d8 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -27,6 +27,14 @@ env:
   TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
   PG_TEST_EXTRA: kerberos ldap ssl
 
+  # The commit that this branch is rebased on.  There's no easy way to find this.
+  # This does the right thing for cfbot, which always squishes all patches into a single commit.
+  # And does the right thing for any 1-patch commits.
+  # Patch series manually submitted to cirrus would benefit from setting this to the
+  # number of patches in the series (or directly to the commit the series was rebased on).
+  #BASE_COMMIT: HEAD~1
+  # For demo purposes:
+  BASE_COMMIT: 3d14e171e9e2236139e8976f3309a588bcc8683b
 
 # What files to preserve in case tests fail
 on_failure_ac: &on_failure_ac
@@ -756,7 +764,7 @@ task:
 
 
 ###
-# Verify docs can be built
+# Verify docs can be built, and upload changed docs as artifacts
 # changesInclude() will skip this task if none of the commits since
 # CIRRUS_LAST_GREEN_CHANGE touched any relevant files. The comparison appears
 # to be like "git log a..b -- ./file", not "git diff a..b -- ./file"
@@ -764,18 +772,21 @@ task:
 
 task:
   name: Documentation
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(docs|html).*'
+  #skip: "!changesInclude('.cirrus.yml', 'doc/**', 'src/tools/ci/copy-changed-docs')"
 
   env:
     CPUS: 1
     BUILD_JOBS: 1
 
-  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(docs|html).*'
-
   container:
     image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
     cpu: $CPUS
     memory: 2G
 
+  sysinfo_script: |
+    git diff --name-only "$BASE_COMMIT"
+
   # Exercise HTML and other docs:
   ninja_docs_build_script: |
     mkdir build-ninja
@@ -787,3 +798,17 @@ task:
   docs_build_script: |
     time ./configure
     make -s -j${BUILD_JOBS} -C doc
+    cp -r doc new-docs
+
+    # Build HTML docs from the base commit.
+    git checkout "$BASE_COMMIT" -- doc
+    #html-stamp
+    make -s -C doc clean
+    make -s -C doc html
+    cp -r doc old-docs
+
+  copy_changed_docs_script: |
+    src/tools/ci/copy-changed-docs "old-docs" "new-docs" "html_docs"
+
+  html_docs_artifacts:
+    paths: ['html_docs/*.html', 'html_docs/*.png', 'html_docs/*.css']
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
new file mode 100755
index 00000000000..1c921a8df6f
--- /dev/null
+++ b/src/tools/ci/copy-changed-docs
@@ -0,0 +1,29 @@
+#! /bin/sh
+# Copy HTML which differ between $old and $new into $outdir
+set -e
+
+old=$1
+new=$2
+outdir=$3
+
+# The index is large and changes often
+skippages="bookindex.html"
+
+mkdir "$outdir"
+cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
+
+changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
+	[ $? -eq 1 ]
+
+for f in $changed
+do
+	# Avoid removed files
+	[ -f "$f" ] || continue
+
+	echo "$f" |grep -Ew "$skippages" >/dev/null &&
+		continue
+
+	cp -v "$f" "$outdir"
+done
+
+exit 0
-- 
2.25.1

From 3bd2db1aba704bb2fd1d8ebd5cec349770ee5e21 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 19 Oct 2022 20:33:28 -0500
Subject: [PATCH 09/10] WIP: show changed docs with meson

ci-os-only: html docs
---
 .cirrus.yml                    | 27 ++++++++++-----------------
 src/tools/ci/copy-changed-docs |  3 ++-
 2 files changed, 12 insertions(+), 18 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 2f2c09027d8..338c6c2be16 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -776,8 +776,7 @@ task:
   #skip: "!changesInclude('.cirrus.yml', 'doc/**', 'src/tools/ci/copy-changed-docs')"
 
   env:
-    CPUS: 1
-    BUILD_JOBS: 1
+    CPUS: 2
 
   container:
     image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
@@ -793,22 +792,16 @@ task:
     cd build-ninja
     time meson setup
     time ninja docs
+    cp -r doc ../new-docs
 
-  # Again, with autoconf
-  docs_build_script: |
-    time ./configure
-    make -s -j${BUILD_JOBS} -C doc
-    cp -r doc new-docs
-
-    # Build HTML docs from the base commit.
-    git checkout "$BASE_COMMIT" -- doc
-    #html-stamp
-    make -s -C doc clean
-    make -s -C doc html
-    cp -r doc old-docs
-
-  copy_changed_docs_script: |
-    src/tools/ci/copy-changed-docs "old-docs" "new-docs" "html_docs"
+    # Re-build docs from the base commit.
+    git checkout "$BASE_COMMIT" -- ../doc
+    ninja clean
+    time ninja doc/src/sgml/html
+    cp -r doc ../old-docs
+
+  copy_changed_docs_script:
+    - src/tools/ci/copy-changed-docs "old-docs" "new-docs" "html_docs"
 
   html_docs_artifacts:
     paths: ['html_docs/*.html', 'html_docs/*.png', 'html_docs/*.css']
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
index 1c921a8df6f..598e639f695 100755
--- a/src/tools/ci/copy-changed-docs
+++ b/src/tools/ci/copy-changed-docs
@@ -10,7 +10,8 @@ outdir=$3
 skippages="bookindex.html"
 
 mkdir "$outdir"
-cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
+cp doc/src/sgml/*.css "$outdir"
+cp doc/src/sgml/images/*.svg "$outdir"
 
 changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
 	[ $? -eq 1 ]
-- 
2.25.1

From 8086a65429b81fbbb2886f5bfa4495ff019b59bf Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Mon, 28 Feb 2022 23:18:19 -0600
Subject: [PATCH 10/10] +html index file

This allows linking to the artifacts from the last successful build, which
itself allows *not* rebuilding when sources haven't changed.

//freebsd
//-os-only: html,
---
 .cirrus.yml                    |  2 +-
 src/tools/ci/copy-changed-docs | 42 ++++++++++++++++++++++++++++++++--
 2 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 338c6c2be16..5e1612b583e 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -773,7 +773,7 @@ task:
 task:
   name: Documentation
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(docs|html).*'
-  #skip: "!changesInclude('.cirrus.yml', 'doc/**', 'src/tools/ci/copy-changed-docs')"
+  skip: "!changesInclude('.cirrus.yml', 'doc/**', 'src/tools/ci/copy-changed-docs')"
 
   env:
     CPUS: 2
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
index 598e639f695..c60428e1ae1 100755
--- a/src/tools/ci/copy-changed-docs
+++ b/src/tools/ci/copy-changed-docs
@@ -5,6 +5,10 @@ set -e
 old=$1
 new=$2
 outdir=$3
+branch=$CIRRUS_BRANCH
+
+# The index is large and changes often
+skippages="bookindex.html"
 
 # The index is large and changes often
 skippages="bookindex.html"
@@ -13,6 +17,19 @@ mkdir "$outdir"
 cp doc/src/sgml/*.css "$outdir"
 cp doc/src/sgml/images/*.svg "$outdir"
 
+# The index is useful to allow a static link (not specific to a cirrus run) to the artifacts for the most-recent, successful CI run for a branch
+branchurl=https://api.cirrus-ci.com/v1/artifact/github/$CIRRUS_REPO_FULL_NAME/$CIRRUS_TASK_NAME/html_docs/html_docs/00-index.html?branch=$branch
+
+index="$outdir/00-index.html"
+cat >"$index" <<EOF
+<html>
+<head><title>Index of docs changed since: $BASE_COMMIT</title></head>
+<body>
+<!-- A link to documentation for the most recent successful build: $branchurl -->
+<h1>Index of docs changed since: $BASE_COMMIT</h1>
+<ul>
+EOF
+
 changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
 	[ $? -eq 1 ]
 
@@ -24,7 +41,28 @@ do
 	echo "$f" |grep -Ew "$skippages" >/dev/null &&
 		continue
 
-	cp -v "$f" "$outdir"
-done
+	cp -v "$f" "$outdir" >&2
+	fn=${f##*/}
+	# ?branch=... is needed when accessing the artifacts for the static link for the branch
+	# It's not needed and ignored if accessing artifacts for *this* CI run
+	echo "<li><a href='$fn?branch=$branch'>$fn</a>"
+done >>"$index"
+
+github=https://github.com/$CIRRUS_REPO_FULL_NAME/commit
+cirrus=https://cirrus-ci.com/build
+
+cat >>"$index" <<EOF
+</ul>
+<hr>
+<code>
+<br>This file was written on: `date --rfc-822 --utc`
+<br>CIRRUS_CHANGE_TITLE: $CIRRUS_CHANGE_TITLE
+<br>CIRRUS_CHANGE_IN_REPO: <a href="$github/$CIRRUS_CHANGE_IN_REPO">$CIRRUS_CHANGE_IN_REPO</a>
+<br>CIRRUS_BUILD_ID: <a href="$cirrus/$CIRRUS_BUILD_ID">$CIRRUS_BUILD_ID</a>
+<br>CIRRUS_LAST_GREEN_CHANGE: <a href="$github/$CIRRUS_LAST_GREEN_CHANGE">$CIRRUS_LAST_GREEN_CHANGE</a>
+<br>CIRRUS_LAST_GREEN_BUILD_ID: <a href="$cirrus/$CIRRUS_LAST_GREEN_BUILD_ID">$CIRRUS_LAST_GREEN_BUILD_ID</a>
+</code>
+</body></html>
+EOF
 
 exit 0
-- 
2.25.1

From a355638fd9c1c16e54418d8374a374967081e0a0 Mon Sep 17 00:00:00 2001
From: Thomas Munro <thomas.munro@gmail.com>
Date: Wed, 28 Dec 2022 16:58:09 +1300
Subject: [PATCH] ci: Change macOS builds from Intel to ARM.

Cirrus is about to shut down its macOS-on-Intel support, so it's time to
move our CI testing over to ARM instances.  The Homebrew package manager
changed its default installation prefix for the new architecture, so a
couple of tests need tweaks to find binaries.

Author: Justin Pryzby <pryzby@telsasoft.com>
Discussion: https://postgr.es/m/20221122225744.GF11463%40telsasoft.com
---
 .cirrus.yml                     | 8 ++++----
 src/test/kerberos/t/001_auth.pl | 9 ++++++++-
 src/test/ldap/t/001_auth.pl     | 8 +++++++-
 3 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 993af88865..354102613d 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -407,7 +407,7 @@ task:
   name: macOS - Monterey - Meson
 
   env:
-    CPUS: 12 # always get that much for cirrusci macOS instances
+    CPUS: 4 # always get that much for cirrusci macOS instances
     BUILD_JOBS: $CPUS
     # Test performance regresses noticably when using all cores. 8 seems to
     # work OK. See
@@ -428,8 +428,8 @@ task:
   depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
 
-  osx_instance:
-    image: monterey-base
+  macos_instance:
+    image: ghcr.io/cirruslabs/macos-monterey-base:latest
 
   sysinfo_script: |
     id
@@ -475,7 +475,7 @@ task:
   ccache_cache:
     folder: $CCACHE_DIR
   configure_script: |
-    brewpath="/usr/local"
+    brewpath="/opt/homebrew"
     PKG_CONFIG_PATH="${brewpath}/lib/pkgconfig:${PKG_CONFIG_PATH}"
 
     for pkg in icu4c krb5 openldap openssl zstd ; do
diff --git a/src/test/kerberos/t/001_auth.pl b/src/test/kerberos/t/001_auth.pl
index a2bc8a5351..298dc0c62b 100644
--- a/src/test/kerberos/t/001_auth.pl
+++ b/src/test/kerberos/t/001_auth.pl
@@ -32,8 +32,15 @@ elsif ($ENV{PG_TEST_EXTRA} !~ /\bkerberos\b/)
 
 my ($krb5_bin_dir, $krb5_sbin_dir);
 
-if ($^O eq 'darwin')
+if ($^O eq 'darwin' && -d "/opt/homebrew" )
 {
+	# typical paths for Homebrew on ARM
+	$krb5_bin_dir  = '/opt/homebrew/opt/krb5/bin';
+	$krb5_sbin_dir = '/opt/homebrew/opt/krb5/sbin';
+}
+elsif ($^O eq 'darwin')
+{
+	# typical paths for Homebrew on Intel
 	$krb5_bin_dir  = '/usr/local/opt/krb5/bin';
 	$krb5_sbin_dir = '/usr/local/opt/krb5/sbin';
 }
diff --git a/src/test/ldap/t/001_auth.pl b/src/test/ldap/t/001_auth.pl
index 0ea274c383..39736e5116 100644
--- a/src/test/ldap/t/001_auth.pl
+++ b/src/test/ldap/t/001_auth.pl
@@ -21,9 +21,15 @@ elsif ($ENV{PG_TEST_EXTRA} !~ /\bldap\b/)
 {
 	plan skip_all => 'Potentially unsafe test LDAP not enabled in PG_TEST_EXTRA';
 }
+elsif ($^O eq 'darwin' && -d '/opt/homebrew/opt/openldap')
+{
+	# typical paths for Homebrew on ARM
+	$slapd           = '/opt/homebrew/opt/openldap/libexec/slapd';
+	$ldap_schema_dir = '/opt/homebrew/etc/openldap/schema';
+}
 elsif ($^O eq 'darwin' && -d '/usr/local/opt/openldap')
 {
-	# typical paths for Homebrew
+	# typical paths for Homebrew on Intel
 	$slapd           = '/usr/local/opt/openldap/libexec/slapd';
 	$ldap_schema_dir = '/usr/local/etc/openldap/schema';
 }
-- 
2.35.1

From 59bbbce620fdf3c4f228c0569b1fac29beb06988 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 25 May 2022 21:53:22 -0500
Subject: [PATCH 1/7] cirrus/windows: add compiler_warnings_script

I'm not sure how to write this test in windows shell; it's also easy to
write something that doesn't work in posix sh, since windows shell is
interpretting && and ||...

https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de

See also:
8a1ce5e54f6d144e4f8e19af7c767b026ee0c956
https://cirrus-ci.com/task/6241060062494720
https://cirrus-ci.com/task/6496366607204352

ci-os-only: windows
---
 .cirrus.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 69837bcd5ad..fd461048372 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -564,12 +564,21 @@ task:
 
   build_script: |
     vcvarsall x64
-    ninja -C build
+    ninja -C build |tee build.txt
+    REM Since pipes lose the exit status of the preceding command, rerun the compilation
+    REM without the pipe, exiting now if it fails, to avoid trying to run checks
+    ninja -C build > nul
 
   check_world_script: |
     vcvarsall x64
     meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
 
+  # This should be last, so check_world is run even if there are warnings
+  always:
+    compiler_warnings_script:
+      # this avoids using metachars which would be interpretted by the windows shell
+      - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0'
+
   on_failure:
     <<: *on_failure_meson
     crashlog_artifacts:
-- 
2.25.1

From 4edbdf0337c75bd69e536b542a4cd06dac87c33d Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 4 Nov 2022 15:43:56 -0500
Subject: [PATCH 2/7] cirrus/macos: update to macos ventura

ci-os-only: macos
---
 .cirrus.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index fd461048372..858454a3d08 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -404,7 +404,7 @@ task:
 
 
 task:
-  name: macOS - Monterey - Meson
+  name: macOS - Ventura - Meson
 
   env:
     CPUS: 4 # always get that much for cirrusci macOS instances
@@ -429,7 +429,7 @@ task:
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
 
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-monterey-base:latest
+    image: ghcr.io/cirruslabs/macos-ventura-base:latest
 
   sysinfo_script: |
     id
-- 
2.25.1

From 2f6a7102f717d7fe5e291d6e439fd32b122180a1 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 24 Jun 2022 00:09:12 -0500
Subject: [PATCH 3/7] cirrus/freebsd: run with more CPUs+RAM and do not
 repartition

There was some historic problem where tests under freebsd took 8+ minutes (and
before 4a288a37f took 15 minutes).

This reduces test time from 10min to 3min.
4 CPUs 4 tests https://cirrus-ci.com/task/4880240739614720
4 CPUs 6 tests https://cirrus-ci.com/task/4664440120410112 https://cirrus-ci.com/task/4586784884523008
4 CPUs 8 tests https://cirrus-ci.com/task/5001995491737600

6 CPUs https://cirrus-ci.com/task/6678321684545536
8 CPUs https://cirrus-ci.com/task/6264854121021440

See also:
https://www.postgresql.org/message-id/flat/20220310033347.hgxk4pyarzq4hxwp@alap3.anarazel.de#f36c0b17e33e31e7925e7e5812998686
8 jobs 7min https://cirrus-ci.com/task/6186376667332608

//-os-only: freebsd
---
 .cirrus.yml | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 858454a3d08..2ba6b7cc2d8 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -131,11 +131,9 @@ task:
   name: FreeBSD - 13 - Meson
 
   env:
-    # FreeBSD on GCP is slow when running with larger number of CPUS /
-    # jobs. Using one more job than cpus seems to work best.
-    CPUS: 2
-    BUILD_JOBS: 3
-    TEST_JOBS: 3
+    CPUS: 4
+    BUILD_JOBS: 4
+    TEST_JOBS: 6
 
     CCACHE_DIR: /tmp/ccache_dir
     CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST
@@ -160,8 +158,6 @@ task:
 
   ccache_cache:
     folder: $CCACHE_DIR
-  # Work around performance issues due to 32KB block size
-  repartition_script: src/tools/ci/gcp_freebsd_repartition.sh
   create_user_script: |
     pw useradd postgres
     chown -R postgres:postgres .
-- 
2.25.1

From b55c042ec467a03984a5e2132dc4e7fe0f49fef3 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 9 Dec 2022 15:32:50 -0600
Subject: [PATCH 4/7] cirrus/freebsd: define
 ENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS

See also: 54100f5c6052404f68de9ce7310ceb61f1c291f8

ci-os-only: freebsd
---
 .cirrus.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 2ba6b7cc2d8..34c9d52b624 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -136,7 +136,7 @@ task:
     TEST_JOBS: 6
 
     CCACHE_DIR: /tmp/ccache_dir
-    CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST
+    CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST -DENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS
     CFLAGS: -Og -ggdb
 
   depends_on: SanityCheck
-- 
2.25.1

From a4ecb1a1bf008052108382f5833345680eec156c Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Tue, 19 Jul 2022 12:38:45 -0500
Subject: [PATCH 5/7] cirrus/warnings: use a single/common 'always' block

ci-os-only: warnings
---
 .cirrus.yml | 35 ++++++++++++++---------------------
 1 file changed, 14 insertions(+), 21 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 34c9d52b624..123f6957513 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -696,8 +696,8 @@ task:
   # different compilers to build with different combinations of dtrace on/off
   # and cassert on/off.
 
-  # gcc, cassert off, dtrace on
   always:
+    # gcc, cassert off, dtrace on
     gcc_warning_script: |
       time ./configure \
         --cache gcc.cache \
@@ -707,8 +707,7 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  # gcc, cassert on, dtrace off
-  always:
+    # gcc, cassert on, dtrace off
     gcc_a_warning_script: |
       time ./configure \
         --cache gcc.cache \
@@ -718,8 +717,7 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  # clang, cassert off, dtrace off
-  always:
+    # clang, cassert off, dtrace off
     clang_warning_script: |
       time ./configure \
         --cache clang.cache \
@@ -728,8 +726,7 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  # clang, cassert on, dtrace on
-  always:
+    # clang, cassert on, dtrace on
     clang_a_warning_script: |
       time ./configure \
         --cache clang.cache \
@@ -740,8 +737,7 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
-  # cross-compile to windows
-  always:
+    # cross-compile to windows
     mingw_cross_warning_script: |
       time ./configure \
         --host=x86_64-w64-mingw32 \
@@ -755,7 +751,6 @@ task:
   # Verify docs can be built
   ###
   # XXX: Only do this if there have been changes in doc/ since last build
-  always:
     docs_build_script: |
       time ./configure \
         --cache gcc.cache \
@@ -765,16 +760,15 @@ task:
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} -C doc
 
-  ###
-  # Verify headerscheck / cpluspluscheck succeed
-  #
-  # - Don't use ccache, the files are uncacheable, polluting ccache's
-  #   cache
-  # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
-  # - XXX have to disable ICU to avoid errors:
-  #   https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
-  ###
-  always:
+    ###
+    # Verify headerscheck / cpluspluscheck succeed
+    #
+    # - Don't use ccache, the files are uncacheable, polluting ccache's
+    #   cache
+    # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
+    # - XXX have to disable ICU to avoid errors:
+    #   https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
+    ###
     headers_headerscheck_script: |
       time ./configure \
         ${LINUX_CONFIGURE_FEATURES} \
@@ -786,5 +780,4 @@ task:
     headers_cpluspluscheck_script: |
       time make -s cpluspluscheck EXTRAFLAGS='-fmax-errors=10'
 
-  always:
     upload_caches: ccache
-- 
2.25.1

From e523bd786c034587d6c45079393ba1563cd16eff Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 26 Feb 2022 19:39:10 -0600
Subject: [PATCH 6/7] cirrus: upload changed html docs as artifacts

This could be done on the client side (cfbot).  One advantage of doing
it here is that fewer docs are uploaded - many patches won't upload docs
at all.

https://www.postgresql.org/message-id/flat/20220409021853.GP24419@telsasoft.com
https://www.postgresql.org/message-id/CAB8KJ=i4qmEuopQ+PCSMBzGd4O-Xv0FCnC+q1x7hN9hsdvkBug@mail.gmail.com

https://cirrus-ci.com/task/5396696388599808

ci-os-only: html
---
 .cirrus.yml                    | 24 +++++++++++++++++++++++-
 src/tools/ci/copy-changed-docs | 29 +++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 1 deletion(-)
 create mode 100755 src/tools/ci/copy-changed-docs

diff --git a/.cirrus.yml b/.cirrus.yml
index 123f6957513..a6a246882c9 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -27,6 +27,14 @@ env:
   TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
   PG_TEST_EXTRA: kerberos ldap ssl
 
+  # The commit that this branch is rebased on.  There's no easy way to find this.
+  # This does the right thing for cfbot, which always squishes all patches into a single commit.
+  # And does the right thing for any 1-patch commits.
+  # Patch series manually submitted to cirrus would benefit from setting this to the
+  # number of patches in the series (or directly to the commit the series was rebased on).
+  #BASE_COMMIT: HEAD~1
+  # For demo purposes:
+  BASE_COMMIT: HEAD~11
 
 # What files to preserve in case tests fail
 on_failure_ac: &on_failure_ac
@@ -678,6 +686,7 @@ task:
     gcc -v
     clang -v
     export
+    git diff --name-only "$BASE_COMMIT"
 
   ccache_cache:
     folder: $CCACHE_DIR
@@ -748,7 +757,7 @@ task:
       time make -s -j${BUILD_JOBS} world-bin
 
   ###
-  # Verify docs can be built
+  # Verify docs can be built, and upload changed docs as artifacts
   ###
   # XXX: Only do this if there have been changes in doc/ since last build
     docs_build_script: |
@@ -759,6 +768,19 @@ task:
         CLANG="ccache clang"
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} -C doc
+      cp -r doc new-docs
+
+      # Re-build HTML docs from the base commit.
+      git checkout "$BASE_COMMIT" -- doc
+      make -s -C doc clean
+      time make -s -C doc html
+      cp -r doc old-docs
+
+    copy_changed_docs_script:
+      - src/tools/ci/copy-changed-docs "old-docs" "new-docs" "html_docs"
+
+    html_docs_artifacts:
+      paths: ['html_docs/*.html', 'html_docs/*.png', 'html_docs/*.css']
 
     ###
     # Verify headerscheck / cpluspluscheck succeed
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
new file mode 100755
index 00000000000..1c921a8df6f
--- /dev/null
+++ b/src/tools/ci/copy-changed-docs
@@ -0,0 +1,29 @@
+#! /bin/sh
+# Copy HTML which differ between $old and $new into $outdir
+set -e
+
+old=$1
+new=$2
+outdir=$3
+
+# The index is large and changes often
+skippages="bookindex.html"
+
+mkdir "$outdir"
+cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
+
+changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
+	[ $? -eq 1 ]
+
+for f in $changed
+do
+	# Avoid removed files
+	[ -f "$f" ] || continue
+
+	echo "$f" |grep -Ew "$skippages" >/dev/null &&
+		continue
+
+	cp -v "$f" "$outdir"
+done
+
+exit 0
-- 
2.25.1

From e0ef15ae513d62a42887dcbe135e2bc2c477074d Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 25 Nov 2022 13:57:17 -0600
Subject: [PATCH 7/7] WIP: ci/meson: allow showing only failed tests ..

It's simpler and seems to make more sense to integrate this with
testwrap, rather than to run it after check-world, but only if it
failed, and finding a way to preserve the exit code.

https://www.postgresql.org/message-id/20221114235328.lxdj3puenfhirhqm@awork3.anarazel.de

> It is wasteful to upload thousdands of logfiles to show a single
> failure.  That would make our cirrus tasks faster - compressing and
> uploading the logs takes over a minute.
>
> It's also a lot friendlier to show fewer than 8 pages of test folders to
> search through to find the one that failed.

macos
ci-os-only: linux-meson freebsd
---
 .cirrus.yml        | 13 +++++++------
 src/tools/testwrap |  6 ++++++
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index a6a246882c9..f64aad64756 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -23,6 +23,7 @@ env:
   CHECKFLAGS: -Otarget
   PROVE_FLAGS: --timer
   MTEST_ARGS: --print-errorlogs --no-rebuild -C build
+  PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
   PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
   TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
   PG_TEST_EXTRA: kerberos ldap ssl
@@ -48,9 +49,9 @@ on_failure_ac: &on_failure_ac
 on_failure_meson: &on_failure_meson
   testrun_artifacts:
     paths:
-      - "build*/testrun/**/*.log"
-      - "build*/testrun/**/*.diffs"
-      - "build*/testrun/**/regress_log_*"
+      - "failed.build*/**/*.log"
+      - "failed.build*/**/*.diffs"
+      - "failed.build*/**/regress_log_*"
     type: text/plain
 
   # In theory it'd be nice to upload the junit files meson generates, so that
@@ -206,10 +207,10 @@ task:
       ulimit -c unlimited
       meson test $MTEST_ARGS --quiet --suite setup
       export LD_LIBRARY_PATH="$(pwd)/build/tmp_install/usr/local/pgsql/lib/:$LD_LIBRARY_PATH"
-      mkdir -p build/testrun
+      mkdir -p build/testrun ${PG_FAILED_TESTDIR}
       build/tmp_install/usr/local/pgsql/bin/initdb -N build/runningcheck --no-instructions -A trust
       echo "include '$(pwd)/src/tools/ci/pg_ci_base.conf'" >> build/runningcheck/postgresql.conf
-      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
+      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l ${PG_FAILED_TESTDIR}/runningcheck.log start
       meson test $MTEST_ARGS --num-processes ${TEST_JOBS} --setup running
       build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
     EOF
@@ -397,7 +398,7 @@ task:
       test_world_32_script: |
         su postgres <<-EOF
           ulimit -c unlimited
-          PYTHONCOERCECLOCALE=0 LANG=C meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
+          PYTHONCOERCECLOCALE=0 LANG=C PG_FAILED_TESTDIR=`pwd`/failed.build-32 meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
         EOF
 
       on_failure:
diff --git a/src/tools/testwrap b/src/tools/testwrap
index 7a64fe76a2d..445ac595afc 100755
--- a/src/tools/testwrap
+++ b/src/tools/testwrap
@@ -44,4 +44,10 @@ if sp.returncode == 0:
 else:
     print('# test failed')
     open(os.path.join(testdir, 'test.fail'), 'x')
+    faileddir = os.getenv('PG_FAILED_TESTDIR')
+    if faileddir:
+        parentdir = os.path.dirname(testdir)
+        newdest = os.path.join(faileddir, os.path.basename(parentdir), os.path.basename(testdir))
+        shutil.copytree(testdir, newdest)
+
 sys.exit(sp.returncode)
-- 
2.25.1

From a5b3e50d922a6dcff22feabbcae741b23b2347c6 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 25 May 2022 21:53:22 -0500
Subject: [PATCH 1/9] cirrus/windows: add compiler_warnings_script

I'm not sure how to write this test in windows shell; it's also easy to
write something that doesn't work in posix sh, since windows shell is
interpretting && and ||...

https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de

See also:
8a1ce5e54f6d144e4f8e19af7c767b026ee0c956
https://cirrus-ci.com/task/6241060062494720
https://cirrus-ci.com/task/6496366607204352

ci-os-only: windows
---
 .cirrus.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 1204824d2eb..eefc5c21fe6 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -566,12 +566,21 @@ task:
 
   build_script: |
     vcvarsall x64
-    ninja -C build
+    ninja -C build |tee build.txt
+    REM Since pipes lose the exit status of the preceding command, rerun the compilation
+    REM without the pipe, exiting now if it fails, to avoid trying to run checks
+    ninja -C build > nul
 
   check_world_script: |
     vcvarsall x64
     meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
 
+  # This should be last, so check_world is run even if there are warnings
+  always:
+    compiler_warnings_script:
+      # this avoids using metachars which would be interpretted by the windows shell
+      - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0'
+
   on_failure:
     <<: *on_failure_meson
     crashlog_artifacts:
-- 
2.25.1

From 4c98dcb0e033ca12c6a6093c5e94e4231756fa4d Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 24 Jun 2022 00:09:12 -0500
Subject: [PATCH 2/9] cirrus/freebsd: run with more CPUs+RAM and do not
 repartition

There was some historic problem where tests under freebsd took 8+ minutes (and
before 4a288a37f took 15 minutes).

This reduces test time from 10min to 3min.
4 CPUs 4 tests https://cirrus-ci.com/task/4880240739614720
4 CPUs 6 tests https://cirrus-ci.com/task/4664440120410112 https://cirrus-ci.com/task/4586784884523008
4 CPUs 8 tests https://cirrus-ci.com/task/5001995491737600

6 CPUs https://cirrus-ci.com/task/6678321684545536
8 CPUs https://cirrus-ci.com/task/6264854121021440

See also:
https://www.postgresql.org/message-id/flat/20220310033347.hgxk4pyarzq4hxwp@alap3.anarazel.de#f36c0b17e33e31e7925e7e5812998686
8 jobs 7min https://cirrus-ci.com/task/6186376667332608

//-os-only: freebsd
---
 .cirrus.yml | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index eefc5c21fe6..0c12ca04fd9 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -131,11 +131,9 @@ task:
   name: FreeBSD - 13 - Meson
 
   env:
-    # FreeBSD on GCP is slow when running with larger number of CPUS /
-    # jobs. Using one more job than cpus seems to work best.
-    CPUS: 2
-    BUILD_JOBS: 3
-    TEST_JOBS: 3
+    CPUS: 4
+    BUILD_JOBS: 4
+    TEST_JOBS: 6
 
     CCACHE_DIR: /tmp/ccache_dir
     CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST
@@ -160,8 +158,6 @@ task:
 
   ccache_cache:
     folder: $CCACHE_DIR
-  # Work around performance issues due to 32KB block size
-  repartition_script: src/tools/ci/gcp_freebsd_repartition.sh
   create_user_script: |
     pw useradd postgres
     chown -R postgres:postgres .
@@ -175,8 +171,7 @@ task:
     #pkg install -y ...
 
   # NB: Intentionally build without -Dllvm. The freebsd image size is already
-  # large enough to make VM startup slow, and even without llvm freebsd
-  # already takes longer than other platforms except for windows.
+  # large enough to make VM startup slow
   configure_script: |
     su postgres <<-EOF
       meson setup \
-- 
2.25.1

From aaeef938ed4a1b604c41d3973a49d790ad58af20 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 9 Dec 2022 15:32:50 -0600
Subject: [PATCH 3/9] cirrus/freebsd: define
 ENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS

See also: 54100f5c6052404f68de9ce7310ceb61f1c291f8

ci-os-only: freebsd
---
 .cirrus.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 0c12ca04fd9..6186505ccd7 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -136,7 +136,7 @@ task:
     TEST_JOBS: 6
 
     CCACHE_DIR: /tmp/ccache_dir
-    CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST
+    CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST -DENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS
     CFLAGS: -Og -ggdb
 
   depends_on: SanityCheck
-- 
2.25.1

From 9baf41674ad28d68f2c657363079643955252fee Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 30 Jul 2022 19:25:51 -0500
Subject: [PATCH 4/9] pg_upgrade: tap test: exercise --link and --clone

This both increases code coverage and accelerates tests.

See also: b059a2409faf5833b3ba7792e247d6466c9e8090

linux,
macos,
//-os-only: freebsd
---
 .cirrus.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.cirrus.yml b/.cirrus.yml
index 6186505ccd7..a8a514ed178 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -138,6 +138,7 @@ task:
     CCACHE_DIR: /tmp/ccache_dir
     CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST -DENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS
     CFLAGS: -Og -ggdb
+    PG_TEST_PG_UPGRADE_MODE: --link
 
   depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
@@ -269,6 +270,7 @@ task:
     LDFLAGS: $SANITIZER_FLAGS
     CC: ccache gcc
     CXX: ccache g++
+    PG_TEST_PG_UPGRADE_MODE: --link
 
     LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
     LINUX_MESON_FEATURES: *LINUX_MESON_FEATURES
@@ -419,6 +421,7 @@ task:
     CPPFLAGS: -DRANDOMIZE_ALLOCATED_MEMORY
     CFLAGS: -Og -ggdb
     CXXFLAGS: -Og -ggdb
+    PG_TEST_PG_UPGRADE_MODE: --clone
 
   depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
-- 
2.25.1

From 7e09035f5882a0262fe339eb305e64f3bbaa2bb9 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 25 Nov 2022 13:57:17 -0600
Subject: [PATCH 5/9] WIP: ci/meson: allow showing only failed tests ..

It's simpler and seems to make more sense to integrate this with
testwrap, rather than to run it after check-world, but only if it
failed, and finding a way to preserve the exit code.

https://www.postgresql.org/message-id/20221114235328.lxdj3puenfhirhqm@awork3.anarazel.de

> It is wasteful to upload thousdands of logfiles to show a single
> failure.  That would make our cirrus tasks faster - compressing and
> uploading the logs takes over a minute.
>
> It's also a lot friendlier to show fewer than 8 pages of test folders to
> search through to find the one that failed.

macos
ci-os-only: linux-meson freebsd
---
 .cirrus.yml        | 13 +++++++------
 src/tools/testwrap |  6 ++++++
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index a8a514ed178..fd5585e5818 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -23,6 +23,7 @@ env:
   CHECKFLAGS: -Otarget
   PROVE_FLAGS: --timer
   MTEST_ARGS: --print-errorlogs --no-rebuild -C build
+  PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
   PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
   TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
   PG_TEST_EXTRA: kerberos ldap ssl
@@ -40,9 +41,9 @@ on_failure_ac: &on_failure_ac
 on_failure_meson: &on_failure_meson
   testrun_artifacts:
     paths:
-      - "build*/testrun/**/*.log"
-      - "build*/testrun/**/*.diffs"
-      - "build*/testrun/**/regress_log_*"
+      - "failed.build*/**/*.log"
+      - "failed.build*/**/*.diffs"
+      - "failed.build*/**/regress_log_*"
     type: text/plain
 
   # In theory it'd be nice to upload the junit files meson generates, so that
@@ -198,10 +199,10 @@ task:
       ulimit -c unlimited
       meson test $MTEST_ARGS --quiet --suite setup
       export LD_LIBRARY_PATH="$(pwd)/build/tmp_install/usr/local/pgsql/lib/:$LD_LIBRARY_PATH"
-      mkdir -p build/testrun
+      mkdir -p build/testrun ${PG_FAILED_TESTDIR}
       build/tmp_install/usr/local/pgsql/bin/initdb -N build/runningcheck --no-instructions -A trust
       echo "include '$(pwd)/src/tools/ci/pg_ci_base.conf'" >> build/runningcheck/postgresql.conf
-      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
+      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l ${PG_FAILED_TESTDIR}/runningcheck.log start
       meson test $MTEST_ARGS --num-processes ${TEST_JOBS} --setup running
       build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
     EOF
@@ -390,7 +391,7 @@ task:
       test_world_32_script: |
         su postgres <<-EOF
           ulimit -c unlimited
-          PYTHONCOERCECLOCALE=0 LANG=C meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
+          PYTHONCOERCECLOCALE=0 LANG=C PG_FAILED_TESTDIR=`pwd`/failed.build-32 meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
         EOF
 
       on_failure:
diff --git a/src/tools/testwrap b/src/tools/testwrap
index 7a64fe76a2d..445ac595afc 100755
--- a/src/tools/testwrap
+++ b/src/tools/testwrap
@@ -44,4 +44,10 @@ if sp.returncode == 0:
 else:
     print('# test failed')
     open(os.path.join(testdir, 'test.fail'), 'x')
+    faileddir = os.getenv('PG_FAILED_TESTDIR')
+    if faileddir:
+        parentdir = os.path.dirname(testdir)
+        newdest = os.path.join(faileddir, os.path.basename(parentdir), os.path.basename(testdir))
+        shutil.copytree(testdir, newdest)
+
 sys.exit(sp.returncode)
-- 
2.25.1

From e4534821ef5c1997ff34a2515884c5592f171e0b Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sun, 13 Nov 2022 14:03:27 -0600
Subject: [PATCH 6/9] cirrus/ccache: use G rather than GB suffix..

the former being the documented spelling.
---
 .cirrus.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index fd5585e5818..a9fa4b5af27 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -667,7 +667,7 @@ task:
 
     # Use larger ccache cache, as this task compiles with multiple compilers /
     # flag combinations
-    CCACHE_MAXSIZE: "1GB"
+    CCACHE_MAXSIZE: "1G"
     CCACHE_DIR: "/tmp/ccache_dir"
 
     LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
-- 
2.25.1

From 185d1c3ed13f5472fea32fd15f55158721b7245b Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Mon, 17 Jan 2022 00:54:28 -0600
Subject: [PATCH 7/9] cirrus: code coverage

Some alternatives:
- could build with "--coverage -fprofile-filter-files=", but that means
  that ccache will never work (both because it doesn't support that option
  and also because the arguments will be different for every patch).
- could use ninja coverage-html, but that can't filter only changed
  files, and would take a long time to upload a lot of useless files.

https://www.postgresql.org/message-id/202202111821.w3gqblvfp4pr%40alvherre.pgsql
https://www.postgresql.org/message-id/flat/20220409021853.GP24419@telsasoft.com

ci-os-only: freebsd
---
 .cirrus.yml                       | 18 +++++++++++++++-
 src/tools/ci/code-coverage-report | 35 +++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 1 deletion(-)
 create mode 100755 src/tools/ci/code-coverage-report

diff --git a/.cirrus.yml b/.cirrus.yml
index a9fa4b5af27..ec15263a665 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -28,6 +28,14 @@ env:
   TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
   PG_TEST_EXTRA: kerberos ldap ssl
 
+  # The commit that this branch is rebased on.  There's no easy way to find this.
+  # This does the right thing for cfbot, which always squishes all patches into a single commit.
+  # And does the right thing for any 1-patch commits.
+  # Patch series manually submitted to cirrus would benefit from setting this to the
+  # number of patches in the series (or directly to the commit the series was rebased on).
+  #BASE_COMMIT: HEAD~1
+  # For demo purposes:
+  BASE_COMMIT: HEAD~11
 
 # What files to preserve in case tests fail
 on_failure_ac: &on_failure_ac
@@ -157,6 +165,7 @@ task:
     uname -a
     ulimit -a -H && ulimit -a -S
     export
+    git diff --name-only "$BASE_COMMIT"
 
   ccache_cache:
     folder: $CCACHE_DIR
@@ -170,7 +179,7 @@ task:
     chown root:postgres /tmp/cores
     sysctl kern.corefile='/tmp/cores/%N.%P.core'
   setup_additional_packages_script: |
-    #pkg install -y ...
+    pkg install -y lcov
 
   # NB: Intentionally build without -Dllvm. The freebsd image size is already
   # large enough to make VM startup slow
@@ -178,6 +187,7 @@ task:
     su postgres <<-EOF
       meson setup \
         --buildtype=debug \
+        -Db_coverage=true \
         -Dcassert=true -Dssl=openssl -Duuid=bsd -Dtcl_version=tcl86 -Ddtrace=auto \
         -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
         -Dextra_lib_dirs=/usr/local/lib -Dextra_include_dirs=/usr/local/include/ \
@@ -188,10 +198,16 @@ task:
 
   test_world_script: |
     su postgres <<-EOF
+      set -e
       ulimit -c unlimited
       meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
+      # Create coverage report for files changed since the base commit.
+      time ./src/tools/ci/code-coverage-report "$BASE_COMMIT" ./build ./coverage
     EOF
 
+  coverage_artifacts:
+    path: 'coverage/**'
+
   # test runningcheck, freebsd chosen because it's currently fast enough
   test_running_script: |
     su postgres <<-EOF
diff --git a/src/tools/ci/code-coverage-report b/src/tools/ci/code-coverage-report
new file mode 100755
index 00000000000..4f29d3f17a7
--- /dev/null
+++ b/src/tools/ci/code-coverage-report
@@ -0,0 +1,35 @@
+#! /bin/sh
+# Called during CI to generate a code coverage report of changed files.
+set -e
+
+base_branch=$1
+build_dir=$2
+outdir=$3
+
+changed=`git diff --name-only "$base_branch" '*.c'`
+[ -z "$changed" ] && exit 0 # Nothing changed
+
+mkdir "$outdir"
+
+# Coverage is shown only for changed files
+# This is useful to see coverage of newly-added code, but won't
+# show added/lost coverage in files which this patch doesn't modify.
+
+# This could be used to map from object file back to source file:
+# readelf --debug-dump=info src/backend/postgres_lib.a.p/utils_adt_array_userfuncs.c.o |awk '/DW_AT_name/{print $NF;exit}'
+# gcov ./src/port/libpgport_shlib.a.p/inet_net_ntop.c.gcno --stdout
+
+gcov=$outdir/coverage.gcov
+lcov --quiet --capture --directory "$build_dir" >"$gcov.all"
+
+# Filter to include only changed files
+echo "$changed" |sed 's,^,*/,' |
+	xargs -rt lcov --extract "$gcov.all" >"$gcov"
+
+ls -l "$outdir"
+
+# Exit successfully if no relevant files were changed
+[ -s "$gcov" ] || exit 0
+
+genhtml "$gcov" --show-details --legend --quiet --num-spaces=4 --output-directory "$outdir" --title="Coverage report of files changed since: $base_branch"
+cp "$outdir"/index.html "$outdir"/00-index.html
-- 
2.25.1

From 5dace84a03882f7c0a274102fe497bfe9f606a76 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 26 Feb 2022 19:39:10 -0600
Subject: [PATCH 8/9] cirrus: upload changed html docs as artifacts

This could be done on the client side (cfbot).  One advantage of doing
it here is that fewer docs are uploaded - many patches won't upload docs
at all.

https://www.postgresql.org/message-id/flat/20220409021853.GP24419@telsasoft.com
https://www.postgresql.org/message-id/CAB8KJ=i4qmEuopQ+PCSMBzGd4O-Xv0FCnC+q1x7hN9hsdvkBug@mail.gmail.com

https://cirrus-ci.com/task/5396696388599808

ci-os-only: html
---
 .cirrus.yml                    | 16 +++++++++++++++-
 src/tools/ci/copy-changed-docs | 29 +++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)
 create mode 100755 src/tools/ci/copy-changed-docs

diff --git a/.cirrus.yml b/.cirrus.yml
index ec15263a665..57359f43867 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -701,6 +701,7 @@ task:
     gcc -v
     clang -v
     export
+    git diff --name-only "$BASE_COMMIT"
 
   ccache_cache:
     folder: $CCACHE_DIR
@@ -775,7 +776,7 @@ task:
       time make -s -j${BUILD_JOBS} world-bin
 
   ###
-  # Verify docs can be built
+  # Verify docs can be built, and upload changed docs as artifacts
   ###
   # XXX: Only do this if there have been changes in doc/ since last build
   always:
@@ -787,6 +788,19 @@ task:
         CLANG="ccache clang"
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} -C doc
+      cp -r doc new-docs
+
+      # Re-build HTML docs from the base commit.
+      git checkout "$BASE_COMMIT" -- doc
+      make -s -C doc clean
+      time make -s -C doc html
+      cp -r doc old-docs
+
+    copy_changed_docs_script:
+      - src/tools/ci/copy-changed-docs "old-docs" "new-docs" "html_docs"
+
+    html_docs_artifacts:
+      paths: ['html_docs/*.html', 'html_docs/*.png', 'html_docs/*.css']
 
   ###
   # Verify headerscheck / cpluspluscheck succeed
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
new file mode 100755
index 00000000000..1c921a8df6f
--- /dev/null
+++ b/src/tools/ci/copy-changed-docs
@@ -0,0 +1,29 @@
+#! /bin/sh
+# Copy HTML which differ between $old and $new into $outdir
+set -e
+
+old=$1
+new=$2
+outdir=$3
+
+# The index is large and changes often
+skippages="bookindex.html"
+
+mkdir "$outdir"
+cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
+
+changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
+	[ $? -eq 1 ]
+
+for f in $changed
+do
+	# Avoid removed files
+	[ -f "$f" ] || continue
+
+	echo "$f" |grep -Ew "$skippages" >/dev/null &&
+		continue
+
+	cp -v "$f" "$outdir"
+done
+
+exit 0
-- 
2.25.1

From 852360330ef7e41edfb4d591ef246435aa956731 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Mon, 28 Feb 2022 23:18:19 -0600
Subject: [PATCH 9/9] +html index file

This allows linking to the artifacts from the last successful build.

//freebsd
ci-os-only: warnings
---
 .cirrus.yml                    |  4 ++--
 src/tools/ci/copy-changed-docs | 42 ++++++++++++++++++++++++++++++++--
 2 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 57359f43867..f36d7d110e0 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -674,8 +674,8 @@ task:
   # succeeds. This is particularly important for this task as we intentionally
   # use always: to continue after failures. Task that did not run count as a
   # success, so we need to recheck SanityChecks's condition here ...
-  depends_on: SanityCheck
-  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*'
+  #depends_on: SanityCheck
+  only_if: $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:.*warnings'
 
   env:
     CPUS: 4
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
index 1c921a8df6f..57dd2823a50 100755
--- a/src/tools/ci/copy-changed-docs
+++ b/src/tools/ci/copy-changed-docs
@@ -5,6 +5,10 @@ set -e
 old=$1
 new=$2
 outdir=$3
+branch=$CIRRUS_BRANCH
+
+# The index is large and changes often
+skippages="bookindex.html"
 
 # The index is large and changes often
 skippages="bookindex.html"
@@ -12,6 +16,19 @@ skippages="bookindex.html"
 mkdir "$outdir"
 cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
 
+# The index is useful to allow a static link (not specific to a cirrus run) to the artifacts for the most-recent, successful CI run for a branch
+branchurl=https://api.cirrus-ci.com/v1/artifact/github/$CIRRUS_REPO_FULL_NAME/$CIRRUS_TASK_NAME/html_docs/html_docs/00-index.html?branch=$branch
+
+index="$outdir/00-index.html"
+cat >"$index" <<EOF
+<html>
+<head><title>Index of docs changed since: $BASE_COMMIT</title></head>
+<body>
+<!-- A link to documentation for the most recent successful build: $branchurl -->
+<h1>Index of docs changed since: $BASE_COMMIT</h1>
+<ul>
+EOF
+
 changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
 	[ $? -eq 1 ]
 
@@ -23,7 +40,28 @@ do
 	echo "$f" |grep -Ew "$skippages" >/dev/null &&
 		continue
 
-	cp -v "$f" "$outdir"
-done
+	cp -v "$f" "$outdir" >&2
+	fn=${f##*/}
+	# ?branch=... is needed when accessing the artifacts for the static link for the branch
+	# It's not needed and ignored if accessing artifacts for *this* CI run
+	echo "<li><a href='$fn?branch=$branch'>$fn</a>"
+done >>"$index"
+
+github=https://github.com/$CIRRUS_REPO_FULL_NAME/commit
+cirrus=https://cirrus-ci.com/build
+
+cat >>"$index" <<EOF
+</ul>
+<hr>
+<code>
+<br>This file was written on: `date --rfc-822 --utc`
+<br>CIRRUS_CHANGE_TITLE: $CIRRUS_CHANGE_TITLE
+<br>CIRRUS_CHANGE_IN_REPO: <a href="$github/$CIRRUS_CHANGE_IN_REPO">$CIRRUS_CHANGE_IN_REPO</a>
+<br>CIRRUS_BUILD_ID: <a href="$cirrus/$CIRRUS_BUILD_ID">$CIRRUS_BUILD_ID</a>
+<br>CIRRUS_LAST_GREEN_CHANGE: <a href="$github/$CIRRUS_LAST_GREEN_CHANGE">$CIRRUS_LAST_GREEN_CHANGE</a>
+<br>CIRRUS_LAST_GREEN_BUILD_ID: <a href="$cirrus/$CIRRUS_LAST_GREEN_BUILD_ID">$CIRRUS_LAST_GREEN_BUILD_ID</a>
+</code>
+</body></html>
+EOF
 
 exit 0
-- 
2.25.1

From f6174c446b0ee4f69239524ecc1506edaf41b33b Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 25 May 2022 21:53:22 -0500
Subject: [PATCH 1/8] cirrus/windows: add compiler_warnings_script

I'm not sure how to write this test in windows shell; it's also easy to
write something that doesn't work in posix sh, since windows shell is
interpretting && and ||...

https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de

See also:
8a1ce5e54f6d144e4f8e19af7c767b026ee0c956
https://cirrus-ci.com/task/6241060062494720
https://cirrus-ci.com/task/6496366607204352

ci-os-only: windows
---
 .cirrus.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 505c50f3285..60c0efc2e63 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -563,28 +563,37 @@ task:
 
   setup_additional_packages_script: |
     REM choco install -y --no-progress ...
 
   # Use /DEBUG:FASTLINK to avoid high memory usage during linking
   configure_script: |
     vcvarsall x64
     meson setup --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" build
 
   build_script: |
     vcvarsall x64
-    ninja -C build
+    ninja -C build |tee build.txt
+    REM Since pipes lose the exit status of the preceding command, rerun the compilation
+    REM without the pipe, exiting now if it fails, to avoid trying to run checks
+    ninja -C build > nul
 
   check_world_script: |
     vcvarsall x64
     meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
 
+  # This should be last, so check_world is run even if there are warnings
+  always:
+    compiler_warnings_script:
+      # this avoids using metachars which would be interpretted by the windows shell
+      - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0'
+
   on_failure:
     <<: *on_failure_meson
     crashlog_artifacts:
       path: "crashlog-*.txt"
       type: text/plain
 
 
 task:
   << : *WINDOWS_ENVIRONMENT_BASE
   name: Windows - Server 2019, MinGW64 - Meson
 
-- 
2.34.1

From 60378805b5b2d634f426850b6ce4d1c64b8aabf4 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 24 Jun 2022 00:09:12 -0500
Subject: [PATCH 2/8] cirrus/freebsd: run with more CPUs+RAM and do not
 repartition

There was some historic problem where tests under freebsd took 8+ minutes (and
before 4a288a37f took 15 minutes).

This reduces test time from 10min to 3min.
4 CPUs 4 tests https://cirrus-ci.com/task/4880240739614720
4 CPUs 6 tests https://cirrus-ci.com/task/4664440120410112 https://cirrus-ci.com/task/4586784884523008
4 CPUs 8 tests https://cirrus-ci.com/task/5001995491737600

6 CPUs https://cirrus-ci.com/task/6678321684545536
8 CPUs https://cirrus-ci.com/task/6264854121021440

See also:
https://www.postgresql.org/message-id/flat/20220310033347.hgxk4pyarzq4hxwp@alap3.anarazel.de#f36c0b17e33e31e7925e7e5812998686
8 jobs 7min https://cirrus-ci.com/task/6186376667332608

//-os-only: freebsd
---
 .cirrus.yml | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 60c0efc2e63..99276481e57 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -123,27 +123,25 @@ task:
     <<: *on_failure_meson
     cores_script: |
       mkdir -m 770 /tmp/cores
       find / -maxdepth 1 -type f -name 'core*' -exec mv '{}' /tmp/cores/ \;
       src/tools/ci/cores_backtrace.sh linux /tmp/cores
 
 
 task:
   name: FreeBSD - 13 - Meson
 
   env:
-    # FreeBSD on GCP is slow when running with larger number of CPUS /
-    # jobs. Using one more job than cpus seems to work best.
-    CPUS: 2
-    BUILD_JOBS: 3
-    TEST_JOBS: 3
+    CPUS: 4
+    BUILD_JOBS: 4
+    TEST_JOBS: 6
 
     CCACHE_DIR: /tmp/ccache_dir
     CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST
     CFLAGS: -Og -ggdb
 
   depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
 
   compute_engine_instance:
     image_project: $IMAGE_PROJECT
     image: family/pg-ci-freebsd-13
@@ -152,39 +150,36 @@ task:
     memory: 4G
     disk: 50
 
   sysinfo_script: |
     id
     uname -a
     ulimit -a -H && ulimit -a -S
     export
 
   ccache_cache:
     folder: $CCACHE_DIR
-  # Work around performance issues due to 32KB block size
-  repartition_script: src/tools/ci/gcp_freebsd_repartition.sh
   create_user_script: |
     pw useradd postgres
     chown -R postgres:postgres .
     mkdir -p ${CCACHE_DIR}
     chown -R postgres:postgres ${CCACHE_DIR}
   setup_core_files_script: |
     mkdir -m 770 /tmp/cores
     chown root:postgres /tmp/cores
     sysctl kern.corefile='/tmp/cores/%N.%P.core'
   setup_additional_packages_script: |
     #pkg install -y ...
 
   # NB: Intentionally build without -Dllvm. The freebsd image size is already
-  # large enough to make VM startup slow, and even without llvm freebsd
-  # already takes longer than other platforms except for windows.
+  # large enough to make VM startup slow
   configure_script: |
     su postgres <<-EOF
       meson setup \
         --buildtype=debug \
         -Dcassert=true -Duuid=bsd -Dtcl_version=tcl86 -Ddtrace=auto \
         -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
         -Dextra_lib_dirs=/usr/local/lib -Dextra_include_dirs=/usr/local/include/ \
         build
     EOF
   build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS}'
   upload_caches: ccache
-- 
2.34.1

From 5b376acde8f6e7eff2f4a846620d2bb68091c681 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 9 Dec 2022 15:32:50 -0600
Subject: [PATCH 3/8] cirrus/freebsd: define
 ENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS

See also: 54100f5c6052404f68de9ce7310ceb61f1c291f8

ci-os-only: freebsd
---
 .cirrus.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 99276481e57..f8238498613 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -128,23 +128,23 @@ task:
 
 
 task:
   name: FreeBSD - 13 - Meson
 
   env:
     CPUS: 4
     BUILD_JOBS: 4
     TEST_JOBS: 6
 
     CCACHE_DIR: /tmp/ccache_dir
-    CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST
+    CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST -DENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS
     CFLAGS: -Og -ggdb
 
   depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
 
   compute_engine_instance:
     image_project: $IMAGE_PROJECT
     image: family/pg-ci-freebsd-13
     platform: freebsd
     cpu: $CPUS
     memory: 4G
-- 
2.34.1

From 473991f8246a2e710dc0c3b614b769cfa1efc31d Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 30 Jul 2022 19:25:51 -0500
Subject: [PATCH 4/8] pg_upgrade: tap test: exercise --link and --clone

This both increases code coverage and accelerates tests.

See also: b059a2409faf5833b3ba7792e247d6466c9e8090

linux,
macos,
//-os-only: freebsd
---
 .cirrus.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.cirrus.yml b/.cirrus.yml
index f8238498613..4910a0fceba 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -130,22 +130,23 @@ task:
 task:
   name: FreeBSD - 13 - Meson
 
   env:
     CPUS: 4
     BUILD_JOBS: 4
     TEST_JOBS: 6
 
     CCACHE_DIR: /tmp/ccache_dir
     CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST -DENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS
     CFLAGS: -Og -ggdb
+    PG_TEST_PG_UPGRADE_MODE: --link
 
   depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
 
   compute_engine_instance:
     image_project: $IMAGE_PROJECT
     image: family/pg-ci-freebsd-13
     platform: freebsd
     cpu: $CPUS
     memory: 4G
     disk: 50
@@ -266,22 +267,23 @@ task:
     # print_stacktraces=1,verbosity=2, duh
     # detect_leaks=0: too many uninteresting leak errors in short-lived binaries
     UBSAN_OPTIONS: print_stacktrace=1:disable_coredump=0:abort_on_error=1:verbosity=2
     ASAN_OPTIONS: print_stacktrace=1:disable_coredump=0:abort_on_error=1:detect_leaks=0
 
     # SANITIZER_FLAGS is set in the tasks below
     CFLAGS: -Og -ggdb -fno-sanitize-recover=all $SANITIZER_FLAGS
     CXXFLAGS: $CFLAGS
     LDFLAGS: $SANITIZER_FLAGS
     CC: ccache gcc
     CXX: ccache g++
+    PG_TEST_PG_UPGRADE_MODE: --link
 
     LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
     LINUX_MESON_FEATURES: *LINUX_MESON_FEATURES
 
   depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
 
   compute_engine_instance:
     image_project: $IMAGE_PROJECT
     image: family/pg-ci-bullseye
     platform: linux
@@ -416,22 +418,23 @@ task:
 
     CIRRUS_WORKING_DIR: ${HOME}/pgsql/
     CCACHE_DIR: ${HOME}/ccache
     HOMEBREW_CACHE: ${HOME}/homebrew-cache
     PERL5LIB: ${HOME}/perl5/lib/perl5
 
     CC: ccache cc
     CXX: ccache c++
     CPPFLAGS: -DRANDOMIZE_ALLOCATED_MEMORY
     CFLAGS: -Og -ggdb
     CXXFLAGS: -Og -ggdb
+    PG_TEST_PG_UPGRADE_MODE: --clone
 
   depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
 
   macos_instance:
     image: ghcr.io/cirruslabs/macos-ventura-base:latest
 
   sysinfo_script: |
     id
     uname -a
     ulimit -a -H && ulimit -a -S
-- 
2.34.1

From 187302a1932b753fc175fbbfd2d17cd288cf1e5f Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 25 Nov 2022 13:57:17 -0600
Subject: [PATCH 5/8] WIP: ci/meson: allow showing only failed tests ..

It's simpler and seems to make more sense to integrate this with
testwrap, rather than to run it after check-world, but only if it
failed, and finding a way to preserve the exit code.

https://www.postgresql.org/message-id/20221114235328.lxdj3puenfhirhqm@awork3.anarazel.de

> It is wasteful to upload thousdands of logfiles to show a single
> failure.  That would make our cirrus tasks faster - compressing and
> uploading the logs takes over a minute.
>
> It's also a lot friendlier to show fewer than 8 pages of test folders to
> search through to find the one that failed.

macos
ci-os-only: linux-meson freebsd
---
 .cirrus.yml        | 13 +++++++------
 src/tools/testwrap |  6 ++++++
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 4910a0fceba..6cbbea2f1e1 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -15,42 +15,43 @@ env:
   CIRRUS_CLONE_DEPTH: 500
   # Useful to be able to analyse what in a script takes long
   CIRRUS_LOG_TIMESTAMP: true
 
   CCACHE_MAXSIZE: "250M"
 
   # target to test, for all but windows
   CHECK: check-world PROVE_FLAGS=$PROVE_FLAGS
   CHECKFLAGS: -Otarget
   PROVE_FLAGS: --timer
   MTEST_ARGS: --print-errorlogs --no-rebuild -C build
+  PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
   PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
   TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
   PG_TEST_EXTRA: kerberos ldap ssl
 
 
 # What files to preserve in case tests fail
 on_failure_ac: &on_failure_ac
   log_artifacts:
     paths:
       - "**/*.log"
       - "**/*.diffs"
       - "**/regress_log_*"
     type: text/plain
 
 on_failure_meson: &on_failure_meson
   testrun_artifacts:
     paths:
-      - "build*/testrun/**/*.log"
-      - "build*/testrun/**/*.diffs"
-      - "build*/testrun/**/regress_log_*"
+      - "failed.build*/**/*.log"
+      - "failed.build*/**/*.diffs"
+      - "failed.build*/**/regress_log_*"
     type: text/plain
 
   # In theory it'd be nice to upload the junit files meson generates, so that
   # cirrus will nicely annotate the commit. Unfortunately the files don't
   # contain identifiable file + line numbers right now, so the annotations
   # don't end up useful. We could probably improve on that with a some custom
   # conversion script, but ...
   meson_log_artifacts:
     path: "build*/meson-logs/*.txt"
     type: text/plain
 
@@ -190,26 +191,26 @@ task:
       ulimit -c unlimited
       meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
     EOF
 
   # test runningcheck, freebsd chosen because it's currently fast enough
   test_running_script: |
     su postgres <<-EOF
       set -e
       ulimit -c unlimited
       meson test $MTEST_ARGS --quiet --suite setup
       export LD_LIBRARY_PATH="$(pwd)/build/tmp_install/usr/local/pgsql/lib/:$LD_LIBRARY_PATH"
-      mkdir -p build/testrun
+      mkdir -p build/testrun ${PG_FAILED_TESTDIR}
       build/tmp_install/usr/local/pgsql/bin/initdb -N build/runningcheck --no-instructions -A trust
       echo "include '$(pwd)/src/tools/ci/pg_ci_base.conf'" >> build/runningcheck/postgresql.conf
-      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
+      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l ${PG_FAILED_TESTDIR}/runningcheck.log start
       meson test $MTEST_ARGS --num-processes ${TEST_JOBS} --setup running
       build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
     EOF
 
   on_failure:
     # if the server continues running, it often causes cirrus-ci to fail
     # during upload, as it doesn't expect artifacts to change size
     stop_running_script: |
       su postgres <<-EOF
         build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop || true
       EOF
@@ -387,23 +388,23 @@ task:
         EOF
         # so that we don't upload 64bit logs if 32bit fails
         rm -rf build/
 
       # There's currently no coverage of icu with LANG=C in the buildfarm. We
       # can easily provide some here by running one of the sets of tests that
       # way. Newer versions of python insist on changing the LC_CTYPE away
       # from C, prevent that with PYTHONCOERCECLOCALE.
       test_world_32_script: |
         su postgres <<-EOF
           ulimit -c unlimited
-          PYTHONCOERCECLOCALE=0 LANG=C meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
+          PYTHONCOERCECLOCALE=0 LANG=C PG_FAILED_TESTDIR=`pwd`/failed.build-32 meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
         EOF
 
       on_failure:
         <<: *on_failure_meson
 
   on_failure:
     cores_script: src/tools/ci/cores_backtrace.sh linux /tmp/cores
 
 
 task:
   name: macOS - Ventura - Meson
diff --git a/src/tools/testwrap b/src/tools/testwrap
index 7a64fe76a2d..445ac595afc 100755
--- a/src/tools/testwrap
+++ b/src/tools/testwrap
@@ -36,12 +36,18 @@ env_dict = {**os.environ,
             'TESTDATADIR': os.path.join(testdir, 'data'),
             'TESTLOGDIR': os.path.join(testdir, 'log')}
 
 sp = subprocess.run(args.test_command, env=env_dict)
 
 if sp.returncode == 0:
     print('# test succeeded')
     open(os.path.join(testdir, 'test.success'), 'x')
 else:
     print('# test failed')
     open(os.path.join(testdir, 'test.fail'), 'x')
+    faileddir = os.getenv('PG_FAILED_TESTDIR')
+    if faileddir:
+        parentdir = os.path.dirname(testdir)
+        newdest = os.path.join(faileddir, os.path.basename(parentdir), os.path.basename(testdir))
+        shutil.copytree(testdir, newdest)
+
 sys.exit(sp.returncode)
-- 
2.34.1

From fe982a4652518c6b346009873284ed0fbd60a205 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Mon, 17 Jan 2022 00:54:28 -0600
Subject: [PATCH 6/8] cirrus: code coverage

Coverage is shown only for changed files.  This is useful to see
coverage of newly-added code, but won't show added/lost coverage in
files which this patch doesn't modify.

Some alternatives:
- could build with "--coverage -fprofile-filter-files=", but that means
  that ccache will never work (both because it doesn't support that option
  and also because the arguments will be different for every patch).
- could use ninja coverage-html, but that can't filter only changed
  files, and would take a long time and a lot of space to upload a lot
  of useless files.

https://www.postgresql.org/message-id/202202111821.w3gqblvfp4pr%40alvherre.pgsql
https://www.postgresql.org/message-id/flat/20220409021853.GP24419@telsasoft.com

ci-os-only: freebsd
---
 .cirrus.yml                       | 20 ++++++++++++-
 src/tools/ci/code-coverage-report | 48 +++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+), 1 deletion(-)
 create mode 100755 src/tools/ci/code-coverage-report

diff --git a/.cirrus.yml b/.cirrus.yml
index 6cbbea2f1e1..cea8b6fef2b 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -20,22 +20,30 @@ env:
 
   # target to test, for all but windows
   CHECK: check-world PROVE_FLAGS=$PROVE_FLAGS
   CHECKFLAGS: -Otarget
   PROVE_FLAGS: --timer
   MTEST_ARGS: --print-errorlogs --no-rebuild -C build
   PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
   PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
   TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
   PG_TEST_EXTRA: kerberos ldap ssl
 
+  # The commit that this branch is rebased on.  There's no easy way to find this.
+  # This does the right thing for cfbot, which always squishes all patches into a single commit.
+  # And does the right thing for any 1-patch commits.
+  # Patch series manually submitted to cirrus would benefit from setting this to the
+  # number of patches in the series (or directly to the commit the series was rebased on).
+  #BASE_COMMIT: HEAD~1
+  # For demo purposes:
+  BASE_COMMIT: HEAD~11
 
 # What files to preserve in case tests fail
 on_failure_ac: &on_failure_ac
   log_artifacts:
     paths:
       - "**/*.log"
       - "**/*.diffs"
       - "**/regress_log_*"
     type: text/plain
 
 on_failure_meson: &on_failure_meson
@@ -149,57 +157,67 @@ task:
     image: family/pg-ci-freebsd-13
     platform: freebsd
     cpu: $CPUS
     memory: 4G
     disk: 50
 
   sysinfo_script: |
     id
     uname -a
     ulimit -a -H && ulimit -a -S
     export
+    git diff --name-only "$BASE_COMMIT"
 
   ccache_cache:
     folder: $CCACHE_DIR
   create_user_script: |
     pw useradd postgres
     chown -R postgres:postgres .
     mkdir -p ${CCACHE_DIR}
     chown -R postgres:postgres ${CCACHE_DIR}
   setup_core_files_script: |
     mkdir -m 770 /tmp/cores
     chown root:postgres /tmp/cores
     sysctl kern.corefile='/tmp/cores/%N.%P.core'
   setup_additional_packages_script: |
-    #pkg install -y ...
+    pkg install -y lcov
 
   # NB: Intentionally build without -Dllvm. The freebsd image size is already
   # large enough to make VM startup slow
   configure_script: |
     su postgres <<-EOF
       meson setup \
         --buildtype=debug \
+        -Db_coverage=true \
         -Dcassert=true -Duuid=bsd -Dtcl_version=tcl86 -Ddtrace=auto \
         -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
         -Dextra_lib_dirs=/usr/local/lib -Dextra_include_dirs=/usr/local/include/ \
         build
     EOF
   build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS}'
   upload_caches: ccache
 
   test_world_script: |
     su postgres <<-EOF
+      set -e
       ulimit -c unlimited
+      # Write initial coverage files before running tests:
+      time ./src/tools/ci/code-coverage-report "$BASE_COMMIT" ./build ./coverage "--initial"
       meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
+      # Create coverage report for files changed since the base commit.
+      time ./src/tools/ci/code-coverage-report "$BASE_COMMIT" ./build ./coverage
     EOF
 
+  coverage_artifacts:
+    path: 'coverage/**'
+
   # test runningcheck, freebsd chosen because it's currently fast enough
   test_running_script: |
     su postgres <<-EOF
       set -e
       ulimit -c unlimited
       meson test $MTEST_ARGS --quiet --suite setup
       export LD_LIBRARY_PATH="$(pwd)/build/tmp_install/usr/local/pgsql/lib/:$LD_LIBRARY_PATH"
       mkdir -p build/testrun ${PG_FAILED_TESTDIR}
       build/tmp_install/usr/local/pgsql/bin/initdb -N build/runningcheck --no-instructions -A trust
       echo "include '$(pwd)/src/tools/ci/pg_ci_base.conf'" >> build/runningcheck/postgresql.conf
       build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l ${PG_FAILED_TESTDIR}/runningcheck.log start
diff --git a/src/tools/ci/code-coverage-report b/src/tools/ci/code-coverage-report
new file mode 100755
index 00000000000..db448e802ba
--- /dev/null
+++ b/src/tools/ci/code-coverage-report
@@ -0,0 +1,48 @@
+#! /bin/sh
+# Called during CI to generate a code coverage report of changed files.
+set -e
+
+base_branch=$1
+build_dir=$2
+outdir=$3
+args=$4
+
+changed=`git diff --name-only "$base_branch" '*.c'`
+[ -z "$changed" ] && exit 0 # Nothing changed
+
+[ -d "$outdir" ] ||
+	mkdir "$outdir"
+
+# This could be used to map from object file back to source file:
+# readelf --debug-dump=info src/backend/postgres_lib.a.p/utils_adt_array_userfuncs.c.o |awk '/DW_AT_name/{print $NF;exit}'
+# gcov ./src/port/libpgport_shlib.a.p/inet_net_ntop.c.gcno --stdout
+
+gcov=$outdir/coverage.gcov
+lcov --quiet --capture --directory "$build_dir" $args >"$gcov.new"
+
+# Filter to include only changed files
+echo "$changed" |sed 's,^,*/,' |
+	xargs -rt lcov --extract "$gcov.new" >"$gcov.filtered"
+rm "$gcov.new"
+
+echo "$args" |grep initial >/dev/null && {
+	mv "$gcov.filtered" "$gcov.init"
+	exit 0
+}
+
+# Exit successfully if no relevant files were changed
+[ -s "$gcov.filtered" ] || {
+	rm "$gcov.init"
+	exit 0
+}
+
+# Otherwise combine with the init file:
+lcov -a "$gcov.init" -a "$gcov.filtered" >"$gcov"
+
+genhtml "$gcov" --show-details --legend --quiet --num-spaces=4 --output-directory "$outdir" \
+	--title="Coverage report of files changed since: $base_branch"
+cp "$outdir"/index.html "$outdir"/00-index.html
+
+gzip "$gcov" "$gcov.init" "$gcov.filtered"
+ls -l "$outdir"
+du -sh "$outdir"
-- 
2.34.1

From df877bcb5a4e9cc73d9d278b2ccd3e7b8ad0bdee Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 26 Feb 2022 19:39:10 -0600
Subject: [PATCH 7/8] cirrus: upload changed html docs as artifacts

This could be done on the client side (cfbot).  One advantage of doing
it here is that fewer docs are uploaded - many patches won't upload docs
at all.

https://www.postgresql.org/message-id/flat/20220409021853.GP24419@telsasoft.com
https://www.postgresql.org/message-id/CAB8KJ=i4qmEuopQ+PCSMBzGd4O-Xv0FCnC+q1x7hN9hsdvkBug@mail.gmail.com

https://cirrus-ci.com/task/5396696388599808

ci-os-only: html
---
 .cirrus.yml                    | 16 +++++++++++++++-
 src/tools/ci/copy-changed-docs | 29 +++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)
 create mode 100755 src/tools/ci/copy-changed-docs

diff --git a/.cirrus.yml b/.cirrus.yml
index cea8b6fef2b..a0929f7eb5a 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -700,22 +700,23 @@ task:
     image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
     cpu: $CPUS
 
   sysinfo_script: |
     id
     uname -a
     cat /proc/cmdline
     ulimit -a -H && ulimit -a -S
     gcc -v
     clang -v
     export
+    git diff --name-only "$BASE_COMMIT"
 
   ccache_cache:
     folder: $CCACHE_DIR
 
   setup_additional_packages_script: |
     #apt-get update
     #DEBIAN_FRONTEND=noninteractive apt-get -y install ...
 
   ###
   # Test that code can be built with gcc/clang without warnings
   ###
@@ -774,34 +775,47 @@ task:
   always:
     mingw_cross_warning_script: |
       time ./configure \
         --host=x86_64-w64-mingw32 \
         --enable-cassert \
         CC="ccache x86_64-w64-mingw32-gcc" \
         CXX="ccache x86_64-w64-mingw32-g++"
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
   ###
-  # Verify docs can be built
+  # Verify docs can be built, and upload changed docs as artifacts
   ###
   # XXX: Only do this if there have been changes in doc/ since last build
   always:
     docs_build_script: |
       time ./configure \
         --cache gcc.cache \
         CC="ccache gcc" \
         CXX="ccache g++" \
         CLANG="ccache clang"
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} -C doc
+      cp -r doc new-docs
+
+      # Re-build HTML docs from the base commit.
+      git checkout "$BASE_COMMIT" -- doc
+      make -s -C doc clean
+      time make -s -C doc html
+      cp -r doc old-docs
+
+    copy_changed_docs_script:
+      - src/tools/ci/copy-changed-docs "old-docs" "new-docs" "html_docs"
+
+    html_docs_artifacts:
+      paths: ['html_docs/*.html', 'html_docs/*.png', 'html_docs/*.css']
 
   ###
   # Verify headerscheck / cpluspluscheck succeed
   #
   # - Don't use ccache, the files are uncacheable, polluting ccache's
   #   cache
   # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
   # - XXX have to disable ICU to avoid errors:
   #   https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
   ###
   always:
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
new file mode 100755
index 00000000000..1c921a8df6f
--- /dev/null
+++ b/src/tools/ci/copy-changed-docs
@@ -0,0 +1,29 @@
+#! /bin/sh
+# Copy HTML which differ between $old and $new into $outdir
+set -e
+
+old=$1
+new=$2
+outdir=$3
+
+# The index is large and changes often
+skippages="bookindex.html"
+
+mkdir "$outdir"
+cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
+
+changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
+	[ $? -eq 1 ]
+
+for f in $changed
+do
+	# Avoid removed files
+	[ -f "$f" ] || continue
+
+	echo "$f" |grep -Ew "$skippages" >/dev/null &&
+		continue
+
+	cp -v "$f" "$outdir"
+done
+
+exit 0
-- 
2.34.1

From 6ff86c991dabad62dbe8e69c53db37e081529e6a Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Mon, 28 Feb 2022 23:18:19 -0600
Subject: [PATCH 8/8] +html index file

This allows linking to the artifacts from the last successful build.

//freebsd
ci-os-only: warnings
---
 src/tools/ci/copy-changed-docs | 42 ++++++++++++++++++++++++++++++++--
 1 file changed, 40 insertions(+), 2 deletions(-)

diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
index 1c921a8df6f..57dd2823a50 100755
--- a/src/tools/ci/copy-changed-docs
+++ b/src/tools/ci/copy-changed-docs
@@ -1,29 +1,67 @@
 #! /bin/sh
 # Copy HTML which differ between $old and $new into $outdir
 set -e
 
 old=$1
 new=$2
 outdir=$3
+branch=$CIRRUS_BRANCH
+
+# The index is large and changes often
+skippages="bookindex.html"
 
 # The index is large and changes often
 skippages="bookindex.html"
 
 mkdir "$outdir"
 cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
 
+# The index is useful to allow a static link (not specific to a cirrus run) to the artifacts for the most-recent, successful CI run for a branch
+branchurl=https://api.cirrus-ci.com/v1/artifact/github/$CIRRUS_REPO_FULL_NAME/$CIRRUS_TASK_NAME/html_docs/html_docs/00-index.html?branch=$branch
+
+index="$outdir/00-index.html"
+cat >"$index" <<EOF
+<html>
+<head><title>Index of docs changed since: $BASE_COMMIT</title></head>
+<body>
+<!-- A link to documentation for the most recent successful build: $branchurl -->
+<h1>Index of docs changed since: $BASE_COMMIT</h1>
+<ul>
+EOF
+
 changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
 	[ $? -eq 1 ]
 
 for f in $changed
 do
 	# Avoid removed files
 	[ -f "$f" ] || continue
 
 	echo "$f" |grep -Ew "$skippages" >/dev/null &&
 		continue
 
-	cp -v "$f" "$outdir"
-done
+	cp -v "$f" "$outdir" >&2
+	fn=${f##*/}
+	# ?branch=... is needed when accessing the artifacts for the static link for the branch
+	# It's not needed and ignored if accessing artifacts for *this* CI run
+	echo "<li><a href='$fn?branch=$branch'>$fn</a>"
+done >>"$index"
+
+github=https://github.com/$CIRRUS_REPO_FULL_NAME/commit
+cirrus=https://cirrus-ci.com/build
+
+cat >>"$index" <<EOF
+</ul>
+<hr>
+<code>
+<br>This file was written on: `date --rfc-822 --utc`
+<br>CIRRUS_CHANGE_TITLE: $CIRRUS_CHANGE_TITLE
+<br>CIRRUS_CHANGE_IN_REPO: <a href="$github/$CIRRUS_CHANGE_IN_REPO">$CIRRUS_CHANGE_IN_REPO</a>
+<br>CIRRUS_BUILD_ID: <a href="$cirrus/$CIRRUS_BUILD_ID">$CIRRUS_BUILD_ID</a>
+<br>CIRRUS_LAST_GREEN_CHANGE: <a href="$github/$CIRRUS_LAST_GREEN_CHANGE">$CIRRUS_LAST_GREEN_CHANGE</a>
+<br>CIRRUS_LAST_GREEN_BUILD_ID: <a href="$cirrus/$CIRRUS_LAST_GREEN_BUILD_ID">$CIRRUS_LAST_GREEN_BUILD_ID</a>
+</code>
+</body></html>
+EOF
 
 exit 0
-- 
2.34.1

From 4695da5b426731a651d90fc41f41434207596848 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 25 May 2022 21:53:22 -0500
Subject: [PATCH 1/8] cirrus/windows: add compiler_warnings_script

The goal is to fail due to warnings only after running tests.
(At least historically, it's too slow to run a separate windows VM to
compile with -Werror.)

https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de

I'm not sure how to write this test in windows shell; it's also easy to
write something that doesn't work in posix sh, since windows shell is
interpretting && and ||...

See also:
8a1ce5e54f6d144e4f8e19af7c767b026ee0c956
https://cirrus-ci.com/task/6241060062494720
https://cirrus-ci.com/task/6496366607204352

ci-os-only: windows
---
 .cirrus.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index d3f88821a85..13213ffd304 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -589,28 +589,37 @@ task:
     echo 127.0.0.2 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts
     echo 127.0.0.3 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts
     type c:\Windows\System32\Drivers\etc\hosts
 
   # Use /DEBUG:FASTLINK to avoid high memory usage during linking
   configure_script: |
     vcvarsall x64
     meson setup --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" build
 
   build_script: |
     vcvarsall x64
-    ninja -C build
+    ninja -C build |tee build.txt
+    REM Since pipes lose the exit status of the preceding command, rerun the compilation
+    REM without the pipe, exiting now if it fails, to avoid trying to run checks
+    ninja -C build > nul
 
   check_world_script: |
     vcvarsall x64
     meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
 
+  # This should be last, so check_world is run even if there are warnings
+  always:
+    compiler_warnings_script:
+      # this avoids using metachars which would be interpretted by the windows shell
+      - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0'
+
   on_failure:
     <<: *on_failure_meson
     crashlog_artifacts:
       path: "crashlog-*.txt"
       type: text/plain
 
 
 task:
   << : *WINDOWS_ENVIRONMENT_BASE
   name: Windows - Server 2019, MinGW64 - Meson
 
-- 
2.34.1

From 62e04da9b12fc8a421957facee60d80c3c4f2ad8 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 24 Jun 2022 00:09:12 -0500
Subject: [PATCH 2/8] cirrus/freebsd: run with more CPUs+RAM and do not
 repartition

There was some historic problem where tests under freebsd took 8+ minutes (and
before 4a288a37f took 15 minutes).

This reduces test time from 10min to 3min.
4 CPUs 4 tests https://cirrus-ci.com/task/4880240739614720
4 CPUs 6 tests https://cirrus-ci.com/task/4664440120410112 https://cirrus-ci.com/task/4586784884523008
4 CPUs 8 tests https://cirrus-ci.com/task/5001995491737600

6 CPUs https://cirrus-ci.com/task/6678321684545536
8 CPUs https://cirrus-ci.com/task/6264854121021440

See also:
https://www.postgresql.org/message-id/flat/20220310033347.hgxk4pyarzq4hxwp@alap3.anarazel.de#f36c0b17e33e31e7925e7e5812998686
8 jobs 7min https://cirrus-ci.com/task/6186376667332608

//-os-only: freebsd
---
 .cirrus.yml | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 13213ffd304..21608ac2e0a 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -123,27 +123,25 @@ task:
     <<: *on_failure_meson
     cores_script: |
       mkdir -m 770 /tmp/cores
       find / -maxdepth 1 -type f -name 'core*' -exec mv '{}' /tmp/cores/ \;
       src/tools/ci/cores_backtrace.sh linux /tmp/cores
 
 
 task:
   name: FreeBSD - 13 - Meson
 
   env:
-    # FreeBSD on GCP is slow when running with larger number of CPUS /
-    # jobs. Using one more job than cpus seems to work best.
-    CPUS: 2
-    BUILD_JOBS: 3
-    TEST_JOBS: 3
+    CPUS: 4
+    BUILD_JOBS: 4
+    TEST_JOBS: 6
 
     CCACHE_DIR: /tmp/ccache_dir
     CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST
     CFLAGS: -Og -ggdb
 
   depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
 
   compute_engine_instance:
     image_project: $IMAGE_PROJECT
     image: family/pg-ci-freebsd-13
@@ -152,39 +150,36 @@ task:
     memory: 4G
     disk: 50
 
   sysinfo_script: |
     id
     uname -a
     ulimit -a -H && ulimit -a -S
     export
 
   ccache_cache:
     folder: $CCACHE_DIR
-  # Work around performance issues due to 32KB block size
-  repartition_script: src/tools/ci/gcp_freebsd_repartition.sh
   create_user_script: |
     pw useradd postgres
     chown -R postgres:postgres .
     mkdir -p ${CCACHE_DIR}
     chown -R postgres:postgres ${CCACHE_DIR}
   setup_core_files_script: |
     mkdir -m 770 /tmp/cores
     chown root:postgres /tmp/cores
     sysctl kern.corefile='/tmp/cores/%N.%P.core'
   setup_additional_packages_script: |
     #pkg install -y ...
 
   # NB: Intentionally build without -Dllvm. The freebsd image size is already
-  # large enough to make VM startup slow, and even without llvm freebsd
-  # already takes longer than other platforms except for windows.
+  # large enough to make VM startup slow
   configure_script: |
     su postgres <<-EOF
       meson setup \
         --buildtype=debug \
         -Dcassert=true -Duuid=bsd -Dtcl_version=tcl86 -Ddtrace=auto \
         -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
         -Dextra_lib_dirs=/usr/local/lib -Dextra_include_dirs=/usr/local/include/ \
         build
     EOF
   build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS}'
   upload_caches: ccache
-- 
2.34.1

From 5c1691867c3098bb0ededb358d8ee329193b1bcb Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 9 Dec 2022 15:32:50 -0600
Subject: [PATCH 3/8] cirrus/freebsd: define
 ENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS

See also: 54100f5c6052404f68de9ce7310ceb61f1c291f8

ci-os-only: freebsd
---
 .cirrus.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 21608ac2e0a..b9a70a05cfa 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -128,23 +128,23 @@ task:
 
 
 task:
   name: FreeBSD - 13 - Meson
 
   env:
     CPUS: 4
     BUILD_JOBS: 4
     TEST_JOBS: 6
 
     CCACHE_DIR: /tmp/ccache_dir
-    CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST
+    CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST -DENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS
     CFLAGS: -Og -ggdb
 
   depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
 
   compute_engine_instance:
     image_project: $IMAGE_PROJECT
     image: family/pg-ci-freebsd-13
     platform: freebsd
     cpu: $CPUS
     memory: 4G
-- 
2.34.1

From b0cbff02c3a211f35df5be1ba63a5d55a3bfda8f Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 30 Jul 2022 19:25:51 -0500
Subject: [PATCH 4/8] cirrus/002_pg_upgrade: exercise --link and --clone

This increases code coverage (and maybe accelerates the test).

See also: b059a2409faf5833b3ba7792e247d6466c9e8090

linux,
macos,
//-os-only: freebsd
---
 .cirrus.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.cirrus.yml b/.cirrus.yml
index b9a70a05cfa..4cc90484eaf 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -130,22 +130,23 @@ task:
 task:
   name: FreeBSD - 13 - Meson
 
   env:
     CPUS: 4
     BUILD_JOBS: 4
     TEST_JOBS: 6
 
     CCACHE_DIR: /tmp/ccache_dir
     CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST -DENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS
     CFLAGS: -Og -ggdb
+    PG_TEST_PG_UPGRADE_MODE: --link
 
   depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
 
   compute_engine_instance:
     image_project: $IMAGE_PROJECT
     image: family/pg-ci-freebsd-13
     platform: freebsd
     cpu: $CPUS
     memory: 4G
     disk: 50
@@ -424,22 +425,23 @@ task:
 
     CIRRUS_WORKING_DIR: ${HOME}/pgsql/
     CCACHE_DIR: ${HOME}/ccache
     HOMEBREW_CACHE: ${HOME}/homebrew-cache
     PERL5LIB: ${HOME}/perl5/lib/perl5
 
     CC: ccache cc
     CXX: ccache c++
     CPPFLAGS: -DRANDOMIZE_ALLOCATED_MEMORY
     CFLAGS: -Og -ggdb
     CXXFLAGS: -Og -ggdb
+    PG_TEST_PG_UPGRADE_MODE: --clone
 
   depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
 
   macos_instance:
     image: ghcr.io/cirruslabs/macos-ventura-base:latest
 
   sysinfo_script: |
     id
     uname -a
     ulimit -a -H && ulimit -a -S
-- 
2.34.1

From 0e079b20520fa001bb7dd10658acf061a3026733 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 25 Nov 2022 13:57:17 -0600
Subject: [PATCH 5/8] WIP: ci/meson: allow showing only failed tests ..

It's simpler and seems to make more sense to integrate this with
testwrap, rather than to run it after check-world, but only if it
failed, and finding a way to preserve the exit code.

https://www.postgresql.org/message-id/20221114235328.lxdj3puenfhirhqm@awork3.anarazel.de

> It is wasteful to upload thousdands of logfiles to show a single
> failure.  That would make our cirrus tasks faster - compressing and
> uploading the logs takes over a minute.
>
> It's also a lot friendlier to show fewer than 8 pages of test folders to
> search through to find the one that failed.

macos
ci-os-only: linux-meson freebsd
---
 .cirrus.yml        | 13 +++++++------
 src/tools/testwrap |  6 ++++++
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 4cc90484eaf..82ae18145b1 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -15,42 +15,43 @@ env:
   CIRRUS_CLONE_DEPTH: 500
   # Useful to be able to analyse what in a script takes long
   CIRRUS_LOG_TIMESTAMP: true
 
   CCACHE_MAXSIZE: "250M"
 
   # target to test, for all but windows
   CHECK: check-world PROVE_FLAGS=$PROVE_FLAGS
   CHECKFLAGS: -Otarget
   PROVE_FLAGS: --timer
   MTEST_ARGS: --print-errorlogs --no-rebuild -C build
+  PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
   PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
   TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
   PG_TEST_EXTRA: kerberos ldap ssl load_balance
 
 
 # What files to preserve in case tests fail
 on_failure_ac: &on_failure_ac
   log_artifacts:
     paths:
       - "**/*.log"
       - "**/*.diffs"
       - "**/regress_log_*"
     type: text/plain
 
 on_failure_meson: &on_failure_meson
   testrun_artifacts:
     paths:
-      - "build*/testrun/**/*.log"
-      - "build*/testrun/**/*.diffs"
-      - "build*/testrun/**/regress_log_*"
+      - "failed.build*/**/*.log"
+      - "failed.build*/**/*.diffs"
+      - "failed.build*/**/regress_log_*"
     type: text/plain
 
   # In theory it'd be nice to upload the junit files meson generates, so that
   # cirrus will nicely annotate the commit. Unfortunately the files don't
   # contain identifiable file + line numbers right now, so the annotations
   # don't end up useful. We could probably improve on that with a some custom
   # conversion script, but ...
   meson_log_artifacts:
     path: "build*/meson-logs/*.txt"
     type: text/plain
 
@@ -190,26 +191,26 @@ task:
       ulimit -c unlimited
       meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
     EOF
 
   # test runningcheck, freebsd chosen because it's currently fast enough
   test_running_script: |
     su postgres <<-EOF
       set -e
       ulimit -c unlimited
       meson test $MTEST_ARGS --quiet --suite setup
       export LD_LIBRARY_PATH="$(pwd)/build/tmp_install/usr/local/pgsql/lib/:$LD_LIBRARY_PATH"
-      mkdir -p build/testrun
+      mkdir -p build/testrun ${PG_FAILED_TESTDIR}
       build/tmp_install/usr/local/pgsql/bin/initdb -N build/runningcheck --no-instructions -A trust
       echo "include '$(pwd)/src/tools/ci/pg_ci_base.conf'" >> build/runningcheck/postgresql.conf
-      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
+      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l ${PG_FAILED_TESTDIR}/runningcheck.log start
       meson test $MTEST_ARGS --num-processes ${TEST_JOBS} --setup running
       build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
     EOF
 
   on_failure:
     # if the server continues running, it often causes cirrus-ci to fail
     # during upload, as it doesn't expect artifacts to change size
     stop_running_script: |
       su postgres <<-EOF
         build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop || true
       EOF
@@ -394,23 +395,23 @@ task:
         EOF
         # so that we don't upload 64bit logs if 32bit fails
         rm -rf build/
 
       # There's currently no coverage of icu with LANG=C in the buildfarm. We
       # can easily provide some here by running one of the sets of tests that
       # way. Newer versions of python insist on changing the LC_CTYPE away
       # from C, prevent that with PYTHONCOERCECLOCALE.
       test_world_32_script: |
         su postgres <<-EOF
           ulimit -c unlimited
-          PYTHONCOERCECLOCALE=0 LANG=C meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
+          PYTHONCOERCECLOCALE=0 LANG=C PG_FAILED_TESTDIR=`pwd`/failed.build-32 meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
         EOF
 
       on_failure:
         <<: *on_failure_meson
 
   on_failure:
     cores_script: src/tools/ci/cores_backtrace.sh linux /tmp/cores
 
 
 task:
   name: macOS - Ventura - Meson
diff --git a/src/tools/testwrap b/src/tools/testwrap
index 7a64fe76a2d..445ac595afc 100755
--- a/src/tools/testwrap
+++ b/src/tools/testwrap
@@ -36,12 +36,18 @@ env_dict = {**os.environ,
             'TESTDATADIR': os.path.join(testdir, 'data'),
             'TESTLOGDIR': os.path.join(testdir, 'log')}
 
 sp = subprocess.run(args.test_command, env=env_dict)
 
 if sp.returncode == 0:
     print('# test succeeded')
     open(os.path.join(testdir, 'test.success'), 'x')
 else:
     print('# test failed')
     open(os.path.join(testdir, 'test.fail'), 'x')
+    faileddir = os.getenv('PG_FAILED_TESTDIR')
+    if faileddir:
+        parentdir = os.path.dirname(testdir)
+        newdest = os.path.join(faileddir, os.path.basename(parentdir), os.path.basename(testdir))
+        shutil.copytree(testdir, newdest)
+
 sys.exit(sp.returncode)
-- 
2.34.1

From 651fce87b7bda15029c5b55c4fca2a1e63dfaf29 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Mon, 17 Jan 2022 00:54:28 -0600
Subject: [PATCH 6/8] cirrus: show coverage report of new code for every patch

Coverage is shown only for changed files.  This is useful to see
coverage of newly-added code, but won't show added/lost coverage in
files which this patch doesn't modify.

Some alternatives:
- could build with "--coverage -fprofile-filter-files=", but that means
  that ccache will never work (both because it doesn't support that option
  and also because the arguments will be different for every patch).
- could use ninja coverage-html, but that can't filter only changed
  files, and would take a long time and a lot of space to upload a lot
  of useless files.

https://www.postgresql.org/message-id/202202111821.w3gqblvfp4pr%40alvherre.pgsql
https://www.postgresql.org/message-id/flat/20220409021853.GP24419@telsasoft.com

ci-os-only: freebsd
---
 .cirrus.yml                       | 20 ++++++++++++-
 src/tools/ci/code-coverage-report | 48 +++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+), 1 deletion(-)
 create mode 100755 src/tools/ci/code-coverage-report

diff --git a/.cirrus.yml b/.cirrus.yml
index 82ae18145b1..0f48bd2ab9d 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -20,22 +20,30 @@ env:
 
   # target to test, for all but windows
   CHECK: check-world PROVE_FLAGS=$PROVE_FLAGS
   CHECKFLAGS: -Otarget
   PROVE_FLAGS: --timer
   MTEST_ARGS: --print-errorlogs --no-rebuild -C build
   PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
   PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
   TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
   PG_TEST_EXTRA: kerberos ldap ssl load_balance
 
+  # The commit that this branch is rebased on.  There's no easy way to find this.
+  # This does the right thing for cfbot, which always squishes all patches into a single commit.
+  # And does the right thing for any 1-patch commits.
+  # Patch series manually submitted to cirrus would benefit from setting this to the
+  # number of patches in the series (or directly to the commit the series was rebased on).
+  #BASE_COMMIT: HEAD~1
+  # For demo purposes:
+  BASE_COMMIT: HEAD~11
 
 # What files to preserve in case tests fail
 on_failure_ac: &on_failure_ac
   log_artifacts:
     paths:
       - "**/*.log"
       - "**/*.diffs"
       - "**/regress_log_*"
     type: text/plain
 
 on_failure_meson: &on_failure_meson
@@ -149,57 +157,67 @@ task:
     image: family/pg-ci-freebsd-13
     platform: freebsd
     cpu: $CPUS
     memory: 4G
     disk: 50
 
   sysinfo_script: |
     id
     uname -a
     ulimit -a -H && ulimit -a -S
     export
+    git diff --name-only "$BASE_COMMIT"
 
   ccache_cache:
     folder: $CCACHE_DIR
   create_user_script: |
     pw useradd postgres
     chown -R postgres:postgres .
     mkdir -p ${CCACHE_DIR}
     chown -R postgres:postgres ${CCACHE_DIR}
   setup_core_files_script: |
     mkdir -m 770 /tmp/cores
     chown root:postgres /tmp/cores
     sysctl kern.corefile='/tmp/cores/%N.%P.core'
   setup_additional_packages_script: |
-    #pkg install -y ...
+    pkg install -y lcov
 
   # NB: Intentionally build without -Dllvm. The freebsd image size is already
   # large enough to make VM startup slow
   configure_script: |
     su postgres <<-EOF
       meson setup \
         --buildtype=debug \
+        -Db_coverage=true \
         -Dcassert=true -Duuid=bsd -Dtcl_version=tcl86 -Ddtrace=auto \
         -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
         -Dextra_lib_dirs=/usr/local/lib -Dextra_include_dirs=/usr/local/include/ \
         build
     EOF
   build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS}'
   upload_caches: ccache
 
   test_world_script: |
     su postgres <<-EOF
+      set -e
       ulimit -c unlimited
+      # Write initial coverage files before running tests:
+      time ./src/tools/ci/code-coverage-report "$BASE_COMMIT" ./build ./coverage "--initial"
       meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
+      # Create coverage report for files changed since the base commit.
+      time ./src/tools/ci/code-coverage-report "$BASE_COMMIT" ./build ./coverage
     EOF
 
+  coverage_artifacts:
+    path: 'coverage/**'
+
   # test runningcheck, freebsd chosen because it's currently fast enough
   test_running_script: |
     su postgres <<-EOF
       set -e
       ulimit -c unlimited
       meson test $MTEST_ARGS --quiet --suite setup
       export LD_LIBRARY_PATH="$(pwd)/build/tmp_install/usr/local/pgsql/lib/:$LD_LIBRARY_PATH"
       mkdir -p build/testrun ${PG_FAILED_TESTDIR}
       build/tmp_install/usr/local/pgsql/bin/initdb -N build/runningcheck --no-instructions -A trust
       echo "include '$(pwd)/src/tools/ci/pg_ci_base.conf'" >> build/runningcheck/postgresql.conf
       build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l ${PG_FAILED_TESTDIR}/runningcheck.log start
diff --git a/src/tools/ci/code-coverage-report b/src/tools/ci/code-coverage-report
new file mode 100755
index 00000000000..db448e802ba
--- /dev/null
+++ b/src/tools/ci/code-coverage-report
@@ -0,0 +1,48 @@
+#! /bin/sh
+# Called during CI to generate a code coverage report of changed files.
+set -e
+
+base_branch=$1
+build_dir=$2
+outdir=$3
+args=$4
+
+changed=`git diff --name-only "$base_branch" '*.c'`
+[ -z "$changed" ] && exit 0 # Nothing changed
+
+[ -d "$outdir" ] ||
+	mkdir "$outdir"
+
+# This could be used to map from object file back to source file:
+# readelf --debug-dump=info src/backend/postgres_lib.a.p/utils_adt_array_userfuncs.c.o |awk '/DW_AT_name/{print $NF;exit}'
+# gcov ./src/port/libpgport_shlib.a.p/inet_net_ntop.c.gcno --stdout
+
+gcov=$outdir/coverage.gcov
+lcov --quiet --capture --directory "$build_dir" $args >"$gcov.new"
+
+# Filter to include only changed files
+echo "$changed" |sed 's,^,*/,' |
+	xargs -rt lcov --extract "$gcov.new" >"$gcov.filtered"
+rm "$gcov.new"
+
+echo "$args" |grep initial >/dev/null && {
+	mv "$gcov.filtered" "$gcov.init"
+	exit 0
+}
+
+# Exit successfully if no relevant files were changed
+[ -s "$gcov.filtered" ] || {
+	rm "$gcov.init"
+	exit 0
+}
+
+# Otherwise combine with the init file:
+lcov -a "$gcov.init" -a "$gcov.filtered" >"$gcov"
+
+genhtml "$gcov" --show-details --legend --quiet --num-spaces=4 --output-directory "$outdir" \
+	--title="Coverage report of files changed since: $base_branch"
+cp "$outdir"/index.html "$outdir"/00-index.html
+
+gzip "$gcov" "$gcov.init" "$gcov.filtered"
+ls -l "$outdir"
+du -sh "$outdir"
-- 
2.34.1

From a18875c0da01c2a9cd9a7e0a9b9edbe88883129c Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 26 Feb 2022 19:39:10 -0600
Subject: [PATCH 7/8] cirrus: upload changed html docs as artifacts

This could be done on the client side (cfbot).  One advantage of doing
it here is that fewer docs are uploaded - many patches won't upload docs
at all.

https://www.postgresql.org/message-id/flat/20220409021853.GP24419@telsasoft.com
https://www.postgresql.org/message-id/CAB8KJ=i4qmEuopQ+PCSMBzGd4O-Xv0FCnC+q1x7hN9hsdvkBug@mail.gmail.com

https://cirrus-ci.com/task/5396696388599808

ci-os-only: html
---
 .cirrus.yml                    | 16 +++++++++++++++-
 src/tools/ci/copy-changed-docs | 29 +++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)
 create mode 100755 src/tools/ci/copy-changed-docs

diff --git a/.cirrus.yml b/.cirrus.yml
index 0f48bd2ab9d..68b7229d117 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -725,22 +725,23 @@ task:
     image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
     cpu: $CPUS
 
   sysinfo_script: |
     id
     uname -a
     cat /proc/cmdline
     ulimit -a -H && ulimit -a -S
     gcc -v
     clang -v
     export
+    git diff --name-only "$BASE_COMMIT"
 
   ccache_cache:
     folder: $CCACHE_DIR
 
   setup_additional_packages_script: |
     #apt-get update
     #DEBIAN_FRONTEND=noninteractive apt-get -y install ...
 
   ###
   # Test that code can be built with gcc/clang without warnings
   ###
@@ -799,34 +800,47 @@ task:
   always:
     mingw_cross_warning_script: |
       time ./configure \
         --host=x86_64-w64-mingw32 \
         --enable-cassert \
         CC="ccache x86_64-w64-mingw32-gcc" \
         CXX="ccache x86_64-w64-mingw32-g++"
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} world-bin
 
   ###
-  # Verify docs can be built
+  # Verify docs can be built, and upload changed docs as artifacts
   ###
   # XXX: Only do this if there have been changes in doc/ since last build
   always:
     docs_build_script: |
       time ./configure \
         --cache gcc.cache \
         CC="ccache gcc" \
         CXX="ccache g++" \
         CLANG="ccache clang"
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} -C doc
+      cp -r doc new-docs
+
+      # Re-build HTML docs from the base commit.
+      git checkout "$BASE_COMMIT" -- doc
+      make -s -C doc clean
+      time make -s -C doc html
+      cp -r doc old-docs
+
+    copy_changed_docs_script:
+      - src/tools/ci/copy-changed-docs "old-docs" "new-docs" "html_docs"
+
+    html_docs_artifacts:
+      paths: ['html_docs/*.html', 'html_docs/*.png', 'html_docs/*.css']
 
   ###
   # Verify headerscheck / cpluspluscheck succeed
   #
   # - Don't use ccache, the files are uncacheable, polluting ccache's
   #   cache
   # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
   # - XXX have to disable ICU to avoid errors:
   #   https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
   ###
   always:
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
new file mode 100755
index 00000000000..1c921a8df6f
--- /dev/null
+++ b/src/tools/ci/copy-changed-docs
@@ -0,0 +1,29 @@
+#! /bin/sh
+# Copy HTML which differ between $old and $new into $outdir
+set -e
+
+old=$1
+new=$2
+outdir=$3
+
+# The index is large and changes often
+skippages="bookindex.html"
+
+mkdir "$outdir"
+cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
+
+changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
+	[ $? -eq 1 ]
+
+for f in $changed
+do
+	# Avoid removed files
+	[ -f "$f" ] || continue
+
+	echo "$f" |grep -Ew "$skippages" >/dev/null &&
+		continue
+
+	cp -v "$f" "$outdir"
+done
+
+exit 0
-- 
2.34.1

From 5287027a1bceb719075a681fc130c246dc2a5d38 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Mon, 28 Feb 2022 23:18:19 -0600
Subject: [PATCH 8/8] +html index file

This allows linking to the artifacts from the last successful build.

//freebsd
ci-os-only: warnings
---
 .cirrus.yml                    |  4 ++--
 src/tools/ci/copy-changed-docs | 39 ++++++++++++++++++++++++++++++++--
 2 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 68b7229d117..e6f435b6981 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -27,23 +27,23 @@ env:
   PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
   TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
   PG_TEST_EXTRA: kerberos ldap ssl load_balance
 
   # The commit that this branch is rebased on.  There's no easy way to find this.
   # This does the right thing for cfbot, which always squishes all patches into a single commit.
   # And does the right thing for any 1-patch commits.
   # Patch series manually submitted to cirrus would benefit from setting this to the
   # number of patches in the series (or directly to the commit the series was rebased on).
   #BASE_COMMIT: HEAD~1
   # For demo purposes:
-  BASE_COMMIT: HEAD~11
+  BASE_COMMIT: HEAD~22
 
 # What files to preserve in case tests fail
 on_failure_ac: &on_failure_ac
   log_artifacts:
     paths:
       - "**/*.log"
       - "**/*.diffs"
       - "**/regress_log_*"
     type: text/plain
 
 on_failure_meson: &on_failure_meson
@@ -699,23 +699,23 @@ task:
       type: text/plain
 
 
 task:
   name: CompilerWarnings
 
   # To limit unnecessary work only run this once the SanityCheck
   # succeeds. This is particularly important for this task as we intentionally
   # use always: to continue after failures. Task that did not run count as a
   # success, so we need to recheck SanityChecks's condition here ...
   depends_on: SanityCheck
-  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*'
+  # only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*'
 
   env:
     CPUS: 4
     BUILD_JOBS: 4
 
     # Use larger ccache cache, as this task compiles with multiple compilers /
     # flag combinations
     CCACHE_MAXSIZE: "1G"
     CCACHE_DIR: "/tmp/ccache_dir"
 
     LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
index 1c921a8df6f..0efad386cca 100755
--- a/src/tools/ci/copy-changed-docs
+++ b/src/tools/ci/copy-changed-docs
@@ -1,29 +1,64 @@
 #! /bin/sh
 # Copy HTML which differ between $old and $new into $outdir
 set -e
 
 old=$1
 new=$2
 outdir=$3
+branch=$CIRRUS_BRANCH
 
 # The index is large and changes often
 skippages="bookindex.html"
 
 mkdir "$outdir"
 cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
 
+# The index is useful to allow a static link (not specific to a cirrus run) to the artifacts for the most-recent, successful CI run for a branch
+branchurl=https://api.cirrus-ci.com/v1/artifact/github/$CIRRUS_REPO_FULL_NAME/$CIRRUS_TASK_NAME/html_docs/html_docs/00-index.html?branch=$branch
+
+index="$outdir/00-index.html"
+cat >"$index" <<EOF
+<html>
+<head><title>Index of docs changed since: $BASE_COMMIT</title></head>
+<body>
+<!-- A link to documentation for the most recent successful build: $branchurl -->
+<h1>Index of docs changed since: $BASE_COMMIT</h1>
+<ul>
+EOF
+
 changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
 	[ $? -eq 1 ]
 
 for f in $changed
 do
 	# Avoid removed files
 	[ -f "$f" ] || continue
 
 	echo "$f" |grep -Ew "$skippages" >/dev/null &&
 		continue
 
-	cp -v "$f" "$outdir"
-done
+	cp -v "$f" "$outdir" >&2
+	fn=${f##*/}
+	# ?branch=... is needed when accessing the artifacts for the static link for the branch
+	# It's not needed and ignored if accessing artifacts for *this* CI run
+	echo "<li><a href='$fn?branch=$branch'>$fn</a>"
+done >>"$index"
+
+github=https://github.com/$CIRRUS_REPO_FULL_NAME/commit
+cirrus=https://cirrus-ci.com/build
+
+cat >>"$index" <<EOF
+</ul>
+<hr>
+<code>
+<br>This file was written on: `date --rfc-822 --utc`
+<br>CIRRUS_CHANGE_TITLE: $CIRRUS_CHANGE_TITLE
+<br>CIRRUS_CHANGE_IN_REPO: <a href="$github/$CIRRUS_CHANGE_IN_REPO">$CIRRUS_CHANGE_IN_REPO</a>
+<br>CIRRUS_BUILD_ID: <a href="$cirrus/$CIRRUS_BUILD_ID">$CIRRUS_BUILD_ID</a>
+<br>CIRRUS_LAST_GREEN_CHANGE: <a href="$github/$CIRRUS_LAST_GREEN_CHANGE">$CIRRUS_LAST_GREEN_CHANGE</a>
+<br>CIRRUS_LAST_GREEN_BUILD_ID: <a href="$cirrus/$CIRRUS_LAST_GREEN_BUILD_ID">$CIRRUS_LAST_GREEN_BUILD_ID</a>
+</code>
+</body></html>
+EOF
 
 exit 0
-- 
2.34.1

From 37a697bf2ff2e9e24c7b8cb2df289f21e1fca924 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 25 May 2022 21:53:22 -0500
Subject: [PATCH 1/6] cirrus/windows: add compiler_warnings_script

The goal is to fail due to warnings only after running tests.
(At least historically, it's too slow to run a separate windows VM to
compile with -Werror.)

https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de

I'm not sure how to write this test in windows shell; it's also easy to
write something that doesn't work in posix sh, since windows shell is
interpretting && and ||...

See also:
8a1ce5e54f6d144e4f8e19af7c767b026ee0c956
https://cirrus-ci.com/task/6241060062494720
https://cirrus-ci.com/task/6496366607204352

ci-os-only: windows
---
 .cirrus.tasks.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index e4e1bcfeb99..2d397448851 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -561,12 +561,21 @@ task:
 
   build_script: |
     vcvarsall x64
-    ninja -C build
+    ninja -C build |tee build.txt
+    REM Since pipes lose the exit status of the preceding command, rerun the compilation
+    REM without the pipe, exiting now if it fails, to avoid trying to run checks
+    ninja -C build > nul
 
   check_world_script: |
     vcvarsall x64
     meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
 
+  # This should be last, so check_world is run even if there are warnings
+  always:
+    compiler_warnings_script:
+      # this avoids using metachars which would be interpretted by the windows shell
+      - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0'
+
   on_failure:
     <<: *on_failure_meson
     crashlog_artifacts:
-- 
2.42.0

From 71d3bdad00bd0a4967db5a394247a54eb0f8a1fa Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 30 Jul 2022 19:25:51 -0500
Subject: [PATCH 2/6] cirrus/002_pg_upgrade: exercise --link and --clone

This increases code coverage (and maybe accelerates the test).

See also: b059a2409faf5833b3ba7792e247d6466c9e8090

linux,
macos,
//-os-only: freebsd
---
 .cirrus.tasks.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 2d397448851..53be0ce15e4 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -138,6 +138,8 @@ task:
     CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST -DENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS
     CFLAGS: -Og -ggdb
 
+    PG_TEST_PG_UPGRADE_MODE: --link
+
   <<: *freebsd_task_template
 
   depends_on: SanityCheck
@@ -431,6 +433,8 @@ task:
     CFLAGS: -Og -ggdb
     CXXFLAGS: -Og -ggdb
 
+    PG_TEST_PG_UPGRADE_MODE: --clone
+
   <<: *macos_task_template
 
   depends_on: SanityCheck
-- 
2.42.0

From 78dd05a15f1b0f476eaa29c8f8ceb70964e009a0 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 25 Nov 2022 13:57:17 -0600
Subject: [PATCH 3/6] WIP: ci/meson: allow showing only failed tests ..

It's simpler and seems to make more sense to integrate this with
testwrap, rather than to run it after check-world, but only if it
failed, and finding a way to preserve the exit code.

https://www.postgresql.org/message-id/20221114235328.lxdj3puenfhirhqm@awork3.anarazel.de

> It is wasteful to upload thousdands of logfiles to show a single
> failure.  That would make our cirrus tasks faster - compressing and
> uploading the logs takes over a minute.
>
> It's also a lot friendlier to show fewer than 8 pages of test folders to
> search through to find the one that failed.

macos
linux-meson
ci-os-only: freebsd
---
 .cirrus.tasks.yml  | 24 ++++++++++++++++++------
 src/tools/testwrap |  6 ++++++
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 53be0ce15e4..c0d72998199 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -22,6 +22,17 @@ env:
   TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
   PG_TEST_EXTRA: kerberos ldap ssl load_balance
 
+  PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
+
+  # The commit that this branch is rebased on.  There's no easy way to find this.
+  # This does the right thing for cfbot, which always squishes all patches into a single commit.
+  # And does the right thing for any 1-patch commits.
+  # Patch series manually submitted to cirrus would benefit from setting this to the
+  # number of patches in the series (or directly to the commit the series was rebased on).
+  #BASE_COMMIT: HEAD~1
+  # For demo purposes:
+  BASE_COMMIT: HEAD~11
+
 
 # What files to preserve in case tests fail
 on_failure_ac: &on_failure_ac
@@ -35,9 +46,9 @@ on_failure_ac: &on_failure_ac
 on_failure_meson: &on_failure_meson
   testrun_artifacts:
     paths:
-      - "build*/testrun/**/*.log"
-      - "build*/testrun/**/*.diffs"
-      - "build*/testrun/**/regress_log_*"
+      - failed.build*/**/*.log
+      - failed.build*/**/*.diffs
+      - failed.build*/**/regress_log_*
     type: text/plain
 
   # In theory it'd be nice to upload the junit files meson generates, so that
@@ -138,6 +149,7 @@ task:
     CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST -DENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS
     CFLAGS: -Og -ggdb
 
+    PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
     PG_TEST_PG_UPGRADE_MODE: --link
 
   <<: *freebsd_task_template
@@ -196,10 +208,10 @@ task:
       ulimit -c unlimited
       meson test $MTEST_ARGS --quiet --suite setup
       export LD_LIBRARY_PATH="$(pwd)/build/tmp_install/usr/local/pgsql/lib/:$LD_LIBRARY_PATH"
-      mkdir -p build/testrun
+      mkdir -p build/testrun ${PG_FAILED_TESTDIR}
       build/tmp_install/usr/local/pgsql/bin/initdb -N build/runningcheck --no-instructions -A trust
       echo "include '$(pwd)/src/tools/ci/pg_ci_base.conf'" >> build/runningcheck/postgresql.conf
-      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
+      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l ${PG_FAILED_TESTDIR}/runningcheck.log start
       meson test $MTEST_ARGS --num-processes ${TEST_JOBS} --setup running
       build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
     EOF
@@ -402,7 +414,7 @@ task:
       test_world_32_script: |
         su postgres <<-EOF
           ulimit -c unlimited
-          PYTHONCOERCECLOCALE=0 LANG=C meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
+          PYTHONCOERCECLOCALE=0 LANG=C PG_FAILED_TESTDIR=`pwd`/failed.build-32 meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
         EOF
 
       on_failure:
diff --git a/src/tools/testwrap b/src/tools/testwrap
index d01e61051cb..7e5a17dbf94 100755
--- a/src/tools/testwrap
+++ b/src/tools/testwrap
@@ -49,4 +49,10 @@ if sp.returncode == 0:
 else:
     print('# test failed')
     open(os.path.join(testdir, 'test.fail'), 'x')
+    faileddir = os.getenv('PG_FAILED_TESTDIR')
+    if faileddir:
+        parentdir = os.path.dirname(testdir)
+        newdest = os.path.join(faileddir, os.path.basename(parentdir), os.path.basename(testdir))
+        shutil.copytree(testdir, newdest)
+
 sys.exit(sp.returncode)
-- 
2.42.0

From 896a72c9e193a3ad0bdb5d1d8872156c18c38d73 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 26 Feb 2022 19:39:10 -0600
Subject: [PATCH 4/6] WIP: cirrus: upload changed html docs as artifacts

This could be done on the client side (cfbot).  One advantage of doing
it here is that fewer docs are uploaded - many patches won't upload docs
at all.

https://www.postgresql.org/message-id/flat/20220409021853.GP24419@telsasoft.com
https://www.postgresql.org/message-id/CAB8KJ=i4qmEuopQ+PCSMBzGd4O-Xv0FCnC+q1x7hN9hsdvkBug@mail.gmail.com

https://cirrus-ci.com/task/5396696388599808

//-os-only:
---
 .cirrus.tasks.yml              | 18 ++++++++++++++++--
 src/tools/ci/copy-changed-docs | 29 +++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+), 2 deletions(-)
 create mode 100755 src/tools/ci/copy-changed-docs

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index c0d72998199..8774db82291 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -31,7 +31,7 @@ env:
   # number of patches in the series (or directly to the commit the series was rebased on).
   #BASE_COMMIT: HEAD~1
   # For demo purposes:
-  BASE_COMMIT: HEAD~11
+  BASE_COMMIT: HEAD~55
 
 
 # What files to preserve in case tests fail
@@ -767,7 +767,7 @@ task:
       time make -s -j${BUILD_JOBS} world-bin
 
   ###
-  # Verify docs can be built
+  # Verify docs can be built, and upload changed docs as artifacts
   ###
   # XXX: Only do this if there have been changes in doc/ since last build
   always:
@@ -779,6 +779,20 @@ task:
         CLANG="ccache clang"
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} -C doc
+      cp -r doc new-docs
+
+      # Re-build HTML docs from the base commit.
+      git checkout "$BASE_COMMIT" -- doc
+      make -s -C doc clean
+      time make -s -C doc html
+      cp -r doc old-docs
+
+    copy_changed_docs_script:
+      - src/tools/ci/copy-changed-docs "old-docs" "new-docs" "html_docs"
+
+    html_docs_artifacts:
+      paths: ['html_docs/*.html', 'html_docs/*.png', 'html_docs/*.css']
+
 
   ###
   # Verify headerscheck / cpluspluscheck succeed
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
new file mode 100755
index 00000000000..1c921a8df6f
--- /dev/null
+++ b/src/tools/ci/copy-changed-docs
@@ -0,0 +1,29 @@
+#! /bin/sh
+# Copy HTML which differ between $old and $new into $outdir
+set -e
+
+old=$1
+new=$2
+outdir=$3
+
+# The index is large and changes often
+skippages="bookindex.html"
+
+mkdir "$outdir"
+cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
+
+changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
+	[ $? -eq 1 ]
+
+for f in $changed
+do
+	# Avoid removed files
+	[ -f "$f" ] || continue
+
+	echo "$f" |grep -Ew "$skippages" >/dev/null &&
+		continue
+
+	cp -v "$f" "$outdir"
+done
+
+exit 0
-- 
2.42.0

From 0c7778bd46f295b01fedd83daf1b57c73ead1b26 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Mon, 28 Feb 2022 23:18:19 -0600
Subject: [PATCH 5/6] +html index file

This allows linking to the artifacts from the last successful build.

//freebsd
ci-os-only: warnings
---
 src/tools/ci/copy-changed-docs | 39 ++++++++++++++++++++++++++++++++--
 1 file changed, 37 insertions(+), 2 deletions(-)

diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
index 1c921a8df6f..0efad386cca 100755
--- a/src/tools/ci/copy-changed-docs
+++ b/src/tools/ci/copy-changed-docs
@@ -5,6 +5,7 @@ set -e
 old=$1
 new=$2
 outdir=$3
+branch=$CIRRUS_BRANCH
 
 # The index is large and changes often
 skippages="bookindex.html"
@@ -12,6 +13,19 @@ skippages="bookindex.html"
 mkdir "$outdir"
 cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
 
+# The index is useful to allow a static link (not specific to a cirrus run) to the artifacts for the most-recent, successful CI run for a branch
+branchurl=https://api.cirrus-ci.com/v1/artifact/github/$CIRRUS_REPO_FULL_NAME/$CIRRUS_TASK_NAME/html_docs/html_docs/00-index.html?branch=$branch
+
+index="$outdir/00-index.html"
+cat >"$index" <<EOF
+<html>
+<head><title>Index of docs changed since: $BASE_COMMIT</title></head>
+<body>
+<!-- A link to documentation for the most recent successful build: $branchurl -->
+<h1>Index of docs changed since: $BASE_COMMIT</h1>
+<ul>
+EOF
+
 changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
 	[ $? -eq 1 ]
 
@@ -23,7 +37,28 @@ do
 	echo "$f" |grep -Ew "$skippages" >/dev/null &&
 		continue
 
-	cp -v "$f" "$outdir"
-done
+	cp -v "$f" "$outdir" >&2
+	fn=${f##*/}
+	# ?branch=... is needed when accessing the artifacts for the static link for the branch
+	# It's not needed and ignored if accessing artifacts for *this* CI run
+	echo "<li><a href='$fn?branch=$branch'>$fn</a>"
+done >>"$index"
+
+github=https://github.com/$CIRRUS_REPO_FULL_NAME/commit
+cirrus=https://cirrus-ci.com/build
+
+cat >>"$index" <<EOF
+</ul>
+<hr>
+<code>
+<br>This file was written on: `date --rfc-822 --utc`
+<br>CIRRUS_CHANGE_TITLE: $CIRRUS_CHANGE_TITLE
+<br>CIRRUS_CHANGE_IN_REPO: <a href="$github/$CIRRUS_CHANGE_IN_REPO">$CIRRUS_CHANGE_IN_REPO</a>
+<br>CIRRUS_BUILD_ID: <a href="$cirrus/$CIRRUS_BUILD_ID">$CIRRUS_BUILD_ID</a>
+<br>CIRRUS_LAST_GREEN_CHANGE: <a href="$github/$CIRRUS_LAST_GREEN_CHANGE">$CIRRUS_LAST_GREEN_CHANGE</a>
+<br>CIRRUS_LAST_GREEN_BUILD_ID: <a href="$cirrus/$CIRRUS_LAST_GREEN_BUILD_ID">$CIRRUS_LAST_GREEN_BUILD_ID</a>
+</code>
+</body></html>
+EOF
 
 exit 0
-- 
2.42.0

From d1aa01727dc4e0caee74d54c1c4b5061c627ffa0 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 19 Oct 2022 20:33:28 -0500
Subject: [PATCH 6/6] WIP: show changed docs with meson

//-os-only:
---
 .cirrus.tasks.yml              | 26 ++++++++++++++++++++++----
 src/tools/ci/copy-changed-docs |  3 ++-
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 8774db82291..342761c146e 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -772,20 +772,38 @@ task:
   # XXX: Only do this if there have been changes in doc/ since last build
   always:
     docs_build_script: |
-      time ./configure \
+      mkdir build-autoconf
+      cd build-autoconf
+      time ../configure \
         --cache gcc.cache \
+        --without-icu \
         CC="ccache gcc" \
         CXX="ccache g++" \
         CLANG="ccache clang"
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} -C doc
-      cp -r doc new-docs
+      cp -r doc ../new-docs
 
       # Re-build HTML docs from the base commit.
-      git checkout "$BASE_COMMIT" -- doc
+      git checkout "$BASE_COMMIT" -- ../doc
       make -s -C doc clean
       time make -s -C doc html
-      cp -r doc old-docs
+      cp -r doc ../old-docs
+
+    # Exercise HTML and other docs:
+    ninja_docs_build_script: |
+      make maintainer-clean # XXX not needed once compiler-warnings is switched meson
+      mkdir build-ninja
+      cd build-ninja
+      time meson setup
+      time ninja docs
+      cp -r doc ../new-docs
+
+      # Re-build HTML docs from the base commit.
+      git checkout "$BASE_COMMIT" -- ../doc
+      ninja clean
+      time ninja doc/src/sgml/html
+      cp -r doc ../old-docs
 
     copy_changed_docs_script:
       - src/tools/ci/copy-changed-docs "old-docs" "new-docs" "html_docs"
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
index 0efad386cca..ee3135c298b 100755
--- a/src/tools/ci/copy-changed-docs
+++ b/src/tools/ci/copy-changed-docs
@@ -11,7 +11,8 @@ branch=$CIRRUS_BRANCH
 skippages="bookindex.html"
 
 mkdir "$outdir"
-cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
+cp doc/src/sgml/*.css "$outdir"
+cp doc/src/sgml/images/*.svg "$outdir"
 
 # The index is useful to allow a static link (not specific to a cirrus run) to the artifacts for the most-recent, successful CI run for a branch
 branchurl=https://api.cirrus-ci.com/v1/artifact/github/$CIRRUS_REPO_FULL_NAME/$CIRRUS_TASK_NAME/html_docs/html_docs/00-index.html?branch=$branch
-- 
2.42.0

From 8cebe912127ee5c5d03b62b231811c2670698da0 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 25 May 2022 21:53:22 -0500
Subject: [PATCH 1/6] cirrus/windows: add compiler_warnings_script

The goal is to fail due to warnings only after running tests.
(At least historically, it's too slow to run a separate windows VM to
compile with -Werror.)

https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de

I'm not sure how to write this test in windows shell; it's also easy to
write something that doesn't work in posix sh, since windows shell is
interpretting && and ||...

See also:
8a1ce5e54f6d144e4f8e19af7c767b026ee0c956
https://cirrus-ci.com/task/6241060062494720
https://cirrus-ci.com/task/6496366607204352

20221104161934.GB16921@telsasoft.com
20221031223744.GT16921@telsasoft.com
20221104161934.GB16921@telsasoft.com
20221123054329.GG11463@telsasoft.com
20230224002029.GQ1653@telsasoft.com

ci-os-only: windows
---
 .cirrus.tasks.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 33646faeadf..5a2b64f64c2 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -566,12 +566,21 @@ task:
 
   build_script: |
     vcvarsall x64
-    ninja -C build
+    ninja -C build |tee build.txt
+    REM Since pipes lose the exit status of the preceding command, rerun the compilation
+    REM without the pipe, exiting now if it fails, to avoid trying to run checks
+    ninja -C build > nul
 
   check_world_script: |
     vcvarsall x64
     meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
 
+  # This should be last, so check_world is run even if there are warnings
+  always:
+    compiler_warnings_script:
+      # this avoids using metachars which would be interpretted by the windows shell
+      - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0'
+
   on_failure:
     <<: *on_failure_meson
     crashlog_artifacts:
-- 
2.42.0

From 7f8e1c244b24f04a28d5a1a03da85f00419717e1 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 25 May 2022 22:05:13 -0500
Subject: [PATCH 2/6] cirrus/windows: ccache

https://www.postgresql.org/message-id/flat/20220522232606.GZ19626%40telsasoft.com

https://cirrus-ci.com/task/5213936495099904 build 29sec

, linux
linux-meson
ci-os-only: windows, windows-msvc
---
 .cirrus.tasks.yml | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 5a2b64f64c2..7a3fcc36fae 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -536,6 +536,11 @@ task:
   env:
     TEST_JOBS: 8 # wild guess, data based value welcome
 
+    CCACHE_DIR: $CIRRUS_WORKING_DIR/.ccache
+    CCACHE_MAXSIZE: "500M"
+    CCACHE_SLOPPINESS: pch_defines,time_macros
+    CCACHE_DEPEND: 1
+
     # Cirrus defaults to SetErrorMode(SEM_NOGPFAULTERRORBOX | ...). That
     # prevents crash reporting from working unless binaries do SetErrorMode()
     # themselves. Furthermore, it appears that either python or, more likely,
@@ -550,8 +555,11 @@ task:
   depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
 
-  setup_additional_packages_script: |
-    REM choco install -y --no-progress ...
+  setup_additional_packages_script:
+    - choco install -y --no-progress ccache --version 4.10.0
+
+  ccache_cache:
+    folder: $CCACHE_DIR
 
   setup_hosts_file_script: |
     echo 127.0.0.1 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts
@@ -562,7 +570,8 @@ task:
   # Use /DEBUG:FASTLINK to avoid high memory usage during linking
   configure_script: |
     vcvarsall x64
-    meson setup --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Dinjection_points=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" build
+    set CC=c:\ProgramData\chocolatey\lib\ccache\tools\ccache-4.10-windows-x86_64\ccache.exe cl.exe
+    meson setup build --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Dinjection_points=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" -Dc_args="/Z7"
 
   build_script: |
     vcvarsall x64
@@ -571,6 +580,12 @@ task:
     REM without the pipe, exiting now if it fails, to avoid trying to run checks
     ninja -C build > nul
 
+    ccache --show-log-stats -v
+    ccache --show-stats -v
+
+  always:
+    upload_caches: [ccache]
+
   check_world_script: |
     vcvarsall x64
     meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
-- 
2.42.0

From 7ad1d5c1999210fc2a4298d4afc756dac78eb4a6 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 25 Nov 2022 13:57:17 -0600
Subject: [PATCH 3/6] WIP: ci/meson: allow showing only failed tests ..

It's simpler and seems to make more sense to integrate this with
testwrap, rather than to run it after check-world, but only if it
failed, and finding a way to preserve the exit code.

https://www.postgresql.org/message-id/20221114235328.lxdj3puenfhirhqm@awork3.anarazel.de

> It is wasteful to upload thousdands of logfiles to show a single
> failure.  That would make our cirrus tasks faster - compressing and
> uploading the logs takes over a minute.
>
> It's also a lot friendlier to show fewer than 8 pages of test folders to
> search through to find the one that failed.

macos
linux-meson
ci-os-only: freebsd
---
 .cirrus.tasks.yml  | 14 ++++++++------
 src/tools/testwrap |  6 ++++++
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 7a3fcc36fae..e9f3a4fb8f9 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -21,6 +21,7 @@ env:
   PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
   TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
   PG_TEST_EXTRA: kerberos ldap ssl libpq_encryption load_balance
+  PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
 
 
 # What files to preserve in case tests fail
@@ -35,9 +36,9 @@ on_failure_ac: &on_failure_ac
 on_failure_meson: &on_failure_meson
   testrun_artifacts:
     paths:
-      - "build*/testrun/**/*.log"
-      - "build*/testrun/**/*.diffs"
-      - "build*/testrun/**/regress_log_*"
+      - failed.build*/**/*.log
+      - failed.build*/**/*.diffs
+      - failed.build*/**/regress_log_*
     type: text/plain
 
   # In theory it'd be nice to upload the junit files meson generates, so that
@@ -136,6 +137,7 @@ task:
     CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST -DENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS
     CFLAGS: -Og -ggdb
 
+    PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
     PG_TEST_PG_UPGRADE_MODE: --link
 
   <<: *freebsd_task_template
@@ -194,10 +196,10 @@ task:
       ulimit -c unlimited
       meson test $MTEST_ARGS --quiet --suite setup
       export LD_LIBRARY_PATH="$(pwd)/build/tmp_install/usr/local/pgsql/lib/:$LD_LIBRARY_PATH"
-      mkdir -p build/testrun
+      mkdir -p build/testrun ${PG_FAILED_TESTDIR}
       build/tmp_install/usr/local/pgsql/bin/initdb -N build/runningcheck --no-instructions -A trust
       echo "include '$(pwd)/src/tools/ci/pg_ci_base.conf'" >> build/runningcheck/postgresql.conf
-      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
+      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l ${PG_FAILED_TESTDIR}/runningcheck.log start
       meson test $MTEST_ARGS --num-processes ${TEST_JOBS} --setup running
       build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
     EOF
@@ -400,7 +402,7 @@ task:
       test_world_32_script: |
         su postgres <<-EOF
           ulimit -c unlimited
-          PYTHONCOERCECLOCALE=0 LANG=C meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
+          PYTHONCOERCECLOCALE=0 LANG=C PG_FAILED_TESTDIR=`pwd`/failed.build-32 meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
         EOF
 
       on_failure:
diff --git a/src/tools/testwrap b/src/tools/testwrap
index d01e61051cb..7e5a17dbf94 100755
--- a/src/tools/testwrap
+++ b/src/tools/testwrap
@@ -49,4 +49,10 @@ if sp.returncode == 0:
 else:
     print('# test failed')
     open(os.path.join(testdir, 'test.fail'), 'x')
+    faileddir = os.getenv('PG_FAILED_TESTDIR')
+    if faileddir:
+        parentdir = os.path.dirname(testdir)
+        newdest = os.path.join(faileddir, os.path.basename(parentdir), os.path.basename(testdir))
+        shutil.copytree(testdir, newdest)
+
 sys.exit(sp.returncode)
-- 
2.42.0

From dcc133e30bca653e31f8672c6e822a55b9c0e550 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 26 Feb 2022 19:39:10 -0600
Subject: [PATCH 4/6] WIP: cirrus: upload changed html docs as artifacts

This could be done on the client side (cfbot).  One advantage of doing
it here is that fewer docs are uploaded - many patches won't upload docs
at all.

https://www.postgresql.org/message-id/flat/20220409021853.GP24419@telsasoft.com
https://www.postgresql.org/message-id/CAB8KJ=i4qmEuopQ+PCSMBzGd4O-Xv0FCnC+q1x7hN9hsdvkBug@mail.gmail.com

https://cirrus-ci.com/task/5396696388599808

//-os-only:
---
 .cirrus.tasks.yml              | 25 ++++++++++++++++++++++++-
 src/tools/ci/copy-changed-docs | 29 +++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 1 deletion(-)
 create mode 100755 src/tools/ci/copy-changed-docs

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index e9f3a4fb8f9..77dca9e3483 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -23,6 +23,15 @@ env:
   PG_TEST_EXTRA: kerberos ldap ssl libpq_encryption load_balance
   PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
 
+  # The commit that this branch is rebased on.  There's no easy way to find this.
+  # This does the right thing for cfbot, which always squishes all patches into a single commit.
+  # And does the right thing for any 1-patch commits.
+  # Patch series manually submitted to cirrus would benefit from setting this to the
+  # number of patches in the series (or directly to the commit the series was rebased on).
+  #BASE_COMMIT: HEAD~1
+  # For demo purposes:
+  BASE_COMMIT: HEAD~55
+
 
 # What files to preserve in case tests fail
 on_failure_ac: &on_failure_ac
@@ -772,7 +781,7 @@ task:
       time make -s -j${BUILD_JOBS} world-bin
 
   ###
-  # Verify docs can be built
+  # Verify docs can be built, and upload changed docs as artifacts
   ###
   # XXX: Only do this if there have been changes in doc/ since last build
   always:
@@ -784,6 +793,20 @@ task:
         CLANG="ccache clang"
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} -C doc
+      cp -r doc new-docs
+
+      # Re-build HTML docs from the base commit.
+      git checkout "$BASE_COMMIT" -- doc
+      make -s -C doc clean
+      time make -s -C doc html
+      cp -r doc old-docs
+
+    copy_changed_docs_script:
+      - src/tools/ci/copy-changed-docs "old-docs" "new-docs" "html_docs"
+
+    html_docs_artifacts:
+      paths: ['html_docs/*.html', 'html_docs/*.png', 'html_docs/*.css']
+
 
   ###
   # Verify headerscheck / cpluspluscheck succeed
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
new file mode 100755
index 00000000000..1c921a8df6f
--- /dev/null
+++ b/src/tools/ci/copy-changed-docs
@@ -0,0 +1,29 @@
+#! /bin/sh
+# Copy HTML which differ between $old and $new into $outdir
+set -e
+
+old=$1
+new=$2
+outdir=$3
+
+# The index is large and changes often
+skippages="bookindex.html"
+
+mkdir "$outdir"
+cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
+
+changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
+	[ $? -eq 1 ]
+
+for f in $changed
+do
+	# Avoid removed files
+	[ -f "$f" ] || continue
+
+	echo "$f" |grep -Ew "$skippages" >/dev/null &&
+		continue
+
+	cp -v "$f" "$outdir"
+done
+
+exit 0
-- 
2.42.0

From 5cedbf6bbfecf0cd82f6116850528c2cd75f4b0a Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Mon, 28 Feb 2022 23:18:19 -0600
Subject: [PATCH 5/6] +html index file

This allows linking to the artifacts from the last successful build.

//freebsd
ci-os-only: warnings
---
 src/tools/ci/copy-changed-docs | 39 ++++++++++++++++++++++++++++++++--
 1 file changed, 37 insertions(+), 2 deletions(-)

diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
index 1c921a8df6f..0efad386cca 100755
--- a/src/tools/ci/copy-changed-docs
+++ b/src/tools/ci/copy-changed-docs
@@ -5,6 +5,7 @@ set -e
 old=$1
 new=$2
 outdir=$3
+branch=$CIRRUS_BRANCH
 
 # The index is large and changes often
 skippages="bookindex.html"
@@ -12,6 +13,19 @@ skippages="bookindex.html"
 mkdir "$outdir"
 cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
 
+# The index is useful to allow a static link (not specific to a cirrus run) to the artifacts for the most-recent, successful CI run for a branch
+branchurl=https://api.cirrus-ci.com/v1/artifact/github/$CIRRUS_REPO_FULL_NAME/$CIRRUS_TASK_NAME/html_docs/html_docs/00-index.html?branch=$branch
+
+index="$outdir/00-index.html"
+cat >"$index" <<EOF
+<html>
+<head><title>Index of docs changed since: $BASE_COMMIT</title></head>
+<body>
+<!-- A link to documentation for the most recent successful build: $branchurl -->
+<h1>Index of docs changed since: $BASE_COMMIT</h1>
+<ul>
+EOF
+
 changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
 	[ $? -eq 1 ]
 
@@ -23,7 +37,28 @@ do
 	echo "$f" |grep -Ew "$skippages" >/dev/null &&
 		continue
 
-	cp -v "$f" "$outdir"
-done
+	cp -v "$f" "$outdir" >&2
+	fn=${f##*/}
+	# ?branch=... is needed when accessing the artifacts for the static link for the branch
+	# It's not needed and ignored if accessing artifacts for *this* CI run
+	echo "<li><a href='$fn?branch=$branch'>$fn</a>"
+done >>"$index"
+
+github=https://github.com/$CIRRUS_REPO_FULL_NAME/commit
+cirrus=https://cirrus-ci.com/build
+
+cat >>"$index" <<EOF
+</ul>
+<hr>
+<code>
+<br>This file was written on: `date --rfc-822 --utc`
+<br>CIRRUS_CHANGE_TITLE: $CIRRUS_CHANGE_TITLE
+<br>CIRRUS_CHANGE_IN_REPO: <a href="$github/$CIRRUS_CHANGE_IN_REPO">$CIRRUS_CHANGE_IN_REPO</a>
+<br>CIRRUS_BUILD_ID: <a href="$cirrus/$CIRRUS_BUILD_ID">$CIRRUS_BUILD_ID</a>
+<br>CIRRUS_LAST_GREEN_CHANGE: <a href="$github/$CIRRUS_LAST_GREEN_CHANGE">$CIRRUS_LAST_GREEN_CHANGE</a>
+<br>CIRRUS_LAST_GREEN_BUILD_ID: <a href="$cirrus/$CIRRUS_LAST_GREEN_BUILD_ID">$CIRRUS_LAST_GREEN_BUILD_ID</a>
+</code>
+</body></html>
+EOF
 
 exit 0
-- 
2.42.0

From b084963b13e34a977c54a9b8f9e02543006720c1 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 19 Oct 2022 20:33:28 -0500
Subject: [PATCH 6/6] WIP: show changed docs with meson

//-os-only:
---
 .cirrus.tasks.yml              | 26 ++++++++++++++++++++++----
 src/tools/ci/copy-changed-docs |  3 ++-
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 77dca9e3483..f4a176103e3 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -786,20 +786,38 @@ task:
   # XXX: Only do this if there have been changes in doc/ since last build
   always:
     docs_build_script: |
-      time ./configure \
+      mkdir build-autoconf
+      cd build-autoconf
+      time ../configure \
         --cache gcc.cache \
+        --without-icu \
         CC="ccache gcc" \
         CXX="ccache g++" \
         CLANG="ccache clang"
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} -C doc
-      cp -r doc new-docs
+      cp -r doc ../new-docs
 
       # Re-build HTML docs from the base commit.
-      git checkout "$BASE_COMMIT" -- doc
+      git checkout "$BASE_COMMIT" -- ../doc
       make -s -C doc clean
       time make -s -C doc html
-      cp -r doc old-docs
+      cp -r doc ../old-docs
+
+    # Exercise HTML and other docs:
+    ninja_docs_build_script: |
+      make maintainer-clean # XXX not needed once compiler-warnings is switched meson
+      mkdir build-ninja
+      cd build-ninja
+      time meson setup
+      time ninja docs
+      cp -r doc ../new-docs
+
+      # Re-build HTML docs from the base commit.
+      git checkout "$BASE_COMMIT" -- ../doc
+      ninja clean
+      time ninja doc/src/sgml/html
+      cp -r doc ../old-docs
 
     copy_changed_docs_script:
       - src/tools/ci/copy-changed-docs "old-docs" "new-docs" "html_docs"
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
index 0efad386cca..ee3135c298b 100755
--- a/src/tools/ci/copy-changed-docs
+++ b/src/tools/ci/copy-changed-docs
@@ -11,7 +11,8 @@ branch=$CIRRUS_BRANCH
 skippages="bookindex.html"
 
 mkdir "$outdir"
-cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
+cp doc/src/sgml/*.css "$outdir"
+cp doc/src/sgml/images/*.svg "$outdir"
 
 # The index is useful to allow a static link (not specific to a cirrus run) to the artifacts for the most-recent, successful CI run for a branch
 branchurl=https://api.cirrus-ci.com/v1/artifact/github/$CIRRUS_REPO_FULL_NAME/$CIRRUS_TASK_NAME/html_docs/html_docs/00-index.html?branch=$branch
-- 
2.42.0

From ad03da2cfa3ecf23334f8b31f2e4529ba3094512 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 25 May 2022 21:53:22 -0500
Subject: [PATCH 1/6] cirrus/windows: add compiler_warnings_script

The goal is to fail due to warnings only after running tests.
(At least historically, it's too slow to run a separate windows VM to
compile with -Werror.)

https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de

I'm not sure how to write this test in windows shell; it's also easy to
write something that doesn't work in posix sh, since windows shell is
interpretting && and ||...

See also:
8a1ce5e54f6d144e4f8e19af7c767b026ee0c956
https://cirrus-ci.com/task/6241060062494720
https://cirrus-ci.com/task/6496366607204352

20221104161934.GB16921@telsasoft.com
20221031223744.GT16921@telsasoft.com
20221104161934.GB16921@telsasoft.com
20221123054329.GG11463@telsasoft.com
20230224002029.GQ1653@telsasoft.com

ci-os-only: windows
---
 .cirrus.tasks.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 33646faeadf..5a2b64f64c2 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -566,12 +566,21 @@ task:
 
   build_script: |
     vcvarsall x64
-    ninja -C build
+    ninja -C build |tee build.txt
+    REM Since pipes lose the exit status of the preceding command, rerun the compilation
+    REM without the pipe, exiting now if it fails, to avoid trying to run checks
+    ninja -C build > nul
 
   check_world_script: |
     vcvarsall x64
     meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
 
+  # This should be last, so check_world is run even if there are warnings
+  always:
+    compiler_warnings_script:
+      # this avoids using metachars which would be interpretted by the windows shell
+      - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0'
+
   on_failure:
     <<: *on_failure_meson
     crashlog_artifacts:
-- 
2.42.0

From 31629f2f413ceb5f2704ac71e24d0d8e82cc26ab Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 25 May 2022 22:05:13 -0500
Subject: [PATCH 2/6] WIP: cirrus/windows: ccache

ccache v4.10 supports PCH, so this now seems viable.
(ccache 4.7 added support for depend mode with MSVC).

https://www.postgresql.org/message-id/flat/20220522232606.GZ19626%40telsasoft.com

ccache should be installed in the VM rather than with choco in each
invocation.

https://cirrus-ci.com/task/5213936495099904 build 29sec

, linux
linux-meson
ci-os-only: windows, windows-msvc
---
 .cirrus.tasks.yml | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 5a2b64f64c2..d8c2a396a20 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -536,6 +536,12 @@ task:
   env:
     TEST_JOBS: 8 # wild guess, data based value welcome
 
+    CCACHE_DIR: $CIRRUS_WORKING_DIR/.ccache
+    CCACHE_MAXSIZE: "500M"
+    CCACHE_SLOPPINESS: pch_defines,time_macros
+    CCACHE_DEPEND: 1
+    CCACHE_STATSLOG: $CCACHE_DIR.stats.log
+
     # Cirrus defaults to SetErrorMode(SEM_NOGPFAULTERRORBOX | ...). That
     # prevents crash reporting from working unless binaries do SetErrorMode()
     # themselves. Furthermore, it appears that either python or, more likely,
@@ -550,8 +556,11 @@ task:
   depends_on: SanityCheck
   only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
 
-  setup_additional_packages_script: |
-    REM choco install -y --no-progress ...
+  setup_additional_packages_script:
+    - choco install -y --no-progress ccache --version 4.10.0
+
+  ccache_cache:
+    folder: $CCACHE_DIR
 
   setup_hosts_file_script: |
     echo 127.0.0.1 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts
@@ -562,7 +571,8 @@ task:
   # Use /DEBUG:FASTLINK to avoid high memory usage during linking
   configure_script: |
     vcvarsall x64
-    meson setup --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Dinjection_points=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" build
+    set CC=c:\ProgramData\chocolatey\lib\ccache\tools\ccache-4.10-windows-x86_64\ccache.exe cl.exe
+    meson setup build --backend ninja --buildtype debugoptimized -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Dinjection_points=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" -Dc_args="/Z7"
 
   build_script: |
     vcvarsall x64
@@ -571,6 +581,12 @@ task:
     REM without the pipe, exiting now if it fails, to avoid trying to run checks
     ninja -C build > nul
 
+    ccache --show-log-stats -v
+    ccache --show-stats -v
+
+  always:
+    upload_caches: [ccache]
+
   check_world_script: |
     vcvarsall x64
     meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
-- 
2.42.0

From 24280295d5c4c5c62242de1db1ab8e7582097899 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 25 Nov 2022 13:57:17 -0600
Subject: [PATCH 3/6] WIP: ci/meson: allow showing only failed tests ..

It's simpler and seems to make more sense to integrate this with
testwrap, rather than to run it after check-world, but only if it
failed, and finding a way to preserve the exit code.

https://www.postgresql.org/message-id/20221114235328.lxdj3puenfhirhqm@awork3.anarazel.de

> It is wasteful to upload thousdands of logfiles to show a single
> failure.  That would make our cirrus tasks faster - compressing and
> uploading the logs takes over a minute.
>
> It's also a lot friendlier to show fewer than 8 pages of test folders to
> search through to find the one that failed.

macos
linux-meson
ci-os-only: freebsd
---
 .cirrus.tasks.yml  | 14 ++++++++------
 src/tools/testwrap |  6 ++++++
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index d8c2a396a20..3df51e6c896 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -21,6 +21,7 @@ env:
   PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
   TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
   PG_TEST_EXTRA: kerberos ldap ssl libpq_encryption load_balance
+  PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
 
 
 # What files to preserve in case tests fail
@@ -35,9 +36,9 @@ on_failure_ac: &on_failure_ac
 on_failure_meson: &on_failure_meson
   testrun_artifacts:
     paths:
-      - "build*/testrun/**/*.log"
-      - "build*/testrun/**/*.diffs"
-      - "build*/testrun/**/regress_log_*"
+      - failed.build*/**/*.log
+      - failed.build*/**/*.diffs
+      - failed.build*/**/regress_log_*
     type: text/plain
 
   # In theory it'd be nice to upload the junit files meson generates, so that
@@ -136,6 +137,7 @@ task:
     CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST -DENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS
     CFLAGS: -Og -ggdb
 
+    PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
     PG_TEST_PG_UPGRADE_MODE: --link
 
   <<: *freebsd_task_template
@@ -194,10 +196,10 @@ task:
       ulimit -c unlimited
       meson test $MTEST_ARGS --quiet --suite setup
       export LD_LIBRARY_PATH="$(pwd)/build/tmp_install/usr/local/pgsql/lib/:$LD_LIBRARY_PATH"
-      mkdir -p build/testrun
+      mkdir -p build/testrun ${PG_FAILED_TESTDIR}
       build/tmp_install/usr/local/pgsql/bin/initdb -N build/runningcheck --no-instructions -A trust
       echo "include '$(pwd)/src/tools/ci/pg_ci_base.conf'" >> build/runningcheck/postgresql.conf
-      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
+      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l ${PG_FAILED_TESTDIR}/runningcheck.log start
       meson test $MTEST_ARGS --num-processes ${TEST_JOBS} --setup running
       build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
     EOF
@@ -400,7 +402,7 @@ task:
       test_world_32_script: |
         su postgres <<-EOF
           ulimit -c unlimited
-          PYTHONCOERCECLOCALE=0 LANG=C meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
+          PYTHONCOERCECLOCALE=0 LANG=C PG_FAILED_TESTDIR=`pwd`/failed.build-32 meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
         EOF
 
       on_failure:
diff --git a/src/tools/testwrap b/src/tools/testwrap
index d01e61051cb..7e5a17dbf94 100755
--- a/src/tools/testwrap
+++ b/src/tools/testwrap
@@ -49,4 +49,10 @@ if sp.returncode == 0:
 else:
     print('# test failed')
     open(os.path.join(testdir, 'test.fail'), 'x')
+    faileddir = os.getenv('PG_FAILED_TESTDIR')
+    if faileddir:
+        parentdir = os.path.dirname(testdir)
+        newdest = os.path.join(faileddir, os.path.basename(parentdir), os.path.basename(testdir))
+        shutil.copytree(testdir, newdest)
+
 sys.exit(sp.returncode)
-- 
2.42.0

From b164129479b56d2b56ee8bacecb3ad31e5c4a914 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 26 Feb 2022 19:39:10 -0600
Subject: [PATCH 4/6] WIP: cirrus: upload changed html docs as artifacts

This could be done on the client side (cfbot).  One advantage of doing
it here is that fewer docs are uploaded - many patches won't upload docs
at all.

https://www.postgresql.org/message-id/flat/20220409021853.GP24419@telsasoft.com
https://www.postgresql.org/message-id/CAB8KJ=i4qmEuopQ+PCSMBzGd4O-Xv0FCnC+q1x7hN9hsdvkBug@mail.gmail.com

https://cirrus-ci.com/task/5396696388599808

//-os-only:
---
 .cirrus.tasks.yml              | 25 ++++++++++++++++++++++++-
 src/tools/ci/copy-changed-docs | 29 +++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 1 deletion(-)
 create mode 100755 src/tools/ci/copy-changed-docs

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 3df51e6c896..3f1960c4cf9 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -23,6 +23,15 @@ env:
   PG_TEST_EXTRA: kerberos ldap ssl libpq_encryption load_balance
   PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
 
+  # The commit that this branch is rebased on.  There's no easy way to find this.
+  # This does the right thing for cfbot, which always squishes all patches into a single commit.
+  # And does the right thing for any 1-patch commits.
+  # Patch series manually submitted to cirrus would benefit from setting this to the
+  # number of patches in the series (or directly to the commit the series was rebased on).
+  #BASE_COMMIT: HEAD~1
+  # For demo purposes:
+  BASE_COMMIT: HEAD~55
+
 
 # What files to preserve in case tests fail
 on_failure_ac: &on_failure_ac
@@ -773,7 +782,7 @@ task:
       time make -s -j${BUILD_JOBS} world-bin
 
   ###
-  # Verify docs can be built
+  # Verify docs can be built, and upload changed docs as artifacts
   ###
   # XXX: Only do this if there have been changes in doc/ since last build
   always:
@@ -785,6 +794,20 @@ task:
         CLANG="ccache clang"
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} -C doc
+      cp -r doc new-docs
+
+      # Re-build HTML docs from the base commit.
+      git checkout "$BASE_COMMIT" -- doc
+      make -s -C doc clean
+      time make -s -C doc html
+      cp -r doc old-docs
+
+    copy_changed_docs_script:
+      - src/tools/ci/copy-changed-docs "old-docs" "new-docs" "html_docs"
+
+    html_docs_artifacts:
+      paths: ['html_docs/*.html', 'html_docs/*.png', 'html_docs/*.css']
+
 
   ###
   # Verify headerscheck / cpluspluscheck succeed
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
new file mode 100755
index 00000000000..1c921a8df6f
--- /dev/null
+++ b/src/tools/ci/copy-changed-docs
@@ -0,0 +1,29 @@
+#! /bin/sh
+# Copy HTML which differ between $old and $new into $outdir
+set -e
+
+old=$1
+new=$2
+outdir=$3
+
+# The index is large and changes often
+skippages="bookindex.html"
+
+mkdir "$outdir"
+cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
+
+changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
+	[ $? -eq 1 ]
+
+for f in $changed
+do
+	# Avoid removed files
+	[ -f "$f" ] || continue
+
+	echo "$f" |grep -Ew "$skippages" >/dev/null &&
+		continue
+
+	cp -v "$f" "$outdir"
+done
+
+exit 0
-- 
2.42.0

From 2fa279d3596976b0face797d16e4f28a357bb73e Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Mon, 28 Feb 2022 23:18:19 -0600
Subject: [PATCH 5/6] +html index file

This allows linking to the artifacts from the last successful build.

//freebsd
ci-os-only: warnings
---
 src/tools/ci/copy-changed-docs | 39 ++++++++++++++++++++++++++++++++--
 1 file changed, 37 insertions(+), 2 deletions(-)

diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
index 1c921a8df6f..0efad386cca 100755
--- a/src/tools/ci/copy-changed-docs
+++ b/src/tools/ci/copy-changed-docs
@@ -5,6 +5,7 @@ set -e
 old=$1
 new=$2
 outdir=$3
+branch=$CIRRUS_BRANCH
 
 # The index is large and changes often
 skippages="bookindex.html"
@@ -12,6 +13,19 @@ skippages="bookindex.html"
 mkdir "$outdir"
 cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
 
+# The index is useful to allow a static link (not specific to a cirrus run) to the artifacts for the most-recent, successful CI run for a branch
+branchurl=https://api.cirrus-ci.com/v1/artifact/github/$CIRRUS_REPO_FULL_NAME/$CIRRUS_TASK_NAME/html_docs/html_docs/00-index.html?branch=$branch
+
+index="$outdir/00-index.html"
+cat >"$index" <<EOF
+<html>
+<head><title>Index of docs changed since: $BASE_COMMIT</title></head>
+<body>
+<!-- A link to documentation for the most recent successful build: $branchurl -->
+<h1>Index of docs changed since: $BASE_COMMIT</h1>
+<ul>
+EOF
+
 changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
 	[ $? -eq 1 ]
 
@@ -23,7 +37,28 @@ do
 	echo "$f" |grep -Ew "$skippages" >/dev/null &&
 		continue
 
-	cp -v "$f" "$outdir"
-done
+	cp -v "$f" "$outdir" >&2
+	fn=${f##*/}
+	# ?branch=... is needed when accessing the artifacts for the static link for the branch
+	# It's not needed and ignored if accessing artifacts for *this* CI run
+	echo "<li><a href='$fn?branch=$branch'>$fn</a>"
+done >>"$index"
+
+github=https://github.com/$CIRRUS_REPO_FULL_NAME/commit
+cirrus=https://cirrus-ci.com/build
+
+cat >>"$index" <<EOF
+</ul>
+<hr>
+<code>
+<br>This file was written on: `date --rfc-822 --utc`
+<br>CIRRUS_CHANGE_TITLE: $CIRRUS_CHANGE_TITLE
+<br>CIRRUS_CHANGE_IN_REPO: <a href="$github/$CIRRUS_CHANGE_IN_REPO">$CIRRUS_CHANGE_IN_REPO</a>
+<br>CIRRUS_BUILD_ID: <a href="$cirrus/$CIRRUS_BUILD_ID">$CIRRUS_BUILD_ID</a>
+<br>CIRRUS_LAST_GREEN_CHANGE: <a href="$github/$CIRRUS_LAST_GREEN_CHANGE">$CIRRUS_LAST_GREEN_CHANGE</a>
+<br>CIRRUS_LAST_GREEN_BUILD_ID: <a href="$cirrus/$CIRRUS_LAST_GREEN_BUILD_ID">$CIRRUS_LAST_GREEN_BUILD_ID</a>
+</code>
+</body></html>
+EOF
 
 exit 0
-- 
2.42.0

From a3e1a122e90cfc6b76623240c45a81e67a760670 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 19 Oct 2022 20:33:28 -0500
Subject: [PATCH 6/6] WIP: show changed docs with meson

//-os-only:
---
 .cirrus.tasks.yml              | 26 ++++++++++++++++++++++----
 src/tools/ci/copy-changed-docs |  3 ++-
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 3f1960c4cf9..8fa1bf54985 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -787,20 +787,38 @@ task:
   # XXX: Only do this if there have been changes in doc/ since last build
   always:
     docs_build_script: |
-      time ./configure \
+      mkdir build-autoconf
+      cd build-autoconf
+      time ../configure \
         --cache gcc.cache \
+        --without-icu \
         CC="ccache gcc" \
         CXX="ccache g++" \
         CLANG="ccache clang"
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} -C doc
-      cp -r doc new-docs
+      cp -r doc ../new-docs
 
       # Re-build HTML docs from the base commit.
-      git checkout "$BASE_COMMIT" -- doc
+      git checkout "$BASE_COMMIT" -- ../doc
       make -s -C doc clean
       time make -s -C doc html
-      cp -r doc old-docs
+      cp -r doc ../old-docs
+
+    # Exercise HTML and other docs:
+    ninja_docs_build_script: |
+      make maintainer-clean # XXX not needed once compiler-warnings is switched meson
+      mkdir build-ninja
+      cd build-ninja
+      time meson setup
+      time ninja docs
+      cp -r doc ../new-docs
+
+      # Re-build HTML docs from the base commit.
+      git checkout "$BASE_COMMIT" -- ../doc
+      ninja clean
+      time ninja doc/src/sgml/html
+      cp -r doc ../old-docs
 
     copy_changed_docs_script:
       - src/tools/ci/copy-changed-docs "old-docs" "new-docs" "html_docs"
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
index 0efad386cca..ee3135c298b 100755
--- a/src/tools/ci/copy-changed-docs
+++ b/src/tools/ci/copy-changed-docs
@@ -11,7 +11,8 @@ branch=$CIRRUS_BRANCH
 skippages="bookindex.html"
 
 mkdir "$outdir"
-cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
+cp doc/src/sgml/*.css "$outdir"
+cp doc/src/sgml/images/*.svg "$outdir"
 
 # The index is useful to allow a static link (not specific to a cirrus run) to the artifacts for the most-recent, successful CI run for a branch
 branchurl=https://api.cirrus-ci.com/v1/artifact/github/$CIRRUS_REPO_FULL_NAME/$CIRRUS_TASK_NAME/html_docs/html_docs/00-index.html?branch=$branch
-- 
2.42.0

From ad03da2cfa3ecf23334f8b31f2e4529ba3094512 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 25 May 2022 21:53:22 -0500
Subject: [PATCH 1/7] cirrus/windows: add compiler_warnings_script

The goal is to fail due to warnings only after running tests.
(At least historically, it's too slow to run a separate windows VM to
compile with -Werror.)

https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de

I'm not sure how to write this test in windows shell; it's also easy to
write something that doesn't work in posix sh, since windows shell is
interpretting && and ||...

See also:
8a1ce5e54f6d144e4f8e19af7c767b026ee0c956
https://cirrus-ci.com/task/6241060062494720
https://cirrus-ci.com/task/6496366607204352

20221104161934.GB16921@telsasoft.com
20221031223744.GT16921@telsasoft.com
20221104161934.GB16921@telsasoft.com
20221123054329.GG11463@telsasoft.com
20230224002029.GQ1653@telsasoft.com

ci-os-only: windows
---
 .cirrus.tasks.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 33646faeadf..5a2b64f64c2 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -566,12 +566,21 @@ task:
 
   build_script: |
     vcvarsall x64
-    ninja -C build
+    ninja -C build |tee build.txt
+    REM Since pipes lose the exit status of the preceding command, rerun the compilation
+    REM without the pipe, exiting now if it fails, to avoid trying to run checks
+    ninja -C build > nul
 
   check_world_script: |
     vcvarsall x64
     meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
 
+  # This should be last, so check_world is run even if there are warnings
+  always:
+    compiler_warnings_script:
+      # this avoids using metachars which would be interpretted by the windows shell
+      - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0'
+
   on_failure:
     <<: *on_failure_meson
     crashlog_artifacts:
-- 
2.42.0

From 78031c45afad402397e4efa1389e6396557cd405 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 25 May 2022 22:05:13 -0500
Subject: [PATCH 2/7] cirrus/windows: ccache

ccache 4.7 added support for depend mode with MSVC, and ccache v4.10
supports PCH, so this is now viable.

https://www.postgresql.org/message-id/flat/20220522232606.GZ19626%40telsasoft.com

https://cirrus-ci.com/task/5213936495099904 build 29sec
---
 .cirrus.tasks.yml | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 5a2b64f64c2..6ebbc1ccd81 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -536,6 +536,12 @@ task:
   env:
     TEST_JOBS: 8 # wild guess, data based value welcome
 
+    CC: c:\msys64\ucrt64\bin\ccache.exe cl.exe
+    CCACHE_DIR: $CIRRUS_WORKING_DIR/.ccache
+    CCACHE_MAXSIZE: "500M"
+    CCACHE_SLOPPINESS: pch_defines,time_macros
+    CCACHE_DEPEND: 1
+
     # Cirrus defaults to SetErrorMode(SEM_NOGPFAULTERRORBOX | ...). That
     # prevents crash reporting from working unless binaries do SetErrorMode()
     # themselves. Furthermore, it appears that either python or, more likely,
@@ -553,6 +559,9 @@ task:
   setup_additional_packages_script: |
     REM choco install -y --no-progress ...
 
+  ccache_cache:
+    folder: $CCACHE_DIR
+
   setup_hosts_file_script: |
     echo 127.0.0.1 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts
     echo 127.0.0.2 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts
@@ -562,7 +571,7 @@ task:
   # Use /DEBUG:FASTLINK to avoid high memory usage during linking
   configure_script: |
     vcvarsall x64
-    meson setup --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Dinjection_points=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" build
+    meson setup build --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Dinjection_points=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" -Dc_args="/Z7"
 
   build_script: |
     vcvarsall x64
@@ -571,6 +580,9 @@ task:
     REM without the pipe, exiting now if it fails, to avoid trying to run checks
     ninja -C build > nul
 
+  always:
+    upload_caches: [ccache]
+
   check_world_script: |
     vcvarsall x64
     meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
-- 
2.42.0

From cc09e8363d56933b8d62ebafb8319c91fed3b03f Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 25 Nov 2022 13:57:17 -0600
Subject: [PATCH 3/7] WIP: ci/meson: allow showing only failed tests ..

It's simpler and seems to make more sense to integrate this with
testwrap, rather than to run it after check-world, but only if it
failed, and finding a way to preserve the exit code.

https://www.postgresql.org/message-id/20221114235328.lxdj3puenfhirhqm@awork3.anarazel.de

> It is wasteful to upload thousdands of logfiles to show a single
> failure.  That would make our cirrus tasks faster - compressing and
> uploading the logs takes over a minute.
>
> It's also a lot friendlier to show fewer than 8 pages of test folders to
> search through to find the one that failed.

macos
linux-meson
ci-os-only: freebsd
---
 .cirrus.tasks.yml  | 14 ++++++++------
 src/tools/testwrap |  6 ++++++
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 6ebbc1ccd81..56061e61a3e 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -21,6 +21,7 @@ env:
   PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
   TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
   PG_TEST_EXTRA: kerberos ldap ssl libpq_encryption load_balance
+  PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
 
 
 # What files to preserve in case tests fail
@@ -35,9 +36,9 @@ on_failure_ac: &on_failure_ac
 on_failure_meson: &on_failure_meson
   testrun_artifacts:
     paths:
-      - "build*/testrun/**/*.log"
-      - "build*/testrun/**/*.diffs"
-      - "build*/testrun/**/regress_log_*"
+      - failed.build*/**/*.log
+      - failed.build*/**/*.diffs
+      - failed.build*/**/regress_log_*
     type: text/plain
 
   # In theory it'd be nice to upload the junit files meson generates, so that
@@ -136,6 +137,7 @@ task:
     CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST -DENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS
     CFLAGS: -Og -ggdb
 
+    PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
     PG_TEST_PG_UPGRADE_MODE: --link
 
   <<: *freebsd_task_template
@@ -194,10 +196,10 @@ task:
       ulimit -c unlimited
       meson test $MTEST_ARGS --quiet --suite setup
       export LD_LIBRARY_PATH="$(pwd)/build/tmp_install/usr/local/pgsql/lib/:$LD_LIBRARY_PATH"
-      mkdir -p build/testrun
+      mkdir -p build/testrun ${PG_FAILED_TESTDIR}
       build/tmp_install/usr/local/pgsql/bin/initdb -N build/runningcheck --no-instructions -A trust
       echo "include '$(pwd)/src/tools/ci/pg_ci_base.conf'" >> build/runningcheck/postgresql.conf
-      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
+      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l ${PG_FAILED_TESTDIR}/runningcheck.log start
       meson test $MTEST_ARGS --num-processes ${TEST_JOBS} --setup running
       build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
     EOF
@@ -400,7 +402,7 @@ task:
       test_world_32_script: |
         su postgres <<-EOF
           ulimit -c unlimited
-          PYTHONCOERCECLOCALE=0 LANG=C meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
+          PYTHONCOERCECLOCALE=0 LANG=C PG_FAILED_TESTDIR=`pwd`/failed.build-32 meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
         EOF
 
       on_failure:
diff --git a/src/tools/testwrap b/src/tools/testwrap
index d01e61051cb..7e5a17dbf94 100755
--- a/src/tools/testwrap
+++ b/src/tools/testwrap
@@ -49,4 +49,10 @@ if sp.returncode == 0:
 else:
     print('# test failed')
     open(os.path.join(testdir, 'test.fail'), 'x')
+    faileddir = os.getenv('PG_FAILED_TESTDIR')
+    if faileddir:
+        parentdir = os.path.dirname(testdir)
+        newdest = os.path.join(faileddir, os.path.basename(parentdir), os.path.basename(testdir))
+        shutil.copytree(testdir, newdest)
+
 sys.exit(sp.returncode)
-- 
2.42.0

From 3a399c6350ed2f341425431f184e382c3f46d981 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 26 Feb 2022 19:39:10 -0600
Subject: [PATCH 4/7] WIP: cirrus: upload changed html docs as artifacts

This could be done on the client side (cfbot).  One advantage of doing
it here is that fewer docs are uploaded - many patches won't upload docs
at all.

https://www.postgresql.org/message-id/flat/20220409021853.GP24419@telsasoft.com
https://www.postgresql.org/message-id/CAB8KJ=i4qmEuopQ+PCSMBzGd4O-Xv0FCnC+q1x7hN9hsdvkBug@mail.gmail.com

https://cirrus-ci.com/task/5396696388599808

//-os-only:
---
 .cirrus.tasks.yml              | 25 ++++++++++++++++++++++++-
 src/tools/ci/copy-changed-docs | 29 +++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 1 deletion(-)
 create mode 100755 src/tools/ci/copy-changed-docs

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 56061e61a3e..0ce5b172331 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -23,6 +23,15 @@ env:
   PG_TEST_EXTRA: kerberos ldap ssl libpq_encryption load_balance
   PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
 
+  # The commit that this branch is rebased on.  There's no easy way to find this.
+  # This does the right thing for cfbot, which always squishes all patches into a single commit.
+  # And does the right thing for any 1-patch commits.
+  # Patch series manually submitted to cirrus would benefit from setting this to the
+  # number of patches in the series (or directly to the commit the series was rebased on).
+  #BASE_COMMIT: HEAD~1
+  # For demo purposes:
+  BASE_COMMIT: HEAD~55
+
 
 # What files to preserve in case tests fail
 on_failure_ac: &on_failure_ac
@@ -769,7 +778,7 @@ task:
       time make -s -j${BUILD_JOBS} world-bin
 
   ###
-  # Verify docs can be built
+  # Verify docs can be built, and upload changed docs as artifacts
   ###
   # XXX: Only do this if there have been changes in doc/ since last build
   always:
@@ -781,6 +790,20 @@ task:
         CLANG="ccache clang"
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} -C doc
+      cp -r doc new-docs
+
+      # Re-build HTML docs from the base commit.
+      git checkout "$BASE_COMMIT" -- doc
+      make -s -C doc clean
+      time make -s -C doc html
+      cp -r doc old-docs
+
+    copy_changed_docs_script:
+      - src/tools/ci/copy-changed-docs "old-docs" "new-docs" "html_docs"
+
+    html_docs_artifacts:
+      paths: ['html_docs/*.html', 'html_docs/*.png', 'html_docs/*.css']
+
 
   ###
   # Verify headerscheck / cpluspluscheck succeed
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
new file mode 100755
index 00000000000..1c921a8df6f
--- /dev/null
+++ b/src/tools/ci/copy-changed-docs
@@ -0,0 +1,29 @@
+#! /bin/sh
+# Copy HTML which differ between $old and $new into $outdir
+set -e
+
+old=$1
+new=$2
+outdir=$3
+
+# The index is large and changes often
+skippages="bookindex.html"
+
+mkdir "$outdir"
+cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
+
+changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
+	[ $? -eq 1 ]
+
+for f in $changed
+do
+	# Avoid removed files
+	[ -f "$f" ] || continue
+
+	echo "$f" |grep -Ew "$skippages" >/dev/null &&
+		continue
+
+	cp -v "$f" "$outdir"
+done
+
+exit 0
-- 
2.42.0

From c0dd484060e18c4221e7ec883e621720eec230ed Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Mon, 28 Feb 2022 23:18:19 -0600
Subject: [PATCH 5/7] +html index file

This allows linking to the artifacts from the last successful build.

//freebsd
ci-os-only: warnings
---
 src/tools/ci/copy-changed-docs | 39 ++++++++++++++++++++++++++++++++--
 1 file changed, 37 insertions(+), 2 deletions(-)

diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
index 1c921a8df6f..0efad386cca 100755
--- a/src/tools/ci/copy-changed-docs
+++ b/src/tools/ci/copy-changed-docs
@@ -5,6 +5,7 @@ set -e
 old=$1
 new=$2
 outdir=$3
+branch=$CIRRUS_BRANCH
 
 # The index is large and changes often
 skippages="bookindex.html"
@@ -12,6 +13,19 @@ skippages="bookindex.html"
 mkdir "$outdir"
 cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
 
+# The index is useful to allow a static link (not specific to a cirrus run) to the artifacts for the most-recent, successful CI run for a branch
+branchurl=https://api.cirrus-ci.com/v1/artifact/github/$CIRRUS_REPO_FULL_NAME/$CIRRUS_TASK_NAME/html_docs/html_docs/00-index.html?branch=$branch
+
+index="$outdir/00-index.html"
+cat >"$index" <<EOF
+<html>
+<head><title>Index of docs changed since: $BASE_COMMIT</title></head>
+<body>
+<!-- A link to documentation for the most recent successful build: $branchurl -->
+<h1>Index of docs changed since: $BASE_COMMIT</h1>
+<ul>
+EOF
+
 changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
 	[ $? -eq 1 ]
 
@@ -23,7 +37,28 @@ do
 	echo "$f" |grep -Ew "$skippages" >/dev/null &&
 		continue
 
-	cp -v "$f" "$outdir"
-done
+	cp -v "$f" "$outdir" >&2
+	fn=${f##*/}
+	# ?branch=... is needed when accessing the artifacts for the static link for the branch
+	# It's not needed and ignored if accessing artifacts for *this* CI run
+	echo "<li><a href='$fn?branch=$branch'>$fn</a>"
+done >>"$index"
+
+github=https://github.com/$CIRRUS_REPO_FULL_NAME/commit
+cirrus=https://cirrus-ci.com/build
+
+cat >>"$index" <<EOF
+</ul>
+<hr>
+<code>
+<br>This file was written on: `date --rfc-822 --utc`
+<br>CIRRUS_CHANGE_TITLE: $CIRRUS_CHANGE_TITLE
+<br>CIRRUS_CHANGE_IN_REPO: <a href="$github/$CIRRUS_CHANGE_IN_REPO">$CIRRUS_CHANGE_IN_REPO</a>
+<br>CIRRUS_BUILD_ID: <a href="$cirrus/$CIRRUS_BUILD_ID">$CIRRUS_BUILD_ID</a>
+<br>CIRRUS_LAST_GREEN_CHANGE: <a href="$github/$CIRRUS_LAST_GREEN_CHANGE">$CIRRUS_LAST_GREEN_CHANGE</a>
+<br>CIRRUS_LAST_GREEN_BUILD_ID: <a href="$cirrus/$CIRRUS_LAST_GREEN_BUILD_ID">$CIRRUS_LAST_GREEN_BUILD_ID</a>
+</code>
+</body></html>
+EOF
 
 exit 0
-- 
2.42.0

From c68c8e6923328a9a2b0fb461a5ecc089e9d47450 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 19 Oct 2022 20:33:28 -0500
Subject: [PATCH 6/7] WIP: show changed docs with meson

//-os-only:
---
 .cirrus.tasks.yml              | 26 ++++++++++++++++++++++----
 src/tools/ci/copy-changed-docs |  3 ++-
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 0ce5b172331..c53408197d4 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -783,20 +783,38 @@ task:
   # XXX: Only do this if there have been changes in doc/ since last build
   always:
     docs_build_script: |
-      time ./configure \
+      mkdir build-autoconf
+      cd build-autoconf
+      time ../configure \
         --cache gcc.cache \
+        --without-icu \
         CC="ccache gcc" \
         CXX="ccache g++" \
         CLANG="ccache clang"
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} -C doc
-      cp -r doc new-docs
+      cp -r doc ../new-docs
 
       # Re-build HTML docs from the base commit.
-      git checkout "$BASE_COMMIT" -- doc
+      git checkout "$BASE_COMMIT" -- ../doc
       make -s -C doc clean
       time make -s -C doc html
-      cp -r doc old-docs
+      cp -r doc ../old-docs
+
+    # Exercise HTML and other docs:
+    ninja_docs_build_script: |
+      make maintainer-clean # XXX not needed once compiler-warnings is switched meson
+      mkdir build-ninja
+      cd build-ninja
+      time meson setup
+      time ninja docs
+      cp -r doc ../new-docs
+
+      # Re-build HTML docs from the base commit.
+      git checkout "$BASE_COMMIT" -- ../doc
+      ninja clean
+      time ninja doc/src/sgml/html
+      cp -r doc ../old-docs
 
     copy_changed_docs_script:
       - src/tools/ci/copy-changed-docs "old-docs" "new-docs" "html_docs"
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
index 0efad386cca..ee3135c298b 100755
--- a/src/tools/ci/copy-changed-docs
+++ b/src/tools/ci/copy-changed-docs
@@ -11,7 +11,8 @@ branch=$CIRRUS_BRANCH
 skippages="bookindex.html"
 
 mkdir "$outdir"
-cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
+cp doc/src/sgml/*.css "$outdir"
+cp doc/src/sgml/images/*.svg "$outdir"
 
 # The index is useful to allow a static link (not specific to a cirrus run) to the artifacts for the most-recent, successful CI run for a branch
 branchurl=https://api.cirrus-ci.com/v1/artifact/github/$CIRRUS_REPO_FULL_NAME/$CIRRUS_TASK_NAME/html_docs/html_docs/00-index.html?branch=$branch
-- 
2.42.0

From 0cf4fa490b19aea8e536506e975f1e9b61be6574 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 25 May 2022 21:53:22 -0500
Subject: [PATCH 1/6] cirrus/windows: add compiler_warnings_script

The goal is to fail due to warnings only after running tests.
(At least historically, it's too slow to run a separate windows VM to
compile with -Werror.)

https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de

I'm not sure how to write this test in windows shell; it's also easy to
write something that doesn't work in posix sh, since windows shell is
interpretting && and ||...

See also:
8a1ce5e54f6d144e4f8e19af7c767b026ee0c956
https://cirrus-ci.com/task/6241060062494720
https://cirrus-ci.com/task/6496366607204352

20221104161934.GB16921@telsasoft.com
20221031223744.GT16921@telsasoft.com
20221104161934.GB16921@telsasoft.com
20221123054329.GG11463@telsasoft.com
20230224002029.GQ1653@telsasoft.com

ci-os-only: windows
---
 .cirrus.tasks.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 1ce6c443a8c..5dd37e07aae 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -576,12 +576,21 @@ task:
 
   build_script: |
     vcvarsall x64
-    ninja -C build
+    ninja -C build |tee build.txt
+    REM Since pipes lose the exit status of the preceding command, rerun the compilation
+    REM without the pipe, exiting now if it fails, to avoid trying to run checks
+    ninja -C build > nul
 
   check_world_script: |
     vcvarsall x64
     meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
 
+  # This should be last, so check_world is run even if there are warnings
+  always:
+    compiler_warnings_script:
+      # this avoids using metachars which would be interpretted by the windows shell
+      - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0'
+
   on_failure:
     <<: *on_failure_meson
     crashlog_artifacts:
-- 
2.42.0

From 764987ff4093783e967c13f55dee60fb6d89b4a0 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 25 May 2022 22:05:13 -0500
Subject: [PATCH 2/6] cirrus/windows: ccache

ccache 4.7 added support for depend mode with MSVC, and ccache v4.10
supports PCH, so this is now viable.

https://www.postgresql.org/message-id/flat/20220522232606.GZ19626%40telsasoft.com

https://cirrus-ci.com/task/5213936495099904 build 29sec
---
 .cirrus.tasks.yml | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 5dd37e07aae..fb4ea3d666a 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -546,6 +546,12 @@ task:
   env:
     TEST_JOBS: 8 # wild guess, data based value welcome
 
+    CC: c:\msys64\ucrt64\bin\ccache.exe cl.exe
+    CCACHE_DIR: $CIRRUS_WORKING_DIR/.ccache
+    CCACHE_MAXSIZE: "500M"
+    CCACHE_SLOPPINESS: pch_defines,time_macros
+    CCACHE_DEPEND: 1
+
     # Cirrus defaults to SetErrorMode(SEM_NOGPFAULTERRORBOX | ...). That
     # prevents crash reporting from working unless binaries do SetErrorMode()
     # themselves. Furthermore, it appears that either python or, more likely,
@@ -563,6 +569,9 @@ task:
   setup_additional_packages_script: |
     REM choco install -y --no-progress ...
 
+  ccache_cache:
+    folder: $CCACHE_DIR
+
   setup_hosts_file_script: |
     echo 127.0.0.1 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts
     echo 127.0.0.2 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts
@@ -570,9 +579,10 @@ task:
     type c:\Windows\System32\Drivers\etc\hosts
 
   # Use /DEBUG:FASTLINK to avoid high memory usage during linking
+  # Use /Z7 to write separate debug files, to allow ccache to work
   configure_script: |
     vcvarsall x64
-    meson setup --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Dinjection_points=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" build
+    meson setup build --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Dinjection_points=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" -Dc_args="/Z7"
 
   build_script: |
     vcvarsall x64
@@ -581,6 +591,9 @@ task:
     REM without the pipe, exiting now if it fails, to avoid trying to run checks
     ninja -C build > nul
 
+  always:
+    upload_caches: [ccache]
+
   check_world_script: |
     vcvarsall x64
     meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
-- 
2.42.0

From 75c304b2282d4252c7ad41c95bf8ee77b08f53e4 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Fri, 25 Nov 2022 13:57:17 -0600
Subject: [PATCH 3/6] WIP: ci/meson: allow showing only failed tests ..

It's simpler and seems to make more sense to integrate this with
testwrap, rather than to run it after check-world, but only if it
failed, and finding a way to preserve the exit code.

https://www.postgresql.org/message-id/20221114235328.lxdj3puenfhirhqm@awork3.anarazel.de

> It is wasteful to upload thousdands of logfiles to show a single
> failure.  That would make our cirrus tasks faster - compressing and
> uploading the logs takes over a minute.
>
> It's also a lot friendlier to show fewer than 8 pages of test folders to
> search through to find the one that failed.

macos
linux-meson
ci-os-only: freebsd
---
 .cirrus.tasks.yml  | 14 ++++++++------
 src/tools/testwrap |  6 ++++++
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index fb4ea3d666a..8ed7f7e5dd7 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -21,6 +21,7 @@ env:
   PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
   TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
   PG_TEST_EXTRA: kerberos ldap ssl libpq_encryption load_balance
+  PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
 
 
 # What files to preserve in case tests fail
@@ -35,9 +36,9 @@ on_failure_ac: &on_failure_ac
 on_failure_meson: &on_failure_meson
   testrun_artifacts:
     paths:
-      - "build*/testrun/**/*.log"
-      - "build*/testrun/**/*.diffs"
-      - "build*/testrun/**/regress_log_*"
+      - failed.build*/**/*.log
+      - failed.build*/**/*.diffs
+      - failed.build*/**/regress_log_*
     type: text/plain
 
   # In theory it'd be nice to upload the junit files meson generates, so that
@@ -138,6 +139,7 @@ task:
 
     PG_TEST_INITDB_EXTRA_OPTS: -c debug_copy_parse_plan_trees=on -c debug_write_read_parse_plan_trees=on -c debug_raw_expression_coverage_test=on
     PG_TEST_PG_UPGRADE_MODE: --link
+    PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
 
   <<: *freebsd_task_template
 
@@ -195,10 +197,10 @@ task:
       ulimit -c unlimited
       meson test $MTEST_ARGS --quiet --suite setup
       export LD_LIBRARY_PATH="$(pwd)/build/tmp_install/usr/local/pgsql/lib/:$LD_LIBRARY_PATH"
-      mkdir -p build/testrun
+      mkdir -p build/testrun ${PG_FAILED_TESTDIR}
       build/tmp_install/usr/local/pgsql/bin/initdb -N build/runningcheck --no-instructions -A trust
       echo "include '$(pwd)/src/tools/ci/pg_ci_base.conf'" >> build/runningcheck/postgresql.conf
-      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
+      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l ${PG_FAILED_TESTDIR}/runningcheck.log start
       meson test $MTEST_ARGS --num-processes ${TEST_JOBS} --setup running
       build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
     EOF
@@ -403,7 +405,7 @@ task:
       test_world_32_script: |
         su postgres <<-EOF
           ulimit -c unlimited
-          PYTHONCOERCECLOCALE=0 LANG=C meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
+          PYTHONCOERCECLOCALE=0 LANG=C PG_FAILED_TESTDIR=`pwd`/failed.build-32 meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
         EOF
 
       on_failure:
diff --git a/src/tools/testwrap b/src/tools/testwrap
index 9a270beb72d..0d0121e62da 100755
--- a/src/tools/testwrap
+++ b/src/tools/testwrap
@@ -59,4 +59,10 @@ if returncode == 0:
 else:
     print('# test failed')
     open(os.path.join(testdir, 'test.fail'), 'x')
+    faileddir = os.getenv('PG_FAILED_TESTDIR')
+    if faileddir:
+        parentdir = os.path.dirname(testdir)
+        newdest = os.path.join(faileddir, os.path.basename(parentdir), os.path.basename(testdir))
+        shutil.copytree(testdir, newdest)
+
 sys.exit(returncode)
-- 
2.42.0

From e028901375221cc6bcdc102132fcf884fdf51fae Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 26 Feb 2022 19:39:10 -0600
Subject: [PATCH 4/6] WIP: cirrus: upload changed html docs as artifacts

This could be done on the client side (cfbot).  One advantage of doing
it here is that fewer docs are uploaded - many patches won't upload docs
at all.

https://www.postgresql.org/message-id/flat/20220409021853.GP24419@telsasoft.com
https://www.postgresql.org/message-id/CAB8KJ=i4qmEuopQ+PCSMBzGd4O-Xv0FCnC+q1x7hN9hsdvkBug@mail.gmail.com

https://cirrus-ci.com/task/5396696388599808

//-os-only:
---
 .cirrus.tasks.yml              | 25 ++++++++++++++++++++++++-
 src/tools/ci/copy-changed-docs | 29 +++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 1 deletion(-)
 create mode 100755 src/tools/ci/copy-changed-docs

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 8ed7f7e5dd7..8b07fb2e538 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -23,6 +23,15 @@ env:
   PG_TEST_EXTRA: kerberos ldap ssl libpq_encryption load_balance
   PG_FAILED_TESTDIR: ${CIRRUS_WORKING_DIR}/failed.build
 
+  # The commit that this branch is rebased on.  There's no easy way to find this.
+  # This does the right thing for cfbot, which always squishes all patches into a single commit.
+  # And does the right thing for any 1-patch commits.
+  # Patch series manually submitted to cirrus would benefit from setting this to the
+  # number of patches in the series (or directly to the commit the series was rebased on).
+  #BASE_COMMIT: HEAD~1
+  # For demo purposes:
+  BASE_COMMIT: HEAD~55
+
 
 # What files to preserve in case tests fail
 on_failure_ac: &on_failure_ac
@@ -783,7 +792,7 @@ task:
       time make -s -j${BUILD_JOBS} world-bin
 
   ###
-  # Verify docs can be built
+  # Verify docs can be built, and upload changed docs as artifacts
   ###
   # XXX: Only do this if there have been changes in doc/ since last build
   always:
@@ -793,6 +802,20 @@ task:
         CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang-16"
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} -C doc
+      cp -r doc new-docs
+
+      # Re-build HTML docs from the base commit.
+      git checkout "$BASE_COMMIT" -- doc
+      make -s -C doc clean
+      time make -s -C doc html
+      cp -r doc old-docs
+
+    copy_changed_docs_script:
+      - src/tools/ci/copy-changed-docs "old-docs" "new-docs" "html_docs"
+
+    html_docs_artifacts:
+      paths: ['html_docs/*.html', 'html_docs/*.png', 'html_docs/*.css']
+
 
   ###
   # Verify headerscheck / cpluspluscheck succeed
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
new file mode 100755
index 00000000000..1c921a8df6f
--- /dev/null
+++ b/src/tools/ci/copy-changed-docs
@@ -0,0 +1,29 @@
+#! /bin/sh
+# Copy HTML which differ between $old and $new into $outdir
+set -e
+
+old=$1
+new=$2
+outdir=$3
+
+# The index is large and changes often
+skippages="bookindex.html"
+
+mkdir "$outdir"
+cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
+
+changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
+	[ $? -eq 1 ]
+
+for f in $changed
+do
+	# Avoid removed files
+	[ -f "$f" ] || continue
+
+	echo "$f" |grep -Ew "$skippages" >/dev/null &&
+		continue
+
+	cp -v "$f" "$outdir"
+done
+
+exit 0
-- 
2.42.0

From c5466195f82ad000abb68fa12b7269f5e53c4465 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Mon, 28 Feb 2022 23:18:19 -0600
Subject: [PATCH 5/6] +html index file

This allows linking to the artifacts from the last successful build.

//freebsd
ci-os-only: warnings
---
 src/tools/ci/copy-changed-docs | 39 ++++++++++++++++++++++++++++++++--
 1 file changed, 37 insertions(+), 2 deletions(-)

diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
index 1c921a8df6f..0efad386cca 100755
--- a/src/tools/ci/copy-changed-docs
+++ b/src/tools/ci/copy-changed-docs
@@ -5,6 +5,7 @@ set -e
 old=$1
 new=$2
 outdir=$3
+branch=$CIRRUS_BRANCH
 
 # The index is large and changes often
 skippages="bookindex.html"
@@ -12,6 +13,19 @@ skippages="bookindex.html"
 mkdir "$outdir"
 cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
 
+# The index is useful to allow a static link (not specific to a cirrus run) to the artifacts for the most-recent, successful CI run for a branch
+branchurl=https://api.cirrus-ci.com/v1/artifact/github/$CIRRUS_REPO_FULL_NAME/$CIRRUS_TASK_NAME/html_docs/html_docs/00-index.html?branch=$branch
+
+index="$outdir/00-index.html"
+cat >"$index" <<EOF
+<html>
+<head><title>Index of docs changed since: $BASE_COMMIT</title></head>
+<body>
+<!-- A link to documentation for the most recent successful build: $branchurl -->
+<h1>Index of docs changed since: $BASE_COMMIT</h1>
+<ul>
+EOF
+
 changed=`git diff --no-index --name-only "$old"/src/sgml/html "$new"/src/sgml/html` ||
 	[ $? -eq 1 ]
 
@@ -23,7 +37,28 @@ do
 	echo "$f" |grep -Ew "$skippages" >/dev/null &&
 		continue
 
-	cp -v "$f" "$outdir"
-done
+	cp -v "$f" "$outdir" >&2
+	fn=${f##*/}
+	# ?branch=... is needed when accessing the artifacts for the static link for the branch
+	# It's not needed and ignored if accessing artifacts for *this* CI run
+	echo "<li><a href='$fn?branch=$branch'>$fn</a>"
+done >>"$index"
+
+github=https://github.com/$CIRRUS_REPO_FULL_NAME/commit
+cirrus=https://cirrus-ci.com/build
+
+cat >>"$index" <<EOF
+</ul>
+<hr>
+<code>
+<br>This file was written on: `date --rfc-822 --utc`
+<br>CIRRUS_CHANGE_TITLE: $CIRRUS_CHANGE_TITLE
+<br>CIRRUS_CHANGE_IN_REPO: <a href="$github/$CIRRUS_CHANGE_IN_REPO">$CIRRUS_CHANGE_IN_REPO</a>
+<br>CIRRUS_BUILD_ID: <a href="$cirrus/$CIRRUS_BUILD_ID">$CIRRUS_BUILD_ID</a>
+<br>CIRRUS_LAST_GREEN_CHANGE: <a href="$github/$CIRRUS_LAST_GREEN_CHANGE">$CIRRUS_LAST_GREEN_CHANGE</a>
+<br>CIRRUS_LAST_GREEN_BUILD_ID: <a href="$cirrus/$CIRRUS_LAST_GREEN_BUILD_ID">$CIRRUS_LAST_GREEN_BUILD_ID</a>
+</code>
+</body></html>
+EOF
 
 exit 0
-- 
2.42.0

From 7112295112dff33e541ea680084ca97aa01ec7f4 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Wed, 19 Oct 2022 20:33:28 -0500
Subject: [PATCH 6/6] WIP: show changed docs with meson

//-os-only:
---
 .cirrus.tasks.yml              | 26 ++++++++++++++++++++++----
 src/tools/ci/copy-changed-docs |  3 ++-
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 8b07fb2e538..aea79f29d7e 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -797,18 +797,36 @@ task:
   # XXX: Only do this if there have been changes in doc/ since last build
   always:
     docs_build_script: |
-      time ./configure \
+      mkdir build-autoconf
+      cd build-autoconf
+      time ../configure \
         --cache gcc.cache \
         CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang-16"
+        --without-icu \
       make -s -j${BUILD_JOBS} clean
       time make -s -j${BUILD_JOBS} -C doc
-      cp -r doc new-docs
+      cp -r doc ../new-docs
 
       # Re-build HTML docs from the base commit.
-      git checkout "$BASE_COMMIT" -- doc
+      git checkout "$BASE_COMMIT" -- ../doc
       make -s -C doc clean
       time make -s -C doc html
-      cp -r doc old-docs
+      cp -r doc ../old-docs
+
+    # Exercise HTML and other docs:
+    ninja_docs_build_script: |
+      make maintainer-clean # XXX not needed once compiler-warnings is switched meson
+      mkdir build-ninja
+      cd build-ninja
+      time meson setup
+      time ninja docs
+      cp -r doc ../new-docs
+
+      # Re-build HTML docs from the base commit.
+      git checkout "$BASE_COMMIT" -- ../doc
+      ninja clean
+      time ninja doc/src/sgml/html
+      cp -r doc ../old-docs
 
     copy_changed_docs_script:
       - src/tools/ci/copy-changed-docs "old-docs" "new-docs" "html_docs"
diff --git a/src/tools/ci/copy-changed-docs b/src/tools/ci/copy-changed-docs
index 0efad386cca..ee3135c298b 100755
--- a/src/tools/ci/copy-changed-docs
+++ b/src/tools/ci/copy-changed-docs
@@ -11,7 +11,8 @@ branch=$CIRRUS_BRANCH
 skippages="bookindex.html"
 
 mkdir "$outdir"
-cp "$new"/src/sgml/html/*.css "$new"/src/sgml/html/*.svg "$outdir"
+cp doc/src/sgml/*.css "$outdir"
+cp doc/src/sgml/images/*.svg "$outdir"
 
 # The index is useful to allow a static link (not specific to a cirrus run) to the artifacts for the most-recent, successful CI run for a branch
 branchurl=https://api.cirrus-ci.com/v1/artifact/github/$CIRRUS_REPO_FULL_NAME/$CIRRUS_TASK_NAME/html_docs/html_docs/00-index.html?branch=$branch
-- 
2.42.0