slowest tap tests - split or accelerate?

From 4abed85e9085786f82d87667f2451821c01d37c0 Mon Sep 17 00:00:00 2001
From: Andres Freund <andres@anarazel.de>
Date: Mon, 17 Jan 2022 00:51:43 -0800
Subject: [PATCH v1 1/4] tests: Consistently use pg_basebackup -cfast --no-sync
 to accelerate tests.

Most tests invoking pg_basebackup themselves did not yet use -cfast, which
makes pg_basebackup take considerably longer. The only reason this didn't
cause the tests to take many minutes is that spread checkpoints only throttle
when writing out a buffer and there aren't that many dirty buffers...

Author:
Reviewed-By:
Discussion: https://postgr.es/m/
Backpatch:
---
 src/bin/pg_basebackup/t/010_pg_basebackup.pl | 104 ++++++++++---------
 src/bin/pg_verifybackup/t/002_algorithm.pl   |   2 +-
 src/bin/pg_verifybackup/t/003_corruption.pl  |   2 +-
 src/bin/pg_verifybackup/t/004_options.pl     |   2 +-
 src/bin/pg_verifybackup/t/006_encoding.pl    |   2 +-
 src/bin/pg_verifybackup/t/007_wal.pl         |   4 +-
 6 files changed, 61 insertions(+), 55 deletions(-)

diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
index 872fec3d350..ebd102cd098 100644
--- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl
+++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
@@ -20,6 +20,13 @@ my $tempdir = PostgreSQL::Test::Utils::tempdir;
 
 my $node = PostgreSQL::Test::Cluster->new('main');
 
+# For nearly all tests some options should be specified, to keep test times
+# reasonable. Using @pg_basebackup_defs as the first element of the array
+# passed to to IPC::Run interpolates the array (as it is not a reference to an
+# array)..
+my @pg_basebackup_defs = ('pg_basebackup', '--no-sync', '-cfast');
+
+
 # Set umask so test directories and files are created with default permissions
 umask(0077);
 
@@ -43,7 +50,7 @@ $node->set_replication_conf();
 system_or_bail 'pg_ctl', '-D', $pgdata, 'reload';
 
 $node->command_fails(
-	[ 'pg_basebackup', '-D', "$tempdir/backup" ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/backup" ],
 	'pg_basebackup fails because of WAL configuration');
 
 ok(!-d "$tempdir/backup", 'backup directory was cleaned up');
@@ -54,7 +61,7 @@ mkdir("$tempdir/backup")
   or BAIL_OUT("unable to create $tempdir/backup");
 append_to_file("$tempdir/backup/dir-not-empty.txt", "Some data");
 
-$node->command_fails([ 'pg_basebackup', '-D', "$tempdir/backup", '-n' ],
+$node->command_fails([ @pg_basebackup_defs, '-D', "$tempdir/backup", '-n' ],
 	'failing run with no-clean option');
 
 ok(-d "$tempdir/backup", 'backup directory was created and left behind');
@@ -105,7 +112,7 @@ foreach my $filename (@tempRelationFiles)
 }
 
 # Run base backup.
-$node->command_ok([ 'pg_basebackup', '-D', "$tempdir/backup", '-X', 'none' ],
+$node->command_ok([ @pg_basebackup_defs, '-D', "$tempdir/backup", '-X', 'none' ],
 	'pg_basebackup runs');
 ok(-f "$tempdir/backup/PG_VERSION",      'backup was created');
 ok(-f "$tempdir/backup/backup_manifest", 'backup manifest included');
@@ -165,9 +172,9 @@ rmtree("$tempdir/backup");
 
 $node->command_ok(
 	[
-		'pg_basebackup',    '-D',
-		"$tempdir/backup2", '--no-manifest',
-		'--waldir',         "$tempdir/xlog2"
+		@pg_basebackup_defs, '-D',
+		"$tempdir/backup2",  '--no-manifest',
+		'--waldir',          "$tempdir/xlog2"
 	],
 	'separate xlog directory');
 ok(-f "$tempdir/backup2/PG_VERSION",       'backup was created');
@@ -176,31 +183,31 @@ ok(-d "$tempdir/xlog2/",                   'xlog directory was created');
 rmtree("$tempdir/backup2");
 rmtree("$tempdir/xlog2");
 
-$node->command_ok([ 'pg_basebackup', '-D', "$tempdir/tarbackup", '-Ft' ],
+$node->command_ok([ @pg_basebackup_defs, '-D', "$tempdir/tarbackup", '-Ft' ],
 	'tar format');
 ok(-f "$tempdir/tarbackup/base.tar", 'backup tar was created');
 rmtree("$tempdir/tarbackup");
 
 $node->command_fails(
-	[ 'pg_basebackup', '-D', "$tempdir/backup_foo", '-Fp', "-T=/foo" ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/backup_foo", '-Fp', "-T=/foo" ],
 	'-T with empty old directory fails');
 $node->command_fails(
-	[ 'pg_basebackup', '-D', "$tempdir/backup_foo", '-Fp', "-T/foo=" ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/backup_foo", '-Fp', "-T/foo=" ],
 	'-T with empty new directory fails');
 $node->command_fails(
 	[
-		'pg_basebackup', '-D', "$tempdir/backup_foo", '-Fp',
+		@pg_basebackup_defs, '-D', "$tempdir/backup_foo", '-Fp',
 		"-T/foo=/bar=/baz"
 	],
 	'-T with multiple = fails');
 $node->command_fails(
-	[ 'pg_basebackup', '-D', "$tempdir/backup_foo", '-Fp', "-Tfoo=/bar" ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/backup_foo", '-Fp', "-Tfoo=/bar" ],
 	'-T with old directory not absolute fails');
 $node->command_fails(
-	[ 'pg_basebackup', '-D', "$tempdir/backup_foo", '-Fp', "-T/foo=bar" ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/backup_foo", '-Fp', "-T/foo=bar" ],
 	'-T with new directory not absolute fails');
 $node->command_fails(
-	[ 'pg_basebackup', '-D', "$tempdir/backup_foo", '-Fp', "-Tfoo" ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/backup_foo", '-Fp', "-Tfoo" ],
 	'-T with invalid format fails');
 
 # Tar format doesn't support filenames longer than 100 bytes.
@@ -211,7 +218,7 @@ open my $file, '>', "$superlongpath"
   or die "unable to create file $superlongpath";
 close $file;
 $node->command_fails(
-	[ 'pg_basebackup', '-D', "$tempdir/tarbackup_l1", '-Ft' ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/tarbackup_l1", '-Ft' ],
 	'pg_basebackup tar with long name fails');
 unlink "$pgdata/$superlongname";
 
@@ -329,14 +336,14 @@ foreach my $filename (@tempRelationFiles)
 }
 
 $node->command_fails(
-	[ 'pg_basebackup', '-D', "$tempdir/backup1", '-Fp' ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/backup1", '-Fp' ],
 	'plain format with tablespaces fails without tablespace mapping');
 
 $node->command_ok(
 	[
-		'pg_basebackup',    '-D',
-		"$tempdir/backup1", '-Fp',
-		"-T$realTsDir=$real_tempdir/tbackup/tblspc1"
+		@pg_basebackup_defs, '-D',
+		"$tempdir/backup1",  '-Fp',
+		"-T$realTsDir=$real_tempdir/tbackup/tblspc1",
 	],
 	'plain format with tablespaces succeeds with tablespace mapping');
 ok(-d "$tempdir/tbackup/tblspc1", 'tablespace was relocated');
@@ -404,9 +411,9 @@ $node->safe_psql('postgres',
 $realTsDir =~ s/=/\\=/;
 $node->command_ok(
 	[
-		'pg_basebackup',    '-D',
-		"$tempdir/backup3", '-Fp',
-		"-T$realTsDir=$real_tempdir/tbackup/tbl\\=spc2"
+		@pg_basebackup_defs, '-D',
+		"$tempdir/backup3",  '-Fp',
+		"-T$realTsDir=$real_tempdir/tbackup/tbl\\=spc2",
 	],
 	'mapping tablespace with = sign in path');
 ok(-d "$tempdir/tbackup/tbl=spc2", 'tablespace with = sign was relocated');
@@ -417,12 +424,12 @@ mkdir "$tempdir/$superlongname";
 $realTsDir = "$real_sys_tempdir/$superlongname";
 $node->safe_psql('postgres',
 	"CREATE TABLESPACE tblspc3 LOCATION '$realTsDir';");
-$node->command_ok([ 'pg_basebackup', '-D', "$tempdir/tarbackup_l3", '-Ft' ],
+$node->command_ok([ @pg_basebackup_defs, '-D', "$tempdir/tarbackup_l3", '-Ft' ],
 	'pg_basebackup tar with long symlink target');
 $node->safe_psql('postgres', "DROP TABLESPACE tblspc3;");
 rmtree("$tempdir/tarbackup_l3");
 
-$node->command_ok([ 'pg_basebackup', '-D', "$tempdir/backupR", '-R' ],
+$node->command_ok([ @pg_basebackup_defs, '-D', "$tempdir/backupR", '-R' ],
 	'pg_basebackup -R runs');
 ok(-f "$tempdir/backupR/postgresql.auto.conf", 'postgresql.auto.conf exists');
 ok(-f "$tempdir/backupR/standby.signal",       'standby.signal was created');
@@ -436,32 +443,32 @@ like(
 	'postgresql.auto.conf sets primary_conninfo');
 
 $node->command_ok(
-	[ 'pg_basebackup', '-D', "$tempdir/backupxd" ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/backupxd" ],
 	'pg_basebackup runs in default xlog mode');
 ok(grep(/^[0-9A-F]{24}$/, slurp_dir("$tempdir/backupxd/pg_wal")),
 	'WAL files copied');
 rmtree("$tempdir/backupxd");
 
 $node->command_ok(
-	[ 'pg_basebackup', '-D', "$tempdir/backupxf", '-X', 'fetch' ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/backupxf", '-X', 'fetch' ],
 	'pg_basebackup -X fetch runs');
 ok(grep(/^[0-9A-F]{24}$/, slurp_dir("$tempdir/backupxf/pg_wal")),
 	'WAL files copied');
 rmtree("$tempdir/backupxf");
 $node->command_ok(
-	[ 'pg_basebackup', '-D', "$tempdir/backupxs", '-X', 'stream' ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/backupxs", '-X', 'stream' ],
 	'pg_basebackup -X stream runs');
 ok(grep(/^[0-9A-F]{24}$/, slurp_dir("$tempdir/backupxs/pg_wal")),
 	'WAL files copied');
 rmtree("$tempdir/backupxs");
 $node->command_ok(
-	[ 'pg_basebackup', '-D', "$tempdir/backupxst", '-X', 'stream', '-Ft' ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/backupxst", '-X', 'stream', '-Ft' ],
 	'pg_basebackup -X stream runs in tar mode');
 ok(-f "$tempdir/backupxst/pg_wal.tar", "tar file was created");
 rmtree("$tempdir/backupxst");
 $node->command_ok(
 	[
-		'pg_basebackup',         '-D',
+		@pg_basebackup_defs,     '-D',
 		"$tempdir/backupnoslot", '-X',
 		'stream',                '--no-slot'
 	],
@@ -470,7 +477,7 @@ rmtree("$tempdir/backupnoslot");
 
 $node->command_fails(
 	[
-		'pg_basebackup',             '-D',
+		@pg_basebackup_defs,         '-D',
 		"$tempdir/backupxs_sl_fail", '-X',
 		'stream',                    '-S',
 		'slot0'
@@ -478,12 +485,12 @@ $node->command_fails(
 	'pg_basebackup fails with nonexistent replication slot');
 
 $node->command_fails(
-	[ 'pg_basebackup', '-D', "$tempdir/backupxs_slot", '-C' ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/backupxs_slot", '-C' ],
 	'pg_basebackup -C fails without slot name');
 
 $node->command_fails(
 	[
-		'pg_basebackup',          '-D',
+		@pg_basebackup_defs,      '-D',
 		"$tempdir/backupxs_slot", '-C',
 		'-S',                     'slot0',
 		'--no-slot'
@@ -491,7 +498,7 @@ $node->command_fails(
 	'pg_basebackup fails with -C -S --no-slot');
 
 $node->command_ok(
-	[ 'pg_basebackup', '-D', "$tempdir/backupxs_slot", '-C', '-S', 'slot0' ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/backupxs_slot", '-C', '-S', 'slot0' ],
 	'pg_basebackup -C runs');
 rmtree("$tempdir/backupxs_slot");
 
@@ -510,7 +517,7 @@ isnt(
 	'restart LSN of new slot is not null');
 
 $node->command_fails(
-	[ 'pg_basebackup', '-D', "$tempdir/backupxs_slot1", '-C', '-S', 'slot0' ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/backupxs_slot1", '-C', '-S', 'slot0' ],
 	'pg_basebackup fails with -C -S and a previously existing slot');
 
 $node->safe_psql('postgres',
@@ -520,12 +527,12 @@ my $lsn = $node->safe_psql('postgres',
 );
 is($lsn, '', 'restart LSN of new slot is null');
 $node->command_fails(
-	[ 'pg_basebackup', '-D', "$tempdir/fail", '-S', 'slot1', '-X', 'none' ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/fail", '-S', 'slot1', '-X', 'none' ],
 	'pg_basebackup with replication slot fails without WAL streaming');
 $node->command_ok(
 	[
-		'pg_basebackup', '-D', "$tempdir/backupxs_sl", '-X',
-		'stream',        '-S', 'slot1'
+		@pg_basebackup_defs, '-D', "$tempdir/backupxs_sl", '-X',
+		'stream',            '-S', 'slot1'
 	],
 	'pg_basebackup -X stream with replication slot runs');
 $lsn = $node->safe_psql('postgres',
@@ -536,8 +543,8 @@ rmtree("$tempdir/backupxs_sl");
 
 $node->command_ok(
 	[
-		'pg_basebackup', '-D', "$tempdir/backupxs_sl_R", '-X',
-		'stream',        '-S', 'slot1',                  '-R'
+		@pg_basebackup_defs, '-D', "$tempdir/backupxs_sl_R", '-X',
+		'stream',            '-S', 'slot1',                  '-R',
 	],
 	'pg_basebackup with replication slot and -R runs');
 like(
@@ -570,7 +577,7 @@ close $file;
 system_or_bail 'pg_ctl', '-D', $pgdata, 'start';
 
 $node->command_checks_all(
-	[ 'pg_basebackup', '-D', "$tempdir/backup_corrupt" ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/backup_corrupt" ],
 	1,
 	[qr{^$}],
 	[qr/^WARNING.*checksum verification failed/s],
@@ -590,7 +597,7 @@ close $file;
 system_or_bail 'pg_ctl', '-D', $pgdata, 'start';
 
 $node->command_checks_all(
-	[ 'pg_basebackup', '-D', "$tempdir/backup_corrupt2" ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/backup_corrupt2" ],
 	1,
 	[qr{^$}],
 	[qr/^WARNING.*further.*failures.*will.not.be.reported/s],
@@ -606,7 +613,7 @@ close $file;
 system_or_bail 'pg_ctl', '-D', $pgdata, 'start';
 
 $node->command_checks_all(
-	[ 'pg_basebackup', '-D', "$tempdir/backup_corrupt3" ],
+	[ @pg_basebackup_defs, '-D', "$tempdir/backup_corrupt3" ],
 	1,
 	[qr{^$}],
 	[qr/^WARNING.*7 total checksum verification failures/s],
@@ -616,8 +623,8 @@ rmtree("$tempdir/backup_corrupt3");
 # do not verify checksums, should return ok
 $node->command_ok(
 	[
-		'pg_basebackup',            '-D',
-		"$tempdir/backup_corrupt4", '--no-verify-checksums'
+		@pg_basebackup_defs,        '-D',
+		"$tempdir/backup_corrupt4", '--no-verify-checksums',
 	],
 	'pg_basebackup with -k does not report checksum mismatch');
 rmtree("$tempdir/backup_corrupt4");
@@ -635,18 +642,17 @@ SKIP:
 
 	$node->command_ok(
 		[
-			'pg_basebackup',        '-D',
+			@pg_basebackup_defs,    '-D',
 			"$tempdir/backup_gzip", '--compress',
-			'1',                    '--no-sync',
-			'--format',             't'
+			'1',                    '--format',
+			't'
 		],
 		'pg_basebackup with --compress');
 	$node->command_ok(
 		[
-			'pg_basebackup',         '-D',
+			@pg_basebackup_defs,     '-D',
 			"$tempdir/backup_gzip2", '--gzip',
-			'--no-sync',             '--format',
-			't'
+			'--format',              't'
 		],
 		'pg_basebackup with --gzip');
 
diff --git a/src/bin/pg_verifybackup/t/002_algorithm.pl b/src/bin/pg_verifybackup/t/002_algorithm.pl
index 06333c6c0d5..5d9965ba8f4 100644
--- a/src/bin/pg_verifybackup/t/002_algorithm.pl
+++ b/src/bin/pg_verifybackup/t/002_algorithm.pl
@@ -21,7 +21,7 @@ for my $algorithm (qw(bogus none crc32c sha224 sha256 sha384 sha512))
 	my $backup_path = $primary->backup_dir . '/' . $algorithm;
 	my @backup      = (
 		'pg_basebackup', '-D', $backup_path,
-		'--manifest-checksums', $algorithm, '--no-sync');
+		'--manifest-checksums', $algorithm, '--no-sync', '-cfast');
 	my @verify = ('pg_verifybackup', '-e', $backup_path);
 
 	# A backup with a bogus algorithm should fail.
diff --git a/src/bin/pg_verifybackup/t/003_corruption.pl b/src/bin/pg_verifybackup/t/003_corruption.pl
index 63cad6647b5..53be2efd87d 100644
--- a/src/bin/pg_verifybackup/t/003_corruption.pl
+++ b/src/bin/pg_verifybackup/t/003_corruption.pl
@@ -114,7 +114,7 @@ for my $scenario (@scenario)
 		local $ENV{MSYS2_ARG_CONV_EXCL} = $source_ts_prefix;
 		$primary->command_ok(
 			[
-				'pg_basebackup', '-D', $backup_path, '--no-sync',
+				'pg_basebackup', '-D', $backup_path, '--no-sync', '-cfast',
 				'-T', "${source_ts_path}=${backup_ts_path}"
 			],
 			"base backup ok");
diff --git a/src/bin/pg_verifybackup/t/004_options.pl b/src/bin/pg_verifybackup/t/004_options.pl
index 3ec4651b680..5a8fd9b0ecb 100644
--- a/src/bin/pg_verifybackup/t/004_options.pl
+++ b/src/bin/pg_verifybackup/t/004_options.pl
@@ -17,7 +17,7 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 my $backup_path = $primary->backup_dir . '/test_options';
-$primary->command_ok([ 'pg_basebackup', '-D', $backup_path, '--no-sync' ],
+$primary->command_ok([ 'pg_basebackup', '-D', $backup_path, '--no-sync', '-cfast' ],
 	"base backup ok");
 
 # Verify that pg_verifybackup -q succeeds and produces no output.
diff --git a/src/bin/pg_verifybackup/t/006_encoding.pl b/src/bin/pg_verifybackup/t/006_encoding.pl
index e746ae6dcc4..b869cd8fe6e 100644
--- a/src/bin/pg_verifybackup/t/006_encoding.pl
+++ b/src/bin/pg_verifybackup/t/006_encoding.pl
@@ -19,7 +19,7 @@ $primary->command_ok(
 	[
 		'pg_basebackup', '-D',
 		$backup_path,    '--no-sync',
-		'--manifest-force-encode'
+		'-cfast',        '--manifest-force-encode'
 	],
 	"backup ok with forced hex encoding");
 
diff --git a/src/bin/pg_verifybackup/t/007_wal.pl b/src/bin/pg_verifybackup/t/007_wal.pl
index b9282da7078..4723047d73d 100644
--- a/src/bin/pg_verifybackup/t/007_wal.pl
+++ b/src/bin/pg_verifybackup/t/007_wal.pl
@@ -17,7 +17,7 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 my $backup_path = $primary->backup_dir . '/test_wal';
-$primary->command_ok([ 'pg_basebackup', '-D', $backup_path, '--no-sync' ],
+$primary->command_ok([ 'pg_basebackup', '-D', $backup_path, '--no-sync', '-cfast' ],
 	"base backup ok");
 
 # Rename pg_wal.
@@ -71,7 +71,7 @@ $primary->safe_psql('postgres', 'SELECT pg_switch_wal()');
 my $backup_path2 = $primary->backup_dir . '/test_tli';
 # The base backup run below does a checkpoint, that removes the first segment
 # of the current timeline.
-$primary->command_ok([ 'pg_basebackup', '-D', $backup_path2, '--no-sync' ],
+$primary->command_ok([ 'pg_basebackup', '-D', $backup_path2, '--no-sync', '-cfast' ],
 	"base backup 2 ok");
 command_ok(
 	[ 'pg_verifybackup', $backup_path2 ],
-- 
2.34.0

From aeaa4ba4629d1b48bbf13fd7512a9f1171631d70 Mon Sep 17 00:00:00 2001
From: Robert Haas <rhaas@postgresql.org>
Date: Tue, 18 Jan 2022 12:38:10 -0500
Subject: [PATCH v1] Split 010_pg_basebackup.pl.

---
 .../t/010_pg_basebackup_basic.pl              |  15 +
 ...ckup.pl => 011_pg_basebackup_integrity.pl} | 287 +---------------
 .../t/012_pg_basebackup_options.pl            | 318 ++++++++++++++++++
 3 files changed, 334 insertions(+), 286 deletions(-)
 create mode 100644 src/bin/pg_basebackup/t/010_pg_basebackup_basic.pl
 rename src/bin/pg_basebackup/t/{010_pg_basebackup.pl => 011_pg_basebackup_integrity.pl} (58%)
 create mode 100644 src/bin/pg_basebackup/t/012_pg_basebackup_options.pl

diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup_basic.pl b/src/bin/pg_basebackup/t/010_pg_basebackup_basic.pl
new file mode 100644
index 0000000000..48863dc213
--- /dev/null
+++ b/src/bin/pg_basebackup/t/010_pg_basebackup_basic.pl
@@ -0,0 +1,15 @@
+
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More tests => 9;
+
+program_help_ok('pg_basebackup');
+program_version_ok('pg_basebackup');
+program_options_handling_ok('pg_basebackup');
+
+command_fails(['pg_basebackup'],
+			  'pg_basebackup needs target directory specified');
diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/011_pg_basebackup_integrity.pl
similarity index 58%
rename from src/bin/pg_basebackup/t/010_pg_basebackup.pl
rename to src/bin/pg_basebackup/t/011_pg_basebackup_integrity.pl
index f0243f28d4..68332931ac 100644
--- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl
+++ b/src/bin/pg_basebackup/t/011_pg_basebackup_integrity.pl
@@ -10,11 +10,7 @@ use File::Path qw(rmtree);
 use Fcntl qw(:seek);
 use PostgreSQL::Test::Cluster;
 use PostgreSQL::Test::Utils;
-use Test::More tests => 115;
-
-program_help_ok('pg_basebackup');
-program_version_ok('pg_basebackup');
-program_options_handling_ok('pg_basebackup');
+use Test::More tests => 61;
 
 my $tempdir = PostgreSQL::Test::Utils::tempdir;
 
@@ -35,9 +31,6 @@ $node->init(extra => ['--data-checksums']);
 $node->start;
 my $pgdata = $node->data_dir;
 
-$node->command_fails(['pg_basebackup'],
-	'pg_basebackup needs target directory specified');
-
 # Some Windows ANSI code pages may reject this filename, in which case we
 # quietly proceed without this bit of test coverage.
 if (open my $badchars, '>>', "$tempdir/pgdata/FOO\xe0\xe0\xe0BAR")
@@ -400,281 +393,3 @@ foreach my $filename (@tempRelationFiles)
 ok( -d "$tempdir/backup1/pg_replslot",
 	'pg_replslot symlink copied as directory');
 rmtree("$tempdir/backup1");
-
-mkdir "$tempdir/tbl=spc2";
-$realTsDir = "$real_sys_tempdir/tbl=spc2";
-$node->safe_psql('postgres', "DROP TABLE test1;");
-$node->safe_psql('postgres', "DROP TABLE tblspc1_unlogged;");
-$node->safe_psql('postgres', "DROP TABLESPACE tblspc1;");
-$node->safe_psql('postgres',
-	"CREATE TABLESPACE tblspc2 LOCATION '$realTsDir';");
-$realTsDir =~ s/=/\\=/;
-$node->command_ok(
-	[
-		@pg_basebackup_defs, '-D',
-		"$tempdir/backup3",  '-Fp',
-		"-T$realTsDir=$real_tempdir/tbackup/tbl\\=spc2",
-	],
-	'mapping tablespace with = sign in path');
-ok(-d "$tempdir/tbackup/tbl=spc2", 'tablespace with = sign was relocated');
-$node->safe_psql('postgres', "DROP TABLESPACE tblspc2;");
-rmtree("$tempdir/backup3");
-
-mkdir "$tempdir/$superlongname";
-$realTsDir = "$real_sys_tempdir/$superlongname";
-$node->safe_psql('postgres',
-	"CREATE TABLESPACE tblspc3 LOCATION '$realTsDir';");
-$node->command_ok([ @pg_basebackup_defs, '-D', "$tempdir/tarbackup_l3", '-Ft' ],
-	'pg_basebackup tar with long symlink target');
-$node->safe_psql('postgres', "DROP TABLESPACE tblspc3;");
-rmtree("$tempdir/tarbackup_l3");
-
-$node->command_ok([ @pg_basebackup_defs, '-D', "$tempdir/backupR", '-R' ],
-	'pg_basebackup -R runs');
-ok(-f "$tempdir/backupR/postgresql.auto.conf", 'postgresql.auto.conf exists');
-ok(-f "$tempdir/backupR/standby.signal",       'standby.signal was created');
-my $recovery_conf = slurp_file "$tempdir/backupR/postgresql.auto.conf";
-rmtree("$tempdir/backupR");
-
-my $port = $node->port;
-like(
-	$recovery_conf,
-	qr/^primary_conninfo = '.*port=$port.*'\n/m,
-	'postgresql.auto.conf sets primary_conninfo');
-
-$node->command_ok(
-	[ @pg_basebackup_defs, '-D', "$tempdir/backupxd" ],
-	'pg_basebackup runs in default xlog mode');
-ok(grep(/^[0-9A-F]{24}$/, slurp_dir("$tempdir/backupxd/pg_wal")),
-	'WAL files copied');
-rmtree("$tempdir/backupxd");
-
-$node->command_ok(
-	[ @pg_basebackup_defs, '-D', "$tempdir/backupxf", '-X', 'fetch' ],
-	'pg_basebackup -X fetch runs');
-ok(grep(/^[0-9A-F]{24}$/, slurp_dir("$tempdir/backupxf/pg_wal")),
-	'WAL files copied');
-rmtree("$tempdir/backupxf");
-$node->command_ok(
-	[ @pg_basebackup_defs, '-D', "$tempdir/backupxs", '-X', 'stream' ],
-	'pg_basebackup -X stream runs');
-ok(grep(/^[0-9A-F]{24}$/, slurp_dir("$tempdir/backupxs/pg_wal")),
-	'WAL files copied');
-rmtree("$tempdir/backupxs");
-$node->command_ok(
-	[ @pg_basebackup_defs, '-D', "$tempdir/backupxst", '-X', 'stream', '-Ft' ],
-	'pg_basebackup -X stream runs in tar mode');
-ok(-f "$tempdir/backupxst/pg_wal.tar", "tar file was created");
-rmtree("$tempdir/backupxst");
-$node->command_ok(
-	[
-		@pg_basebackup_defs,     '-D',
-		"$tempdir/backupnoslot", '-X',
-		'stream',                '--no-slot'
-	],
-	'pg_basebackup -X stream runs with --no-slot');
-rmtree("$tempdir/backupnoslot");
-
-$node->command_fails(
-	[
-		@pg_basebackup_defs,         '-D',
-		"$tempdir/backupxs_sl_fail", '-X',
-		'stream',                    '-S',
-		'slot0'
-	],
-	'pg_basebackup fails with nonexistent replication slot');
-
-$node->command_fails(
-	[ @pg_basebackup_defs, '-D', "$tempdir/backupxs_slot", '-C' ],
-	'pg_basebackup -C fails without slot name');
-
-$node->command_fails(
-	[
-		@pg_basebackup_defs,      '-D',
-		"$tempdir/backupxs_slot", '-C',
-		'-S',                     'slot0',
-		'--no-slot'
-	],
-	'pg_basebackup fails with -C -S --no-slot');
-
-$node->command_ok(
-	[ @pg_basebackup_defs, '-D', "$tempdir/backupxs_slot", '-C', '-S', 'slot0' ],
-	'pg_basebackup -C runs');
-rmtree("$tempdir/backupxs_slot");
-
-is( $node->safe_psql(
-		'postgres',
-		q{SELECT slot_name FROM pg_replication_slots WHERE slot_name = 'slot0'}
-	),
-	'slot0',
-	'replication slot was created');
-isnt(
-	$node->safe_psql(
-		'postgres',
-		q{SELECT restart_lsn FROM pg_replication_slots WHERE slot_name = 'slot0'}
-	),
-	'',
-	'restart LSN of new slot is not null');
-
-$node->command_fails(
-	[ @pg_basebackup_defs, '-D', "$tempdir/backupxs_slot1", '-C', '-S', 'slot0' ],
-	'pg_basebackup fails with -C -S and a previously existing slot');
-
-$node->safe_psql('postgres',
-	q{SELECT * FROM pg_create_physical_replication_slot('slot1')});
-my $lsn = $node->safe_psql('postgres',
-	q{SELECT restart_lsn FROM pg_replication_slots WHERE slot_name = 'slot1'}
-);
-is($lsn, '', 'restart LSN of new slot is null');
-$node->command_fails(
-	[ @pg_basebackup_defs, '-D', "$tempdir/fail", '-S', 'slot1', '-X', 'none' ],
-	'pg_basebackup with replication slot fails without WAL streaming');
-$node->command_ok(
-	[
-		@pg_basebackup_defs, '-D', "$tempdir/backupxs_sl", '-X',
-		'stream',            '-S', 'slot1'
-	],
-	'pg_basebackup -X stream with replication slot runs');
-$lsn = $node->safe_psql('postgres',
-	q{SELECT restart_lsn FROM pg_replication_slots WHERE slot_name = 'slot1'}
-);
-like($lsn, qr!^0/[0-9A-Z]{7,8}$!, 'restart LSN of slot has advanced');
-rmtree("$tempdir/backupxs_sl");
-
-$node->command_ok(
-	[
-		@pg_basebackup_defs, '-D', "$tempdir/backupxs_sl_R", '-X',
-		'stream',            '-S', 'slot1',                  '-R',
-	],
-	'pg_basebackup with replication slot and -R runs');
-like(
-	slurp_file("$tempdir/backupxs_sl_R/postgresql.auto.conf"),
-	qr/^primary_slot_name = 'slot1'\n/m,
-	'recovery conf file sets primary_slot_name');
-
-my $checksum = $node->safe_psql('postgres', 'SHOW data_checksums;');
-is($checksum, 'on', 'checksums are enabled');
-rmtree("$tempdir/backupxs_sl_R");
-
-# create tables to corrupt and get their relfilenodes
-my $file_corrupt1 = $node->safe_psql('postgres',
-	q{CREATE TABLE corrupt1 AS SELECT a FROM generate_series(1,10000) AS a; ALTER TABLE corrupt1 SET (autovacuum_enabled=false); SELECT pg_relation_filepath('corrupt1')}
-);
-my $file_corrupt2 = $node->safe_psql('postgres',
-	q{CREATE TABLE corrupt2 AS SELECT b FROM generate_series(1,2) AS b; ALTER TABLE corrupt2 SET (autovacuum_enabled=false); SELECT pg_relation_filepath('corrupt2')}
-);
-
-# set page header and block sizes
-my $pageheader_size = 24;
-my $block_size = $node->safe_psql('postgres', 'SHOW block_size;');
-
-# induce corruption
-system_or_bail 'pg_ctl', '-D', $pgdata, 'stop';
-open $file, '+<', "$pgdata/$file_corrupt1";
-seek($file, $pageheader_size, SEEK_SET);
-syswrite($file, "\0\0\0\0\0\0\0\0\0");
-close $file;
-system_or_bail 'pg_ctl', '-D', $pgdata, 'start';
-
-$node->command_checks_all(
-	[ @pg_basebackup_defs, '-D', "$tempdir/backup_corrupt" ],
-	1,
-	[qr{^$}],
-	[qr/^WARNING.*checksum verification failed/s],
-	'pg_basebackup reports checksum mismatch');
-rmtree("$tempdir/backup_corrupt");
-
-# induce further corruption in 5 more blocks
-system_or_bail 'pg_ctl', '-D', $pgdata, 'stop';
-open $file, '+<', "$pgdata/$file_corrupt1";
-for my $i (1 .. 5)
-{
-	my $offset = $pageheader_size + $i * $block_size;
-	seek($file, $offset, SEEK_SET);
-	syswrite($file, "\0\0\0\0\0\0\0\0\0");
-}
-close $file;
-system_or_bail 'pg_ctl', '-D', $pgdata, 'start';
-
-$node->command_checks_all(
-	[ @pg_basebackup_defs, '-D', "$tempdir/backup_corrupt2" ],
-	1,
-	[qr{^$}],
-	[qr/^WARNING.*further.*failures.*will.not.be.reported/s],
-	'pg_basebackup does not report more than 5 checksum mismatches');
-rmtree("$tempdir/backup_corrupt2");
-
-# induce corruption in a second file
-system_or_bail 'pg_ctl', '-D', $pgdata, 'stop';
-open $file, '+<', "$pgdata/$file_corrupt2";
-seek($file, $pageheader_size, SEEK_SET);
-syswrite($file, "\0\0\0\0\0\0\0\0\0");
-close $file;
-system_or_bail 'pg_ctl', '-D', $pgdata, 'start';
-
-$node->command_checks_all(
-	[ @pg_basebackup_defs, '-D', "$tempdir/backup_corrupt3" ],
-	1,
-	[qr{^$}],
-	[qr/^WARNING.*7 total checksum verification failures/s],
-	'pg_basebackup correctly report the total number of checksum mismatches');
-rmtree("$tempdir/backup_corrupt3");
-
-# do not verify checksums, should return ok
-$node->command_ok(
-	[
-		@pg_basebackup_defs,        '-D',
-		"$tempdir/backup_corrupt4", '--no-verify-checksums',
-	],
-	'pg_basebackup with -k does not report checksum mismatch');
-rmtree("$tempdir/backup_corrupt4");
-
-$node->safe_psql('postgres', "DROP TABLE corrupt1;");
-$node->safe_psql('postgres', "DROP TABLE corrupt2;");
-
-note "Testing pg_basebackup with compression methods";
-
-# Check ZLIB compression if available.
-SKIP:
-{
-	skip "postgres was not built with ZLIB support", 5
-	  if (!check_pg_config("#define HAVE_LIBZ 1"));
-
-	$node->command_ok(
-		[
-			@pg_basebackup_defs,    '-D',
-			"$tempdir/backup_gzip", '--compress',
-			'1',                    '--format',
-			't'
-		],
-		'pg_basebackup with --compress');
-	$node->command_ok(
-		[
-			@pg_basebackup_defs,     '-D',
-			"$tempdir/backup_gzip2", '--gzip',
-			'--format',              't'
-		],
-		'pg_basebackup with --gzip');
-
-	# Verify that the stored files are generated with their expected
-	# names.
-	my @zlib_files = glob "$tempdir/backup_gzip/*.tar.gz";
-	is(scalar(@zlib_files), 2,
-		"two files created with --compress (base.tar.gz and pg_wal.tar.gz)");
-	my @zlib_files2 = glob "$tempdir/backup_gzip2/*.tar.gz";
-	is(scalar(@zlib_files2), 2,
-		"two files created with --gzip (base.tar.gz and pg_wal.tar.gz)");
-
-	# Check the integrity of the files generated.
-	my $gzip = $ENV{GZIP_PROGRAM};
-	skip "program gzip is not found in your system", 1
-	  if ( !defined $gzip
-		|| $gzip eq ''
-		|| system_log($gzip, '--version') != 0);
-
-	my $gzip_is_valid =
-	  system_log($gzip, '--test', @zlib_files, @zlib_files2);
-	is($gzip_is_valid, 0, "gzip verified the integrity of compressed data");
-	rmtree("$tempdir/backup_gzip");
-	rmtree("$tempdir/backup_gzip2");
-}
diff --git a/src/bin/pg_basebackup/t/012_pg_basebackup_options.pl b/src/bin/pg_basebackup/t/012_pg_basebackup_options.pl
new file mode 100644
index 0000000000..35c13f9e97
--- /dev/null
+++ b/src/bin/pg_basebackup/t/012_pg_basebackup_options.pl
@@ -0,0 +1,318 @@
+
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use Cwd;
+use Config;
+use File::Basename qw(basename dirname);
+use File::Path qw(rmtree);
+use Fcntl qw(:seek);
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More tests => 45;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir;
+my $real_tempdir = PostgreSQL::Test::Utils::perl2host($tempdir);
+
+my $node = PostgreSQL::Test::Cluster->new('main');
+
+# For nearly all pg_basebackup invocations some options should be specified,
+# to keep test times reasonable. Using @pg_basebackup_defs as the first
+# element of the array passed to to IPC::Run interpolate the array (as it is
+# not a reference to an array)...
+my @pg_basebackup_defs = ('pg_basebackup', '--no-sync', '-cfast');
+
+
+# Set umask so test directories and files are created with default permissions
+umask(0077);
+
+# Initialize node
+$node->init(allows_streaming => 1, extra => ['--data-checksums']);
+$node->start;
+my $pgdata = $node->data_dir;
+
+# Create a temporary directory in the system location and symlink it
+# to our physical temp location.  That way we can use shorter names
+# for the tablespace directories, which hopefully won't run afoul of
+# the 99 character length limit.
+my $sys_tempdir = PostgreSQL::Test::Utils::tempdir_short;
+my $real_sys_tempdir = PostgreSQL::Test::Utils::perl2host($sys_tempdir) . "/tempdir";
+my $shorter_tempdir =  $sys_tempdir . "/tempdir";
+dir_symlink "$tempdir", $shorter_tempdir;
+
+mkdir "$tempdir/tbl=spc2";
+my $realTsDir = "$real_sys_tempdir/tbl=spc2";
+$node->safe_psql('postgres',
+	"CREATE TABLESPACE tblspc2 LOCATION '$realTsDir';");
+$realTsDir =~ s/=/\\=/;
+$node->command_ok(
+	[
+		@pg_basebackup_defs, '-D',
+		"$tempdir/backup3",  '-Fp',
+		"-T$realTsDir=$real_tempdir/tbackup/tbl\\=spc2",
+	],
+	'mapping tablespace with = sign in path');
+ok(-d "$tempdir/tbackup/tbl=spc2", 'tablespace with = sign was relocated');
+$node->safe_psql('postgres', "DROP TABLESPACE tblspc2;");
+rmtree("$tempdir/backup3");
+
+my $superlongname = "superlongname_" . ("x" x 100);
+mkdir "$tempdir/$superlongname";
+$realTsDir = "$real_sys_tempdir/$superlongname";
+$node->safe_psql('postgres',
+	"CREATE TABLESPACE tblspc3 LOCATION '$realTsDir';");
+$node->command_ok([ @pg_basebackup_defs, '-D', "$tempdir/tarbackup_l3", '-Ft' ],
+	'pg_basebackup tar with long symlink target');
+$node->safe_psql('postgres', "DROP TABLESPACE tblspc3;");
+rmtree("$tempdir/tarbackup_l3");
+
+$node->command_ok([ @pg_basebackup_defs, '-D', "$tempdir/backupR", '-R' ],
+	'pg_basebackup -R runs');
+ok(-f "$tempdir/backupR/postgresql.auto.conf", 'postgresql.auto.conf exists');
+ok(-f "$tempdir/backupR/standby.signal",       'standby.signal was created');
+my $recovery_conf = slurp_file "$tempdir/backupR/postgresql.auto.conf";
+rmtree("$tempdir/backupR");
+
+my $port = $node->port;
+like(
+	$recovery_conf,
+	qr/^primary_conninfo = '.*port=$port.*'\n/m,
+	'postgresql.auto.conf sets primary_conninfo');
+
+$node->command_ok(
+	[ @pg_basebackup_defs, '-D', "$tempdir/backupxd" ],
+	'pg_basebackup runs in default xlog mode');
+ok(grep(/^[0-9A-F]{24}$/, slurp_dir("$tempdir/backupxd/pg_wal")),
+	'WAL files copied');
+rmtree("$tempdir/backupxd");
+
+$node->command_ok(
+	[ @pg_basebackup_defs, '-D', "$tempdir/backupxf", '-X', 'fetch' ],
+	'pg_basebackup -X fetch runs');
+ok(grep(/^[0-9A-F]{24}$/, slurp_dir("$tempdir/backupxf/pg_wal")),
+	'WAL files copied');
+rmtree("$tempdir/backupxf");
+$node->command_ok(
+	[ @pg_basebackup_defs, '-D', "$tempdir/backupxs", '-X', 'stream' ],
+	'pg_basebackup -X stream runs');
+ok(grep(/^[0-9A-F]{24}$/, slurp_dir("$tempdir/backupxs/pg_wal")),
+	'WAL files copied');
+rmtree("$tempdir/backupxs");
+$node->command_ok(
+	[ @pg_basebackup_defs, '-D', "$tempdir/backupxst", '-X', 'stream', '-Ft' ],
+	'pg_basebackup -X stream runs in tar mode');
+ok(-f "$tempdir/backupxst/pg_wal.tar", "tar file was created");
+rmtree("$tempdir/backupxst");
+$node->command_ok(
+	[
+		@pg_basebackup_defs,     '-D',
+		"$tempdir/backupnoslot", '-X',
+		'stream',                '--no-slot'
+	],
+	'pg_basebackup -X stream runs with --no-slot');
+rmtree("$tempdir/backupnoslot");
+
+$node->command_fails(
+	[
+		@pg_basebackup_defs,         '-D',
+		"$tempdir/backupxs_sl_fail", '-X',
+		'stream',                    '-S',
+		'slot0'
+	],
+	'pg_basebackup fails with nonexistent replication slot');
+
+$node->command_fails(
+	[ @pg_basebackup_defs, '-D', "$tempdir/backupxs_slot", '-C' ],
+	'pg_basebackup -C fails without slot name');
+
+$node->command_fails(
+	[
+		@pg_basebackup_defs,      '-D',
+		"$tempdir/backupxs_slot", '-C',
+		'-S',                     'slot0',
+		'--no-slot'
+	],
+	'pg_basebackup fails with -C -S --no-slot');
+
+$node->command_ok(
+	[ @pg_basebackup_defs, '-D', "$tempdir/backupxs_slot", '-C', '-S', 'slot0' ],
+	'pg_basebackup -C runs');
+rmtree("$tempdir/backupxs_slot");
+
+is( $node->safe_psql(
+		'postgres',
+		q{SELECT slot_name FROM pg_replication_slots WHERE slot_name = 'slot0'}
+	),
+	'slot0',
+	'replication slot was created');
+isnt(
+	$node->safe_psql(
+		'postgres',
+		q{SELECT restart_lsn FROM pg_replication_slots WHERE slot_name = 'slot0'}
+	),
+	'',
+	'restart LSN of new slot is not null');
+
+$node->command_fails(
+	[ @pg_basebackup_defs, '-D', "$tempdir/backupxs_slot1", '-C', '-S', 'slot0' ],
+	'pg_basebackup fails with -C -S and a previously existing slot');
+
+$node->safe_psql('postgres',
+	q{SELECT * FROM pg_create_physical_replication_slot('slot1')});
+my $lsn = $node->safe_psql('postgres',
+	q{SELECT restart_lsn FROM pg_replication_slots WHERE slot_name = 'slot1'}
+);
+is($lsn, '', 'restart LSN of new slot is null');
+$node->command_fails(
+	[ @pg_basebackup_defs, '-D', "$tempdir/fail", '-S', 'slot1', '-X', 'none' ],
+	'pg_basebackup with replication slot fails without WAL streaming');
+$node->command_ok(
+	[
+		@pg_basebackup_defs, '-D', "$tempdir/backupxs_sl", '-X',
+		'stream',            '-S', 'slot1'
+	],
+	'pg_basebackup -X stream with replication slot runs');
+$lsn = $node->safe_psql('postgres',
+	q{SELECT restart_lsn FROM pg_replication_slots WHERE slot_name = 'slot1'}
+);
+like($lsn, qr!^0/[0-9A-Z]{7,8}$!, 'restart LSN of slot has advanced');
+rmtree("$tempdir/backupxs_sl");
+
+$node->command_ok(
+	[
+		@pg_basebackup_defs, '-D', "$tempdir/backupxs_sl_R", '-X',
+		'stream',            '-S', 'slot1',                  '-R',
+	],
+	'pg_basebackup with replication slot and -R runs');
+like(
+	slurp_file("$tempdir/backupxs_sl_R/postgresql.auto.conf"),
+	qr/^primary_slot_name = 'slot1'\n/m,
+	'recovery conf file sets primary_slot_name');
+
+my $checksum = $node->safe_psql('postgres', 'SHOW data_checksums;');
+is($checksum, 'on', 'checksums are enabled');
+rmtree("$tempdir/backupxs_sl_R");
+
+# create tables to corrupt and get their relfilenodes
+my $file_corrupt1 = $node->safe_psql('postgres',
+	q{CREATE TABLE corrupt1 AS SELECT a FROM generate_series(1,10000) AS a; ALTER TABLE corrupt1 SET (autovacuum_enabled=false); SELECT pg_relation_filepath('corrupt1')}
+);
+my $file_corrupt2 = $node->safe_psql('postgres',
+	q{CREATE TABLE corrupt2 AS SELECT b FROM generate_series(1,2) AS b; ALTER TABLE corrupt2 SET (autovacuum_enabled=false); SELECT pg_relation_filepath('corrupt2')}
+);
+
+# set page header and block sizes
+my $pageheader_size = 24;
+my $block_size = $node->safe_psql('postgres', 'SHOW block_size;');
+
+# induce corruption
+system_or_bail 'pg_ctl', '-D', $pgdata, 'stop';
+open my $file, '+<', "$pgdata/$file_corrupt1";
+seek($file, $pageheader_size, SEEK_SET);
+syswrite($file, "\0\0\0\0\0\0\0\0\0");
+close $file;
+system_or_bail 'pg_ctl', '-D', $pgdata, 'start';
+
+$node->command_checks_all(
+	[ @pg_basebackup_defs, '-D', "$tempdir/backup_corrupt" ],
+	1,
+	[qr{^$}],
+	[qr/^WARNING.*checksum verification failed/s],
+	'pg_basebackup reports checksum mismatch');
+rmtree("$tempdir/backup_corrupt");
+
+# induce further corruption in 5 more blocks
+system_or_bail 'pg_ctl', '-D', $pgdata, 'stop';
+open $file, '+<', "$pgdata/$file_corrupt1";
+for my $i (1 .. 5)
+{
+	my $offset = $pageheader_size + $i * $block_size;
+	seek($file, $offset, SEEK_SET);
+	syswrite($file, "\0\0\0\0\0\0\0\0\0");
+}
+close $file;
+system_or_bail 'pg_ctl', '-D', $pgdata, 'start';
+
+$node->command_checks_all(
+	[ @pg_basebackup_defs, '-D', "$tempdir/backup_corrupt2" ],
+	1,
+	[qr{^$}],
+	[qr/^WARNING.*further.*failures.*will.not.be.reported/s],
+	'pg_basebackup does not report more than 5 checksum mismatches');
+rmtree("$tempdir/backup_corrupt2");
+
+# induce corruption in a second file
+system_or_bail 'pg_ctl', '-D', $pgdata, 'stop';
+open $file, '+<', "$pgdata/$file_corrupt2";
+seek($file, $pageheader_size, SEEK_SET);
+syswrite($file, "\0\0\0\0\0\0\0\0\0");
+close $file;
+system_or_bail 'pg_ctl', '-D', $pgdata, 'start';
+
+$node->command_checks_all(
+	[ @pg_basebackup_defs, '-D', "$tempdir/backup_corrupt3" ],
+	1,
+	[qr{^$}],
+	[qr/^WARNING.*7 total checksum verification failures/s],
+	'pg_basebackup correctly report the total number of checksum mismatches');
+rmtree("$tempdir/backup_corrupt3");
+
+# do not verify checksums, should return ok
+$node->command_ok(
+	[
+		@pg_basebackup_defs,        '-D',
+		"$tempdir/backup_corrupt4", '--no-verify-checksums',
+	],
+	'pg_basebackup with -k does not report checksum mismatch');
+rmtree("$tempdir/backup_corrupt4");
+
+$node->safe_psql('postgres', "DROP TABLE corrupt1;");
+$node->safe_psql('postgres', "DROP TABLE corrupt2;");
+
+note "Testing pg_basebackup with compression methods";
+
+# Check ZLIB compression if available.
+SKIP:
+{
+	skip "postgres was not built with ZLIB support", 5
+	  if (!check_pg_config("#define HAVE_LIBZ 1"));
+
+	$node->command_ok(
+		[
+			@pg_basebackup_defs,    '-D',
+			"$tempdir/backup_gzip", '--compress',
+			'1',                    '--format',
+			't'
+		],
+		'pg_basebackup with --compress');
+	$node->command_ok(
+		[
+			@pg_basebackup_defs,     '-D',
+			"$tempdir/backup_gzip2", '--gzip',
+			'--format',              't'
+		],
+		'pg_basebackup with --gzip');
+
+	# Verify that the stored files are generated with their expected
+	# names.
+	my @zlib_files = glob "$tempdir/backup_gzip/*.tar.gz";
+	is(scalar(@zlib_files), 2,
+		"two files created with --compress (base.tar.gz and pg_wal.tar.gz)");
+	my @zlib_files2 = glob "$tempdir/backup_gzip2/*.tar.gz";
+	is(scalar(@zlib_files2), 2,
+		"two files created with --gzip (base.tar.gz and pg_wal.tar.gz)");
+
+	# Check the integrity of the files generated.
+	my $gzip = $ENV{GZIP_PROGRAM};
+	skip "program gzip is not found in your system", 1
+	  if ( !defined $gzip
+		|| $gzip eq ''
+		|| system_log($gzip, '--version') != 0);
+
+	my $gzip_is_valid =
+	  system_log($gzip, '--test', @zlib_files, @zlib_files2);
+	is($gzip_is_valid, 0, "gzip verified the integrity of compressed data");
+	rmtree("$tempdir/backup_gzip");
+	rmtree("$tempdir/backup_gzip2");
+}
-- 
2.24.3 (Apple Git-128)

From a0818b6c8d03f152baa5df231b27aa7b8a7fde45 Mon Sep 17 00:00:00 2001
From: Andres Freund <andres@anarazel.de>
Date: Tue, 18 Jan 2022 15:25:27 -0800
Subject: [PATCH v2] hack: use "template" initdb in tap tests.

---
 src/test/perl/PostgreSQL/Test/Cluster.pm | 31 +++++++++++++-
 src/test/regress/pg_regress.c            | 45 ++++++++++++++------
 src/Makefile.global.in                   | 52 +++++++++++++-----------
 src/tools/msvc/Install.pm                |  4 ++
 src/tools/msvc/vcregress.pl              |  1 +
 5 files changed, 94 insertions(+), 39 deletions(-)

diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 7af0f8db139..7e235c90d8c 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -430,8 +430,35 @@ sub init
 	mkdir $self->backup_dir;
 	mkdir $self->archive_dir;
 
-	PostgreSQL::Test::Utils::system_or_bail('initdb', '-D', $pgdata, '-A', 'trust', '-N',
-		@{ $params{extra} });
+	if (defined $params{extra} or !defined $ENV{INITDB_TEMPLATE})
+	{
+		diag("*not* using initdb template");
+		PostgreSQL::Test::Utils::system_or_bail('initdb', '-D', $pgdata, '-A', 'trust', '-N',
+			@{ $params{extra} });
+	}
+	else
+	{
+		my $old_umask;
+
+		diag("using initdb template");
+
+		if (!$PostgreSQL::Test::Utils::windows_os)
+		{
+			$old_umask = umask;
+			umask 0077;
+		}
+
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$ENV{INITDB_TEMPLATE},
+			$pgdata,
+		  );
+
+		if (!$PostgreSQL::Test::Utils::windows_os)
+		{
+			umask $old_umask;
+		}
+	}
+
 	PostgreSQL::Test::Utils::system_or_bail($ENV{PG_REGRESS}, '--config-auth', $pgdata,
 		@{ $params{auth_extra} });
 
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index e6f71c7582e..07f1beae6c6 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2232,6 +2232,7 @@ regression_main(int argc, char *argv[],
 		FILE	   *pg_conf;
 		const char *env_wait;
 		int			wait_seconds;
+		const char* initdb_template_dir;
 
 		/*
 		 * Prepare the temp instance
@@ -2258,20 +2259,38 @@ regression_main(int argc, char *argv[],
 		if (!directory_exists(buf))
 			make_directory(buf);
 
-		/* initdb */
-		header(_("initializing database system"));
-		snprintf(buf, sizeof(buf),
-				 "\"%s%sinitdb\" -D \"%s/data\" --no-clean --no-sync%s%s > \"%s/log/initdb.log\" 2>&1",
-				 bindir ? bindir : "",
-				 bindir ? "/" : "",
-				 temp_instance,
-				 debug ? " --debug" : "",
-				 nolocale ? " --no-locale" : "",
-				 outputdir);
-		if (system(buf))
+		/* create data directory, either from a template, or by running initdb */
+		initdb_template_dir = getenv("INITDB_TEMPLATE");
+		if (initdb_template_dir == NULL)
 		{
-			fprintf(stderr, _("\n%s: initdb failed\nExamine %s/log/initdb.log for the reason.\nCommand was: %s\n"), progname, outputdir, buf);
-			exit(2);
+			header(_("initializing database system"));
+			snprintf(buf, sizeof(buf),
+					 "\"%s%sinitdb\" -D \"%s/data\" --no-clean --no-sync%s%s > \"%s/log/initdb.log\" 2>&1",
+					 bindir ? bindir : "",
+					 bindir ? "/" : "",
+					 temp_instance,
+					 debug ? " --debug" : "",
+					 nolocale ? " --no-locale" : "",
+					 outputdir);
+			if (system(buf))
+			{
+				fprintf(stderr, _("\n%s: initdb failed\nExamine %s/log/initdb.log for the reason.\nCommand was: %s\n"), progname, outputdir, buf);
+				exit(2);
+			}
+		}
+		else
+		{
+			/* FIXME: this only works on windows when there's a GNU cp in PATH */
+			header(_("initializing database system from template"));
+			snprintf(buf, sizeof(buf),
+					 "cp -a \"%s\" \"%s/data\"",
+					 initdb_template_dir,
+					 temp_instance);
+			if (system(buf))
+			{
+				fprintf(stderr, _("\n%s: copying of initdb template failed\n"), progname);
+				exit(2);
+			}
 		}
 
 		/*
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index 05c54b27def..6d07e76ad68 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -401,30 +401,6 @@ check: temp-install
 
 .PHONY: temp-install
 
-temp-install: | submake-generated-headers
-ifndef NO_TEMP_INSTALL
-ifneq ($(abs_top_builddir),)
-ifeq ($(MAKELEVEL),0)
-	rm -rf '$(abs_top_builddir)'/tmp_install
-	$(MKDIR_P) '$(abs_top_builddir)'/tmp_install/log
-	$(MAKE) -C '$(top_builddir)' DESTDIR='$(abs_top_builddir)'/tmp_install install >'$(abs_top_builddir)'/tmp_install/log/install.log 2>&1
-	$(MAKE) -j1 $(if $(CHECKPREP_TOP),-C $(CHECKPREP_TOP),) checkprep >>'$(abs_top_builddir)'/tmp_install/log/install.log 2>&1
-endif
-endif
-endif
-
-# Tasks to run serially at the end of temp-install.  Some EXTRA_INSTALL
-# entries appear more than once in the tree, and parallel installs of the same
-# file can fail with EEXIST.
-checkprep:
-	$(if $(EXTRA_INSTALL),for extra in $(EXTRA_INSTALL); do $(MAKE) -C '$(top_builddir)'/$$extra DESTDIR='$(abs_top_builddir)'/tmp_install install || exit; done)
-
-PROVE = @PROVE@
-# There are common routines in src/test/perl, and some test suites have
-# extra perl modules in their own directory.
-PG_PROVE_FLAGS = -I $(top_srcdir)/src/test/perl/ -I $(srcdir)
-# User-supplied prove flags such as --verbose can be provided in PROVE_FLAGS.
-PROVE_FLAGS =
 
 # prepend to path if already set, else just set it
 define add_to_path
@@ -441,8 +417,36 @@ ld_library_path_var = LD_LIBRARY_PATH
 with_temp_install = \
 	PATH="$(abs_top_builddir)/tmp_install$(bindir):$(CURDIR):$$PATH" \
 	$(call add_to_path,$(strip $(ld_library_path_var)),$(abs_top_builddir)/tmp_install$(libdir)) \
+	INITDB_TEMPLATE='$(abs_top_builddir)'/tmp_install/initdb-template \
 	$(with_temp_install_extra)
 
+temp-install: | submake-generated-headers
+ifndef NO_TEMP_INSTALL
+ifneq ($(abs_top_builddir),)
+ifeq ($(MAKELEVEL),0)
+	rm -rf '$(abs_top_builddir)'/tmp_install
+	$(MKDIR_P) '$(abs_top_builddir)'/tmp_install/log
+	$(MAKE) -C '$(top_builddir)' DESTDIR='$(abs_top_builddir)'/tmp_install install >'$(abs_top_builddir)'/tmp_install/log/install.log 2>&1
+	$(MAKE) -j1 $(if $(CHECKPREP_TOP),-C $(CHECKPREP_TOP),) checkprep >>'$(abs_top_builddir)'/tmp_install/log/install.log 2>&1
+
+	$(with_temp_install) initdb -A trust -N --no-instructions '$(abs_top_builddir)'/tmp_install/initdb-template >>'$(abs_top_builddir)'/tmp_install/log/initdb-template.log 2>&1
+endif
+endif
+endif
+
+# Tasks to run serially at the end of temp-install.  Some EXTRA_INSTALL
+# entries appear more than once in the tree, and parallel installs of the same
+# file can fail with EEXIST.
+checkprep:
+	$(if $(EXTRA_INSTALL),for extra in $(EXTRA_INSTALL); do $(MAKE) -C '$(top_builddir)'/$$extra DESTDIR='$(abs_top_builddir)'/tmp_install install || exit; done)
+
+PROVE = @PROVE@
+# There are common routines in src/test/perl, and some test suites have
+# extra perl modules in their own directory.
+PG_PROVE_FLAGS = -I $(top_srcdir)/src/test/perl/ -I $(srcdir)
+# User-supplied prove flags such as --verbose can be provided in PROVE_FLAGS.
+PROVE_FLAGS =
+
 ifeq ($(enable_tap_tests),yes)
 
 ifndef PGXS
diff --git a/src/tools/msvc/Install.pm b/src/tools/msvc/Install.pm
index 8de79c618cb..cef069c6162 100644
--- a/src/tools/msvc/Install.pm
+++ b/src/tools/msvc/Install.pm
@@ -14,6 +14,7 @@ use Carp;
 use File::Basename;
 use File::Copy;
 use File::Find ();
+use File::Path qw(rmtree);
 
 use Exporter;
 our (@ISA, @EXPORT_OK);
@@ -183,6 +184,9 @@ sub Install
 
 	GenerateNLSFiles($target, $config->{nls}, $majorver) if ($config->{nls});
 
+	rmtree("$target/initdb_template");
+	system("$target/bin/initdb -D $target/initdb_template -A trust -N") == 0 || die "template initdb failed";
+
 	print "Installation complete.\n";
 	return;
 }
diff --git a/src/tools/msvc/vcregress.pl b/src/tools/msvc/vcregress.pl
index 6dcd742fa6c..d05fc364c55 100644
--- a/src/tools/msvc/vcregress.pl
+++ b/src/tools/msvc/vcregress.pl
@@ -73,6 +73,7 @@ $ENV{with_icu} = $config->{icu} ? 'yes' : 'no';
 $ENV{with_gssapi} = $config->{gss} ? 'yes' : 'no';
 $ENV{with_krb_srvnam} = $config->{krb_srvnam} || 'postgres';
 $ENV{with_readline} = 'no';
+$ENV{INITDB_TEMPLATE} = "$tmp_installdir/initdb_template";
 
 $ENV{PATH} = "$topdir/$Config/libpq;$ENV{PATH}";
 
-- 
2.34.0

diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index dc8a89af8e..a28a03ac72 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -332,6 +332,7 @@ ifeq ($(MAKELEVEL),0)
 	rm -rf '$(abs_top_builddir)'/tmp_install
 	$(MKDIR_P) '$(abs_top_builddir)'/tmp_install/log
 	$(MAKE) -C '$(top_builddir)' DESTDIR='$(abs_top_builddir)'/tmp_install install >'$(abs_top_builddir)'/tmp_install/log/install.log 2>&1
+	'$(abs_top_builddir)/tmp_install$(bindir)/initdb' -D '$(abs_top_builddir)/tmp_install/proto_pgdata' --no-clean --no-sync -A trust >'$(abs_top_builddir)'/tmp_install/log/initdb.log 2>&1
 endif
 	$(if $(EXTRA_INSTALL),for extra in $(EXTRA_INSTALL); do $(MAKE) -C '$(top_builddir)'/$$extra DESTDIR='$(abs_top_builddir)'/tmp_install install >>'$(abs_top_builddir)'/tmp_install/log/install.log || exit; done)
 endif
@@ -355,7 +356,7 @@ $(if $(filter $(PORTNAME),darwin),DYLD_LIBRARY_PATH,$(if $(filter $(PORTNAME),ai
 endef
 
 define with_temp_install
-PATH="$(abs_top_builddir)/tmp_install$(bindir):$$PATH" $(call add_to_path,$(ld_library_path_var),$(abs_top_builddir)/tmp_install$(libdir))
+PATH="$(abs_top_builddir)/tmp_install$(bindir):$$PATH" $(call add_to_path,$(ld_library_path_var),$(abs_top_builddir)/tmp_install$(libdir)) PG_PROTO_DATADIR='$(abs_top_builddir)/tmp_install/proto_pgdata'
 endef
 
 ifeq ($(enable_tap_tests),yes)
diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index f4fa600951..22c1a726c7 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -404,8 +404,23 @@ sub init
 	mkdir $self->backup_dir;
 	mkdir $self->archive_dir;
 
-	TestLib::system_or_bail('initdb', '-D', $pgdata, '-A', 'trust', '-N',
-		@{ $params{extra} });
+	# If we're using default initdb parameters, and the top-level "make check"
+	# created a prototype data directory for us, just copy that.
+	if (!defined($params{extra}) &&
+		defined($ENV{PG_PROTO_DATADIR}) &&
+		-d $ENV{PG_PROTO_DATADIR})
+	{
+		rmdir($pgdata);
+		RecursiveCopy::copypath($ENV{PG_PROTO_DATADIR}, $pgdata);
+		chmod(0700, $pgdata);
+	}
+	else
+	{
+		TestLib::system_or_bail('initdb', '-D', $pgdata,
+								'--no-clean', '--no-sync', '-A', 'trust',
+								@{ $params{extra} });
+	}
+
 	TestLib::system_or_bail($ENV{PG_REGRESS}, '--config-auth', $pgdata);
 
 	open my $conf, '>>', "$pgdata/postgresql.conf";
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index abb742b1ed..04d7fb910b 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2214,6 +2214,8 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 
 	if (temp_instance)
 	{
+		char	   *pg_proto_datadir;
+		struct stat ppst;
 		FILE	   *pg_conf;
 		const char *env_wait;
 		int			wait_seconds;
@@ -2243,20 +2245,43 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		if (!directory_exists(buf))
 			make_directory(buf);
 
-		/* initdb */
-		header(_("initializing database system"));
-		snprintf(buf, sizeof(buf),
-				 "\"%s%sinitdb\" -D \"%s/data\" --no-clean --no-sync%s%s > \"%s/log/initdb.log\" 2>&1",
-				 bindir ? bindir : "",
-				 bindir ? "/" : "",
-				 temp_instance,
-				 debug ? " --debug" : "",
-				 nolocale ? " --no-locale" : "",
-				 outputdir);
-		if (system(buf))
+		/* see if we should run initdb or just copy a prototype datadir */
+		pg_proto_datadir = getenv("PG_PROTO_DATADIR");
+		if (!nolocale &&
+			pg_proto_datadir &&
+			stat(pg_proto_datadir, &ppst) == 0 &&
+			S_ISDIR(ppst.st_mode))
 		{
-			fprintf(stderr, _("\n%s: initdb failed\nExamine %s/log/initdb.log for the reason.\nCommand was: %s\n"), progname, outputdir, buf);
-			exit(2);
+			/* copy prototype */
+			header(_("copying prototype data directory"));
+			snprintf(buf, sizeof(buf),
+					 "cp -a \"%s\" \"%s/data\" > \"%s/log/initdb.log\" 2>&1",
+					 pg_proto_datadir,
+					 temp_instance,
+					 outputdir);
+			if (system(buf))
+			{
+				fprintf(stderr, _("\n%s: cp failed\nExamine %s/log/initdb.log for the reason.\nCommand was: %s\n"), progname, outputdir, buf);
+				exit(2);
+			}
+		}
+		else
+		{
+			/* run initdb */
+			header(_("initializing database system"));
+			snprintf(buf, sizeof(buf),
+					 "\"%s%sinitdb\" -D \"%s/data\" --no-clean --no-sync%s%s > \"%s/log/initdb.log\" 2>&1",
+					 bindir ? bindir : "",
+					 bindir ? "/" : "",
+					 temp_instance,
+					 debug ? " --debug" : "",
+					 nolocale ? " --no-locale" : "",
+					 outputdir);
+			if (system(buf))
+			{
+				fprintf(stderr, _("\n%s: initdb failed\nExamine %s/log/initdb.log for the reason.\nCommand was: %s\n"), progname, outputdir, buf);
+				exit(2);
+			}
 		}
 
 		/*

From 0fd431c277f01284a91999a04368de6b59b6691e Mon Sep 17 00:00:00 2001
From: Andres Freund <andres@anarazel.de>
Date: Thu, 2 Feb 2023 21:51:53 -0800
Subject: [PATCH v3 1/2] Use "template" initdb in tests

Discussion: https://postgr.es/m/20220120021859.3zpsfqn4z7ob7afz@alap3.anarazel.de
---
 meson.build                              | 30 ++++++++++
 .cirrus.yml                              |  3 +-
 src/test/perl/PostgreSQL/Test/Cluster.pm | 46 ++++++++++++++-
 src/test/regress/pg_regress.c            | 74 ++++++++++++++++++------
 src/Makefile.global.in                   | 52 +++++++++--------
 5 files changed, 161 insertions(+), 44 deletions(-)

diff --git a/meson.build b/meson.build
index 04ea3488522..47429a18c3f 100644
--- a/meson.build
+++ b/meson.build
@@ -3056,8 +3056,10 @@ testport = 40000
 test_env = environment()
 
 temp_install_bindir = test_install_location / get_option('bindir')
+test_initdb_template = meson.build_root() / 'tmp_install' / 'initdb-template'
 test_env.set('PG_REGRESS', pg_regress.full_path())
 test_env.set('REGRESS_SHLIB', regress_module.full_path())
+test_env.set('INITDB_TEMPLATE', test_initdb_template)
 
 # Test suites that are not safe by default but can be run if selected
 # by the user via the whitespace-separated list in variable PG_TEST_EXTRA.
@@ -3072,6 +3074,34 @@ if library_path_var != ''
 endif
 
 
+# Create (and remove old) initdb template directory. Tests use that, where
+# possible, to make it cheaper to run tests.
+#
+# Use python to remove the old cached initdb, as we cannot rely on a working
+# 'rm' binary on windows.
+test('initdb_cache',
+     python,
+     args: [
+       '-c', '''
+import shutil
+import sys
+import subprocess
+
+shutil.rmtree(sys.argv[1], ignore_errors=True)
+sp = subprocess.run(sys.argv[2:] + [sys.argv[1]])
+sys.exit(sp.returncode)
+''',
+       test_initdb_template,
+       temp_install_bindir / 'initdb',
+       '-A', 'trust', '-N', '--no-instructions'
+     ],
+     priority: setup_tests_priority - 1,
+     timeout: 300,
+     is_parallel: false,
+     env: test_env,
+     suite: ['setup'])
+
+
 
 ###############################################################
 # Test Generation
diff --git a/.cirrus.yml b/.cirrus.yml
index d260f15c4e2..8fce17dff08 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -115,8 +115,9 @@ task:
   test_minimal_script: |
     su postgres <<-EOF
       ulimit -c unlimited
+      meson test $MTEST_ARGS --suite setup
       meson test $MTEST_ARGS --num-processes ${TEST_JOBS} \
-        tmp_install cube/regress pg_ctl/001_start_stop
+        cube/regress pg_ctl/001_start_stop
     EOF
 
   on_failure:
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 5e161dbee60..4d449c35de9 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -522,8 +522,50 @@ sub init
 	mkdir $self->backup_dir;
 	mkdir $self->archive_dir;
 
-	PostgreSQL::Test::Utils::system_or_bail('initdb', '-D', $pgdata, '-A',
-		'trust', '-N', @{ $params{extra} });
+	# If available and if there aren't any parameters, use a previously
+	# initdb'd cluster as a template by copying it. For a lot of tests, that's
+	# substantially cheaper. Do so only if there aren't parameters, it doesn't
+	# seem worth figuring out whether they affect compatibility.
+	#
+	# There's very similar code in pg_regress.c, but we can't easily
+	# deduplicate it until we require perl at build time.
+	if (defined $params{extra} or !defined $ENV{INITDB_TEMPLATE})
+	{
+		note("initializing database system by running initdb");
+		PostgreSQL::Test::Utils::system_or_bail('initdb', '-D', $pgdata, '-A',
+			'trust', '-N', @{ $params{extra} });
+	}
+	else
+	{
+		my @copycmd;
+		my $expected_exitcode;
+
+		note("initializing database system by copying initdb template");
+
+		if ($PostgreSQL::Test::Utils::windows_os)
+		{
+			@copycmd = qw(robocopy /E /NJS /NJH /NFL /NDL /NP);
+			$expected_exitcode = 1;    # 1 denotes files were copied
+		}
+		else
+		{
+			@copycmd = qw(cp -a);
+			$expected_exitcode = 0;
+		}
+
+		@copycmd = (@copycmd, $ENV{INITDB_TEMPLATE}, $pgdata);
+
+		my $ret = PostgreSQL::Test::Utils::system_log(@copycmd);
+
+		# See http://perldoc.perl.org/perlvar.html#%24CHILD_ERROR
+		if ($ret & 127 or $ret >> 8 != $expected_exitcode)
+		{
+			BAIL_OUT(
+				sprintf("failed to execute command \"%s\": $ret",
+					join(" ", @copycmd)));
+		}
+	}
+
 	PostgreSQL::Test::Utils::system_or_bail($ENV{PG_REGRESS},
 		'--config-auth', $pgdata, @{ $params{auth_extra} });
 
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index b68632320a7..407e3915cec 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2295,6 +2295,7 @@ regression_main(int argc, char *argv[],
 		FILE	   *pg_conf;
 		const char *env_wait;
 		int			wait_seconds;
+		const char *initdb_template_dir;
 
 		/*
 		 * Prepare the temp instance
@@ -2316,25 +2317,64 @@ regression_main(int argc, char *argv[],
 		if (!directory_exists(buf))
 			make_directory(buf);
 
-		/* initdb */
 		initStringInfo(&cmd);
-		appendStringInfo(&cmd,
-						 "\"%s%sinitdb\" -D \"%s/data\" --no-clean --no-sync",
-						 bindir ? bindir : "",
-						 bindir ? "/" : "",
-						 temp_instance);
-		if (debug)
-			appendStringInfo(&cmd, " --debug");
-		if (nolocale)
-			appendStringInfo(&cmd, " --no-locale");
-		appendStringInfo(&cmd, " > \"%s/log/initdb.log\" 2>&1", outputdir);
-		fflush(NULL);
-		if (system(cmd.data))
+
+		/*
+		 * Create data directory.
+		 *
+		 * If available, use a previously initdb'd cluster as a template by
+		 * copying it. For a lot of tests, that's substantially cheaper.
+		 *
+		 * There's very similar code in Cluster.pm, but we can't easily de
+		 * duplicate it until we require perl at build time.
+		 */
+		initdb_template_dir = getenv("INITDB_TEMPLATE");
+		if (initdb_template_dir == NULL || nolocale || debug)
 		{
-			bail("initdb failed\n"
-				 "# Examine \"%s/log/initdb.log\" for the reason.\n"
-				 "# Command was: %s",
-				 outputdir, cmd.data);
+			note("initializing database system by running initdb");
+
+			appendStringInfo(&cmd,
+							 "\"%s%sinitdb\" -D \"%s/data\" --no-clean --no-sync",
+							 bindir ? bindir : "",
+							 bindir ? "/" : "",
+							 temp_instance);
+			if (debug)
+				appendStringInfo(&cmd, " --debug");
+			if (nolocale)
+				appendStringInfo(&cmd, " --no-locale");
+			appendStringInfo(&cmd, " > \"%s/log/initdb.log\" 2>&1", outputdir);
+			fflush(NULL);
+			if (system(cmd.data))
+			{
+				bail("initdb failed\n"
+					 "# Examine \"%s/log/initdb.log\" for the reason.\n"
+					 "# Command was: %s",
+					 outputdir, cmd.data);
+			}
+		}
+		else
+		{
+#ifndef WIN32
+			const char *copycmd = "cp -a \"%s\" \"%s/data\"";
+			int			expected_exitcode = 0;
+#else
+			const char *copycmd = "robocopy /E /NJS /NJH /NFL /NDL /NP \"%s\" \"%s/data\"";
+			int			expected_exitcode = 1;	/* 1 denotes files were copied */
+#endif
+
+			note("initializing database system by copying initdb template");
+
+			appendStringInfo(&cmd,
+							 copycmd,
+							 initdb_template_dir,
+							 temp_instance);
+			if (system(cmd.data) != expected_exitcode)
+			{
+				bail("copying of initdb template failed\n"
+					 "# Examine \"%s/log/initdb.log\" for the reason.\n"
+					 "# Command was: %s",
+					 outputdir, cmd.data);
+			}
 		}
 
 		pfree(cmd.data);
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index df9f721a41a..0b4ca0eb6ae 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -397,30 +397,6 @@ check: temp-install
 
 .PHONY: temp-install
 
-temp-install: | submake-generated-headers
-ifndef NO_TEMP_INSTALL
-ifneq ($(abs_top_builddir),)
-ifeq ($(MAKELEVEL),0)
-	rm -rf '$(abs_top_builddir)'/tmp_install
-	$(MKDIR_P) '$(abs_top_builddir)'/tmp_install/log
-	$(MAKE) -C '$(top_builddir)' DESTDIR='$(abs_top_builddir)'/tmp_install install >'$(abs_top_builddir)'/tmp_install/log/install.log 2>&1
-	$(MAKE) -j1 $(if $(CHECKPREP_TOP),-C $(CHECKPREP_TOP),) checkprep >>'$(abs_top_builddir)'/tmp_install/log/install.log 2>&1
-endif
-endif
-endif
-
-# Tasks to run serially at the end of temp-install.  Some EXTRA_INSTALL
-# entries appear more than once in the tree, and parallel installs of the same
-# file can fail with EEXIST.
-checkprep:
-	$(if $(EXTRA_INSTALL),for extra in $(EXTRA_INSTALL); do $(MAKE) -C '$(top_builddir)'/$$extra DESTDIR='$(abs_top_builddir)'/tmp_install install || exit; done)
-
-PROVE = @PROVE@
-# There are common routines in src/test/perl, and some test suites have
-# extra perl modules in their own directory.
-PG_PROVE_FLAGS = -I $(top_srcdir)/src/test/perl/ -I $(srcdir)
-# User-supplied prove flags such as --verbose can be provided in PROVE_FLAGS.
-PROVE_FLAGS =
 
 # prepend to path if already set, else just set it
 define add_to_path
@@ -437,8 +413,36 @@ ld_library_path_var = LD_LIBRARY_PATH
 with_temp_install = \
 	PATH="$(abs_top_builddir)/tmp_install$(bindir):$(CURDIR):$$PATH" \
 	$(call add_to_path,$(strip $(ld_library_path_var)),$(abs_top_builddir)/tmp_install$(libdir)) \
+	INITDB_TEMPLATE='$(abs_top_builddir)'/tmp_install/initdb-template \
 	$(with_temp_install_extra)
 
+temp-install: | submake-generated-headers
+ifndef NO_TEMP_INSTALL
+ifneq ($(abs_top_builddir),)
+ifeq ($(MAKELEVEL),0)
+	rm -rf '$(abs_top_builddir)'/tmp_install
+	$(MKDIR_P) '$(abs_top_builddir)'/tmp_install/log
+	$(MAKE) -C '$(top_builddir)' DESTDIR='$(abs_top_builddir)'/tmp_install install >'$(abs_top_builddir)'/tmp_install/log/install.log 2>&1
+	$(MAKE) -j1 $(if $(CHECKPREP_TOP),-C $(CHECKPREP_TOP),) checkprep >>'$(abs_top_builddir)'/tmp_install/log/install.log 2>&1
+
+	$(with_temp_install) initdb -A trust -N --no-instructions '$(abs_top_builddir)'/tmp_install/initdb-template >>'$(abs_top_builddir)'/tmp_install/log/initdb-template.log 2>&1
+endif
+endif
+endif
+
+# Tasks to run serially at the end of temp-install.  Some EXTRA_INSTALL
+# entries appear more than once in the tree, and parallel installs of the same
+# file can fail with EEXIST.
+checkprep:
+	$(if $(EXTRA_INSTALL),for extra in $(EXTRA_INSTALL); do $(MAKE) -C '$(top_builddir)'/$$extra DESTDIR='$(abs_top_builddir)'/tmp_install install || exit; done)
+
+PROVE = @PROVE@
+# There are common routines in src/test/perl, and some test suites have
+# extra perl modules in their own directory.
+PG_PROVE_FLAGS = -I $(top_srcdir)/src/test/perl/ -I $(srcdir)
+# User-supplied prove flags such as --verbose can be provided in PROVE_FLAGS.
+PROVE_FLAGS =
+
 ifeq ($(enable_tap_tests),yes)
 
 ifndef PGXS
-- 
2.38.0

From fd6c558e6bd43eef40d633ace763d8a2088c2509 Mon Sep 17 00:00:00 2001
From: Thomas Munro <thomas.munro@gmail.com>
Date: Fri, 25 Aug 2023 17:30:48 +1200
Subject: [PATCH] Avoid non-POSIX cp flags.

Commit 252dcb32 introduced cp -a, but apparently Solaris doesn't like
it.  Use cp -RPp instead.

diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 426f94ff09..227c34ab4d 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -549,7 +549,7 @@ sub init
 		}
 		else
 		{
-			@copycmd = qw(cp -a);
+			@copycmd = qw(cp -RPp);
 			$expected_exitcode = 0;
 		}
 
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index 06674141a3..ec67588cf5 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2355,7 +2355,7 @@ regression_main(int argc, char *argv[],
 		else
 		{
 #ifndef WIN32
-			const char *copycmd = "cp -a \"%s\" \"%s/data\"";
+			const char *copycmd = "cp -RPp \"%s\" \"%s/data\"";
 			int			expected_exitcode = 0;
 #else
 			const char *copycmd = "robocopy /E /NJS /NJH /NFL /NDL /NP \"%s\" \"%s/data\"";
-- 
2.39.2

From bd727a7cd560140a1646e3afaba237dfb6a92dc4 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <boekewurm+postgres@gmail.com>
Date: Thu, 7 Dec 2023 14:47:32 +0100
Subject: [PATCH v1] Don't remove initdb template when initdb fails

pg_regress doesn't do that either, so keep a copy around
to allow us to debug those issues.
---
 meson.build            | 2 +-
 src/Makefile.global.in | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/meson.build b/meson.build
index 0095fb183a..5bae1eb04c 100644
--- a/meson.build
+++ b/meson.build
@@ -3112,7 +3112,7 @@ sys.exit(sp.returncode)
 ''',
        test_initdb_template,
        temp_install_bindir / 'initdb',
-       '-A', 'trust', '-N', '--no-instructions', '--no-locale'
+       '-A', 'trust', '-N', '--no-instructions', '--no-locale', '--no-clean'
      ],
      priority: setup_tests_priority - 1,
      timeout: 300,
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index b3ca6392a6..06588b1c47 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -423,7 +423,7 @@ ifeq ($(MAKELEVEL),0)
 	$(MAKE) -C '$(top_builddir)' DESTDIR='$(abs_top_builddir)'/tmp_install install >'$(abs_top_builddir)'/tmp_install/log/install.log 2>&1
 	$(MAKE) -j1 $(if $(CHECKPREP_TOP),-C $(CHECKPREP_TOP),) checkprep >>'$(abs_top_builddir)'/tmp_install/log/install.log 2>&1
 
-	$(with_temp_install) initdb -A trust -N --no-instructions --no-locale '$(abs_top_builddir)'/tmp_install/initdb-template >>'$(abs_top_builddir)'/tmp_install/log/initdb-template.log 2>&1
+	$(with_temp_install) initdb -A trust -N --no-instructions --no-locale --no-clean '$(abs_top_builddir)'/tmp_install/initdb-template >>'$(abs_top_builddir)'/tmp_install/log/initdb-template.log 2>&1
 endif
 endif
 endif
-- 
2.40.1

From a77293d22bc58fe23bdc9bdec64ac9b2f153718b Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <boekewurm+postgres@gmail.com>
Date: Thu, 7 Dec 2023 14:47:32 +0100
Subject: [PATCH v2] Don't remove initdb template when initdb fails

pg_regress doesn't do that either, so keep a copy around
to allow us to debug those issues.

While we're here, use the long options for initdb, to
make this code more self-documenting.

Reviewed-by: Daniel Gustafsson
---
 meson.build            | 3 ++-
 src/Makefile.global.in | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/meson.build b/meson.build
index 0095fb183a..502f511588 100644
--- a/meson.build
+++ b/meson.build
@@ -3112,7 +3112,8 @@ sys.exit(sp.returncode)
 ''',
        test_initdb_template,
        temp_install_bindir / 'initdb',
-       '-A', 'trust', '-N', '--no-instructions', '--no-locale'
+       '--auth', 'trust', '--no-sync', '--no-instructions', '--no-locale',
+       '--no-clean'
      ],
      priority: setup_tests_priority - 1,
      timeout: 300,
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index b3ca6392a6..104e5de0fe 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -423,7 +423,7 @@ ifeq ($(MAKELEVEL),0)
 	$(MAKE) -C '$(top_builddir)' DESTDIR='$(abs_top_builddir)'/tmp_install install >'$(abs_top_builddir)'/tmp_install/log/install.log 2>&1
 	$(MAKE) -j1 $(if $(CHECKPREP_TOP),-C $(CHECKPREP_TOP),) checkprep >>'$(abs_top_builddir)'/tmp_install/log/install.log 2>&1
 
-	$(with_temp_install) initdb -A trust -N --no-instructions --no-locale '$(abs_top_builddir)'/tmp_install/initdb-template >>'$(abs_top_builddir)'/tmp_install/log/initdb-template.log 2>&1
+	$(with_temp_install) initdb --auth trust --no-sync --no-instructions --no-locale --no-clean '$(abs_top_builddir)'/tmp_install/initdb-template >>'$(abs_top_builddir)'/tmp_install/log/initdb-template.log 2>&1
 endif
 endif
 endif
-- 
2.40.1