pg_verifybackup: TAR format backup verification

From ae9064a6eac9029390ca5c0228fcbcafc5ebc4af Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 10:10:08 +0530
Subject: [PATCH v2 07/10] Refactor: split verify_control_file.

Moved verify_control_file doing the control file checks to a separate
function that can be called from other places as well.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 40 ++++++++++-------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 14 ++++++++
 2 files changed, 32 insertions(+), 22 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 7b845bece71..375f196b300 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,7 +18,6 @@
 #include <sys/stat.h>
 #include <time.h>
 
-#include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
@@ -61,8 +60,6 @@ static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
 							   char *relpath, char *fullpath);
-static void verify_control_file(const char *controlpath,
-								uint64 manifest_system_identifier);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -626,14 +623,20 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 	/* Check whether there's an entry in the manifest hash. */
 	m = verify_manifest_entry(context, relpath, sb.st_size);
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0 &&
-		m->matched && !m->bad)
-		verify_control_file(fullpath, context->manifest->system_identifier);
+	/* Validate the manifest system identifier */
+	if (m != NULL && should_verify_sysid(context->manifest, m))
+	{
+		ControlFileData *control_file;
+		bool		crc_ok;
+
+		pg_log_debug("reading \"%s\"", fullpath);
+		control_file = get_controlfile_by_exact_path(fullpath, &crc_ok);
+
+		verify_control_file_data(control_file, fullpath, crc_ok,
+								 context->manifest->system_identifier);
+		/* Release memory. */
+		pfree(control_file);
+	}
 }
 
 /*
@@ -683,15 +686,11 @@ verify_manifest_entry(verifier_context *context, char *relpath, size_t filesize)
  * Sanity check control file and validate system identifier against manifest
  * system identifier.
  */
-static void
-verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
+void
+verify_control_file_data(ControlFileData *control_file,
+						 const char *controlpath, bool crc_ok,
+						 uint64 manifest_system_identifier)
 {
-	ControlFileData *control_file;
-	bool		crc_ok;
-
-	pg_log_debug("reading \"%s\"", controlpath);
-	control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
-
 	/* Control file contents not meaningful if CRC is bad. */
 	if (!crc_ok)
 		report_fatal_error("%s: CRC is incorrect", controlpath);
@@ -707,9 +706,6 @@ verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
 						   controlpath,
 						   (unsigned long long) manifest_system_identifier,
 						   (unsigned long long) control_file->system_identifier);
-
-	/* Release memory. */
-	pfree(control_file);
 }
 
 /*
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 50a285752aa..64508578290 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -16,6 +16,7 @@
 
 #include "common/controldata_utils.h"
 #include "common/hashfn_unstable.h"
+#include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 
@@ -46,6 +47,16 @@ typedef struct manifest_file
 #define should_verify_checksum(m) \
 	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
 
+/*
+ * Validate the manifest system identifier against the control file; this
+ * feature is not available in manifest version 1.  This validation should be
+ * carried out only if the manifest entry validation is completed without any
+ * errors.
+ */
+#define should_verify_sysid(manifest, m) \
+	(((manifest)->version != 1) && \
+	 ((m)->matched) && !((m)->bad) && (strcmp((m)->pathname, "global/pg_control") == 0))
+
 /*
  * Define a hash table which we can use to store information about the files
  * mentioned in the backup manifest.
@@ -105,6 +116,9 @@ extern bool verify_content_checksum(verifier_context *context,
 									pg_checksum_context *checksum_ctx,
 									manifest_file *m, uint8 *buf,
 									int buf_len, size_t *computed_len);
+extern void verify_control_file_data(ControlFileData *control_file,
+									 const char *controlpath, bool crc_ok,
+									 uint64 manifest_system_identifier);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From c3262b852c3a2020535ee4aee6a3296a400ba4b7 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 10:26:35 +0530
Subject: [PATCH v2 08/10] pg_verifybackup: Add backup format and compression
 option

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the next patch that implements tar format support.
----
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 143 +++++++++++++++++++++-
 1 file changed, 141 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 375f196b300..0d458298f34 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,6 +18,7 @@
 #include <sys/stat.h>
 #include <time.h>
 
+#include "common/compression.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
@@ -56,6 +57,8 @@ static void report_manifest_error(JsonManifestParseContext *context,
 								  const char *fmt,...)
 			pg_attribute_printf(2, 3) pg_attribute_noreturn();
 
+static char find_backup_format(verifier_context *context);
+static pg_compress_algorithm find_backup_compression(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
@@ -82,6 +85,9 @@ bool skip_checksums = false;
 static uint64 total_size = 0;
 static uint64 done_size = 0;
 
+char		format = '\0';		/* p(lain)/t(ar) */
+pg_compress_algorithm compress_algorithm = PG_COMPRESSION_NONE;
+
 /*
  * Main entry point.
  */
@@ -92,11 +98,13 @@ main(int argc, char **argv)
 		{"exit-on-error", no_argument, NULL, 'e'},
 		{"ignore", required_argument, NULL, 'i'},
 		{"manifest-path", required_argument, NULL, 'm'},
+		{"format", required_argument, NULL, 'F'},
 		{"no-parse-wal", no_argument, NULL, 'n'},
 		{"progress", no_argument, NULL, 'P'},
 		{"quiet", no_argument, NULL, 'q'},
 		{"skip-checksums", no_argument, NULL, 's'},
 		{"wal-directory", required_argument, NULL, 'w'},
+		{"compress", required_argument, NULL, 'Z'},
 		{NULL, 0, NULL, 0}
 	};
 
@@ -107,6 +115,7 @@ main(int argc, char **argv)
 	bool		quiet = false;
 	char	   *wal_directory = NULL;
 	char	   *pg_waldump_path = NULL;
+	bool		tar_compression_specified = false;
 
 	pg_logging_init(argv[0]);
 	set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_verifybackup"));
@@ -149,7 +158,7 @@ main(int argc, char **argv)
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
 
-	while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1)
+	while ((c = getopt_long(argc, argv, "eF:i:m:nPqsw:Z:", long_options, NULL)) != -1)
 	{
 		switch (c)
 		{
@@ -168,6 +177,15 @@ main(int argc, char **argv)
 				manifest_path = pstrdup(optarg);
 				canonicalize_path(manifest_path);
 				break;
+			case 'F':
+				if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
+					format = 'p';
+				else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
+					format = 't';
+				else
+					pg_fatal("invalid backup format \"%s\", must be \"plain\" or \"tar\"",
+							 optarg);
+				break;
 			case 'n':
 				no_parse_wal = true;
 				break;
@@ -184,6 +202,12 @@ main(int argc, char **argv)
 				wal_directory = pstrdup(optarg);
 				canonicalize_path(wal_directory);
 				break;
+			case 'Z':
+				if (!parse_compress_algorithm(optarg, &compress_algorithm))
+					pg_fatal("unrecognized compression algorithm: \"%s\"",
+							 optarg);
+				tar_compression_specified = true;
+				break;
 			default:
 				/* getopt_long already emitted a complaint */
 				pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -215,11 +239,41 @@ main(int argc, char **argv)
 		pg_fatal("cannot specify both %s and %s",
 				 "-P/--progress", "-q/--quiet");
 
+	/* Complain if compression method specified but the format isn't tar. */
+	if (format != 't' && tar_compression_specified)
+	{
+		pg_log_error("only tar mode backups can be compressed");
+		pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+		exit(1);
+	}
+
+	/* Determine the backup format if it hasn't been specified. */
+	if (format == '\0')
+		format = find_backup_format(&context);
+
+	/*
+	 * Determine the tar backup compression method if it hasn't been
+	 * specified.
+	 */
+	if (format == 't' && !tar_compression_specified)
+		compress_algorithm = find_backup_compression(&context);
+
 	/* Unless --no-parse-wal was specified, we will need pg_waldump. */
 	if (!no_parse_wal)
 	{
 		int			ret;
 
+		/*
+		 * XXX: In the future, we should consider enhancing pg_waldump to read
+		 * WAL files from the tar archive.
+		 */
+		if (format != 'p')
+		{
+			pg_log_error("pg_waldump does not support parsing WAL files from a tar archive.");
+			pg_log_error_hint("Try \"%s --help\" to skip parse WAL files option.", progname);
+			exit(1);
+		}
+
 		pg_waldump_path = pg_malloc(MAXPGPATH);
 		ret = find_other_exec(argv[0], "pg_waldump",
 							  "pg_waldump (PostgreSQL) " PG_VERSION "\n",
@@ -274,8 +328,13 @@ main(int argc, char **argv)
 	/*
 	 * Now do the expensive work of verifying file checksums, unless we were
 	 * told to skip it.
+	 *
+	 * We were only checking the plain backup here. For the tar backup, file
+	 * checksums verification (if requested) will be done immediately when the
+	 * file is accessed, as we don't have random access to the files like we
+	 * do with plain backups.
 	 */
-	if (!skip_checksums)
+	if (!skip_checksums && format == 'p')
 		verify_backup_checksums(&context);
 
 	/*
@@ -1041,6 +1100,84 @@ progress_report(bool finished)
 	fputc((!finished && isatty(fileno(stderr))) ? '\r' : '\n', stderr);
 }
 
+/*
+ * To detect the backup format, it checks for the PG_VERSION file in the backup
+ * directory. If found, it will be considered a plain-format backup; otherwise,
+ * it will be assumed to be a tar-format backup.
+ */
+static char
+find_backup_format(verifier_context *context)
+{
+	char		result;
+	char	   *path;
+	struct stat sb;
+
+	/* Should be here only if the backup format is unknown */
+	Assert(format == '\0');
+
+	/* Check PG_VERSION file. */
+	path = psprintf("%s/%s", context->backup_directory, "PG_VERSION");
+	result = (stat(path, &sb) == 0) ? 'p' : 't';
+	pfree(path);
+
+	return result;
+}
+
+/*
+ * To determine the compression format, we will search for the main data
+ * directory archive and its extension, which starts with base.tar, as
+ * pg_basebackup writes the main data directory to an archive file named
+ * base.tar followed by a compression type extension like .gz, .lz4, or .zst.
+ */
+static pg_compress_algorithm
+find_backup_compression(verifier_context *context)
+{
+	char	   *path;
+	struct stat sb;
+	bool		found;
+
+	/* Should be here only for tar backup */
+	Assert(format == 't');
+
+	/*
+	 * Is this a tar archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_NONE;
+
+	/*
+	 * Is this a .tar.gz archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar.gz");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_GZIP;
+
+	/*
+	 * Is this a .tar.lz4 archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar.lz4");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_LZ4;
+
+	/*
+	 * Is this a .tar.zst archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar.zst");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_ZSTD;
+
+	return PG_COMPRESSION_NONE; /* placate compiler */
+}
+
 /*
  * Print out usage information and exit.
  */
@@ -1053,11 +1190,13 @@ usage(void)
 	printf(_("  -e, --exit-on-error         exit immediately on error\n"));
 	printf(_("  -i, --ignore=RELATIVE_PATH  ignore indicated path\n"));
 	printf(_("  -m, --manifest-path=PATH    use specified path for manifest\n"));
+	printf(_("  -F, --format=p|t            backup format (plain, tar)\n"));
 	printf(_("  -n, --no-parse-wal          do not try to parse WAL files\n"));
 	printf(_("  -P, --progress              show progress information\n"));
 	printf(_("  -q, --quiet                 do not print any output, except for errors\n"));
 	printf(_("  -s, --skip-checksums        skip checksum verification\n"));
 	printf(_("  -w, --wal-directory=PATH    use specified path for WAL files\n"));
+	printf(_("  -Z, --compress=METHOD       compress method (gzip, lz4, zstd, none) \n"));
 	printf(_("  -V, --version               output version information, then exit\n"));
 	printf(_("  -?, --help                  show this help, then exit\n"));
 	printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
-- 
2.18.0

From ce7198153612faf2afc3b1b45d64f331bbd8bae4 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Mon, 17 Jun 2024 14:22:40 +0530
Subject: [PATCH v2 06/10] Refactor: split verify_file_checksum() function.

Move the core functionality of verify_file_checksum to a new function
to enable incremental checksum computation.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 102 +++++++++++++++-------
 src/bin/pg_verifybackup/pg_verifybackup.h |   4 +
 2 files changed, 73 insertions(+), 33 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index d17b565a604..7b845bece71 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -773,6 +773,72 @@ verify_backup_checksums(verifier_context *context)
 	progress_report(true);
 }
 
+/*
+ * It computes the checksum incrementally for the received bytes, requiring the
+ * caller to pass a properly initialized checksum_ctx parameter. Once the
+ * complete file content is received, which is tracked using the computed_len
+ * parameter, it verifies against the manifest data. If any error occurs, it
+ * returns false; otherwise, it returns true to indicate either the complete
+ * file content is yet to be received or checksum verification is completed
+ * successfully.
+ */
+bool
+verify_content_checksum(verifier_context *context,
+						pg_checksum_context *checksum_ctx,
+						manifest_file *m, uint8 *buffer,
+						int buffer_len, size_t *computed_len)
+{
+	char	   *relpath = m->pathname;
+	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
+	int			checksumlen;
+
+	if (pg_checksum_update(checksum_ctx, buffer, buffer_len) < 0)
+	{
+		report_backup_error(context, "could not update checksum of file \"%s\"",
+							relpath);
+		return false;
+	}
+
+	/* Update the total count of computed checksum bytes. */
+	*computed_len += buffer_len;
+
+	/* Report progress */
+	done_size += buffer_len;
+	progress_report(false);
+
+	/* Yet to receive the full content of the file. */
+	if (*computed_len < m->size)
+		return true;
+
+	/* Get the final checksum. */
+	checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
+	if (checksumlen < 0)
+	{
+		report_backup_error(context,
+							"could not finalize checksum of file \"%s\"",
+							relpath);
+		return false;
+	}
+
+	/* And check it against the manifest. */
+	if (checksumlen != m->checksum_length)
+	{
+		report_backup_error(context,
+							"file \"%s\" has checksum of length %d, but expected %d",
+							relpath, m->checksum_length, checksumlen);
+		return false;
+	}
+	else if (memcmp(checksumbuf, m->checksum_payload, checksumlen) != 0)
+	{
+		report_backup_error(context,
+							"checksum mismatch for file \"%s\"",
+							relpath);
+		return false;
+	}
+
+	return true;
+}
+
 /*
  * Verify the checksum of a single file.
  */
@@ -785,8 +851,6 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	int			fd;
 	int			rc;
 	size_t		bytes_read = 0;
-	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
-	int			checksumlen;
 
 	/* Open the target file. */
 	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
@@ -808,19 +872,14 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	/* Read the file chunk by chunk, updating the checksum as we go. */
 	while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
 	{
-		bytes_read += rc;
-		if (pg_checksum_update(&checksum_ctx, buffer, rc) < 0)
+		if (!verify_content_checksum(context, &checksum_ctx, m, buffer, rc,
+									 &bytes_read))
 		{
-			report_backup_error(context, "could not update checksum of file \"%s\"",
-								relpath);
 			close(fd);
 			return;
 		}
-
-		/* Report progress */
-		done_size += rc;
-		progress_report(false);
 	}
+
 	if (rc < 0)
 		report_backup_error(context, "could not read file \"%s\": %m",
 							relpath);
@@ -845,32 +904,9 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	 * filesystem misbehavior.
 	 */
 	if (bytes_read != m->size)
-	{
 		report_backup_error(context,
 							"file \"%s\" should contain %zu bytes, but read %zu bytes",
 							relpath, m->size, bytes_read);
-		return;
-	}
-
-	/* Get the final checksum. */
-	checksumlen = pg_checksum_final(&checksum_ctx, checksumbuf);
-	if (checksumlen < 0)
-	{
-		report_backup_error(context,
-							"could not finalize checksum of file \"%s\"",
-							relpath);
-		return;
-	}
-
-	/* And check it against the manifest. */
-	if (checksumlen != m->checksum_length)
-		report_backup_error(context,
-							"file \"%s\" has checksum of length %d, but expected %d",
-							relpath, m->checksum_length, checksumlen);
-	else if (memcmp(checksumbuf, m->checksum_payload, checksumlen) != 0)
-		report_backup_error(context,
-							"checksum mismatch for file \"%s\"",
-							relpath);
 }
 
 /*
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index c11ff33a100..50a285752aa 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -101,6 +101,10 @@ typedef struct verifier_context
 
 extern manifest_file *verify_manifest_entry(verifier_context *context,
 											char *relpath, size_t filesize);
+extern bool verify_content_checksum(verifier_context *context,
+									pg_checksum_context *checksum_ctx,
+									manifest_file *m, uint8 *buf,
+									int buf_len, size_t *computed_len);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From 0fd6b72e710e08e96230e9b09974044eccdd4d44 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 17:04:56 +0530
Subject: [PATCH v2 10/10] pg_verifybackup: Tests and document

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the previous patch that implements tar format support.
----
---
 doc/src/sgml/ref/pg_verifybackup.sgml         | 54 +++++++++++++-
 src/bin/pg_verifybackup/t/001_basic.pl        | 18 ++++-
 src/bin/pg_verifybackup/t/004_options.pl      |  2 +-
 src/bin/pg_verifybackup/t/005_bad_manifest.pl |  2 +-
 src/bin/pg_verifybackup/t/008_untar.pl        | 73 ++++++-------------
 src/bin/pg_verifybackup/t/010_client_untar.pl | 48 +-----------
 6 files changed, 96 insertions(+), 101 deletions(-)

diff --git a/doc/src/sgml/ref/pg_verifybackup.sgml b/doc/src/sgml/ref/pg_verifybackup.sgml
index a3f167f9f6e..c743bd89a92 100644
--- a/doc/src/sgml/ref/pg_verifybackup.sgml
+++ b/doc/src/sgml/ref/pg_verifybackup.sgml
@@ -34,8 +34,10 @@ PostgreSQL documentation
    integrity of a database cluster backup taken using
    <command>pg_basebackup</command> against a
    <literal>backup_manifest</literal> generated by the server at the time
-   of the backup.  The backup must be stored in the "plain"
-   format; a "tar" format backup can be checked after extracting it.
+   of the backup.  The backup must be stored in the "plain" or "tar"
+   format.  Verification is supported for <literal>gzip</literal>,
+   <literal>lz4</literal>, and  <literal>zstd</literal> compressed tar backup;
+   any other compressed format backups can be checked after decompressing them.
   </para>
 
   <para>
@@ -168,6 +170,42 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-F <replaceable class="parameter">format</replaceable></option></term>
+      <term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
+      <listitem>
+       <para>
+        Specifies the format of the backup. <replaceable>format</replaceable>
+        can be one of the following:
+
+        <variablelist>
+         <varlistentry>
+          <term><literal>p</literal></term>
+          <term><literal>plain</literal></term>
+          <listitem>
+           <para>
+            Backup consists of plain files with the same layout as the
+            source server's data directory and tablespaces.
+           </para>
+          </listitem>
+         </varlistentry>
+
+         <varlistentry>
+          <term><literal>t</literal></term>
+          <term><literal>tar</literal></term>
+          <listitem>
+           <para>
+            Backup consists of tar files. The main data directory's contents
+            will be written to a file named <filename>base.tar</filename>,
+            and each other tablespace will be written to a separate tar file
+            named after that tablespace's OID.
+           </para>
+           </listitem>
+         </varlistentry>
+        </variablelist></para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-n</option></term>
       <term><option>--no-parse-wal</option></term>
@@ -227,6 +265,18 @@ PostgreSQL documentation
        </para>
       </listitem>
      </varlistentry>
+
+     <varlistentry>
+      <term><option>-Z <replaceable class="parameter">method</replaceable></option></term>
+      <term><option>--compress=<replaceable class="parameter">method</replaceable></option></term>
+      <listitem>
+       <para>
+        The tar backup compression method can be <literal>gzip</literal>,
+        <literal>lz4</literal>, <literal>zstd</literal>, or
+        <literal>none</literal> if no compression.
+       </para>
+      </listitem>
+     </varlistentry>
     </variablelist>
    </para>
 
diff --git a/src/bin/pg_verifybackup/t/001_basic.pl b/src/bin/pg_verifybackup/t/001_basic.pl
index 2f3e52d296f..d47ce1f04fc 100644
--- a/src/bin/pg_verifybackup/t/001_basic.pl
+++ b/src/bin/pg_verifybackup/t/001_basic.pl
@@ -17,13 +17,25 @@ command_fails_like(
 	qr/no backup directory specified/,
 	'target directory must be specified');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir ],
 	qr/could not open file.*\/backup_manifest\"/,
 	'pg_verifybackup requires a manifest');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir, $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir, $tempdir ],
 	qr/too many command-line arguments/,
 	'multiple target directories not allowed');
+command_fails_like(
+	[ 'pg_verifybackup', '-Zgzip', $tempdir ],
+	qr/only tar mode backups can be compressed/,
+	'compression method required format option');
+command_fails_like(
+	[ 'pg_verifybackup', '-Fp', '-Zlz4', $tempdir ],
+	qr/only tar mode backups can be compressed/,
+	'compression method required tar format option');
+command_fails_like(
+	[ 'pg_verifybackup', '-Fp', '-Znon_exist', $tempdir ],
+	qr/unrecognized compression algorithm/,
+	'compression method should be valid');
 
 # create fake manifest file
 open(my $fh, '>', "$tempdir/backup_manifest") || die "open: $!";
@@ -31,7 +43,7 @@ close($fh);
 
 # but then try to use an alternate, nonexisting manifest
 command_fails_like(
-	[ 'pg_verifybackup', '-m', "$tempdir/not_the_manifest", $tempdir ],
+	[ 'pg_verifybackup', '-Fp', '-m', "$tempdir/not_the_manifest", $tempdir ],
 	qr/could not open file.*\/not_the_manifest\"/,
 	'pg_verifybackup respects -m flag');
 
diff --git a/src/bin/pg_verifybackup/t/004_options.pl b/src/bin/pg_verifybackup/t/004_options.pl
index 8ed2214408e..2f197648740 100644
--- a/src/bin/pg_verifybackup/t/004_options.pl
+++ b/src/bin/pg_verifybackup/t/004_options.pl
@@ -108,7 +108,7 @@ unlike(
 # Test valid manifest with nonexistent backup directory.
 command_fails_like(
 	[
-		'pg_verifybackup', '-m',
+		'pg_verifybackup', '-Fp', '-m',
 		"$backup_path/backup_manifest", "$backup_path/fake"
 	],
 	qr/could not open directory/,
diff --git a/src/bin/pg_verifybackup/t/005_bad_manifest.pl b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
index c4ed64b62d5..28c51b6feb0 100644
--- a/src/bin/pg_verifybackup/t/005_bad_manifest.pl
+++ b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
@@ -208,7 +208,7 @@ sub test_bad_manifest
 	print $fh $manifest_contents;
 	close($fh);
 
-	command_fails_like([ 'pg_verifybackup', $tempdir ], $regexp, $test_name);
+	command_fails_like([ 'pg_verifybackup', '-Fp', $tempdir ], $regexp, $test_name);
 	return;
 }
 
diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
index 7a09f3b75b2..9896560adc3 100644
--- a/src/bin/pg_verifybackup/t/008_untar.pl
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -16,6 +16,20 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 
+# Create a couple of directories to use as tablespaces.
+my $TS1_LOCATION = $primary->backup_dir .'/ts1';
+$TS1_LOCATION =~ s/\/\.\//\//g;    # collapse foo/./bar to foo/bar
+mkdir($TS1_LOCATION);
+
+# Create a tablespace with table in it.
+$primary->safe_psql('postgres', qq(
+		CREATE TABLESPACE regress_ts1 LOCATION '$TS1_LOCATION';
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1';
+		CREATE TABLE regress_tbl1(i int) TABLESPACE regress_ts1;
+		INSERT INTO regress_tbl1 VALUES(generate_series(1,5));));
+my $tsoid = $primary->safe_psql('postgres', qq(
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1'));
+
 my $backup_path = $primary->backup_dir . '/server-backup';
 my $extract_path = $primary->backup_dir . '/extracted-backup';
 
@@ -23,39 +37,31 @@ my @test_configuration = (
 	{
 		'compression_method' => 'none',
 		'backup_flags' => [],
-		'backup_archive' => 'base.tar',
+		'backup_archive' => ['base.tar', "$tsoid.tar"],
 		'enabled' => 1
 	},
 	{
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'server-gzip' ],
-		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.gz', "$tsoid.tar.gz" ],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'server-lz4' ],
-		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => [ '-d', '-m' ],
+		'backup_archive' => ['base.tar.lz4', "$tsoid.tar.lz4" ],
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd:level=1,long' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	});
 
@@ -86,47 +92,16 @@ for my $tc (@test_configuration)
 		my $backup_files = join(',',
 			sort grep { $_ ne '.' && $_ ne '..' } slurp_dir($backup_path));
 		my $expected_backup_files =
-		  join(',', sort ('backup_manifest', $tc->{'backup_archive'}));
+		  join(',', sort ('backup_manifest', @{ $tc->{'backup_archive'} }));
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok(['pg_verifybackup', '-n', '-e', $backup_path],
+			"verify backup, compression $method");
 
 		# Cleanup.
-		unlink($backup_path . '/backup_manifest');
-		unlink($backup_path . '/base.tar');
-		unlink($backup_path . '/' . $tc->{'backup_archive'});
+		rmtree($backup_path);
 		rmtree($extract_path);
 	}
 }
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index 8c076d46dee..6b7d7483f6e 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -29,41 +29,30 @@ my @test_configuration = (
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'client-gzip:5' ],
 		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'client-lz4:5' ],
 		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => ['-d'],
-		'output_file' => 'base.tar',
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:5' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:level=1,long' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'parallel zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:workers=3' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1"),
 		'possibly_unsupported' =>
 		  qr/could not set compression worker count to 3: Unsupported parameter/
@@ -118,40 +107,9 @@ for my $tc (@test_configuration)
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			push @decompress, $backup_path . '/' . $tc->{'output_file'}
-			  if $tc->{'output_file'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok( [ 'pg_verifybackup', '-n', '-e', $backup_path ],
+			"verify backup, compression $method");
 
 		# Cleanup.
 		rmtree($extract_path);
-- 
2.18.0

From 15d61fbcceb84d8b8c780309a58893a88a197411 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Mon, 17 Jun 2024 17:11:26 +0530
Subject: [PATCH v2 09/10] pg_verifybackup: Read tar files and verify its
 contents

---
 src/bin/pg_verifybackup/Makefile           |   4 +-
 src/bin/pg_verifybackup/astreamer_verify.c | 250 +++++++++++++++++++++
 src/bin/pg_verifybackup/meson.build        |   6 +-
 src/bin/pg_verifybackup/pg_verifybackup.c  | 220 +++++++++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h  |  10 +
 src/tools/pgindent/typedefs.list           |   1 +
 6 files changed, 480 insertions(+), 11 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/astreamer_verify.c

diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile
index 7c045f142e8..df7aaabd530 100644
--- a/src/bin/pg_verifybackup/Makefile
+++ b/src/bin/pg_verifybackup/Makefile
@@ -17,11 +17,13 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 # We need libpq only because fe_utils does.
+override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 
 OBJS = \
 	$(WIN32RES) \
-	pg_verifybackup.o
+	pg_verifybackup.o \
+	astreamer_verify.o
 
 all: pg_verifybackup
 
diff --git a/src/bin/pg_verifybackup/astreamer_verify.c b/src/bin/pg_verifybackup/astreamer_verify.c
new file mode 100644
index 00000000000..9be9a9bc04a
--- /dev/null
+++ b/src/bin/pg_verifybackup/astreamer_verify.c
@@ -0,0 +1,250 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer_verify.c
+ *
+ * Extend fe_utils/astreamer.h archive streaming facility to verify TAR
+ * backup.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * src/bin/pg_verifybackup/astreamer_verify.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include "common/logging.h"
+#include "fe_utils/astreamer.h"
+#include "pg_verifybackup.h"
+
+typedef struct astreamer_verify
+{
+	astreamer	base;
+	verifier_context *context;
+	char	   *archive_name;
+	Oid			tblspc_oid;
+
+	manifest_file *mfile;
+	size_t		received_bytes;
+	bool		verify_checksums;
+	bool		verify_sysid;
+	pg_checksum_context *checksum_ctx;
+} astreamer_verify;
+
+static void astreamer_verify_content(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len,
+									 astreamer_archive_context context);
+static void astreamer_verify_finalize(astreamer *streamer);
+static void astreamer_verify_free(astreamer *streamer);
+
+static const astreamer_ops astreamer_verify_ops = {
+	.content = astreamer_verify_content,
+	.finalize = astreamer_verify_finalize,
+	.free = astreamer_verify_free
+};
+
+/*
+ * Create a astreamer that can verifies content of a TAR file.
+ */
+astreamer *
+astreamer_verify_content_new(astreamer *next, verifier_context *context,
+							 char *archive_name, Oid tblspc_oid)
+{
+	astreamer_verify *streamer;
+
+	streamer = palloc0(sizeof(astreamer_verify));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_verify_ops;
+
+	streamer->base.bbs_next = next;
+	streamer->context = context;
+	streamer->archive_name = archive_name;
+	streamer->tblspc_oid = tblspc_oid;
+	initStringInfo(&streamer->base.bbs_buffer);
+
+	return &streamer->base;
+}
+
+/*
+ * It verifies each TAR member entry against the manifest data and performs
+ * checksum verification if enabled. Additionally, it validates the backup's
+ * system identifier against the backup_manifest.
+ */
+static void
+astreamer_verify_content(astreamer *streamer,
+						 astreamer_member *member, const char *data,
+						 int len, astreamer_archive_context context)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	Assert(context != ASTREAMER_UNKNOWN);
+
+	switch (context)
+	{
+		case ASTREAMER_MEMBER_HEADER:
+			if (!member->is_directory && !member->is_link &&
+				!should_ignore_relpath(mystreamer->context, member->pathname))
+			{
+				manifest_file *m;
+
+				/*
+				 * The backup_manifest stores a relative path to the base
+				 * directory for files belong tablespace, whereas
+				 * <tablespaceoid>.tar doesn't. Prepare the required path,
+				 * otherwise, the manfiest entry verification will fail.
+				 */
+				if (OidIsValid(mystreamer->tblspc_oid))
+				{
+					char		temp[MAXPGPATH];
+
+					/* Copy original name at temporary space */
+					memcpy(temp, member->pathname, MAXPGPATH);
+
+					snprintf(member->pathname, MAXPGPATH, "%s/%d/%s",
+							 "pg_tblspc", mystreamer->tblspc_oid, temp);
+				}
+
+				/* Check the manifest entry */
+				m = verify_manifest_entry(mystreamer->context, member->pathname,
+										  member->size);
+				mystreamer->mfile = (void *) m;
+
+				/*
+				 * Prepare for checksum and manifest system identifier
+				 * verification.
+				 *
+				 * We could have these checks while receiving contents.
+				 * However, since contents are received in multiple iterations,
+				 * this would result in these lengthy checks being performed
+				 * multiple times. Instead, having a single flag would be more
+				 * efficient.
+				 */
+				if (m != NULL)
+				{
+					mystreamer->verify_checksums =
+						(!skip_checksums && should_verify_checksum(m));
+					mystreamer->verify_sysid =
+						should_verify_sysid(mystreamer->context->manifest, m);
+				}
+			}
+			break;
+
+		case ASTREAMER_MEMBER_CONTENTS:
+
+			/*
+			 * Perform checksum verification as the file content becomes
+			 * available, since the TAR format does not have random access to
+			 * files like a normal backup directory, where checksum verification
+			 * occurs at different points.
+			 */
+			if (mystreamer->verify_checksums)
+			{
+				/* If we were first time for this file */
+				if (!mystreamer->checksum_ctx)
+				{
+					mystreamer->checksum_ctx = pg_malloc(sizeof(pg_checksum_context));
+
+					if (pg_checksum_init(mystreamer->checksum_ctx,
+										 mystreamer->mfile->checksum_type) < 0)
+					{
+						report_backup_error(mystreamer->context,
+											"%s: could not initialize checksum of file \"%s\"",
+											mystreamer->archive_name, member->pathname);
+						mystreamer->verify_checksums = false;
+						return;
+					}
+				}
+
+				/* Compute and do the checksum validation */
+				mystreamer->verify_checksums =
+					verify_content_checksum(mystreamer->context,
+											mystreamer->checksum_ctx,
+											mystreamer->mfile,
+											(uint8 *) data, len,
+											&mystreamer->received_bytes);
+			}
+
+			/* Do the manifest system identifier verification */
+			if (mystreamer->verify_sysid)
+			{
+				ControlFileData control_file;
+				uint64		manifest_system_identifier;
+				pg_crc32c	crc;
+				bool		crc_ok;
+
+				/* Should be here only for control file */
+				Assert(strcmp(member->pathname, "global/pg_control") == 0);
+				Assert(mystreamer->context->manifest->version != 1);
+
+				/* Should have whole control file data. */
+				if (!astreamer_buffer_until(streamer, &data, &len,
+											sizeof(ControlFileData)))
+					return;
+
+				pg_log_debug("%s: reading \"%s\"", mystreamer->archive_name,
+							 member->pathname);
+
+				if (streamer->bbs_buffer.len != sizeof(ControlFileData))
+					report_fatal_error("%s: could not read control file: read %d of %zu",
+									   mystreamer->archive_name, streamer->bbs_buffer.len,
+									   sizeof(ControlFileData));
+
+				memcpy(&control_file, streamer->bbs_buffer.data,
+					   sizeof(ControlFileData));
+
+				/* Check the CRC. */
+				INIT_CRC32C(crc);
+				COMP_CRC32C(crc,
+							(char *) (&control_file),
+							offsetof(ControlFileData, crc));
+				FIN_CRC32C(crc);
+
+				crc_ok = EQ_CRC32C(crc, control_file.crc);
+
+				manifest_system_identifier =
+					mystreamer->context->manifest->system_identifier;
+
+				verify_control_file_data(&control_file, member->pathname,
+										 crc_ok, manifest_system_identifier);
+			}
+			break;
+
+		case ASTREAMER_MEMBER_TRAILER:
+			if (mystreamer->checksum_ctx)
+				pfree(mystreamer->checksum_ctx);
+			mystreamer->checksum_ctx = NULL;
+			mystreamer->mfile = NULL;
+			mystreamer->received_bytes = 0;
+			mystreamer->verify_checksums = false;
+			mystreamer->verify_sysid = false;
+			break;
+
+		case ASTREAMER_ARCHIVE_TRAILER:
+			break;
+
+		default:
+			/* Shouldn't happen. */
+			pg_fatal("unexpected state while parsing tar archive");
+	}
+}
+
+/*
+ * End-of-stream processing for a astreamer_verify stream.
+ */
+static void
+astreamer_verify_finalize(astreamer *streamer)
+{
+	Assert(streamer->bbs_next == NULL);
+}
+
+/*
+ * Free memory associated with a astreamer_verify stream.
+ */
+static void
+astreamer_verify_free(astreamer *streamer)
+{
+	pfree(streamer->bbs_buffer.data);
+	pfree(streamer);
+}
diff --git a/src/bin/pg_verifybackup/meson.build b/src/bin/pg_verifybackup/meson.build
index 7c7d31a0350..1e3fcf7ee5a 100644
--- a/src/bin/pg_verifybackup/meson.build
+++ b/src/bin/pg_verifybackup/meson.build
@@ -1,7 +1,8 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 pg_verifybackup_sources = files(
-  'pg_verifybackup.c'
+  'pg_verifybackup.c',
+  'astreamer_verify.c'
 )
 
 if host_system == 'windows'
@@ -10,9 +11,10 @@ if host_system == 'windows'
     '--FILEDESC', 'pg_verifybackup - verify a backup against using a backup manifest'])
 endif
 
+pg_verifybackup_deps = [frontend_code, libpq, lz4, zlib, zstd]
 pg_verifybackup = executable('pg_verifybackup',
   pg_verifybackup_sources,
-  dependencies: [frontend_code, libpq],
+  dependencies: pg_verifybackup_deps,
   kwargs: default_bin_args,
 )
 bin_targets += pg_verifybackup
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 0d458298f34..21b16c281cb 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -20,9 +20,12 @@
 
 #include "common/compression.h"
 #include "common/parse_manifest.h"
+#include "common/relpath.h"
+#include "fe_utils/astreamer.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
 #include "pg_verifybackup.h"
+#include "pgtar.h"
 #include "pgtime.h"
 
 /*
@@ -39,6 +42,11 @@
  */
 #define ESTIMATED_BYTES_PER_MANIFEST_LINE	100
 
+
+static void (*verify_backup_file_cb) (verifier_context *context,
+									  char *relpath, char *fullpath,
+									  size_t filesize);
+
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -63,6 +71,15 @@ static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
 							   char *relpath, char *fullpath);
+static void verify_plain_file_cb(verifier_context *context,
+								 char *relpath, char *fullpath,
+								 size_t filesize);
+static void verify_tar_file_cb(verifier_context *context,
+							   char *relpath, char *fullpath,
+							   size_t filesize);
+static void verify_tar_content(verifier_context *context,
+							   char *relpath, char *fullpath,
+							   astreamer *streamer);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -71,6 +88,9 @@ static void verify_file_checksum(verifier_context *context,
 static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
+static astreamer *create_archive_verifier(verifier_context *context,
+										  char *archive_name,
+										  Oid tblspc_oid);
 
 static void progress_report(bool finished);
 static void usage(void);
@@ -154,6 +174,10 @@ main(int argc, char **argv)
 	 */
 	simple_string_list_append(&context.ignore_list, "backup_manifest");
 	simple_string_list_append(&context.ignore_list, "pg_wal");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.gz");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.lz4");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.zst");
 	simple_string_list_append(&context.ignore_list, "postgresql.auto.conf");
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
@@ -258,6 +282,15 @@ main(int argc, char **argv)
 	if (format == 't' && !tar_compression_specified)
 		compress_algorithm = find_backup_compression(&context);
 
+	/*
+	 * Setup the required callback function to verify plain or tar backup
+	 * files.
+	 */
+	if (format == 'p')
+		verify_backup_file_cb = verify_plain_file_cb;
+	else
+		verify_backup_file_cb = verify_tar_file_cb;
+
 	/* Unless --no-parse-wal was specified, we will need pg_waldump. */
 	if (!no_parse_wal)
 	{
@@ -637,7 +670,8 @@ verify_backup_directory(verifier_context *context, char *relpath,
 }
 
 /*
- * Verify one file (which might actually be a directory or a symlink).
+ * Verify one file (which might actually be a directory, a symlink or a
+ * archive).
  *
  * The arguments to this function have the same meaning as the arguments to
  * verify_backup_directory.
@@ -646,7 +680,6 @@ static void
 verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 {
 	struct stat sb;
-	manifest_file *m;
 
 	if (stat(fullpath, &sb) != 0)
 	{
@@ -679,8 +712,25 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		return;
 	}
 
+	/* Do the further verifications */
+	verify_backup_file_cb(context, relpath, fullpath, sb.st_size);
+}
+
+/*
+ * Verify one plan file or a symlink.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_backup_directory. The additional argument is the file size for
+ * verifying against manifest entry.
+ */
+static void
+verify_plain_file_cb(verifier_context *context, char *relpath,
+					 char *fullpath, size_t filesize)
+{
+	manifest_file *m;
+
 	/* Check whether there's an entry in the manifest hash. */
-	m = verify_manifest_entry(context, relpath, sb.st_size);
+	m = verify_manifest_entry(context, relpath, filesize);
 
 	/* Validate the manifest system identifier */
 	if (m != NULL && should_verify_sysid(context->manifest, m))
@@ -698,6 +748,124 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 	}
 }
 
+/*
+ * Verify one tar file.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_backup_directory. The additional argument is the file size for
+ * verifying against manifest entry.
+ */
+static void
+verify_tar_file_cb(verifier_context *context, char *relpath,
+				   char *fullpath, size_t filesize)
+{
+	astreamer  *streamer;
+	Oid			tblspc_oid = InvalidOid;
+	int			file_name_len;
+	int			file_extn_len = 0;	/* placate compiler */
+	char	   *file_extn = "";
+
+	/* Should be tar backup */
+	Assert(format == 't');
+
+	/* Find the tar file extension. */
+	if (compress_algorithm == PG_COMPRESSION_NONE)
+	{
+		file_extn = ".tar";
+		file_extn_len = 4;
+
+	}
+	else if (compress_algorithm == PG_COMPRESSION_GZIP)
+	{
+		file_extn = ".tar.gz";
+		file_extn_len = 7;
+
+	}
+	else if (compress_algorithm == PG_COMPRESSION_LZ4)
+	{
+		file_extn = ".tar.lz4";
+		file_extn_len = 8;
+	}
+	else if (compress_algorithm == PG_COMPRESSION_ZSTD)
+	{
+		file_extn = ".tar.zst";
+		file_extn_len = 8;
+	}
+
+	/*
+	 * Ensure that we have the correct file type corresponding to the backup
+	 * format.
+	 */
+	file_name_len = strlen(relpath);
+	if (file_name_len < file_extn_len ||
+		strcmp(relpath + file_name_len - file_extn_len, file_extn) != 0)
+	{
+		if (compress_algorithm == PG_COMPRESSION_NONE)
+			report_backup_error(context,
+								"\"%s\" is not a valid file, expecting tar file",
+								relpath);
+		else
+			report_backup_error(context,
+								"\"%s\" is not a valid file, expecting \"%s\" compressed tar file",
+								relpath,
+								get_compress_algorithm_name(compress_algorithm));
+		return;
+	}
+
+	/*
+	 * For the tablespace, pg_basebackup writes the data out to
+	 * <tablespaceoid>.tar.  If a file matches that format, then extract the
+	 * tablespaceoid, which we need to prepare the paths of the files
+	 * belonging to that tablespace relative to the base directory.
+	 */
+	if (strspn(relpath, "0123456789") == (file_name_len - file_extn_len))
+		tblspc_oid = strtoi64(relpath, NULL, 10);
+
+	streamer = create_archive_verifier(context, relpath, tblspc_oid);
+	verify_tar_content(context, relpath, fullpath, streamer);
+
+	/* Cleanup. */
+	astreamer_finalize(streamer);
+	astreamer_free(streamer);
+}
+
+/*
+ * Reads a given tar file in predefined chunks and pass to astreamer.  Which
+ * initiates routines for decompression (if necessary) then verification
+ * of each member within the tar archive.
+ */
+static void
+verify_tar_content(verifier_context *context, char *relpath, char *fullpath,
+				   astreamer *streamer)
+{
+	int			fd;
+	int			rc;
+	char	   *buffer;
+
+	/* Open the target file. */
+	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
+	{
+		report_backup_error(context, "could not open file \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
+
+	/* Perform the reads */
+	while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
+		astreamer_content(streamer, NULL, buffer, rc, ASTREAMER_UNKNOWN);
+
+	if (rc < 0)
+		report_backup_error(context, "could not read file \"%s\": %m",
+							relpath);
+
+	/* Close the file. */
+	if (close(fd) != 0)
+		report_backup_error(context, "could not close file \"%s\": %m",
+							relpath);
+}
+
 /*
  * Verify file and its size entry in the manifest.
  */
@@ -742,8 +910,8 @@ verify_manifest_entry(verifier_context *context, char *relpath, size_t filesize)
 }
 
 /*
- * Sanity check control file and validate system identifier against manifest
- * system identifier.
+ * Sanity check control file data and validate system identifier against
+ * manifest system identifier.
  */
 void
 verify_control_file_data(ControlFileData *control_file,
@@ -1124,10 +1292,10 @@ find_backup_format(verifier_context *context)
 }
 
 /*
- * To determine the compression format, we will search for the main data
- * directory archive and its extension, which starts with base.tar, as
  * pg_basebackup writes the main data directory to an archive file named
- * base.tar followed by a compression type extension like .gz, .lz4, or .zst.
+ * base.tar, followed by a compression type extension such as .gz, .lz4, or
+ * .zst. To determine the compression format, we need to search for this main
+ * data directory archive file.
  */
 static pg_compress_algorithm
 find_backup_compression(verifier_context *context)
@@ -1178,6 +1346,42 @@ find_backup_compression(verifier_context *context)
 	return PG_COMPRESSION_NONE; /* placate compiler */
 }
 
+/*
+ * Identifies the necessary steps for verifying the contents of the
+ * provided tar file.
+ */
+static astreamer *
+create_archive_verifier(verifier_context *context, char *archive_name,
+						Oid tblspc_oid)
+{
+	astreamer  *streamer = NULL;
+
+	/* Should be here only for tar backup */
+	Assert(format == 't');
+
+	/*
+	 * To verify the contents of the tar file, the initial step is to parse
+	 * its content.
+	 */
+	streamer = astreamer_verify_content_new(streamer, context, archive_name,
+											tblspc_oid);
+	streamer = astreamer_tar_parser_new(streamer);
+
+	/*
+	 * If the tar file is compressed, we must perform the appropriate
+	 * decompression operation before proceeding with the verification of its
+	 * contents.
+	 */
+	if (compress_algorithm == PG_COMPRESSION_GZIP)
+		streamer = astreamer_gzip_decompressor_new(streamer);
+	else if (compress_algorithm == PG_COMPRESSION_LZ4)
+		streamer = astreamer_lz4_decompressor_new(streamer);
+	else if (compress_algorithm == PG_COMPRESSION_ZSTD)
+		streamer = astreamer_zstd_decompressor_new(streamer);
+
+	return streamer;
+}
+
 /*
  * Print out usage information and exit.
  */
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 64508578290..e27fe4da6a2 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -127,4 +127,14 @@ extern void report_fatal_error(const char *pg_restrict fmt,...)
 			pg_attribute_printf(1, 2) pg_attribute_noreturn();
 extern bool should_ignore_relpath(verifier_context *context, const char *relpath);
 
+/* Forward declarations to avoid fe_utils/astreamer.h include. */
+struct astreamer;
+typedef struct astreamer astreamer;
+
+extern astreamer *astreamer_verify_content_new(astreamer *next,
+											   verifier_context *context,
+											   char *archive_name,
+											   Oid tblspc_oid);
+
+
 #endif /* PG_VERIFYBACKUP_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index b982dffa5fc..ab279c9eb39 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3321,6 +3321,7 @@ astreamer_plain_writer
 astreamer_recovery_injector
 astreamer_tar_archiver
 astreamer_tar_parser
+astreamer_verify
 astreamer_zstd_frame
 bgworker_main_type
 bh_node_type
-- 
2.18.0

From b34e3ecffa493b8037416805106c931105d9a9a3 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 9 Jul 2024 10:30:31 +0530
Subject: [PATCH v2 03/10] Refactor: move astreamer* files to fe_utils to make
 common availability of it.

To make it accessible to other code, we need to move the ASTREAMER
code (previously known as BBSTREAMER) to a common location. The
appropriate place would be src/fe_utils, as it is a frontend
infrastructure intended for shared use.
---
 src/bin/pg_basebackup/Makefile                        |  7 +------
 src/bin/pg_basebackup/astreamer_inject.h              |  2 +-
 src/bin/pg_basebackup/meson.build                     |  5 -----
 src/fe_utils/Makefile                                 | 11 +++++++++--
 src/{bin/pg_basebackup => fe_utils}/astreamer_file.c  |  2 +-
 src/{bin/pg_basebackup => fe_utils}/astreamer_gzip.c  |  2 +-
 src/{bin/pg_basebackup => fe_utils}/astreamer_lz4.c   |  2 +-
 src/{bin/pg_basebackup => fe_utils}/astreamer_tar.c   |  2 +-
 src/{bin/pg_basebackup => fe_utils}/astreamer_zstd.c  |  2 +-
 src/fe_utils/meson.build                              |  7 ++++++-
 .../pg_basebackup => include/fe_utils}/astreamer.h    |  0
 11 files changed, 22 insertions(+), 20 deletions(-)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_file.c (99%)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_gzip.c (99%)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_lz4.c (99%)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_tar.c (99%)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_zstd.c (99%)
 rename src/{bin/pg_basebackup => include/fe_utils}/astreamer.h (100%)

diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile
index a71af2d48a7..f1e73058b23 100644
--- a/src/bin/pg_basebackup/Makefile
+++ b/src/bin/pg_basebackup/Makefile
@@ -37,12 +37,7 @@ OBJS = \
 
 BBOBJS = \
 	pg_basebackup.o \
-	astreamer_file.o \
-	astreamer_gzip.o \
-	astreamer_inject.o \
-	astreamer_lz4.o \
-	astreamer_tar.o \
-	astreamer_zstd.o
+	astreamer_inject.o
 
 all: pg_basebackup pg_createsubscriber pg_receivewal pg_recvlogical
 
diff --git a/src/bin/pg_basebackup/astreamer_inject.h b/src/bin/pg_basebackup/astreamer_inject.h
index 8504b3f5e0d..aeed533862b 100644
--- a/src/bin/pg_basebackup/astreamer_inject.h
+++ b/src/bin/pg_basebackup/astreamer_inject.h
@@ -12,7 +12,7 @@
 #ifndef ASTREAMER_INJECT_H
 #define ASTREAMER_INJECT_H
 
-#include "astreamer.h"
+#include "fe_utils/astreamer.h"
 #include "pqexpbuffer.h"
 
 extern astreamer *astreamer_recovery_injector_new(astreamer *next,
diff --git a/src/bin/pg_basebackup/meson.build b/src/bin/pg_basebackup/meson.build
index a68dbd7837d..9101fc18438 100644
--- a/src/bin/pg_basebackup/meson.build
+++ b/src/bin/pg_basebackup/meson.build
@@ -1,12 +1,7 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 common_sources = files(
-  'astreamer_file.c',
-  'astreamer_gzip.c',
   'astreamer_inject.c',
-  'astreamer_lz4.c',
-  'astreamer_tar.c',
-  'astreamer_zstd.c',
   'receivelog.c',
   'streamutil.c',
   'walmethods.c',
diff --git a/src/fe_utils/Makefile b/src/fe_utils/Makefile
index 946c05258f0..ff002f37d57 100644
--- a/src/fe_utils/Makefile
+++ b/src/fe_utils/Makefile
@@ -34,13 +34,20 @@ OBJS = \
 	simple_list.o \
 	string_utils.o
 
+AOBJS = \
+	astreamer_file.o \
+	astreamer_gzip.o \
+	astreamer_lz4.o \
+	astreamer_tar.o \
+	astreamer_zstd.o
+
 ifeq ($(PORTNAME), win32)
 override CPPFLAGS += -DFD_SETSIZE=1024
 endif
 
 all: libpgfeutils.a
 
-libpgfeutils.a: $(OBJS)
+libpgfeutils.a: $(AOBJS) $(OBJS)
 	rm -f $@
 	$(AR) $(AROPT) $@ $^
 
@@ -59,5 +66,5 @@ uninstall:
 	rm -f '$(DESTDIR)$(libdir)/libpgfeutils.a'
 
 clean distclean:
-	rm -f libpgfeutils.a $(OBJS) lex.backup
+	rm -f libpgfeutils.a $(AOBJS) $(OBJS) lex.backup
 	rm -f psqlscan.c
diff --git a/src/bin/pg_basebackup/astreamer_file.c b/src/fe_utils/astreamer_file.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_file.c
rename to src/fe_utils/astreamer_file.c
index 2742385e103..13d1192c6e6 100644
--- a/src/bin/pg_basebackup/astreamer_file.c
+++ b/src/fe_utils/astreamer_file.c
@@ -13,10 +13,10 @@
 
 #include <unistd.h>
 
-#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
+#include "fe_utils/astreamer.h"
 
 typedef struct astreamer_plain_writer
 {
diff --git a/src/bin/pg_basebackup/astreamer_gzip.c b/src/fe_utils/astreamer_gzip.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_gzip.c
rename to src/fe_utils/astreamer_gzip.c
index 6f7c27afbbc..dd28defac7b 100644
--- a/src/bin/pg_basebackup/astreamer_gzip.c
+++ b/src/fe_utils/astreamer_gzip.c
@@ -17,10 +17,10 @@
 #include <zlib.h>
 #endif
 
-#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
+#include "fe_utils/astreamer.h"
 
 #ifdef HAVE_LIBZ
 typedef struct astreamer_gzip_writer
diff --git a/src/bin/pg_basebackup/astreamer_lz4.c b/src/fe_utils/astreamer_lz4.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_lz4.c
rename to src/fe_utils/astreamer_lz4.c
index 1c40d7d8ad5..d8b2a367e47 100644
--- a/src/bin/pg_basebackup/astreamer_lz4.c
+++ b/src/fe_utils/astreamer_lz4.c
@@ -17,10 +17,10 @@
 #include <lz4frame.h>
 #endif
 
-#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
+#include "fe_utils/astreamer.h"
 
 #ifdef USE_LZ4
 typedef struct astreamer_lz4_frame
diff --git a/src/bin/pg_basebackup/astreamer_tar.c b/src/fe_utils/astreamer_tar.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_tar.c
rename to src/fe_utils/astreamer_tar.c
index 673690cd18f..f5d3562d280 100644
--- a/src/bin/pg_basebackup/astreamer_tar.c
+++ b/src/fe_utils/astreamer_tar.c
@@ -23,8 +23,8 @@
 
 #include <time.h>
 
-#include "astreamer.h"
 #include "common/logging.h"
+#include "fe_utils/astreamer.h"
 #include "pgtar.h"
 
 typedef struct astreamer_tar_parser
diff --git a/src/bin/pg_basebackup/astreamer_zstd.c b/src/fe_utils/astreamer_zstd.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_zstd.c
rename to src/fe_utils/astreamer_zstd.c
index 58dc679ef99..45f6cb67363 100644
--- a/src/bin/pg_basebackup/astreamer_zstd.c
+++ b/src/fe_utils/astreamer_zstd.c
@@ -17,8 +17,8 @@
 #include <zstd.h>
 #endif
 
-#include "astreamer.h"
 #include "common/logging.h"
+#include "fe_utils/astreamer.h"
 
 #ifdef USE_ZSTD
 
diff --git a/src/fe_utils/meson.build b/src/fe_utils/meson.build
index 14d0482a2cc..0ec28e86af7 100644
--- a/src/fe_utils/meson.build
+++ b/src/fe_utils/meson.build
@@ -13,6 +13,11 @@ fe_utils_sources = files(
   'recovery_gen.c',
   'simple_list.c',
   'string_utils.c',
+  'astreamer_file.c',
+  'astreamer_gzip.c',
+  'astreamer_lz4.c',
+  'astreamer_tar.c',
+  'astreamer_zstd.c',
 )
 
 psqlscan = custom_target('psqlscan',
@@ -28,6 +33,6 @@ fe_utils = static_library('libpgfeutils',
   c_pch: pch_postgres_fe_h,
   include_directories: [postgres_inc, libpq_inc],
   c_args: host_system == 'windows' ? ['-DFD_SETSIZE=1024'] : [],
-  dependencies: frontend_common_code,
+  dependencies: [frontend_common_code, lz4, zlib, zstd],
   kwargs: default_lib_args,
 )
diff --git a/src/bin/pg_basebackup/astreamer.h b/src/include/fe_utils/astreamer.h
similarity index 100%
rename from src/bin/pg_basebackup/astreamer.h
rename to src/include/fe_utils/astreamer.h
-- 
2.18.0

From 92b5d91ce00087ea30cec35420d0deb124aa1531 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 10:32:11 +0530
Subject: [PATCH v2 04/10] Refactor: move some part of pg_verifybackup.c to
 pg_verifybackup.h

---
 src/bin/pg_verifybackup/pg_verifybackup.c |  99 ++-----------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 112 ++++++++++++++++++++++
 2 files changed, 119 insertions(+), 92 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/pg_verifybackup.h

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index d77e70fbe38..a248db1b28a 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,12 +18,11 @@
 #include <sys/stat.h>
 #include <time.h>
 
-#include "common/controldata_utils.h"
-#include "common/hashfn_unstable.h"
 #include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
+#include "pg_verifybackup.h"
 #include "pgtime.h"
 
 /*
@@ -40,83 +39,6 @@
  */
 #define ESTIMATED_BYTES_PER_MANIFEST_LINE	100
 
-/*
- * How many bytes should we try to read from a file at once?
- */
-#define READ_CHUNK_SIZE				(128 * 1024)
-
-/*
- * Each file described by the manifest file is parsed to produce an object
- * like this.
- */
-typedef struct manifest_file
-{
-	uint32		status;			/* hash status */
-	const char *pathname;
-	size_t		size;
-	pg_checksum_type checksum_type;
-	int			checksum_length;
-	uint8	   *checksum_payload;
-	bool		matched;
-	bool		bad;
-} manifest_file;
-
-#define should_verify_checksum(m) \
-	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
-
-/*
- * Define a hash table which we can use to store information about the files
- * mentioned in the backup manifest.
- */
-#define SH_PREFIX		manifest_files
-#define SH_ELEMENT_TYPE	manifest_file
-#define SH_KEY_TYPE		const char *
-#define	SH_KEY			pathname
-#define SH_HASH_KEY(tb, key)	hash_string(key)
-#define SH_EQUAL(tb, a, b)		(strcmp(a, b) == 0)
-#define	SH_SCOPE		static inline
-#define SH_RAW_ALLOCATOR	pg_malloc0
-#define SH_DECLARE
-#define SH_DEFINE
-#include "lib/simplehash.h"
-
-/*
- * Each WAL range described by the manifest file is parsed to produce an
- * object like this.
- */
-typedef struct manifest_wal_range
-{
-	TimeLineID	tli;
-	XLogRecPtr	start_lsn;
-	XLogRecPtr	end_lsn;
-	struct manifest_wal_range *next;
-	struct manifest_wal_range *prev;
-} manifest_wal_range;
-
-/*
- * All the data parsed from a backup_manifest file.
- */
-typedef struct manifest_data
-{
-	int			version;
-	uint64		system_identifier;
-	manifest_files_hash *files;
-	manifest_wal_range *first_wal_range;
-	manifest_wal_range *last_wal_range;
-} manifest_data;
-
-/*
- * All of the context information we need while checking a backup manifest.
- */
-typedef struct verifier_context
-{
-	manifest_data *manifest;
-	char	   *backup_directory;
-	SimpleStringList ignore_list;
-	bool		exit_on_error;
-	bool		saw_any_error;
-} verifier_context;
-
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -150,21 +72,14 @@ static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
 
-static void report_backup_error(verifier_context *context,
-								const char *pg_restrict fmt,...)
-			pg_attribute_printf(2, 3);
-static void report_fatal_error(const char *pg_restrict fmt,...)
-			pg_attribute_printf(1, 2) pg_attribute_noreturn();
-static bool should_ignore_relpath(verifier_context *context, const char *relpath);
-
 static void progress_report(bool finished);
 static void usage(void);
 
 static const char *progname;
 
 /* options */
-static bool show_progress = false;
-static bool skip_checksums = false;
+bool show_progress = false;
+bool skip_checksums = false;
 
 /* Progress indicators */
 static uint64 total_size = 0;
@@ -556,7 +471,7 @@ verifybackup_per_file_cb(JsonManifestParseContext *context,
 	bool		found;
 
 	/* Make a new entry in the hash table for this file. */
-	m = manifest_files_insert(ht, pathname, &found);
+	m = manifest_files_insert(ht, (char *) pathname, &found);
 	if (found)
 		report_fatal_error("duplicate path name in backup manifest: \"%s\"",
 						   pathname);
@@ -979,7 +894,7 @@ parse_required_wal(verifier_context *context, char *pg_waldump_path,
  * Update the context to indicate that we saw an error, and exit if the
  * context says we should.
  */
-static void
+void
 report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
 {
 	va_list		ap;
@@ -996,7 +911,7 @@ report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
 /*
  * Report a fatal error and exit
  */
-static void
+void
 report_fatal_error(const char *pg_restrict fmt,...)
 {
 	va_list		ap;
@@ -1015,7 +930,7 @@ report_fatal_error(const char *pg_restrict fmt,...)
  * Note that by "prefix" we mean a parent directory; for this purpose,
  * "aa/bb" is not a prefix of "aa/bbb", but it is a prefix of "aa/bb/cc".
  */
-static bool
+bool
 should_ignore_relpath(verifier_context *context, const char *relpath)
 {
 	SimpleStringListCell *cell;
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
new file mode 100644
index 00000000000..c11ff33a100
--- /dev/null
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -0,0 +1,112 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_verifybackup.h
+ *	  Verify a backup against a backup manifest.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_verifybackup/pg_verifybackup.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef PG_VERIFYBACKUP_H
+#define PG_VERIFYBACKUP_H
+
+#include "common/controldata_utils.h"
+#include "common/hashfn_unstable.h"
+#include "common/parse_manifest.h"
+#include "fe_utils/simple_list.h"
+
+extern bool show_progress;
+extern bool skip_checksums;
+
+/*
+ * How many bytes should we try to read from a file at once?
+ */
+#define READ_CHUNK_SIZE				(128 * 1024)
+
+/*
+ * Each file described by the manifest file is parsed to produce an object
+ * like this.
+ */
+typedef struct manifest_file
+{
+	uint32		status;			/* hash status */
+	char	   *pathname;
+	size_t		size;
+	pg_checksum_type checksum_type;
+	int			checksum_length;
+	uint8	   *checksum_payload;
+	bool		matched;
+	bool		bad;
+} manifest_file;
+
+#define should_verify_checksum(m) \
+	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+
+/*
+ * Define a hash table which we can use to store information about the files
+ * mentioned in the backup manifest.
+ */
+#define SH_PREFIX		manifest_files
+#define SH_ELEMENT_TYPE	manifest_file
+#define SH_KEY_TYPE		char *
+#define	SH_KEY			pathname
+#define SH_HASH_KEY(tb, key)	hash_string(key)
+#define SH_EQUAL(tb, a, b)		(strcmp(a, b) == 0)
+#define	SH_SCOPE		static inline
+#define SH_RAW_ALLOCATOR	pg_malloc0
+#define SH_DECLARE
+#define SH_DEFINE
+#include "lib/simplehash.h"
+
+/*
+ * Each WAL range described by the manifest file is parsed to produce an
+ * object like this.
+ */
+typedef struct manifest_wal_range
+{
+	TimeLineID	tli;
+	XLogRecPtr	start_lsn;
+	XLogRecPtr	end_lsn;
+	struct manifest_wal_range *next;
+	struct manifest_wal_range *prev;
+} manifest_wal_range;
+
+/*
+ * All the data parsed from a backup_manifest file.
+ */
+typedef struct manifest_data
+{
+	int			version;
+	uint64		system_identifier;
+	manifest_files_hash *files;
+	manifest_wal_range *first_wal_range;
+	manifest_wal_range *last_wal_range;
+} manifest_data;
+
+/*
+ * All of the context information we need while checking a backup manifest.
+ */
+typedef struct verifier_context
+{
+	manifest_data *manifest;
+	char	   *backup_directory;
+	SimpleStringList ignore_list;
+	bool		exit_on_error;
+	bool		saw_any_error;
+} verifier_context;
+
+extern manifest_file *verify_manifest_entry(verifier_context *context,
+											char *relpath, size_t filesize);
+
+extern void report_backup_error(verifier_context *context,
+								const char *pg_restrict fmt,...)
+			pg_attribute_printf(2, 3);
+extern void report_fatal_error(const char *pg_restrict fmt,...)
+			pg_attribute_printf(1, 2) pg_attribute_noreturn();
+extern bool should_ignore_relpath(verifier_context *context, const char *relpath);
+
+#endif /* PG_VERIFYBACKUP_H */
-- 
2.18.0

From 011ff073e67e9bcc5c3fc5bf30cff996d3a78e03 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 17 Jul 2024 14:23:27 +0530
Subject: [PATCH v2 02/10] Refactor: Add astreamer_inject.h and move related
 declarations to it.

---
 src/bin/pg_basebackup/astreamer.h        |  6 ------
 src/bin/pg_basebackup/astreamer_inject.c |  2 +-
 src/bin/pg_basebackup/astreamer_inject.h | 24 ++++++++++++++++++++++++
 src/bin/pg_basebackup/pg_basebackup.c    |  2 +-
 4 files changed, 26 insertions(+), 8 deletions(-)
 create mode 100644 src/bin/pg_basebackup/astreamer_inject.h

diff --git a/src/bin/pg_basebackup/astreamer.h b/src/bin/pg_basebackup/astreamer.h
index 6b0047418bb..9d0a8c4d0c2 100644
--- a/src/bin/pg_basebackup/astreamer.h
+++ b/src/bin/pg_basebackup/astreamer.h
@@ -217,10 +217,4 @@ extern astreamer *astreamer_tar_parser_new(astreamer *next);
 extern astreamer *astreamer_tar_terminator_new(astreamer *next);
 extern astreamer *astreamer_tar_archiver_new(astreamer *next);
 
-extern astreamer *astreamer_recovery_injector_new(astreamer *next,
-												  bool is_recovery_guc_supported,
-												  PQExpBuffer recoveryconfcontents);
-extern void astreamer_inject_file(astreamer *streamer, char *pathname,
-								  char *data, int len);
-
 #endif
diff --git a/src/bin/pg_basebackup/astreamer_inject.c b/src/bin/pg_basebackup/astreamer_inject.c
index 7f1decded8d..4ad8381f102 100644
--- a/src/bin/pg_basebackup/astreamer_inject.c
+++ b/src/bin/pg_basebackup/astreamer_inject.c
@@ -11,7 +11,7 @@
 
 #include "postgres_fe.h"
 
-#include "astreamer.h"
+#include "astreamer_inject.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 
diff --git a/src/bin/pg_basebackup/astreamer_inject.h b/src/bin/pg_basebackup/astreamer_inject.h
new file mode 100644
index 00000000000..8504b3f5e0d
--- /dev/null
+++ b/src/bin/pg_basebackup/astreamer_inject.h
@@ -0,0 +1,24 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer_inject.h
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *		  src/bin/pg_basebackup/astreamer_inject.h
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef ASTREAMER_INJECT_H
+#define ASTREAMER_INJECT_H
+
+#include "astreamer.h"
+#include "pqexpbuffer.h"
+
+extern astreamer *astreamer_recovery_injector_new(astreamer *next,
+												  bool is_recovery_guc_supported,
+												  PQExpBuffer recoveryconfcontents);
+extern void astreamer_inject_file(astreamer *streamer, char *pathname,
+								  char *data, int len);
+
+#endif
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 4179b064cbc..1e753e40c97 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -26,7 +26,7 @@
 #endif
 
 #include "access/xlog_internal.h"
-#include "astreamer.h"
+#include "astreamer_inject.h"
 #include "backup/basebackup.h"
 #include "common/compression.h"
 #include "common/file_perm.h"
-- 
2.18.0

From f013bc9f93ac0a31b22fcdaa8c57422a15adc552 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Mon, 17 Jun 2024 14:17:22 +0530
Subject: [PATCH v2 05/10] Refactor: split verify_backup_file() function.

Separate the manifest entry verification code into a new function.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 36 ++++++++++++++++-------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index a248db1b28a..d17b565a604 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -624,35 +624,47 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 	}
 
 	/* Check whether there's an entry in the manifest hash. */
+	m = verify_manifest_entry(context, relpath, sb.st_size);
+
+	/*
+	 * Validate the manifest system identifier, not available in manifest
+	 * version 1.
+	 */
+	if (context->manifest->version != 1 &&
+		strcmp(relpath, "global/pg_control") == 0 &&
+		m->matched && !m->bad)
+		verify_control_file(fullpath, context->manifest->system_identifier);
+}
+
+/*
+ * Verify file and its size entry in the manifest.
+ */
+manifest_file *
+verify_manifest_entry(verifier_context *context, char *relpath, size_t filesize)
+{
+	manifest_file *m;
+
 	m = manifest_files_lookup(context->manifest->files, relpath);
 	if (m == NULL)
 	{
 		report_backup_error(context,
 							"\"%s\" is present on disk but not in the manifest",
 							relpath);
-		return;
+		return NULL;
 	}
 
 	/* Flag this entry as having been encountered in the filesystem. */
 	m->matched = true;
 
 	/* Check that the size matches. */
-	if (m->size != sb.st_size)
+	if (m->size != filesize)
 	{
 		report_backup_error(context,
 							"\"%s\" has size %lld on disk but size %zu in the manifest",
-							relpath, (long long int) sb.st_size, m->size);
+							relpath, (long long int) filesize, m->size);
 		m->bad = true;
 	}
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0)
-		verify_control_file(fullpath, context->manifest->system_identifier);
-
 	/* Update statistics for progress report, if necessary */
 	if (show_progress && !skip_checksums && should_verify_checksum(m))
 		total_size += m->size;
@@ -663,6 +675,8 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 	 * afterwards verify the checksums. That's because computing checksums may
 	 * take a while, and we'd like to report more obvious problems quickly.
 	 */
+
+	return m;
 }
 
 /*
-- 
2.18.0

From c83e66ccff090c15979093692f33fcedbc3ccf39 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 09:39:32 +0530
Subject: [PATCH v2 01/10] Refactor: Rename all bbstreamer references to
 astreamer.

BBSTREAMER is specific to pg_basebackup; we need a more generalized
name so it can be placed in a common area, making it accessible for
other modules. Renaming it to ASTREAMER, short for ARCHIVE STREAMER,
makes it more general.
---
 src/bin/pg_basebackup/Makefile                |  12 +-
 src/bin/pg_basebackup/astreamer.h             | 226 +++++++++++++
 .../{bbstreamer_file.c => astreamer_file.c}   | 148 ++++----
 .../{bbstreamer_gzip.c => astreamer_gzip.c}   | 154 ++++-----
 ...bbstreamer_inject.c => astreamer_inject.c} | 152 ++++-----
 .../{bbstreamer_lz4.c => astreamer_lz4.c}     | 172 +++++-----
 .../{bbstreamer_tar.c => astreamer_tar.c}     | 316 +++++++++---------
 .../{bbstreamer_zstd.c => astreamer_zstd.c}   | 160 ++++-----
 src/bin/pg_basebackup/bbstreamer.h            | 226 -------------
 src/bin/pg_basebackup/meson.build             |  12 +-
 src/bin/pg_basebackup/nls.mk                  |  12 +-
 src/bin/pg_basebackup/pg_basebackup.c         |  74 ++--
 src/tools/pgindent/typedefs.list              |  26 +-
 13 files changed, 845 insertions(+), 845 deletions(-)
 create mode 100644 src/bin/pg_basebackup/astreamer.h
 rename src/bin/pg_basebackup/{bbstreamer_file.c => astreamer_file.c} (69%)
 rename src/bin/pg_basebackup/{bbstreamer_gzip.c => astreamer_gzip.c} (62%)
 rename src/bin/pg_basebackup/{bbstreamer_inject.c => astreamer_inject.c} (53%)
 rename src/bin/pg_basebackup/{bbstreamer_lz4.c => astreamer_lz4.c} (69%)
 rename src/bin/pg_basebackup/{bbstreamer_tar.c => astreamer_tar.c} (50%)
 rename src/bin/pg_basebackup/{bbstreamer_zstd.c => astreamer_zstd.c} (64%)
 delete mode 100644 src/bin/pg_basebackup/bbstreamer.h

diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile
index 26c53e473f5..a71af2d48a7 100644
--- a/src/bin/pg_basebackup/Makefile
+++ b/src/bin/pg_basebackup/Makefile
@@ -37,12 +37,12 @@ OBJS = \
 
 BBOBJS = \
 	pg_basebackup.o \
-	bbstreamer_file.o \
-	bbstreamer_gzip.o \
-	bbstreamer_inject.o \
-	bbstreamer_lz4.o \
-	bbstreamer_tar.o \
-	bbstreamer_zstd.o
+	astreamer_file.o \
+	astreamer_gzip.o \
+	astreamer_inject.o \
+	astreamer_lz4.o \
+	astreamer_tar.o \
+	astreamer_zstd.o
 
 all: pg_basebackup pg_createsubscriber pg_receivewal pg_recvlogical
 
diff --git a/src/bin/pg_basebackup/astreamer.h b/src/bin/pg_basebackup/astreamer.h
new file mode 100644
index 00000000000..6b0047418bb
--- /dev/null
+++ b/src/bin/pg_basebackup/astreamer.h
@@ -0,0 +1,226 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer.h
+ *
+ * Each tar archive returned by the server is passed to one or more
+ * astreamer objects for further processing. The astreamer may do
+ * something simple, like write the archive to a file, perhaps after
+ * compressing it, but it can also do more complicated things, like
+ * annotating the byte stream to indicate which parts of the data
+ * correspond to tar headers or trailing padding, vs. which parts are
+ * payload data. A subsequent astreamer may use this information to
+ * make further decisions about how to process the data; for example,
+ * it might choose to modify the archive contents.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *		  src/bin/pg_basebackup/astreamer.h
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef ASTREAMER_H
+#define ASTREAMER_H
+
+#include "common/compression.h"
+#include "lib/stringinfo.h"
+#include "pqexpbuffer.h"
+
+struct astreamer;
+struct astreamer_ops;
+typedef struct astreamer astreamer;
+typedef struct astreamer_ops astreamer_ops;
+
+/*
+ * Each chunk of archive data passed to a astreamer is classified into one
+ * of these categories. When data is first received from the remote server,
+ * each chunk will be categorized as ASTREAMER_UNKNOWN, and the chunks will
+ * be of whatever size the remote server chose to send.
+ *
+ * If the archive is parsed (e.g. see astreamer_tar_parser_new()), then all
+ * chunks should be labelled as one of the other types listed here. In
+ * addition, there should be exactly one ASTREAMER_MEMBER_HEADER chunk and
+ * exactly one ASTREAMER_MEMBER_TRAILER chunk per archive member, even if
+ * that means a zero-length call. There can be any number of
+ * ASTREAMER_MEMBER_CONTENTS chunks in between those calls. There
+ * should exactly ASTREAMER_ARCHIVE_TRAILER chunk, and it should follow the
+ * last ASTREAMER_MEMBER_TRAILER chunk.
+ *
+ * In theory, we could need other classifications here, such as a way of
+ * indicating an archive header, but the "tar" format doesn't need anything
+ * else, so for the time being there's no point.
+ */
+typedef enum
+{
+	ASTREAMER_UNKNOWN,
+	ASTREAMER_MEMBER_HEADER,
+	ASTREAMER_MEMBER_CONTENTS,
+	ASTREAMER_MEMBER_TRAILER,
+	ASTREAMER_ARCHIVE_TRAILER,
+} astreamer_archive_context;
+
+/*
+ * Each chunk of data that is classified as ASTREAMER_MEMBER_HEADER,
+ * ASTREAMER_MEMBER_CONTENTS, or ASTREAMER_MEMBER_TRAILER should also
+ * pass a pointer to an instance of this struct. The details are expected
+ * to be present in the archive header and used to fill the struct, after
+ * which all subsequent calls for the same archive member are expected to
+ * pass the same details.
+ */
+typedef struct
+{
+	char		pathname[MAXPGPATH];
+	pgoff_t		size;
+	mode_t		mode;
+	uid_t		uid;
+	gid_t		gid;
+	bool		is_directory;
+	bool		is_link;
+	char		linktarget[MAXPGPATH];
+} astreamer_member;
+
+/*
+ * Generally, each type of astreamer will define its own struct, but the
+ * first element should be 'astreamer base'. A astreamer that does not
+ * require any additional private data could use this structure directly.
+ *
+ * bbs_ops is a pointer to the astreamer_ops object which contains the
+ * function pointers appropriate to this type of astreamer.
+ *
+ * bbs_next is a pointer to the successor astreamer, for those types of
+ * astreamer which forward data to a successor. It need not be used and
+ * should be set to NULL when not relevant.
+ *
+ * bbs_buffer is a buffer for accumulating data for temporary storage. Each
+ * type of astreamer makes its own decisions about whether and how to use
+ * this buffer.
+ */
+struct astreamer
+{
+	const astreamer_ops *bbs_ops;
+	astreamer *bbs_next;
+	StringInfoData bbs_buffer;
+};
+
+/*
+ * There are three callbacks for a astreamer. The 'content' callback is
+ * called repeatedly, as described in the astreamer_archive_context comments.
+ * Then, the 'finalize' callback is called once at the end, to give the
+ * astreamer a chance to perform cleanup such as closing files. Finally,
+ * because this code is running in a frontend environment where, as of this
+ * writing, there are no memory contexts, the 'free' callback is called to
+ * release memory. These callbacks should always be invoked using the static
+ * inline functions defined below.
+ */
+struct astreamer_ops
+{
+	void		(*content) (astreamer *streamer, astreamer_member *member,
+							const char *data, int len,
+							astreamer_archive_context context);
+	void		(*finalize) (astreamer *streamer);
+	void		(*free) (astreamer *streamer);
+};
+
+/* Send some content to a astreamer. */
+static inline void
+astreamer_content(astreamer *streamer, astreamer_member *member,
+				  const char *data, int len,
+				  astreamer_archive_context context)
+{
+	Assert(streamer != NULL);
+	streamer->bbs_ops->content(streamer, member, data, len, context);
+}
+
+/* Finalize a astreamer. */
+static inline void
+astreamer_finalize(astreamer *streamer)
+{
+	Assert(streamer != NULL);
+	streamer->bbs_ops->finalize(streamer);
+}
+
+/* Free a astreamer. */
+static inline void
+astreamer_free(astreamer *streamer)
+{
+	Assert(streamer != NULL);
+	streamer->bbs_ops->free(streamer);
+}
+
+/*
+ * This is a convenience method for use when implementing a astreamer; it is
+ * not for use by outside callers. It adds the amount of data specified by
+ * 'nbytes' to the astreamer's buffer and adjusts '*len' and '*data'
+ * accordingly.
+ */
+static inline void
+astreamer_buffer_bytes(astreamer *streamer, const char **data, int *len,
+					   int nbytes)
+{
+	Assert(nbytes <= *len);
+
+	appendBinaryStringInfo(&streamer->bbs_buffer, *data, nbytes);
+	*len -= nbytes;
+	*data += nbytes;
+}
+
+/*
+ * This is a convenience method for use when implementing a astreamer; it is
+ * not for use by outsider callers. It attempts to add enough data to the
+ * astreamer's buffer to reach a length of target_bytes and adjusts '*len'
+ * and '*data' accordingly. It returns true if the target length has been
+ * reached and false otherwise.
+ */
+static inline bool
+astreamer_buffer_until(astreamer *streamer, const char **data, int *len,
+					   int target_bytes)
+{
+	int			buflen = streamer->bbs_buffer.len;
+
+	if (buflen >= target_bytes)
+	{
+		/* Target length already reached; nothing to do. */
+		return true;
+	}
+
+	if (buflen + *len < target_bytes)
+	{
+		/* Not enough data to reach target length; buffer all of it. */
+		astreamer_buffer_bytes(streamer, data, len, *len);
+		return false;
+	}
+
+	/* Buffer just enough to reach the target length. */
+	astreamer_buffer_bytes(streamer, data, len, target_bytes - buflen);
+	return true;
+}
+
+/*
+ * Functions for creating astreamer objects of various types. See the header
+ * comments for each of these functions for details.
+ */
+extern astreamer *astreamer_plain_writer_new(char *pathname, FILE *file);
+extern astreamer *astreamer_gzip_writer_new(char *pathname, FILE *file,
+											pg_compress_specification *compress);
+extern astreamer *astreamer_extractor_new(const char *basepath,
+										  const char *(*link_map) (const char *),
+										  void (*report_output_file) (const char *));
+
+extern astreamer *astreamer_gzip_decompressor_new(astreamer *next);
+extern astreamer *astreamer_lz4_compressor_new(astreamer *next,
+											   pg_compress_specification *compress);
+extern astreamer *astreamer_lz4_decompressor_new(astreamer *next);
+extern astreamer *astreamer_zstd_compressor_new(astreamer *next,
+												pg_compress_specification *compress);
+extern astreamer *astreamer_zstd_decompressor_new(astreamer *next);
+extern astreamer *astreamer_tar_parser_new(astreamer *next);
+extern astreamer *astreamer_tar_terminator_new(astreamer *next);
+extern astreamer *astreamer_tar_archiver_new(astreamer *next);
+
+extern astreamer *astreamer_recovery_injector_new(astreamer *next,
+												  bool is_recovery_guc_supported,
+												  PQExpBuffer recoveryconfcontents);
+extern void astreamer_inject_file(astreamer *streamer, char *pathname,
+								  char *data, int len);
+
+#endif
diff --git a/src/bin/pg_basebackup/bbstreamer_file.c b/src/bin/pg_basebackup/astreamer_file.c
similarity index 69%
rename from src/bin/pg_basebackup/bbstreamer_file.c
rename to src/bin/pg_basebackup/astreamer_file.c
index bab6cd4a6b1..2742385e103 100644
--- a/src/bin/pg_basebackup/bbstreamer_file.c
+++ b/src/bin/pg_basebackup/astreamer_file.c
@@ -1,11 +1,11 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_file.c
+ * astreamer_file.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_file.c
+ *		  src/bin/pg_basebackup/astreamer_file.c
  *-------------------------------------------------------------------------
  */
 
@@ -13,60 +13,60 @@
 
 #include <unistd.h>
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
 
-typedef struct bbstreamer_plain_writer
+typedef struct astreamer_plain_writer
 {
-	bbstreamer	base;
+	astreamer	base;
 	char	   *pathname;
 	FILE	   *file;
 	bool		should_close_file;
-} bbstreamer_plain_writer;
+} astreamer_plain_writer;
 
-typedef struct bbstreamer_extractor
+typedef struct astreamer_extractor
 {
-	bbstreamer	base;
+	astreamer	base;
 	char	   *basepath;
 	const char *(*link_map) (const char *);
 	void		(*report_output_file) (const char *);
 	char		filename[MAXPGPATH];
 	FILE	   *file;
-} bbstreamer_extractor;
+} astreamer_extractor;
 
-static void bbstreamer_plain_writer_content(bbstreamer *streamer,
-											bbstreamer_member *member,
-											const char *data, int len,
-											bbstreamer_archive_context context);
-static void bbstreamer_plain_writer_finalize(bbstreamer *streamer);
-static void bbstreamer_plain_writer_free(bbstreamer *streamer);
+static void astreamer_plain_writer_content(astreamer *streamer,
+										   astreamer_member *member,
+										   const char *data, int len,
+										   astreamer_archive_context context);
+static void astreamer_plain_writer_finalize(astreamer *streamer);
+static void astreamer_plain_writer_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_plain_writer_ops = {
-	.content = bbstreamer_plain_writer_content,
-	.finalize = bbstreamer_plain_writer_finalize,
-	.free = bbstreamer_plain_writer_free
+static const astreamer_ops astreamer_plain_writer_ops = {
+	.content = astreamer_plain_writer_content,
+	.finalize = astreamer_plain_writer_finalize,
+	.free = astreamer_plain_writer_free
 };
 
-static void bbstreamer_extractor_content(bbstreamer *streamer,
-										 bbstreamer_member *member,
-										 const char *data, int len,
-										 bbstreamer_archive_context context);
-static void bbstreamer_extractor_finalize(bbstreamer *streamer);
-static void bbstreamer_extractor_free(bbstreamer *streamer);
+static void astreamer_extractor_content(astreamer *streamer,
+										astreamer_member *member,
+										const char *data, int len,
+										astreamer_archive_context context);
+static void astreamer_extractor_finalize(astreamer *streamer);
+static void astreamer_extractor_free(astreamer *streamer);
 static void extract_directory(const char *filename, mode_t mode);
 static void extract_link(const char *filename, const char *linktarget);
 static FILE *create_file_for_extract(const char *filename, mode_t mode);
 
-static const bbstreamer_ops bbstreamer_extractor_ops = {
-	.content = bbstreamer_extractor_content,
-	.finalize = bbstreamer_extractor_finalize,
-	.free = bbstreamer_extractor_free
+static const astreamer_ops astreamer_extractor_ops = {
+	.content = astreamer_extractor_content,
+	.finalize = astreamer_extractor_finalize,
+	.free = astreamer_extractor_free
 };
 
 /*
- * Create a bbstreamer that just writes data to a file.
+ * Create a astreamer that just writes data to a file.
  *
  * The caller must specify a pathname and may specify a file. The pathname is
  * used for error-reporting purposes either way. If file is NULL, the pathname
@@ -74,14 +74,14 @@ static const bbstreamer_ops bbstreamer_extractor_ops = {
  * for writing and closed when done. If file is not NULL, the data is written
  * there.
  */
-bbstreamer *
-bbstreamer_plain_writer_new(char *pathname, FILE *file)
+astreamer *
+astreamer_plain_writer_new(char *pathname, FILE *file)
 {
-	bbstreamer_plain_writer *streamer;
+	astreamer_plain_writer *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_plain_writer));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_plain_writer_ops;
+	streamer = palloc0(sizeof(astreamer_plain_writer));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_plain_writer_ops;
 
 	streamer->pathname = pstrdup(pathname);
 	streamer->file = file;
@@ -101,13 +101,13 @@ bbstreamer_plain_writer_new(char *pathname, FILE *file)
  * Write archive content to file.
  */
 static void
-bbstreamer_plain_writer_content(bbstreamer *streamer,
-								bbstreamer_member *member, const char *data,
-								int len, bbstreamer_archive_context context)
+astreamer_plain_writer_content(astreamer *streamer,
+							   astreamer_member *member, const char *data,
+							   int len, astreamer_archive_context context)
 {
-	bbstreamer_plain_writer *mystreamer;
+	astreamer_plain_writer *mystreamer;
 
-	mystreamer = (bbstreamer_plain_writer *) streamer;
+	mystreamer = (astreamer_plain_writer *) streamer;
 
 	if (len == 0)
 		return;
@@ -128,11 +128,11 @@ bbstreamer_plain_writer_content(bbstreamer *streamer,
  * the file if we opened it, but not if the caller provided it.
  */
 static void
-bbstreamer_plain_writer_finalize(bbstreamer *streamer)
+astreamer_plain_writer_finalize(astreamer *streamer)
 {
-	bbstreamer_plain_writer *mystreamer;
+	astreamer_plain_writer *mystreamer;
 
-	mystreamer = (bbstreamer_plain_writer *) streamer;
+	mystreamer = (astreamer_plain_writer *) streamer;
 
 	if (mystreamer->should_close_file && fclose(mystreamer->file) != 0)
 		pg_fatal("could not close file \"%s\": %m",
@@ -143,14 +143,14 @@ bbstreamer_plain_writer_finalize(bbstreamer *streamer)
 }
 
 /*
- * Free memory associated with this bbstreamer.
+ * Free memory associated with this astreamer.
  */
 static void
-bbstreamer_plain_writer_free(bbstreamer *streamer)
+astreamer_plain_writer_free(astreamer *streamer)
 {
-	bbstreamer_plain_writer *mystreamer;
+	astreamer_plain_writer *mystreamer;
 
-	mystreamer = (bbstreamer_plain_writer *) streamer;
+	mystreamer = (astreamer_plain_writer *) streamer;
 
 	Assert(!mystreamer->should_close_file);
 	Assert(mystreamer->base.bbs_next == NULL);
@@ -160,13 +160,13 @@ bbstreamer_plain_writer_free(bbstreamer *streamer)
 }
 
 /*
- * Create a bbstreamer that extracts an archive.
+ * Create a astreamer that extracts an archive.
  *
  * All pathnames in the archive are interpreted relative to basepath.
  *
- * Unlike e.g. bbstreamer_plain_writer_new() we can't do anything useful here
+ * Unlike e.g. astreamer_plain_writer_new() we can't do anything useful here
  * with untyped chunks; we need typed chunks which follow the rules described
- * in bbstreamer.h. Assuming we have that, we don't need to worry about the
+ * in astreamer.h. Assuming we have that, we don't need to worry about the
  * original archive format; it's enough to just look at the member information
  * provided and write to the corresponding file.
  *
@@ -179,16 +179,16 @@ bbstreamer_plain_writer_free(bbstreamer *streamer)
  * new output file. The pathname to that file is passed as an argument. If
  * NULL, the call is skipped.
  */
-bbstreamer *
-bbstreamer_extractor_new(const char *basepath,
-						 const char *(*link_map) (const char *),
-						 void (*report_output_file) (const char *))
+astreamer *
+astreamer_extractor_new(const char *basepath,
+						const char *(*link_map) (const char *),
+						void (*report_output_file) (const char *))
 {
-	bbstreamer_extractor *streamer;
+	astreamer_extractor *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_extractor));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_extractor_ops;
+	streamer = palloc0(sizeof(astreamer_extractor));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_extractor_ops;
 	streamer->basepath = pstrdup(basepath);
 	streamer->link_map = link_map;
 	streamer->report_output_file = report_output_file;
@@ -200,19 +200,19 @@ bbstreamer_extractor_new(const char *basepath,
  * Extract archive contents to the filesystem.
  */
 static void
-bbstreamer_extractor_content(bbstreamer *streamer, bbstreamer_member *member,
-							 const char *data, int len,
-							 bbstreamer_archive_context context)
+astreamer_extractor_content(astreamer *streamer, astreamer_member *member,
+							const char *data, int len,
+							astreamer_archive_context context)
 {
-	bbstreamer_extractor *mystreamer = (bbstreamer_extractor *) streamer;
+	astreamer_extractor *mystreamer = (astreamer_extractor *) streamer;
 	int			fnamelen;
 
-	Assert(member != NULL || context == BBSTREAMER_ARCHIVE_TRAILER);
-	Assert(context != BBSTREAMER_UNKNOWN);
+	Assert(member != NULL || context == ASTREAMER_ARCHIVE_TRAILER);
+	Assert(context != ASTREAMER_UNKNOWN);
 
 	switch (context)
 	{
-		case BBSTREAMER_MEMBER_HEADER:
+		case ASTREAMER_MEMBER_HEADER:
 			Assert(mystreamer->file == NULL);
 
 			/* Prepend basepath. */
@@ -245,7 +245,7 @@ bbstreamer_extractor_content(bbstreamer *streamer, bbstreamer_member *member,
 				mystreamer->report_output_file(mystreamer->filename);
 			break;
 
-		case BBSTREAMER_MEMBER_CONTENTS:
+		case ASTREAMER_MEMBER_CONTENTS:
 			if (mystreamer->file == NULL)
 				break;
 
@@ -260,14 +260,14 @@ bbstreamer_extractor_content(bbstreamer *streamer, bbstreamer_member *member,
 			}
 			break;
 
-		case BBSTREAMER_MEMBER_TRAILER:
+		case ASTREAMER_MEMBER_TRAILER:
 			if (mystreamer->file == NULL)
 				break;
 			fclose(mystreamer->file);
 			mystreamer->file = NULL;
 			break;
 
-		case BBSTREAMER_ARCHIVE_TRAILER:
+		case ASTREAMER_ARCHIVE_TRAILER:
 			break;
 
 		default:
@@ -375,10 +375,10 @@ create_file_for_extract(const char *filename, mode_t mode)
  * There's nothing to do here but sanity checking.
  */
 static void
-bbstreamer_extractor_finalize(bbstreamer *streamer)
+astreamer_extractor_finalize(astreamer *streamer)
 {
-	bbstreamer_extractor *mystreamer PG_USED_FOR_ASSERTS_ONLY
-	= (bbstreamer_extractor *) streamer;
+	astreamer_extractor *mystreamer PG_USED_FOR_ASSERTS_ONLY
+	= (astreamer_extractor *) streamer;
 
 	Assert(mystreamer->file == NULL);
 }
@@ -387,9 +387,9 @@ bbstreamer_extractor_finalize(bbstreamer *streamer)
  * Free memory.
  */
 static void
-bbstreamer_extractor_free(bbstreamer *streamer)
+astreamer_extractor_free(astreamer *streamer)
 {
-	bbstreamer_extractor *mystreamer = (bbstreamer_extractor *) streamer;
+	astreamer_extractor *mystreamer = (astreamer_extractor *) streamer;
 
 	pfree(mystreamer->basepath);
 	pfree(mystreamer);
diff --git a/src/bin/pg_basebackup/bbstreamer_gzip.c b/src/bin/pg_basebackup/astreamer_gzip.c
similarity index 62%
rename from src/bin/pg_basebackup/bbstreamer_gzip.c
rename to src/bin/pg_basebackup/astreamer_gzip.c
index 0417fd9bc2c..6f7c27afbbc 100644
--- a/src/bin/pg_basebackup/bbstreamer_gzip.c
+++ b/src/bin/pg_basebackup/astreamer_gzip.c
@@ -1,11 +1,11 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_gzip.c
+ * astreamer_gzip.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_gzip.c
+ *		  src/bin/pg_basebackup/astreamer_gzip.c
  *-------------------------------------------------------------------------
  */
 
@@ -17,74 +17,74 @@
 #include <zlib.h>
 #endif
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
 
 #ifdef HAVE_LIBZ
-typedef struct bbstreamer_gzip_writer
+typedef struct astreamer_gzip_writer
 {
-	bbstreamer	base;
+	astreamer	base;
 	char	   *pathname;
 	gzFile		gzfile;
-} bbstreamer_gzip_writer;
+} astreamer_gzip_writer;
 
-typedef struct bbstreamer_gzip_decompressor
+typedef struct astreamer_gzip_decompressor
 {
-	bbstreamer	base;
+	astreamer	base;
 	z_stream	zstream;
 	size_t		bytes_written;
-} bbstreamer_gzip_decompressor;
+} astreamer_gzip_decompressor;
 
-static void bbstreamer_gzip_writer_content(bbstreamer *streamer,
-										   bbstreamer_member *member,
-										   const char *data, int len,
-										   bbstreamer_archive_context context);
-static void bbstreamer_gzip_writer_finalize(bbstreamer *streamer);
-static void bbstreamer_gzip_writer_free(bbstreamer *streamer);
+static void astreamer_gzip_writer_content(astreamer *streamer,
+										  astreamer_member *member,
+										  const char *data, int len,
+										  astreamer_archive_context context);
+static void astreamer_gzip_writer_finalize(astreamer *streamer);
+static void astreamer_gzip_writer_free(astreamer *streamer);
 static const char *get_gz_error(gzFile gzf);
 
-static const bbstreamer_ops bbstreamer_gzip_writer_ops = {
-	.content = bbstreamer_gzip_writer_content,
-	.finalize = bbstreamer_gzip_writer_finalize,
-	.free = bbstreamer_gzip_writer_free
+static const astreamer_ops astreamer_gzip_writer_ops = {
+	.content = astreamer_gzip_writer_content,
+	.finalize = astreamer_gzip_writer_finalize,
+	.free = astreamer_gzip_writer_free
 };
 
-static void bbstreamer_gzip_decompressor_content(bbstreamer *streamer,
-												 bbstreamer_member *member,
-												 const char *data, int len,
-												 bbstreamer_archive_context context);
-static void bbstreamer_gzip_decompressor_finalize(bbstreamer *streamer);
-static void bbstreamer_gzip_decompressor_free(bbstreamer *streamer);
+static void astreamer_gzip_decompressor_content(astreamer *streamer,
+												astreamer_member *member,
+												const char *data, int len,
+												astreamer_archive_context context);
+static void astreamer_gzip_decompressor_finalize(astreamer *streamer);
+static void astreamer_gzip_decompressor_free(astreamer *streamer);
 static void *gzip_palloc(void *opaque, unsigned items, unsigned size);
 static void gzip_pfree(void *opaque, void *address);
 
-static const bbstreamer_ops bbstreamer_gzip_decompressor_ops = {
-	.content = bbstreamer_gzip_decompressor_content,
-	.finalize = bbstreamer_gzip_decompressor_finalize,
-	.free = bbstreamer_gzip_decompressor_free
+static const astreamer_ops astreamer_gzip_decompressor_ops = {
+	.content = astreamer_gzip_decompressor_content,
+	.finalize = astreamer_gzip_decompressor_finalize,
+	.free = astreamer_gzip_decompressor_free
 };
 #endif
 
 /*
- * Create a bbstreamer that just compresses data using gzip, and then writes
+ * Create a astreamer that just compresses data using gzip, and then writes
  * it to a file.
  *
- * As in the case of bbstreamer_plain_writer_new, pathname is always used
+ * As in the case of astreamer_plain_writer_new, pathname is always used
  * for error reporting purposes; if file is NULL, it is also the opened and
  * closed so that the data may be written there.
  */
-bbstreamer *
-bbstreamer_gzip_writer_new(char *pathname, FILE *file,
-						   pg_compress_specification *compress)
+astreamer *
+astreamer_gzip_writer_new(char *pathname, FILE *file,
+						  pg_compress_specification *compress)
 {
 #ifdef HAVE_LIBZ
-	bbstreamer_gzip_writer *streamer;
+	astreamer_gzip_writer *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_gzip_writer));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_gzip_writer_ops;
+	streamer = palloc0(sizeof(astreamer_gzip_writer));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_gzip_writer_ops;
 
 	streamer->pathname = pstrdup(pathname);
 
@@ -123,13 +123,13 @@ bbstreamer_gzip_writer_new(char *pathname, FILE *file,
  * Write archive content to gzip file.
  */
 static void
-bbstreamer_gzip_writer_content(bbstreamer *streamer,
-							   bbstreamer_member *member, const char *data,
-							   int len, bbstreamer_archive_context context)
+astreamer_gzip_writer_content(astreamer *streamer,
+							  astreamer_member *member, const char *data,
+							  int len, astreamer_archive_context context)
 {
-	bbstreamer_gzip_writer *mystreamer;
+	astreamer_gzip_writer *mystreamer;
 
-	mystreamer = (bbstreamer_gzip_writer *) streamer;
+	mystreamer = (astreamer_gzip_writer *) streamer;
 
 	if (len == 0)
 		return;
@@ -151,16 +151,16 @@ bbstreamer_gzip_writer_content(bbstreamer *streamer,
  *
  * It makes no difference whether we opened the file or the caller did it,
  * because libz provides no way of avoiding a close on the underlying file
- * handle. Notice, however, that bbstreamer_gzip_writer_new() uses dup() to
+ * handle. Notice, however, that astreamer_gzip_writer_new() uses dup() to
  * work around this issue, so that the behavior from the caller's viewpoint
- * is the same as for bbstreamer_plain_writer.
+ * is the same as for astreamer_plain_writer.
  */
 static void
-bbstreamer_gzip_writer_finalize(bbstreamer *streamer)
+astreamer_gzip_writer_finalize(astreamer *streamer)
 {
-	bbstreamer_gzip_writer *mystreamer;
+	astreamer_gzip_writer *mystreamer;
 
-	mystreamer = (bbstreamer_gzip_writer *) streamer;
+	mystreamer = (astreamer_gzip_writer *) streamer;
 
 	errno = 0;					/* in case gzclose() doesn't set it */
 	if (gzclose(mystreamer->gzfile) != 0)
@@ -171,14 +171,14 @@ bbstreamer_gzip_writer_finalize(bbstreamer *streamer)
 }
 
 /*
- * Free memory associated with this bbstreamer.
+ * Free memory associated with this astreamer.
  */
 static void
-bbstreamer_gzip_writer_free(bbstreamer *streamer)
+astreamer_gzip_writer_free(astreamer *streamer)
 {
-	bbstreamer_gzip_writer *mystreamer;
+	astreamer_gzip_writer *mystreamer;
 
-	mystreamer = (bbstreamer_gzip_writer *) streamer;
+	mystreamer = (astreamer_gzip_writer *) streamer;
 
 	Assert(mystreamer->base.bbs_next == NULL);
 	Assert(mystreamer->gzfile == NULL);
@@ -208,18 +208,18 @@ get_gz_error(gzFile gzf)
  * Create a new base backup streamer that performs decompression of gzip
  * compressed blocks.
  */
-bbstreamer *
-bbstreamer_gzip_decompressor_new(bbstreamer *next)
+astreamer *
+astreamer_gzip_decompressor_new(astreamer *next)
 {
 #ifdef HAVE_LIBZ
-	bbstreamer_gzip_decompressor *streamer;
+	astreamer_gzip_decompressor *streamer;
 	z_stream   *zs;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_gzip_decompressor));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_gzip_decompressor_ops;
+	streamer = palloc0(sizeof(astreamer_gzip_decompressor));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_gzip_decompressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -258,15 +258,15 @@ bbstreamer_gzip_decompressor_new(bbstreamer *next)
  * to the next streamer.
  */
 static void
-bbstreamer_gzip_decompressor_content(bbstreamer *streamer,
-									 bbstreamer_member *member,
-									 const char *data, int len,
-									 bbstreamer_archive_context context)
+astreamer_gzip_decompressor_content(astreamer *streamer,
+									astreamer_member *member,
+									const char *data, int len,
+									astreamer_archive_context context)
 {
-	bbstreamer_gzip_decompressor *mystreamer;
+	astreamer_gzip_decompressor *mystreamer;
 	z_stream   *zs;
 
-	mystreamer = (bbstreamer_gzip_decompressor *) streamer;
+	mystreamer = (astreamer_gzip_decompressor *) streamer;
 
 	zs = &mystreamer->zstream;
 	zs->next_in = (const uint8 *) data;
@@ -301,9 +301,9 @@ bbstreamer_gzip_decompressor_content(bbstreamer *streamer,
 		/* If output buffer is full then pass data to next streamer */
 		if (mystreamer->bytes_written >= mystreamer->base.bbs_buffer.maxlen)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, member,
-							   mystreamer->base.bbs_buffer.data,
-							   mystreamer->base.bbs_buffer.maxlen, context);
+			astreamer_content(mystreamer->base.bbs_next, member,
+							  mystreamer->base.bbs_buffer.data,
+							  mystreamer->base.bbs_buffer.maxlen, context);
 			mystreamer->bytes_written = 0;
 		}
 	}
@@ -313,31 +313,31 @@ bbstreamer_gzip_decompressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_gzip_decompressor_finalize(bbstreamer *streamer)
+astreamer_gzip_decompressor_finalize(astreamer *streamer)
 {
-	bbstreamer_gzip_decompressor *mystreamer;
+	astreamer_gzip_decompressor *mystreamer;
 
-	mystreamer = (bbstreamer_gzip_decompressor *) streamer;
+	mystreamer = (astreamer_gzip_decompressor *) streamer;
 
 	/*
 	 * End of the stream, if there is some pending data in output buffers then
 	 * we must forward it to next streamer.
 	 */
-	bbstreamer_content(mystreamer->base.bbs_next, NULL,
-					   mystreamer->base.bbs_buffer.data,
-					   mystreamer->base.bbs_buffer.maxlen,
-					   BBSTREAMER_UNKNOWN);
+	astreamer_content(mystreamer->base.bbs_next, NULL,
+					  mystreamer->base.bbs_buffer.data,
+					  mystreamer->base.bbs_buffer.maxlen,
+					  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_gzip_decompressor_free(bbstreamer *streamer)
+astreamer_gzip_decompressor_free(astreamer *streamer)
 {
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
 }
diff --git a/src/bin/pg_basebackup/bbstreamer_inject.c b/src/bin/pg_basebackup/astreamer_inject.c
similarity index 53%
rename from src/bin/pg_basebackup/bbstreamer_inject.c
rename to src/bin/pg_basebackup/astreamer_inject.c
index 194026b56e9..7f1decded8d 100644
--- a/src/bin/pg_basebackup/bbstreamer_inject.c
+++ b/src/bin/pg_basebackup/astreamer_inject.c
@@ -1,51 +1,51 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_inject.c
+ * astreamer_inject.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_inject.c
+ *		  src/bin/pg_basebackup/astreamer_inject.c
  *-------------------------------------------------------------------------
  */
 
 #include "postgres_fe.h"
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 
-typedef struct bbstreamer_recovery_injector
+typedef struct astreamer_recovery_injector
 {
-	bbstreamer	base;
+	astreamer	base;
 	bool		skip_file;
 	bool		is_recovery_guc_supported;
 	bool		is_postgresql_auto_conf;
 	bool		found_postgresql_auto_conf;
 	PQExpBuffer recoveryconfcontents;
-	bbstreamer_member member;
-} bbstreamer_recovery_injector;
+	astreamer_member member;
+} astreamer_recovery_injector;
 
-static void bbstreamer_recovery_injector_content(bbstreamer *streamer,
-												 bbstreamer_member *member,
-												 const char *data, int len,
-												 bbstreamer_archive_context context);
-static void bbstreamer_recovery_injector_finalize(bbstreamer *streamer);
-static void bbstreamer_recovery_injector_free(bbstreamer *streamer);
+static void astreamer_recovery_injector_content(astreamer *streamer,
+												astreamer_member *member,
+												const char *data, int len,
+												astreamer_archive_context context);
+static void astreamer_recovery_injector_finalize(astreamer *streamer);
+static void astreamer_recovery_injector_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_recovery_injector_ops = {
-	.content = bbstreamer_recovery_injector_content,
-	.finalize = bbstreamer_recovery_injector_finalize,
-	.free = bbstreamer_recovery_injector_free
+static const astreamer_ops astreamer_recovery_injector_ops = {
+	.content = astreamer_recovery_injector_content,
+	.finalize = astreamer_recovery_injector_finalize,
+	.free = astreamer_recovery_injector_free
 };
 
 /*
- * Create a bbstreamer that can edit recoverydata into an archive stream.
+ * Create a astreamer that can edit recoverydata into an archive stream.
  *
- * The input should be a series of typed chunks (not BBSTREAMER_UNKNOWN) as
- * per the conventions described in bbstreamer.h; the chunks forwarded to
- * the next bbstreamer will be similarly typed, but the
- * BBSTREAMER_MEMBER_HEADER chunks may be zero-length in cases where we've
+ * The input should be a series of typed chunks (not ASTREAMER_UNKNOWN) as
+ * per the conventions described in astreamer.h; the chunks forwarded to
+ * the next astreamer will be similarly typed, but the
+ * ASTREAMER_MEMBER_HEADER chunks may be zero-length in cases where we've
  * edited the archive stream.
  *
  * Our goal is to do one of the following three things with the content passed
@@ -61,16 +61,16 @@ static const bbstreamer_ops bbstreamer_recovery_injector_ops = {
  * zero-length standby.signal file, dropping any file with that name from
  * the archive.
  */
-bbstreamer *
-bbstreamer_recovery_injector_new(bbstreamer *next,
-								 bool is_recovery_guc_supported,
-								 PQExpBuffer recoveryconfcontents)
+astreamer *
+astreamer_recovery_injector_new(astreamer *next,
+								bool is_recovery_guc_supported,
+								PQExpBuffer recoveryconfcontents)
 {
-	bbstreamer_recovery_injector *streamer;
+	astreamer_recovery_injector *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_recovery_injector));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_recovery_injector_ops;
+	streamer = palloc0(sizeof(astreamer_recovery_injector));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_recovery_injector_ops;
 	streamer->base.bbs_next = next;
 	streamer->is_recovery_guc_supported = is_recovery_guc_supported;
 	streamer->recoveryconfcontents = recoveryconfcontents;
@@ -82,21 +82,21 @@ bbstreamer_recovery_injector_new(bbstreamer *next,
  * Handle each chunk of tar content while injecting recovery configuration.
  */
 static void
-bbstreamer_recovery_injector_content(bbstreamer *streamer,
-									 bbstreamer_member *member,
-									 const char *data, int len,
-									 bbstreamer_archive_context context)
+astreamer_recovery_injector_content(astreamer *streamer,
+									astreamer_member *member,
+									const char *data, int len,
+									astreamer_archive_context context)
 {
-	bbstreamer_recovery_injector *mystreamer;
+	astreamer_recovery_injector *mystreamer;
 
-	mystreamer = (bbstreamer_recovery_injector *) streamer;
-	Assert(member != NULL || context == BBSTREAMER_ARCHIVE_TRAILER);
+	mystreamer = (astreamer_recovery_injector *) streamer;
+	Assert(member != NULL || context == ASTREAMER_ARCHIVE_TRAILER);
 
 	switch (context)
 	{
-		case BBSTREAMER_MEMBER_HEADER:
+		case ASTREAMER_MEMBER_HEADER:
 			/* Must copy provided data so we have the option to modify it. */
-			memcpy(&mystreamer->member, member, sizeof(bbstreamer_member));
+			memcpy(&mystreamer->member, member, sizeof(astreamer_member));
 
 			/*
 			 * On v12+, skip standby.signal and edit postgresql.auto.conf; on
@@ -119,8 +119,8 @@ bbstreamer_recovery_injector_content(bbstreamer *streamer,
 
 					/*
 					 * Zap data and len because the archive header is no
-					 * longer valid; some subsequent bbstreamer must
-					 * regenerate it if it's necessary.
+					 * longer valid; some subsequent astreamer must regenerate
+					 * it if it's necessary.
 					 */
 					data = NULL;
 					len = 0;
@@ -135,26 +135,26 @@ bbstreamer_recovery_injector_content(bbstreamer *streamer,
 				return;
 			break;
 
-		case BBSTREAMER_MEMBER_CONTENTS:
+		case ASTREAMER_MEMBER_CONTENTS:
 			/* Do not forward if the file is to be skipped. */
 			if (mystreamer->skip_file)
 				return;
 			break;
 
-		case BBSTREAMER_MEMBER_TRAILER:
+		case ASTREAMER_MEMBER_TRAILER:
 			/* Do not forward it the file is to be skipped. */
 			if (mystreamer->skip_file)
 				return;
 
 			/* Append provided content to whatever we already sent. */
 			if (mystreamer->is_postgresql_auto_conf)
-				bbstreamer_content(mystreamer->base.bbs_next, member,
-								   mystreamer->recoveryconfcontents->data,
-								   mystreamer->recoveryconfcontents->len,
-								   BBSTREAMER_MEMBER_CONTENTS);
+				astreamer_content(mystreamer->base.bbs_next, member,
+								  mystreamer->recoveryconfcontents->data,
+								  mystreamer->recoveryconfcontents->len,
+								  ASTREAMER_MEMBER_CONTENTS);
 			break;
 
-		case BBSTREAMER_ARCHIVE_TRAILER:
+		case ASTREAMER_ARCHIVE_TRAILER:
 			if (mystreamer->is_recovery_guc_supported)
 			{
 				/*
@@ -163,22 +163,22 @@ bbstreamer_recovery_injector_content(bbstreamer *streamer,
 				 * member now.
 				 */
 				if (!mystreamer->found_postgresql_auto_conf)
-					bbstreamer_inject_file(mystreamer->base.bbs_next,
-										   "postgresql.auto.conf",
-										   mystreamer->recoveryconfcontents->data,
-										   mystreamer->recoveryconfcontents->len);
+					astreamer_inject_file(mystreamer->base.bbs_next,
+										  "postgresql.auto.conf",
+										  mystreamer->recoveryconfcontents->data,
+										  mystreamer->recoveryconfcontents->len);
 
 				/* Inject empty standby.signal file. */
-				bbstreamer_inject_file(mystreamer->base.bbs_next,
-									   "standby.signal", "", 0);
+				astreamer_inject_file(mystreamer->base.bbs_next,
+									  "standby.signal", "", 0);
 			}
 			else
 			{
 				/* Inject recovery.conf file with specified contents. */
-				bbstreamer_inject_file(mystreamer->base.bbs_next,
-									   "recovery.conf",
-									   mystreamer->recoveryconfcontents->data,
-									   mystreamer->recoveryconfcontents->len);
+				astreamer_inject_file(mystreamer->base.bbs_next,
+									  "recovery.conf",
+									  mystreamer->recoveryconfcontents->data,
+									  mystreamer->recoveryconfcontents->len);
 			}
 
 			/* Nothing to do here. */
@@ -189,26 +189,26 @@ bbstreamer_recovery_injector_content(bbstreamer *streamer,
 			pg_fatal("unexpected state while injecting recovery settings");
 	}
 
-	bbstreamer_content(mystreamer->base.bbs_next, &mystreamer->member,
-					   data, len, context);
+	astreamer_content(mystreamer->base.bbs_next, &mystreamer->member,
+					  data, len, context);
 }
 
 /*
- * End-of-stream processing for this bbstreamer.
+ * End-of-stream processing for this astreamer.
  */
 static void
-bbstreamer_recovery_injector_finalize(bbstreamer *streamer)
+astreamer_recovery_injector_finalize(astreamer *streamer)
 {
-	bbstreamer_finalize(streamer->bbs_next);
+	astreamer_finalize(streamer->bbs_next);
 }
 
 /*
- * Free memory associated with this bbstreamer.
+ * Free memory associated with this astreamer.
  */
 static void
-bbstreamer_recovery_injector_free(bbstreamer *streamer)
+astreamer_recovery_injector_free(astreamer *streamer)
 {
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	pfree(streamer);
 }
 
@@ -216,10 +216,10 @@ bbstreamer_recovery_injector_free(bbstreamer *streamer)
  * Inject a member into the archive with specified contents.
  */
 void
-bbstreamer_inject_file(bbstreamer *streamer, char *pathname, char *data,
-					   int len)
+astreamer_inject_file(astreamer *streamer, char *pathname, char *data,
+					  int len)
 {
-	bbstreamer_member member;
+	astreamer_member member;
 
 	strlcpy(member.pathname, pathname, MAXPGPATH);
 	member.size = len;
@@ -238,12 +238,12 @@ bbstreamer_inject_file(bbstreamer *streamer, char *pathname, char *data,
 	/*
 	 * We don't know here how to generate valid member headers and trailers
 	 * for the archiving format in use, so if those are needed, some successor
-	 * bbstreamer will have to generate them using the data from 'member'.
+	 * astreamer will have to generate them using the data from 'member'.
 	 */
-	bbstreamer_content(streamer, &member, NULL, 0,
-					   BBSTREAMER_MEMBER_HEADER);
-	bbstreamer_content(streamer, &member, data, len,
-					   BBSTREAMER_MEMBER_CONTENTS);
-	bbstreamer_content(streamer, &member, NULL, 0,
-					   BBSTREAMER_MEMBER_TRAILER);
+	astreamer_content(streamer, &member, NULL, 0,
+					  ASTREAMER_MEMBER_HEADER);
+	astreamer_content(streamer, &member, data, len,
+					  ASTREAMER_MEMBER_CONTENTS);
+	astreamer_content(streamer, &member, NULL, 0,
+					  ASTREAMER_MEMBER_TRAILER);
 }
diff --git a/src/bin/pg_basebackup/bbstreamer_lz4.c b/src/bin/pg_basebackup/astreamer_lz4.c
similarity index 69%
rename from src/bin/pg_basebackup/bbstreamer_lz4.c
rename to src/bin/pg_basebackup/astreamer_lz4.c
index f5c9e68150c..1c40d7d8ad5 100644
--- a/src/bin/pg_basebackup/bbstreamer_lz4.c
+++ b/src/bin/pg_basebackup/astreamer_lz4.c
@@ -1,11 +1,11 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_lz4.c
+ * astreamer_lz4.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_lz4.c
+ *		  src/bin/pg_basebackup/astreamer_lz4.c
  *-------------------------------------------------------------------------
  */
 
@@ -17,15 +17,15 @@
 #include <lz4frame.h>
 #endif
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
 
 #ifdef USE_LZ4
-typedef struct bbstreamer_lz4_frame
+typedef struct astreamer_lz4_frame
 {
-	bbstreamer	base;
+	astreamer	base;
 
 	LZ4F_compressionContext_t cctx;
 	LZ4F_decompressionContext_t dctx;
@@ -33,32 +33,32 @@ typedef struct bbstreamer_lz4_frame
 
 	size_t		bytes_written;
 	bool		header_written;
-} bbstreamer_lz4_frame;
+} astreamer_lz4_frame;
 
-static void bbstreamer_lz4_compressor_content(bbstreamer *streamer,
-											  bbstreamer_member *member,
-											  const char *data, int len,
-											  bbstreamer_archive_context context);
-static void bbstreamer_lz4_compressor_finalize(bbstreamer *streamer);
-static void bbstreamer_lz4_compressor_free(bbstreamer *streamer);
+static void astreamer_lz4_compressor_content(astreamer *streamer,
+											 astreamer_member *member,
+											 const char *data, int len,
+											 astreamer_archive_context context);
+static void astreamer_lz4_compressor_finalize(astreamer *streamer);
+static void astreamer_lz4_compressor_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_lz4_compressor_ops = {
-	.content = bbstreamer_lz4_compressor_content,
-	.finalize = bbstreamer_lz4_compressor_finalize,
-	.free = bbstreamer_lz4_compressor_free
+static const astreamer_ops astreamer_lz4_compressor_ops = {
+	.content = astreamer_lz4_compressor_content,
+	.finalize = astreamer_lz4_compressor_finalize,
+	.free = astreamer_lz4_compressor_free
 };
 
-static void bbstreamer_lz4_decompressor_content(bbstreamer *streamer,
-												bbstreamer_member *member,
-												const char *data, int len,
-												bbstreamer_archive_context context);
-static void bbstreamer_lz4_decompressor_finalize(bbstreamer *streamer);
-static void bbstreamer_lz4_decompressor_free(bbstreamer *streamer);
+static void astreamer_lz4_decompressor_content(astreamer *streamer,
+											   astreamer_member *member,
+											   const char *data, int len,
+											   astreamer_archive_context context);
+static void astreamer_lz4_decompressor_finalize(astreamer *streamer);
+static void astreamer_lz4_decompressor_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_lz4_decompressor_ops = {
-	.content = bbstreamer_lz4_decompressor_content,
-	.finalize = bbstreamer_lz4_decompressor_finalize,
-	.free = bbstreamer_lz4_decompressor_free
+static const astreamer_ops astreamer_lz4_decompressor_ops = {
+	.content = astreamer_lz4_decompressor_content,
+	.finalize = astreamer_lz4_decompressor_finalize,
+	.free = astreamer_lz4_decompressor_free
 };
 #endif
 
@@ -66,19 +66,19 @@ static const bbstreamer_ops bbstreamer_lz4_decompressor_ops = {
  * Create a new base backup streamer that performs lz4 compression of tar
  * blocks.
  */
-bbstreamer *
-bbstreamer_lz4_compressor_new(bbstreamer *next, pg_compress_specification *compress)
+astreamer *
+astreamer_lz4_compressor_new(astreamer *next, pg_compress_specification *compress)
 {
 #ifdef USE_LZ4
-	bbstreamer_lz4_frame *streamer;
+	astreamer_lz4_frame *streamer;
 	LZ4F_errorCode_t ctxError;
 	LZ4F_preferences_t *prefs;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_lz4_frame));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_lz4_compressor_ops;
+	streamer = palloc0(sizeof(astreamer_lz4_frame));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_lz4_compressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -113,19 +113,19 @@ bbstreamer_lz4_compressor_new(bbstreamer *next, pg_compress_specification *compr
  * of output buffer to next streamer and empty the buffer.
  */
 static void
-bbstreamer_lz4_compressor_content(bbstreamer *streamer,
-								  bbstreamer_member *member,
-								  const char *data, int len,
-								  bbstreamer_archive_context context)
+astreamer_lz4_compressor_content(astreamer *streamer,
+								 astreamer_member *member,
+								 const char *data, int len,
+								 astreamer_archive_context context)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 	uint8	   *next_in,
 			   *next_out;
 	size_t		out_bound,
 				compressed_size,
 				avail_out;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
+	mystreamer = (astreamer_lz4_frame *) streamer;
 	next_in = (uint8 *) data;
 
 	/* Write header before processing the first input chunk. */
@@ -159,10 +159,10 @@ bbstreamer_lz4_compressor_content(bbstreamer *streamer,
 	out_bound = LZ4F_compressBound(len, &mystreamer->prefs);
 	if (avail_out < out_bound)
 	{
-		bbstreamer_content(mystreamer->base.bbs_next, member,
-						   mystreamer->base.bbs_buffer.data,
-						   mystreamer->bytes_written,
-						   context);
+		astreamer_content(mystreamer->base.bbs_next, member,
+						  mystreamer->base.bbs_buffer.data,
+						  mystreamer->bytes_written,
+						  context);
 
 		/* Enlarge buffer if it falls short of out bound. */
 		if (mystreamer->base.bbs_buffer.maxlen < out_bound)
@@ -196,25 +196,25 @@ bbstreamer_lz4_compressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_lz4_compressor_finalize(bbstreamer *streamer)
+astreamer_lz4_compressor_finalize(astreamer *streamer)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 	uint8	   *next_out;
 	size_t		footer_bound,
 				compressed_size,
 				avail_out;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
+	mystreamer = (astreamer_lz4_frame *) streamer;
 
 	/* Find out the footer bound and update the output buffer. */
 	footer_bound = LZ4F_compressBound(0, &mystreamer->prefs);
 	if ((mystreamer->base.bbs_buffer.maxlen - mystreamer->bytes_written) <
 		footer_bound)
 	{
-		bbstreamer_content(mystreamer->base.bbs_next, NULL,
-						   mystreamer->base.bbs_buffer.data,
-						   mystreamer->bytes_written,
-						   BBSTREAMER_UNKNOWN);
+		astreamer_content(mystreamer->base.bbs_next, NULL,
+						  mystreamer->base.bbs_buffer.data,
+						  mystreamer->bytes_written,
+						  ASTREAMER_UNKNOWN);
 
 		/* Enlarge buffer if it falls short of footer bound. */
 		if (mystreamer->base.bbs_buffer.maxlen < footer_bound)
@@ -243,24 +243,24 @@ bbstreamer_lz4_compressor_finalize(bbstreamer *streamer)
 
 	mystreamer->bytes_written += compressed_size;
 
-	bbstreamer_content(mystreamer->base.bbs_next, NULL,
-					   mystreamer->base.bbs_buffer.data,
-					   mystreamer->bytes_written,
-					   BBSTREAMER_UNKNOWN);
+	astreamer_content(mystreamer->base.bbs_next, NULL,
+					  mystreamer->base.bbs_buffer.data,
+					  mystreamer->bytes_written,
+					  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_lz4_compressor_free(bbstreamer *streamer)
+astreamer_lz4_compressor_free(astreamer *streamer)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
-	bbstreamer_free(streamer->bbs_next);
+	mystreamer = (astreamer_lz4_frame *) streamer;
+	astreamer_free(streamer->bbs_next);
 	LZ4F_freeCompressionContext(mystreamer->cctx);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
@@ -271,18 +271,18 @@ bbstreamer_lz4_compressor_free(bbstreamer *streamer)
  * Create a new base backup streamer that performs decompression of lz4
  * compressed blocks.
  */
-bbstreamer *
-bbstreamer_lz4_decompressor_new(bbstreamer *next)
+astreamer *
+astreamer_lz4_decompressor_new(astreamer *next)
 {
 #ifdef USE_LZ4
-	bbstreamer_lz4_frame *streamer;
+	astreamer_lz4_frame *streamer;
 	LZ4F_errorCode_t ctxError;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_lz4_frame));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_lz4_decompressor_ops;
+	streamer = palloc0(sizeof(astreamer_lz4_frame));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_lz4_decompressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -307,18 +307,18 @@ bbstreamer_lz4_decompressor_new(bbstreamer *next)
  * to the next streamer.
  */
 static void
-bbstreamer_lz4_decompressor_content(bbstreamer *streamer,
-									bbstreamer_member *member,
-									const char *data, int len,
-									bbstreamer_archive_context context)
+astreamer_lz4_decompressor_content(astreamer *streamer,
+								   astreamer_member *member,
+								   const char *data, int len,
+								   astreamer_archive_context context)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 	uint8	   *next_in,
 			   *next_out;
 	size_t		avail_in,
 				avail_out;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
+	mystreamer = (astreamer_lz4_frame *) streamer;
 	next_in = (uint8 *) data;
 	next_out = (uint8 *) mystreamer->base.bbs_buffer.data;
 	avail_in = len;
@@ -366,10 +366,10 @@ bbstreamer_lz4_decompressor_content(bbstreamer *streamer,
 		 */
 		if (mystreamer->bytes_written >= mystreamer->base.bbs_buffer.maxlen)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, member,
-							   mystreamer->base.bbs_buffer.data,
-							   mystreamer->base.bbs_buffer.maxlen,
-							   context);
+			astreamer_content(mystreamer->base.bbs_next, member,
+							  mystreamer->base.bbs_buffer.data,
+							  mystreamer->base.bbs_buffer.maxlen,
+							  context);
 
 			avail_out = mystreamer->base.bbs_buffer.maxlen;
 			mystreamer->bytes_written = 0;
@@ -387,34 +387,34 @@ bbstreamer_lz4_decompressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_lz4_decompressor_finalize(bbstreamer *streamer)
+astreamer_lz4_decompressor_finalize(astreamer *streamer)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
+	mystreamer = (astreamer_lz4_frame *) streamer;
 
 	/*
 	 * End of the stream, if there is some pending data in output buffers then
 	 * we must forward it to next streamer.
 	 */
-	bbstreamer_content(mystreamer->base.bbs_next, NULL,
-					   mystreamer->base.bbs_buffer.data,
-					   mystreamer->base.bbs_buffer.maxlen,
-					   BBSTREAMER_UNKNOWN);
+	astreamer_content(mystreamer->base.bbs_next, NULL,
+					  mystreamer->base.bbs_buffer.data,
+					  mystreamer->base.bbs_buffer.maxlen,
+					  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_lz4_decompressor_free(bbstreamer *streamer)
+astreamer_lz4_decompressor_free(astreamer *streamer)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
-	bbstreamer_free(streamer->bbs_next);
+	mystreamer = (astreamer_lz4_frame *) streamer;
+	astreamer_free(streamer->bbs_next);
 	LZ4F_freeDecompressionContext(mystreamer->dctx);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
diff --git a/src/bin/pg_basebackup/bbstreamer_tar.c b/src/bin/pg_basebackup/astreamer_tar.c
similarity index 50%
rename from src/bin/pg_basebackup/bbstreamer_tar.c
rename to src/bin/pg_basebackup/astreamer_tar.c
index 9137d17ddc1..673690cd18f 100644
--- a/src/bin/pg_basebackup/bbstreamer_tar.c
+++ b/src/bin/pg_basebackup/astreamer_tar.c
@@ -1,13 +1,13 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_tar.c
+ * astreamer_tar.c
  *
  * This module implements three types of tar processing. A tar parser
- * expects unlabelled chunks of data (e.g. BBSTREAMER_UNKNOWN) and splits
- * it into labelled chunks (any other value of bbstreamer_archive_context).
+ * expects unlabelled chunks of data (e.g. ASTREAMER_UNKNOWN) and splits
+ * it into labelled chunks (any other value of astreamer_archive_context).
  * A tar archiver does the reverse: it takes a bunch of labelled chunks
  * and produces a tarfile, optionally replacing member headers and trailers
- * so that upstream bbstreamer objects can perform surgery on the tarfile
+ * so that upstream astreamer objects can perform surgery on the tarfile
  * contents without knowing the details of the tar format. A tar terminator
  * just adds two blocks of NUL bytes to the end of the file, since older
  * server versions produce files with this terminator omitted.
@@ -15,7 +15,7 @@
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_tar.c
+ *		  src/bin/pg_basebackup/astreamer_tar.c
  *-------------------------------------------------------------------------
  */
 
@@ -23,83 +23,83 @@
 
 #include <time.h>
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/logging.h"
 #include "pgtar.h"
 
-typedef struct bbstreamer_tar_parser
+typedef struct astreamer_tar_parser
 {
-	bbstreamer	base;
-	bbstreamer_archive_context next_context;
-	bbstreamer_member member;
+	astreamer	base;
+	astreamer_archive_context next_context;
+	astreamer_member member;
 	size_t		file_bytes_sent;
 	size_t		pad_bytes_expected;
-} bbstreamer_tar_parser;
+} astreamer_tar_parser;
 
-typedef struct bbstreamer_tar_archiver
+typedef struct astreamer_tar_archiver
 {
-	bbstreamer	base;
+	astreamer	base;
 	bool		rearchive_member;
-} bbstreamer_tar_archiver;
+} astreamer_tar_archiver;
 
-static void bbstreamer_tar_parser_content(bbstreamer *streamer,
-										  bbstreamer_member *member,
-										  const char *data, int len,
-										  bbstreamer_archive_context context);
-static void bbstreamer_tar_parser_finalize(bbstreamer *streamer);
-static void bbstreamer_tar_parser_free(bbstreamer *streamer);
-static bool bbstreamer_tar_header(bbstreamer_tar_parser *mystreamer);
+static void astreamer_tar_parser_content(astreamer *streamer,
+										 astreamer_member *member,
+										 const char *data, int len,
+										 astreamer_archive_context context);
+static void astreamer_tar_parser_finalize(astreamer *streamer);
+static void astreamer_tar_parser_free(astreamer *streamer);
+static bool astreamer_tar_header(astreamer_tar_parser *mystreamer);
 
-static const bbstreamer_ops bbstreamer_tar_parser_ops = {
-	.content = bbstreamer_tar_parser_content,
-	.finalize = bbstreamer_tar_parser_finalize,
-	.free = bbstreamer_tar_parser_free
+static const astreamer_ops astreamer_tar_parser_ops = {
+	.content = astreamer_tar_parser_content,
+	.finalize = astreamer_tar_parser_finalize,
+	.free = astreamer_tar_parser_free
 };
 
-static void bbstreamer_tar_archiver_content(bbstreamer *streamer,
-											bbstreamer_member *member,
-											const char *data, int len,
-											bbstreamer_archive_context context);
-static void bbstreamer_tar_archiver_finalize(bbstreamer *streamer);
-static void bbstreamer_tar_archiver_free(bbstreamer *streamer);
+static void astreamer_tar_archiver_content(astreamer *streamer,
+										   astreamer_member *member,
+										   const char *data, int len,
+										   astreamer_archive_context context);
+static void astreamer_tar_archiver_finalize(astreamer *streamer);
+static void astreamer_tar_archiver_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_tar_archiver_ops = {
-	.content = bbstreamer_tar_archiver_content,
-	.finalize = bbstreamer_tar_archiver_finalize,
-	.free = bbstreamer_tar_archiver_free
+static const astreamer_ops astreamer_tar_archiver_ops = {
+	.content = astreamer_tar_archiver_content,
+	.finalize = astreamer_tar_archiver_finalize,
+	.free = astreamer_tar_archiver_free
 };
 
-static void bbstreamer_tar_terminator_content(bbstreamer *streamer,
-											  bbstreamer_member *member,
-											  const char *data, int len,
-											  bbstreamer_archive_context context);
-static void bbstreamer_tar_terminator_finalize(bbstreamer *streamer);
-static void bbstreamer_tar_terminator_free(bbstreamer *streamer);
+static void astreamer_tar_terminator_content(astreamer *streamer,
+											 astreamer_member *member,
+											 const char *data, int len,
+											 astreamer_archive_context context);
+static void astreamer_tar_terminator_finalize(astreamer *streamer);
+static void astreamer_tar_terminator_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_tar_terminator_ops = {
-	.content = bbstreamer_tar_terminator_content,
-	.finalize = bbstreamer_tar_terminator_finalize,
-	.free = bbstreamer_tar_terminator_free
+static const astreamer_ops astreamer_tar_terminator_ops = {
+	.content = astreamer_tar_terminator_content,
+	.finalize = astreamer_tar_terminator_finalize,
+	.free = astreamer_tar_terminator_free
 };
 
 /*
- * Create a bbstreamer that can parse a stream of content as tar data.
+ * Create a astreamer that can parse a stream of content as tar data.
  *
- * The input should be a series of BBSTREAMER_UNKNOWN chunks; the bbstreamer
+ * The input should be a series of ASTREAMER_UNKNOWN chunks; the astreamer
  * specified by 'next' will receive a series of typed chunks, as per the
- * conventions described in bbstreamer.h.
+ * conventions described in astreamer.h.
  */
-bbstreamer *
-bbstreamer_tar_parser_new(bbstreamer *next)
+astreamer *
+astreamer_tar_parser_new(astreamer *next)
 {
-	bbstreamer_tar_parser *streamer;
+	astreamer_tar_parser *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_tar_parser));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_tar_parser_ops;
+	streamer = palloc0(sizeof(astreamer_tar_parser));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_tar_parser_ops;
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
-	streamer->next_context = BBSTREAMER_MEMBER_HEADER;
+	streamer->next_context = ASTREAMER_MEMBER_HEADER;
 
 	return &streamer->base;
 }
@@ -108,29 +108,29 @@ bbstreamer_tar_parser_new(bbstreamer *next)
  * Parse unknown content as tar data.
  */
 static void
-bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
-							  const char *data, int len,
-							  bbstreamer_archive_context context)
+astreamer_tar_parser_content(astreamer *streamer, astreamer_member *member,
+							 const char *data, int len,
+							 astreamer_archive_context context)
 {
-	bbstreamer_tar_parser *mystreamer = (bbstreamer_tar_parser *) streamer;
+	astreamer_tar_parser *mystreamer = (astreamer_tar_parser *) streamer;
 	size_t		nbytes;
 
 	/* Expect unparsed input. */
 	Assert(member == NULL);
-	Assert(context == BBSTREAMER_UNKNOWN);
+	Assert(context == ASTREAMER_UNKNOWN);
 
 	while (len > 0)
 	{
 		switch (mystreamer->next_context)
 		{
-			case BBSTREAMER_MEMBER_HEADER:
+			case ASTREAMER_MEMBER_HEADER:
 
 				/*
 				 * If we're expecting an archive member header, accumulate a
 				 * full block of data before doing anything further.
 				 */
-				if (!bbstreamer_buffer_until(streamer, &data, &len,
-											 TAR_BLOCK_SIZE))
+				if (!astreamer_buffer_until(streamer, &data, &len,
+											TAR_BLOCK_SIZE))
 					return;
 
 				/*
@@ -139,32 +139,32 @@ bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
 				 * thought was the next file header is actually the start of
 				 * the archive trailer. Switch modes accordingly.
 				 */
-				if (bbstreamer_tar_header(mystreamer))
+				if (astreamer_tar_header(mystreamer))
 				{
 					if (mystreamer->member.size == 0)
 					{
 						/* No content; trailer is zero-length. */
-						bbstreamer_content(mystreamer->base.bbs_next,
-										   &mystreamer->member,
-										   NULL, 0,
-										   BBSTREAMER_MEMBER_TRAILER);
+						astreamer_content(mystreamer->base.bbs_next,
+										  &mystreamer->member,
+										  NULL, 0,
+										  ASTREAMER_MEMBER_TRAILER);
 
 						/* Expect next header. */
-						mystreamer->next_context = BBSTREAMER_MEMBER_HEADER;
+						mystreamer->next_context = ASTREAMER_MEMBER_HEADER;
 					}
 					else
 					{
 						/* Expect contents. */
-						mystreamer->next_context = BBSTREAMER_MEMBER_CONTENTS;
+						mystreamer->next_context = ASTREAMER_MEMBER_CONTENTS;
 					}
 					mystreamer->base.bbs_buffer.len = 0;
 					mystreamer->file_bytes_sent = 0;
 				}
 				else
-					mystreamer->next_context = BBSTREAMER_ARCHIVE_TRAILER;
+					mystreamer->next_context = ASTREAMER_ARCHIVE_TRAILER;
 				break;
 
-			case BBSTREAMER_MEMBER_CONTENTS:
+			case ASTREAMER_MEMBER_CONTENTS:
 
 				/*
 				 * Send as much content as we have, but not more than the
@@ -174,10 +174,10 @@ bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
 				nbytes = mystreamer->member.size - mystreamer->file_bytes_sent;
 				nbytes = Min(nbytes, len);
 				Assert(nbytes > 0);
-				bbstreamer_content(mystreamer->base.bbs_next,
-								   &mystreamer->member,
-								   data, nbytes,
-								   BBSTREAMER_MEMBER_CONTENTS);
+				astreamer_content(mystreamer->base.bbs_next,
+								  &mystreamer->member,
+								  data, nbytes,
+								  ASTREAMER_MEMBER_CONTENTS);
 				mystreamer->file_bytes_sent += nbytes;
 				data += nbytes;
 				len -= nbytes;
@@ -193,53 +193,53 @@ bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
 					if (mystreamer->pad_bytes_expected == 0)
 					{
 						/* Trailer is zero-length. */
-						bbstreamer_content(mystreamer->base.bbs_next,
-										   &mystreamer->member,
-										   NULL, 0,
-										   BBSTREAMER_MEMBER_TRAILER);
+						astreamer_content(mystreamer->base.bbs_next,
+										  &mystreamer->member,
+										  NULL, 0,
+										  ASTREAMER_MEMBER_TRAILER);
 
 						/* Expect next header. */
-						mystreamer->next_context = BBSTREAMER_MEMBER_HEADER;
+						mystreamer->next_context = ASTREAMER_MEMBER_HEADER;
 					}
 					else
 					{
 						/* Trailer is not zero-length. */
-						mystreamer->next_context = BBSTREAMER_MEMBER_TRAILER;
+						mystreamer->next_context = ASTREAMER_MEMBER_TRAILER;
 					}
 					mystreamer->base.bbs_buffer.len = 0;
 				}
 				break;
 
-			case BBSTREAMER_MEMBER_TRAILER:
+			case ASTREAMER_MEMBER_TRAILER:
 
 				/*
 				 * If we're expecting an archive member trailer, accumulate
 				 * the expected number of padding bytes before sending
 				 * anything onward.
 				 */
-				if (!bbstreamer_buffer_until(streamer, &data, &len,
-											 mystreamer->pad_bytes_expected))
+				if (!astreamer_buffer_until(streamer, &data, &len,
+											mystreamer->pad_bytes_expected))
 					return;
 
 				/* OK, now we can send it. */
-				bbstreamer_content(mystreamer->base.bbs_next,
-								   &mystreamer->member,
-								   data, mystreamer->pad_bytes_expected,
-								   BBSTREAMER_MEMBER_TRAILER);
+				astreamer_content(mystreamer->base.bbs_next,
+								  &mystreamer->member,
+								  data, mystreamer->pad_bytes_expected,
+								  ASTREAMER_MEMBER_TRAILER);
 
 				/* Expect next file header. */
-				mystreamer->next_context = BBSTREAMER_MEMBER_HEADER;
+				mystreamer->next_context = ASTREAMER_MEMBER_HEADER;
 				mystreamer->base.bbs_buffer.len = 0;
 				break;
 
-			case BBSTREAMER_ARCHIVE_TRAILER:
+			case ASTREAMER_ARCHIVE_TRAILER:
 
 				/*
 				 * We've seen an end-of-archive indicator, so anything more is
 				 * buffered and sent as part of the archive trailer. But we
 				 * don't expect more than 2 blocks.
 				 */
-				bbstreamer_buffer_bytes(streamer, &data, &len, len);
+				astreamer_buffer_bytes(streamer, &data, &len, len);
 				if (len > 2 * TAR_BLOCK_SIZE)
 					pg_fatal("tar file trailer exceeds 2 blocks");
 				return;
@@ -255,14 +255,14 @@ bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
  * Parse a file header within a tar stream.
  *
  * The return value is true if we found a file header and passed it on to the
- * next bbstreamer; it is false if we have reached the archive trailer.
+ * next astreamer; it is false if we have reached the archive trailer.
  */
 static bool
-bbstreamer_tar_header(bbstreamer_tar_parser *mystreamer)
+astreamer_tar_header(astreamer_tar_parser *mystreamer)
 {
 	bool		has_nonzero_byte = false;
 	int			i;
-	bbstreamer_member *member = &mystreamer->member;
+	astreamer_member *member = &mystreamer->member;
 	char	   *buffer = mystreamer->base.bbs_buffer.data;
 
 	Assert(mystreamer->base.bbs_buffer.len == TAR_BLOCK_SIZE);
@@ -304,10 +304,10 @@ bbstreamer_tar_header(bbstreamer_tar_parser *mystreamer)
 	/* Compute number of padding bytes. */
 	mystreamer->pad_bytes_expected = tarPaddingBytesRequired(member->size);
 
-	/* Forward the entire header to the next bbstreamer. */
-	bbstreamer_content(mystreamer->base.bbs_next, member,
-					   buffer, TAR_BLOCK_SIZE,
-					   BBSTREAMER_MEMBER_HEADER);
+	/* Forward the entire header to the next astreamer. */
+	astreamer_content(mystreamer->base.bbs_next, member,
+					  buffer, TAR_BLOCK_SIZE,
+					  ASTREAMER_MEMBER_HEADER);
 
 	return true;
 }
@@ -316,50 +316,50 @@ bbstreamer_tar_header(bbstreamer_tar_parser *mystreamer)
  * End-of-stream processing for a tar parser.
  */
 static void
-bbstreamer_tar_parser_finalize(bbstreamer *streamer)
+astreamer_tar_parser_finalize(astreamer *streamer)
 {
-	bbstreamer_tar_parser *mystreamer = (bbstreamer_tar_parser *) streamer;
+	astreamer_tar_parser *mystreamer = (astreamer_tar_parser *) streamer;
 
-	if (mystreamer->next_context != BBSTREAMER_ARCHIVE_TRAILER &&
-		(mystreamer->next_context != BBSTREAMER_MEMBER_HEADER ||
+	if (mystreamer->next_context != ASTREAMER_ARCHIVE_TRAILER &&
+		(mystreamer->next_context != ASTREAMER_MEMBER_HEADER ||
 		 mystreamer->base.bbs_buffer.len > 0))
 		pg_fatal("COPY stream ended before last file was finished");
 
 	/* Send the archive trailer, even if empty. */
-	bbstreamer_content(streamer->bbs_next, NULL,
-					   streamer->bbs_buffer.data, streamer->bbs_buffer.len,
-					   BBSTREAMER_ARCHIVE_TRAILER);
+	astreamer_content(streamer->bbs_next, NULL,
+					  streamer->bbs_buffer.data, streamer->bbs_buffer.len,
+					  ASTREAMER_ARCHIVE_TRAILER);
 
 	/* Now finalize successor. */
-	bbstreamer_finalize(streamer->bbs_next);
+	astreamer_finalize(streamer->bbs_next);
 }
 
 /*
  * Free memory associated with a tar parser.
  */
 static void
-bbstreamer_tar_parser_free(bbstreamer *streamer)
+astreamer_tar_parser_free(astreamer *streamer)
 {
 	pfree(streamer->bbs_buffer.data);
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 }
 
 /*
- * Create a bbstreamer that can generate a tar archive.
+ * Create a astreamer that can generate a tar archive.
  *
  * This is intended to be usable either for generating a brand-new tar archive
  * or for modifying one on the fly. The input should be a series of typed
- * chunks (i.e. not BBSTREAMER_UNKNOWN). See also the comments for
- * bbstreamer_tar_parser_content.
+ * chunks (i.e. not ASTREAMER_UNKNOWN). See also the comments for
+ * astreamer_tar_parser_content.
  */
-bbstreamer *
-bbstreamer_tar_archiver_new(bbstreamer *next)
+astreamer *
+astreamer_tar_archiver_new(astreamer *next)
 {
-	bbstreamer_tar_archiver *streamer;
+	astreamer_tar_archiver *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_tar_archiver));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_tar_archiver_ops;
+	streamer = palloc0(sizeof(astreamer_tar_archiver));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_tar_archiver_ops;
 	streamer->base.bbs_next = next;
 
 	return &streamer->base;
@@ -368,36 +368,36 @@ bbstreamer_tar_archiver_new(bbstreamer *next)
 /*
  * Fix up the stream of input chunks to create a valid tar file.
  *
- * If a BBSTREAMER_MEMBER_HEADER chunk is of size 0, it is replaced with a
+ * If a ASTREAMER_MEMBER_HEADER chunk is of size 0, it is replaced with a
  * newly-constructed tar header. If it is of size TAR_BLOCK_SIZE, it is
  * passed through without change. Any other size is a fatal error (and
  * indicates a bug).
  *
- * Whenever a new BBSTREAMER_MEMBER_HEADER chunk is constructed, the
- * corresponding BBSTREAMER_MEMBER_TRAILER chunk is also constructed from
+ * Whenever a new ASTREAMER_MEMBER_HEADER chunk is constructed, the
+ * corresponding ASTREAMER_MEMBER_TRAILER chunk is also constructed from
  * scratch. Specifically, we construct a block of zero bytes sufficient to
  * pad out to a block boundary, as required by the tar format. Other
- * BBSTREAMER_MEMBER_TRAILER chunks are passed through without change.
+ * ASTREAMER_MEMBER_TRAILER chunks are passed through without change.
  *
- * Any BBSTREAMER_MEMBER_CONTENTS chunks are passed through without change.
+ * Any ASTREAMER_MEMBER_CONTENTS chunks are passed through without change.
  *
- * The BBSTREAMER_ARCHIVE_TRAILER chunk is replaced with two
+ * The ASTREAMER_ARCHIVE_TRAILER chunk is replaced with two
  * blocks of zero bytes. Not all tar programs require this, but apparently
  * some do. The server does not supply this trailer. If no archive trailer is
- * present, one will be added by bbstreamer_tar_parser_finalize.
+ * present, one will be added by astreamer_tar_parser_finalize.
  */
 static void
-bbstreamer_tar_archiver_content(bbstreamer *streamer,
-								bbstreamer_member *member,
-								const char *data, int len,
-								bbstreamer_archive_context context)
+astreamer_tar_archiver_content(astreamer *streamer,
+							   astreamer_member *member,
+							   const char *data, int len,
+							   astreamer_archive_context context)
 {
-	bbstreamer_tar_archiver *mystreamer = (bbstreamer_tar_archiver *) streamer;
+	astreamer_tar_archiver *mystreamer = (astreamer_tar_archiver *) streamer;
 	char		buffer[2 * TAR_BLOCK_SIZE];
 
-	Assert(context != BBSTREAMER_UNKNOWN);
+	Assert(context != ASTREAMER_UNKNOWN);
 
-	if (context == BBSTREAMER_MEMBER_HEADER && len != TAR_BLOCK_SIZE)
+	if (context == ASTREAMER_MEMBER_HEADER && len != TAR_BLOCK_SIZE)
 	{
 		Assert(len == 0);
 
@@ -411,7 +411,7 @@ bbstreamer_tar_archiver_content(bbstreamer *streamer,
 		/* Also make a note to replace padding, in case size changed. */
 		mystreamer->rearchive_member = true;
 	}
-	else if (context == BBSTREAMER_MEMBER_TRAILER &&
+	else if (context == ASTREAMER_MEMBER_TRAILER &&
 			 mystreamer->rearchive_member)
 	{
 		int			pad_bytes = tarPaddingBytesRequired(member->size);
@@ -424,7 +424,7 @@ bbstreamer_tar_archiver_content(bbstreamer *streamer,
 		/* Don't do this again unless we replace another header. */
 		mystreamer->rearchive_member = false;
 	}
-	else if (context == BBSTREAMER_ARCHIVE_TRAILER)
+	else if (context == ASTREAMER_ARCHIVE_TRAILER)
 	{
 		/* Trailer should always be two blocks of zero bytes. */
 		memset(buffer, 0, 2 * TAR_BLOCK_SIZE);
@@ -432,40 +432,40 @@ bbstreamer_tar_archiver_content(bbstreamer *streamer,
 		len = 2 * TAR_BLOCK_SIZE;
 	}
 
-	bbstreamer_content(streamer->bbs_next, member, data, len, context);
+	astreamer_content(streamer->bbs_next, member, data, len, context);
 }
 
 /*
  * End-of-stream processing for a tar archiver.
  */
 static void
-bbstreamer_tar_archiver_finalize(bbstreamer *streamer)
+astreamer_tar_archiver_finalize(astreamer *streamer)
 {
-	bbstreamer_finalize(streamer->bbs_next);
+	astreamer_finalize(streamer->bbs_next);
 }
 
 /*
  * Free memory associated with a tar archiver.
  */
 static void
-bbstreamer_tar_archiver_free(bbstreamer *streamer)
+astreamer_tar_archiver_free(astreamer *streamer)
 {
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	pfree(streamer);
 }
 
 /*
- * Create a bbstreamer that blindly adds two blocks of NUL bytes to the
+ * Create a astreamer that blindly adds two blocks of NUL bytes to the
  * end of an incomplete tarfile that the server might send us.
  */
-bbstreamer *
-bbstreamer_tar_terminator_new(bbstreamer *next)
+astreamer *
+astreamer_tar_terminator_new(astreamer *next)
 {
-	bbstreamer *streamer;
+	astreamer  *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer));
-	*((const bbstreamer_ops **) &streamer->bbs_ops) =
-		&bbstreamer_tar_terminator_ops;
+	streamer = palloc0(sizeof(astreamer));
+	*((const astreamer_ops **) &streamer->bbs_ops) =
+		&astreamer_tar_terminator_ops;
 	streamer->bbs_next = next;
 
 	return streamer;
@@ -475,17 +475,17 @@ bbstreamer_tar_terminator_new(bbstreamer *next)
  * Pass all the content through without change.
  */
 static void
-bbstreamer_tar_terminator_content(bbstreamer *streamer,
-								  bbstreamer_member *member,
-								  const char *data, int len,
-								  bbstreamer_archive_context context)
+astreamer_tar_terminator_content(astreamer *streamer,
+								 astreamer_member *member,
+								 const char *data, int len,
+								 astreamer_archive_context context)
 {
 	/* Expect unparsed input. */
 	Assert(member == NULL);
-	Assert(context == BBSTREAMER_UNKNOWN);
+	Assert(context == ASTREAMER_UNKNOWN);
 
 	/* Just forward it. */
-	bbstreamer_content(streamer->bbs_next, member, data, len, context);
+	astreamer_content(streamer->bbs_next, member, data, len, context);
 }
 
 /*
@@ -493,22 +493,22 @@ bbstreamer_tar_terminator_content(bbstreamer *streamer,
  * to supply.
  */
 static void
-bbstreamer_tar_terminator_finalize(bbstreamer *streamer)
+astreamer_tar_terminator_finalize(astreamer *streamer)
 {
 	char		buffer[2 * TAR_BLOCK_SIZE];
 
 	memset(buffer, 0, 2 * TAR_BLOCK_SIZE);
-	bbstreamer_content(streamer->bbs_next, NULL, buffer,
-					   2 * TAR_BLOCK_SIZE, BBSTREAMER_UNKNOWN);
-	bbstreamer_finalize(streamer->bbs_next);
+	astreamer_content(streamer->bbs_next, NULL, buffer,
+					  2 * TAR_BLOCK_SIZE, ASTREAMER_UNKNOWN);
+	astreamer_finalize(streamer->bbs_next);
 }
 
 /*
  * Free memory associated with a tar terminator.
  */
 static void
-bbstreamer_tar_terminator_free(bbstreamer *streamer)
+astreamer_tar_terminator_free(astreamer *streamer)
 {
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	pfree(streamer);
 }
diff --git a/src/bin/pg_basebackup/bbstreamer_zstd.c b/src/bin/pg_basebackup/astreamer_zstd.c
similarity index 64%
rename from src/bin/pg_basebackup/bbstreamer_zstd.c
rename to src/bin/pg_basebackup/astreamer_zstd.c
index 20f11d4450e..58dc679ef99 100644
--- a/src/bin/pg_basebackup/bbstreamer_zstd.c
+++ b/src/bin/pg_basebackup/astreamer_zstd.c
@@ -1,11 +1,11 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_zstd.c
+ * astreamer_zstd.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_zstd.c
+ *		  src/bin/pg_basebackup/astreamer_zstd.c
  *-------------------------------------------------------------------------
  */
 
@@ -17,44 +17,44 @@
 #include <zstd.h>
 #endif
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/logging.h"
 
 #ifdef USE_ZSTD
 
-typedef struct bbstreamer_zstd_frame
+typedef struct astreamer_zstd_frame
 {
-	bbstreamer	base;
+	astreamer	base;
 
 	ZSTD_CCtx  *cctx;
 	ZSTD_DCtx  *dctx;
 	ZSTD_outBuffer zstd_outBuf;
-} bbstreamer_zstd_frame;
+} astreamer_zstd_frame;
 
-static void bbstreamer_zstd_compressor_content(bbstreamer *streamer,
-											   bbstreamer_member *member,
-											   const char *data, int len,
-											   bbstreamer_archive_context context);
-static void bbstreamer_zstd_compressor_finalize(bbstreamer *streamer);
-static void bbstreamer_zstd_compressor_free(bbstreamer *streamer);
+static void astreamer_zstd_compressor_content(astreamer *streamer,
+											  astreamer_member *member,
+											  const char *data, int len,
+											  astreamer_archive_context context);
+static void astreamer_zstd_compressor_finalize(astreamer *streamer);
+static void astreamer_zstd_compressor_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_zstd_compressor_ops = {
-	.content = bbstreamer_zstd_compressor_content,
-	.finalize = bbstreamer_zstd_compressor_finalize,
-	.free = bbstreamer_zstd_compressor_free
+static const astreamer_ops astreamer_zstd_compressor_ops = {
+	.content = astreamer_zstd_compressor_content,
+	.finalize = astreamer_zstd_compressor_finalize,
+	.free = astreamer_zstd_compressor_free
 };
 
-static void bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
-												 bbstreamer_member *member,
-												 const char *data, int len,
-												 bbstreamer_archive_context context);
-static void bbstreamer_zstd_decompressor_finalize(bbstreamer *streamer);
-static void bbstreamer_zstd_decompressor_free(bbstreamer *streamer);
+static void astreamer_zstd_decompressor_content(astreamer *streamer,
+												astreamer_member *member,
+												const char *data, int len,
+												astreamer_archive_context context);
+static void astreamer_zstd_decompressor_finalize(astreamer *streamer);
+static void astreamer_zstd_decompressor_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_zstd_decompressor_ops = {
-	.content = bbstreamer_zstd_decompressor_content,
-	.finalize = bbstreamer_zstd_decompressor_finalize,
-	.free = bbstreamer_zstd_decompressor_free
+static const astreamer_ops astreamer_zstd_decompressor_ops = {
+	.content = astreamer_zstd_decompressor_content,
+	.finalize = astreamer_zstd_decompressor_finalize,
+	.free = astreamer_zstd_decompressor_free
 };
 #endif
 
@@ -62,19 +62,19 @@ static const bbstreamer_ops bbstreamer_zstd_decompressor_ops = {
  * Create a new base backup streamer that performs zstd compression of tar
  * blocks.
  */
-bbstreamer *
-bbstreamer_zstd_compressor_new(bbstreamer *next, pg_compress_specification *compress)
+astreamer *
+astreamer_zstd_compressor_new(astreamer *next, pg_compress_specification *compress)
 {
 #ifdef USE_ZSTD
-	bbstreamer_zstd_frame *streamer;
+	astreamer_zstd_frame *streamer;
 	size_t		ret;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_zstd_frame));
+	streamer = palloc0(sizeof(astreamer_zstd_frame));
 
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_zstd_compressor_ops;
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_zstd_compressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -142,12 +142,12 @@ bbstreamer_zstd_compressor_new(bbstreamer *next, pg_compress_specification *comp
  * of output buffer to next streamer and empty the buffer.
  */
 static void
-bbstreamer_zstd_compressor_content(bbstreamer *streamer,
-								   bbstreamer_member *member,
-								   const char *data, int len,
-								   bbstreamer_archive_context context)
+astreamer_zstd_compressor_content(astreamer *streamer,
+								  astreamer_member *member,
+								  const char *data, int len,
+								  astreamer_archive_context context)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 	ZSTD_inBuffer inBuf = {data, len, 0};
 
 	while (inBuf.pos < inBuf.size)
@@ -162,10 +162,10 @@ bbstreamer_zstd_compressor_content(bbstreamer *streamer,
 		if (mystreamer->zstd_outBuf.size - mystreamer->zstd_outBuf.pos <
 			max_needed)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, member,
-							   mystreamer->zstd_outBuf.dst,
-							   mystreamer->zstd_outBuf.pos,
-							   context);
+			astreamer_content(mystreamer->base.bbs_next, member,
+							  mystreamer->zstd_outBuf.dst,
+							  mystreamer->zstd_outBuf.pos,
+							  context);
 
 			/* Reset the ZSTD output buffer. */
 			mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data;
@@ -187,9 +187,9 @@ bbstreamer_zstd_compressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_zstd_compressor_finalize(bbstreamer *streamer)
+astreamer_zstd_compressor_finalize(astreamer *streamer)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 	size_t		yet_to_flush;
 
 	do
@@ -204,10 +204,10 @@ bbstreamer_zstd_compressor_finalize(bbstreamer *streamer)
 		if (mystreamer->zstd_outBuf.size - mystreamer->zstd_outBuf.pos <
 			max_needed)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, NULL,
-							   mystreamer->zstd_outBuf.dst,
-							   mystreamer->zstd_outBuf.pos,
-							   BBSTREAMER_UNKNOWN);
+			astreamer_content(mystreamer->base.bbs_next, NULL,
+							  mystreamer->zstd_outBuf.dst,
+							  mystreamer->zstd_outBuf.pos,
+							  ASTREAMER_UNKNOWN);
 
 			/* Reset the ZSTD output buffer. */
 			mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data;
@@ -227,23 +227,23 @@ bbstreamer_zstd_compressor_finalize(bbstreamer *streamer)
 
 	/* Make sure to pass any remaining bytes to the next streamer. */
 	if (mystreamer->zstd_outBuf.pos > 0)
-		bbstreamer_content(mystreamer->base.bbs_next, NULL,
-						   mystreamer->zstd_outBuf.dst,
-						   mystreamer->zstd_outBuf.pos,
-						   BBSTREAMER_UNKNOWN);
+		astreamer_content(mystreamer->base.bbs_next, NULL,
+						  mystreamer->zstd_outBuf.dst,
+						  mystreamer->zstd_outBuf.pos,
+						  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_zstd_compressor_free(bbstreamer *streamer)
+astreamer_zstd_compressor_free(astreamer *streamer)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	ZSTD_freeCCtx(mystreamer->cctx);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
@@ -254,17 +254,17 @@ bbstreamer_zstd_compressor_free(bbstreamer *streamer)
  * Create a new base backup streamer that performs decompression of zstd
  * compressed blocks.
  */
-bbstreamer *
-bbstreamer_zstd_decompressor_new(bbstreamer *next)
+astreamer *
+astreamer_zstd_decompressor_new(astreamer *next)
 {
 #ifdef USE_ZSTD
-	bbstreamer_zstd_frame *streamer;
+	astreamer_zstd_frame *streamer;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_zstd_frame));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_zstd_decompressor_ops;
+	streamer = palloc0(sizeof(astreamer_zstd_frame));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_zstd_decompressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -293,12 +293,12 @@ bbstreamer_zstd_decompressor_new(bbstreamer *next)
  * to the next streamer.
  */
 static void
-bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
-									 bbstreamer_member *member,
-									 const char *data, int len,
-									 bbstreamer_archive_context context)
+astreamer_zstd_decompressor_content(astreamer *streamer,
+									astreamer_member *member,
+									const char *data, int len,
+									astreamer_archive_context context)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 	ZSTD_inBuffer inBuf = {data, len, 0};
 
 	while (inBuf.pos < inBuf.size)
@@ -311,10 +311,10 @@ bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
 		 */
 		if (mystreamer->zstd_outBuf.pos >= mystreamer->zstd_outBuf.size)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, member,
-							   mystreamer->zstd_outBuf.dst,
-							   mystreamer->zstd_outBuf.pos,
-							   context);
+			astreamer_content(mystreamer->base.bbs_next, member,
+							  mystreamer->zstd_outBuf.dst,
+							  mystreamer->zstd_outBuf.pos,
+							  context);
 
 			/* Reset the ZSTD output buffer. */
 			mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data;
@@ -335,32 +335,32 @@ bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_zstd_decompressor_finalize(bbstreamer *streamer)
+astreamer_zstd_decompressor_finalize(astreamer *streamer)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 
 	/*
 	 * End of the stream, if there is some pending data in output buffers then
 	 * we must forward it to next streamer.
 	 */
 	if (mystreamer->zstd_outBuf.pos > 0)
-		bbstreamer_content(mystreamer->base.bbs_next, NULL,
-						   mystreamer->base.bbs_buffer.data,
-						   mystreamer->base.bbs_buffer.maxlen,
-						   BBSTREAMER_UNKNOWN);
+		astreamer_content(mystreamer->base.bbs_next, NULL,
+						  mystreamer->base.bbs_buffer.data,
+						  mystreamer->base.bbs_buffer.maxlen,
+						  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_zstd_decompressor_free(bbstreamer *streamer)
+astreamer_zstd_decompressor_free(astreamer *streamer)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	ZSTD_freeDCtx(mystreamer->dctx);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
diff --git a/src/bin/pg_basebackup/bbstreamer.h b/src/bin/pg_basebackup/bbstreamer.h
deleted file mode 100644
index 3b820f13b51..00000000000
--- a/src/bin/pg_basebackup/bbstreamer.h
+++ /dev/null
@@ -1,226 +0,0 @@
-/*-------------------------------------------------------------------------
- *
- * bbstreamer.h
- *
- * Each tar archive returned by the server is passed to one or more
- * bbstreamer objects for further processing. The bbstreamer may do
- * something simple, like write the archive to a file, perhaps after
- * compressing it, but it can also do more complicated things, like
- * annotating the byte stream to indicate which parts of the data
- * correspond to tar headers or trailing padding, vs. which parts are
- * payload data. A subsequent bbstreamer may use this information to
- * make further decisions about how to process the data; for example,
- * it might choose to modify the archive contents.
- *
- * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
- *
- * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer.h
- *-------------------------------------------------------------------------
- */
-
-#ifndef BBSTREAMER_H
-#define BBSTREAMER_H
-
-#include "common/compression.h"
-#include "lib/stringinfo.h"
-#include "pqexpbuffer.h"
-
-struct bbstreamer;
-struct bbstreamer_ops;
-typedef struct bbstreamer bbstreamer;
-typedef struct bbstreamer_ops bbstreamer_ops;
-
-/*
- * Each chunk of archive data passed to a bbstreamer is classified into one
- * of these categories. When data is first received from the remote server,
- * each chunk will be categorized as BBSTREAMER_UNKNOWN, and the chunks will
- * be of whatever size the remote server chose to send.
- *
- * If the archive is parsed (e.g. see bbstreamer_tar_parser_new()), then all
- * chunks should be labelled as one of the other types listed here. In
- * addition, there should be exactly one BBSTREAMER_MEMBER_HEADER chunk and
- * exactly one BBSTREAMER_MEMBER_TRAILER chunk per archive member, even if
- * that means a zero-length call. There can be any number of
- * BBSTREAMER_MEMBER_CONTENTS chunks in between those calls. There
- * should exactly BBSTREAMER_ARCHIVE_TRAILER chunk, and it should follow the
- * last BBSTREAMER_MEMBER_TRAILER chunk.
- *
- * In theory, we could need other classifications here, such as a way of
- * indicating an archive header, but the "tar" format doesn't need anything
- * else, so for the time being there's no point.
- */
-typedef enum
-{
-	BBSTREAMER_UNKNOWN,
-	BBSTREAMER_MEMBER_HEADER,
-	BBSTREAMER_MEMBER_CONTENTS,
-	BBSTREAMER_MEMBER_TRAILER,
-	BBSTREAMER_ARCHIVE_TRAILER,
-} bbstreamer_archive_context;
-
-/*
- * Each chunk of data that is classified as BBSTREAMER_MEMBER_HEADER,
- * BBSTREAMER_MEMBER_CONTENTS, or BBSTREAMER_MEMBER_TRAILER should also
- * pass a pointer to an instance of this struct. The details are expected
- * to be present in the archive header and used to fill the struct, after
- * which all subsequent calls for the same archive member are expected to
- * pass the same details.
- */
-typedef struct
-{
-	char		pathname[MAXPGPATH];
-	pgoff_t		size;
-	mode_t		mode;
-	uid_t		uid;
-	gid_t		gid;
-	bool		is_directory;
-	bool		is_link;
-	char		linktarget[MAXPGPATH];
-} bbstreamer_member;
-
-/*
- * Generally, each type of bbstreamer will define its own struct, but the
- * first element should be 'bbstreamer base'. A bbstreamer that does not
- * require any additional private data could use this structure directly.
- *
- * bbs_ops is a pointer to the bbstreamer_ops object which contains the
- * function pointers appropriate to this type of bbstreamer.
- *
- * bbs_next is a pointer to the successor bbstreamer, for those types of
- * bbstreamer which forward data to a successor. It need not be used and
- * should be set to NULL when not relevant.
- *
- * bbs_buffer is a buffer for accumulating data for temporary storage. Each
- * type of bbstreamer makes its own decisions about whether and how to use
- * this buffer.
- */
-struct bbstreamer
-{
-	const bbstreamer_ops *bbs_ops;
-	bbstreamer *bbs_next;
-	StringInfoData bbs_buffer;
-};
-
-/*
- * There are three callbacks for a bbstreamer. The 'content' callback is
- * called repeatedly, as described in the bbstreamer_archive_context comments.
- * Then, the 'finalize' callback is called once at the end, to give the
- * bbstreamer a chance to perform cleanup such as closing files. Finally,
- * because this code is running in a frontend environment where, as of this
- * writing, there are no memory contexts, the 'free' callback is called to
- * release memory. These callbacks should always be invoked using the static
- * inline functions defined below.
- */
-struct bbstreamer_ops
-{
-	void		(*content) (bbstreamer *streamer, bbstreamer_member *member,
-							const char *data, int len,
-							bbstreamer_archive_context context);
-	void		(*finalize) (bbstreamer *streamer);
-	void		(*free) (bbstreamer *streamer);
-};
-
-/* Send some content to a bbstreamer. */
-static inline void
-bbstreamer_content(bbstreamer *streamer, bbstreamer_member *member,
-				   const char *data, int len,
-				   bbstreamer_archive_context context)
-{
-	Assert(streamer != NULL);
-	streamer->bbs_ops->content(streamer, member, data, len, context);
-}
-
-/* Finalize a bbstreamer. */
-static inline void
-bbstreamer_finalize(bbstreamer *streamer)
-{
-	Assert(streamer != NULL);
-	streamer->bbs_ops->finalize(streamer);
-}
-
-/* Free a bbstreamer. */
-static inline void
-bbstreamer_free(bbstreamer *streamer)
-{
-	Assert(streamer != NULL);
-	streamer->bbs_ops->free(streamer);
-}
-
-/*
- * This is a convenience method for use when implementing a bbstreamer; it is
- * not for use by outside callers. It adds the amount of data specified by
- * 'nbytes' to the bbstreamer's buffer and adjusts '*len' and '*data'
- * accordingly.
- */
-static inline void
-bbstreamer_buffer_bytes(bbstreamer *streamer, const char **data, int *len,
-						int nbytes)
-{
-	Assert(nbytes <= *len);
-
-	appendBinaryStringInfo(&streamer->bbs_buffer, *data, nbytes);
-	*len -= nbytes;
-	*data += nbytes;
-}
-
-/*
- * This is a convenience method for use when implementing a bbstreamer; it is
- * not for use by outsider callers. It attempts to add enough data to the
- * bbstreamer's buffer to reach a length of target_bytes and adjusts '*len'
- * and '*data' accordingly. It returns true if the target length has been
- * reached and false otherwise.
- */
-static inline bool
-bbstreamer_buffer_until(bbstreamer *streamer, const char **data, int *len,
-						int target_bytes)
-{
-	int			buflen = streamer->bbs_buffer.len;
-
-	if (buflen >= target_bytes)
-	{
-		/* Target length already reached; nothing to do. */
-		return true;
-	}
-
-	if (buflen + *len < target_bytes)
-	{
-		/* Not enough data to reach target length; buffer all of it. */
-		bbstreamer_buffer_bytes(streamer, data, len, *len);
-		return false;
-	}
-
-	/* Buffer just enough to reach the target length. */
-	bbstreamer_buffer_bytes(streamer, data, len, target_bytes - buflen);
-	return true;
-}
-
-/*
- * Functions for creating bbstreamer objects of various types. See the header
- * comments for each of these functions for details.
- */
-extern bbstreamer *bbstreamer_plain_writer_new(char *pathname, FILE *file);
-extern bbstreamer *bbstreamer_gzip_writer_new(char *pathname, FILE *file,
-											  pg_compress_specification *compress);
-extern bbstreamer *bbstreamer_extractor_new(const char *basepath,
-											const char *(*link_map) (const char *),
-											void (*report_output_file) (const char *));
-
-extern bbstreamer *bbstreamer_gzip_decompressor_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_lz4_compressor_new(bbstreamer *next,
-												 pg_compress_specification *compress);
-extern bbstreamer *bbstreamer_lz4_decompressor_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_zstd_compressor_new(bbstreamer *next,
-												  pg_compress_specification *compress);
-extern bbstreamer *bbstreamer_zstd_decompressor_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_tar_parser_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_tar_terminator_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_tar_archiver_new(bbstreamer *next);
-
-extern bbstreamer *bbstreamer_recovery_injector_new(bbstreamer *next,
-													bool is_recovery_guc_supported,
-													PQExpBuffer recoveryconfcontents);
-extern void bbstreamer_inject_file(bbstreamer *streamer, char *pathname,
-								   char *data, int len);
-
-#endif
diff --git a/src/bin/pg_basebackup/meson.build b/src/bin/pg_basebackup/meson.build
index c00acd5e118..a68dbd7837d 100644
--- a/src/bin/pg_basebackup/meson.build
+++ b/src/bin/pg_basebackup/meson.build
@@ -1,12 +1,12 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 common_sources = files(
-  'bbstreamer_file.c',
-  'bbstreamer_gzip.c',
-  'bbstreamer_inject.c',
-  'bbstreamer_lz4.c',
-  'bbstreamer_tar.c',
-  'bbstreamer_zstd.c',
+  'astreamer_file.c',
+  'astreamer_gzip.c',
+  'astreamer_inject.c',
+  'astreamer_lz4.c',
+  'astreamer_tar.c',
+  'astreamer_zstd.c',
   'receivelog.c',
   'streamutil.c',
   'walmethods.c',
diff --git a/src/bin/pg_basebackup/nls.mk b/src/bin/pg_basebackup/nls.mk
index 384dbb021e9..950b9797b1e 100644
--- a/src/bin/pg_basebackup/nls.mk
+++ b/src/bin/pg_basebackup/nls.mk
@@ -1,12 +1,12 @@
 # src/bin/pg_basebackup/nls.mk
 CATALOG_NAME     = pg_basebackup
 GETTEXT_FILES    = $(FRONTEND_COMMON_GETTEXT_FILES) \
-                   bbstreamer_file.c \
-                   bbstreamer_gzip.c \
-                   bbstreamer_inject.c \
-                   bbstreamer_lz4.c \
-                   bbstreamer_tar.c \
-                   bbstreamer_zstd.c \
+                   astreamer_file.c \
+                   astreamer_gzip.c \
+                   astreamer_inject.c \
+                   astreamer_lz4.c \
+                   astreamer_tar.c \
+                   astreamer_zstd.c \
                    pg_basebackup.c \
                    pg_createsubscriber.c \
                    pg_receivewal.c \
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 8f3dd04fd22..4179b064cbc 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -26,8 +26,8 @@
 #endif
 
 #include "access/xlog_internal.h"
+#include "astreamer.h"
 #include "backup/basebackup.h"
-#include "bbstreamer.h"
 #include "common/compression.h"
 #include "common/file_perm.h"
 #include "common/file_utils.h"
@@ -57,8 +57,8 @@ typedef struct ArchiveStreamState
 {
 	int			tablespacenum;
 	pg_compress_specification *compress;
-	bbstreamer *streamer;
-	bbstreamer *manifest_inject_streamer;
+	astreamer *streamer;
+	astreamer *manifest_inject_streamer;
 	PQExpBuffer manifest_buffer;
 	char		manifest_filename[MAXPGPATH];
 	FILE	   *manifest_file;
@@ -67,7 +67,7 @@ typedef struct ArchiveStreamState
 typedef struct WriteTarState
 {
 	int			tablespacenum;
-	bbstreamer *streamer;
+	astreamer *streamer;
 } WriteTarState;
 
 typedef struct WriteManifestState
@@ -199,8 +199,8 @@ static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *fo
 static void progress_update_filename(const char *filename);
 static void progress_report(int tablespacenum, bool force, bool finished);
 
-static bbstreamer *CreateBackupStreamer(char *archive_name, char *spclocation,
-										bbstreamer **manifest_inject_streamer_p,
+static astreamer *CreateBackupStreamer(char *archive_name, char *spclocation,
+										astreamer **manifest_inject_streamer_p,
 										bool is_recovery_guc_supported,
 										bool expect_unterminated_tarfile,
 										pg_compress_specification *compress);
@@ -1053,19 +1053,19 @@ ReceiveCopyData(PGconn *conn, WriteDataCallback callback,
  * the options selected by the user.  We may just write the results directly
  * to a file, or we might compress first, or we might extract the tar file
  * and write each member separately. This function doesn't do any of that
- * directly, but it works out what kind of bbstreamer we need to create so
+ * directly, but it works out what kind of astreamer we need to create so
  * that the right stuff happens when, down the road, we actually receive
  * the data.
  */
-static bbstreamer *
+static astreamer *
 CreateBackupStreamer(char *archive_name, char *spclocation,
-					 bbstreamer **manifest_inject_streamer_p,
+					 astreamer **manifest_inject_streamer_p,
 					 bool is_recovery_guc_supported,
 					 bool expect_unterminated_tarfile,
 					 pg_compress_specification *compress)
 {
-	bbstreamer *streamer = NULL;
-	bbstreamer *manifest_inject_streamer = NULL;
+	astreamer *streamer = NULL;
+	astreamer *manifest_inject_streamer = NULL;
 	bool		inject_manifest;
 	bool		is_tar,
 				is_tar_gz,
@@ -1160,7 +1160,7 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 			directory = psprintf("%s/%s", basedir, spclocation);
 		else
 			directory = get_tablespace_mapping(spclocation);
-		streamer = bbstreamer_extractor_new(directory,
+		streamer = astreamer_extractor_new(directory,
 											get_tablespace_mapping,
 											progress_update_filename);
 	}
@@ -1188,27 +1188,27 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 		}
 
 		if (compress->algorithm == PG_COMPRESSION_NONE)
-			streamer = bbstreamer_plain_writer_new(archive_filename,
+			streamer = astreamer_plain_writer_new(archive_filename,
 												   archive_file);
 		else if (compress->algorithm == PG_COMPRESSION_GZIP)
 		{
 			strlcat(archive_filename, ".gz", sizeof(archive_filename));
-			streamer = bbstreamer_gzip_writer_new(archive_filename,
+			streamer = astreamer_gzip_writer_new(archive_filename,
 												  archive_file, compress);
 		}
 		else if (compress->algorithm == PG_COMPRESSION_LZ4)
 		{
 			strlcat(archive_filename, ".lz4", sizeof(archive_filename));
-			streamer = bbstreamer_plain_writer_new(archive_filename,
+			streamer = astreamer_plain_writer_new(archive_filename,
 												   archive_file);
-			streamer = bbstreamer_lz4_compressor_new(streamer, compress);
+			streamer = astreamer_lz4_compressor_new(streamer, compress);
 		}
 		else if (compress->algorithm == PG_COMPRESSION_ZSTD)
 		{
 			strlcat(archive_filename, ".zst", sizeof(archive_filename));
-			streamer = bbstreamer_plain_writer_new(archive_filename,
+			streamer = astreamer_plain_writer_new(archive_filename,
 												   archive_file);
-			streamer = bbstreamer_zstd_compressor_new(streamer, compress);
+			streamer = astreamer_zstd_compressor_new(streamer, compress);
 		}
 		else
 		{
@@ -1222,7 +1222,7 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 		 * into it.
 		 */
 		if (must_parse_archive)
-			streamer = bbstreamer_tar_archiver_new(streamer);
+			streamer = astreamer_tar_archiver_new(streamer);
 		progress_update_filename(archive_filename);
 	}
 
@@ -1241,7 +1241,7 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 	if (spclocation == NULL && writerecoveryconf)
 	{
 		Assert(must_parse_archive);
-		streamer = bbstreamer_recovery_injector_new(streamer,
+		streamer = astreamer_recovery_injector_new(streamer,
 													is_recovery_guc_supported,
 													recoveryconfcontents);
 	}
@@ -1253,9 +1253,9 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 	 * we're talking to such a server we'll need to add the terminator here.
 	 */
 	if (must_parse_archive)
-		streamer = bbstreamer_tar_parser_new(streamer);
+		streamer = astreamer_tar_parser_new(streamer);
 	else if (expect_unterminated_tarfile)
-		streamer = bbstreamer_tar_terminator_new(streamer);
+		streamer = astreamer_tar_terminator_new(streamer);
 
 	/*
 	 * If the user has requested a server compressed archive along with
@@ -1264,11 +1264,11 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 	if (format == 'p')
 	{
 		if (is_tar_gz)
-			streamer = bbstreamer_gzip_decompressor_new(streamer);
+			streamer = astreamer_gzip_decompressor_new(streamer);
 		else if (is_tar_lz4)
-			streamer = bbstreamer_lz4_decompressor_new(streamer);
+			streamer = astreamer_lz4_decompressor_new(streamer);
 		else if (is_tar_zstd)
-			streamer = bbstreamer_zstd_decompressor_new(streamer);
+			streamer = astreamer_zstd_decompressor_new(streamer);
 	}
 
 	/* Return the results. */
@@ -1307,7 +1307,7 @@ ReceiveArchiveStream(PGconn *conn, pg_compress_specification *compress)
 	if (state.manifest_inject_streamer != NULL &&
 		state.manifest_buffer != NULL)
 	{
-		bbstreamer_inject_file(state.manifest_inject_streamer,
+		astreamer_inject_file(state.manifest_inject_streamer,
 							   "backup_manifest",
 							   state.manifest_buffer->data,
 							   state.manifest_buffer->len);
@@ -1318,8 +1318,8 @@ ReceiveArchiveStream(PGconn *conn, pg_compress_specification *compress)
 	/* If there's still an archive in progress, end processing. */
 	if (state.streamer != NULL)
 	{
-		bbstreamer_finalize(state.streamer);
-		bbstreamer_free(state.streamer);
+		astreamer_finalize(state.streamer);
+		astreamer_free(state.streamer);
 		state.streamer = NULL;
 	}
 }
@@ -1383,8 +1383,8 @@ ReceiveArchiveStreamChunk(size_t r, char *copybuf, void *callback_data)
 				/* End processing of any prior archive. */
 				if (state->streamer != NULL)
 				{
-					bbstreamer_finalize(state->streamer);
-					bbstreamer_free(state->streamer);
+					astreamer_finalize(state->streamer);
+					astreamer_free(state->streamer);
 					state->streamer = NULL;
 				}
 
@@ -1437,8 +1437,8 @@ ReceiveArchiveStreamChunk(size_t r, char *copybuf, void *callback_data)
 				else if (state->streamer != NULL)
 				{
 					/* Archive data. */
-					bbstreamer_content(state->streamer, NULL, copybuf + 1,
-									   r - 1, BBSTREAMER_UNKNOWN);
+					astreamer_content(state->streamer, NULL, copybuf + 1,
+									   r - 1, ASTREAMER_UNKNOWN);
 				}
 				else
 					pg_fatal("unexpected payload data");
@@ -1600,7 +1600,7 @@ ReceiveTarFile(PGconn *conn, char *archive_name, char *spclocation,
 			   bool tablespacenum, pg_compress_specification *compress)
 {
 	WriteTarState state;
-	bbstreamer *manifest_inject_streamer;
+	astreamer *manifest_inject_streamer;
 	bool		is_recovery_guc_supported;
 	bool		expect_unterminated_tarfile;
 
@@ -1636,7 +1636,7 @@ ReceiveTarFile(PGconn *conn, char *archive_name, char *spclocation,
 			pg_fatal("out of memory");
 
 		/* Inject it into the output tarfile. */
-		bbstreamer_inject_file(manifest_inject_streamer, "backup_manifest",
+		astreamer_inject_file(manifest_inject_streamer, "backup_manifest",
 							   buf.data, buf.len);
 
 		/* Free memory. */
@@ -1644,8 +1644,8 @@ ReceiveTarFile(PGconn *conn, char *archive_name, char *spclocation,
 	}
 
 	/* Cleanup. */
-	bbstreamer_finalize(state.streamer);
-	bbstreamer_free(state.streamer);
+	astreamer_finalize(state.streamer);
+	astreamer_free(state.streamer);
 
 	progress_report(tablespacenum, true, false);
 
@@ -1663,7 +1663,7 @@ ReceiveTarCopyChunk(size_t r, char *copybuf, void *callback_data)
 {
 	WriteTarState *state = callback_data;
 
-	bbstreamer_content(state->streamer, NULL, copybuf, r, BBSTREAMER_UNKNOWN);
+	astreamer_content(state->streamer, NULL, copybuf, r, ASTREAMER_UNKNOWN);
 
 	totaldone += r;
 	progress_report(state->tablespacenum, false, false);
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index b4d7f9217ce..b982dffa5fc 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3309,19 +3309,19 @@ bbsink_shell
 bbsink_state
 bbsink_throttle
 bbsink_zstd
-bbstreamer
-bbstreamer_archive_context
-bbstreamer_extractor
-bbstreamer_gzip_decompressor
-bbstreamer_gzip_writer
-bbstreamer_lz4_frame
-bbstreamer_member
-bbstreamer_ops
-bbstreamer_plain_writer
-bbstreamer_recovery_injector
-bbstreamer_tar_archiver
-bbstreamer_tar_parser
-bbstreamer_zstd_frame
+astreamer
+astreamer_archive_context
+astreamer_extractor
+astreamer_gzip_decompressor
+astreamer_gzip_writer
+astreamer_lz4_frame
+astreamer_member
+astreamer_ops
+astreamer_plain_writer
+astreamer_recovery_injector
+astreamer_tar_archiver
+astreamer_tar_parser
+astreamer_zstd_frame
 bgworker_main_type
 bh_node_type
 binaryheap
-- 
2.18.0

From 547b92c51a47fa03b02721d5012c9f7ab3e9bb27 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 17:04:56 +0530
Subject: [PATCH v3 11/11] pg_verifybackup: Tests and document

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the previous patch that implements tar format support.
----
---
 doc/src/sgml/ref/pg_verifybackup.sgml         | 54 +++++++++++++-
 src/bin/pg_verifybackup/t/001_basic.pl        | 18 ++++-
 src/bin/pg_verifybackup/t/004_options.pl      |  2 +-
 src/bin/pg_verifybackup/t/005_bad_manifest.pl |  2 +-
 src/bin/pg_verifybackup/t/008_untar.pl        | 73 ++++++-------------
 src/bin/pg_verifybackup/t/010_client_untar.pl | 48 +-----------
 6 files changed, 96 insertions(+), 101 deletions(-)

diff --git a/doc/src/sgml/ref/pg_verifybackup.sgml b/doc/src/sgml/ref/pg_verifybackup.sgml
index a3f167f9f6e..c743bd89a92 100644
--- a/doc/src/sgml/ref/pg_verifybackup.sgml
+++ b/doc/src/sgml/ref/pg_verifybackup.sgml
@@ -34,8 +34,10 @@ PostgreSQL documentation
    integrity of a database cluster backup taken using
    <command>pg_basebackup</command> against a
    <literal>backup_manifest</literal> generated by the server at the time
-   of the backup.  The backup must be stored in the "plain"
-   format; a "tar" format backup can be checked after extracting it.
+   of the backup.  The backup must be stored in the "plain" or "tar"
+   format.  Verification is supported for <literal>gzip</literal>,
+   <literal>lz4</literal>, and  <literal>zstd</literal> compressed tar backup;
+   any other compressed format backups can be checked after decompressing them.
   </para>
 
   <para>
@@ -168,6 +170,42 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-F <replaceable class="parameter">format</replaceable></option></term>
+      <term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
+      <listitem>
+       <para>
+        Specifies the format of the backup. <replaceable>format</replaceable>
+        can be one of the following:
+
+        <variablelist>
+         <varlistentry>
+          <term><literal>p</literal></term>
+          <term><literal>plain</literal></term>
+          <listitem>
+           <para>
+            Backup consists of plain files with the same layout as the
+            source server's data directory and tablespaces.
+           </para>
+          </listitem>
+         </varlistentry>
+
+         <varlistentry>
+          <term><literal>t</literal></term>
+          <term><literal>tar</literal></term>
+          <listitem>
+           <para>
+            Backup consists of tar files. The main data directory's contents
+            will be written to a file named <filename>base.tar</filename>,
+            and each other tablespace will be written to a separate tar file
+            named after that tablespace's OID.
+           </para>
+           </listitem>
+         </varlistentry>
+        </variablelist></para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-n</option></term>
       <term><option>--no-parse-wal</option></term>
@@ -227,6 +265,18 @@ PostgreSQL documentation
        </para>
       </listitem>
      </varlistentry>
+
+     <varlistentry>
+      <term><option>-Z <replaceable class="parameter">method</replaceable></option></term>
+      <term><option>--compress=<replaceable class="parameter">method</replaceable></option></term>
+      <listitem>
+       <para>
+        The tar backup compression method can be <literal>gzip</literal>,
+        <literal>lz4</literal>, <literal>zstd</literal>, or
+        <literal>none</literal> if no compression.
+       </para>
+      </listitem>
+     </varlistentry>
     </variablelist>
    </para>
 
diff --git a/src/bin/pg_verifybackup/t/001_basic.pl b/src/bin/pg_verifybackup/t/001_basic.pl
index 2f3e52d296f..d47ce1f04fc 100644
--- a/src/bin/pg_verifybackup/t/001_basic.pl
+++ b/src/bin/pg_verifybackup/t/001_basic.pl
@@ -17,13 +17,25 @@ command_fails_like(
 	qr/no backup directory specified/,
 	'target directory must be specified');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir ],
 	qr/could not open file.*\/backup_manifest\"/,
 	'pg_verifybackup requires a manifest');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir, $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir, $tempdir ],
 	qr/too many command-line arguments/,
 	'multiple target directories not allowed');
+command_fails_like(
+	[ 'pg_verifybackup', '-Zgzip', $tempdir ],
+	qr/only tar mode backups can be compressed/,
+	'compression method required format option');
+command_fails_like(
+	[ 'pg_verifybackup', '-Fp', '-Zlz4', $tempdir ],
+	qr/only tar mode backups can be compressed/,
+	'compression method required tar format option');
+command_fails_like(
+	[ 'pg_verifybackup', '-Fp', '-Znon_exist', $tempdir ],
+	qr/unrecognized compression algorithm/,
+	'compression method should be valid');
 
 # create fake manifest file
 open(my $fh, '>', "$tempdir/backup_manifest") || die "open: $!";
@@ -31,7 +43,7 @@ close($fh);
 
 # but then try to use an alternate, nonexisting manifest
 command_fails_like(
-	[ 'pg_verifybackup', '-m', "$tempdir/not_the_manifest", $tempdir ],
+	[ 'pg_verifybackup', '-Fp', '-m', "$tempdir/not_the_manifest", $tempdir ],
 	qr/could not open file.*\/not_the_manifest\"/,
 	'pg_verifybackup respects -m flag');
 
diff --git a/src/bin/pg_verifybackup/t/004_options.pl b/src/bin/pg_verifybackup/t/004_options.pl
index 8ed2214408e..2f197648740 100644
--- a/src/bin/pg_verifybackup/t/004_options.pl
+++ b/src/bin/pg_verifybackup/t/004_options.pl
@@ -108,7 +108,7 @@ unlike(
 # Test valid manifest with nonexistent backup directory.
 command_fails_like(
 	[
-		'pg_verifybackup', '-m',
+		'pg_verifybackup', '-Fp', '-m',
 		"$backup_path/backup_manifest", "$backup_path/fake"
 	],
 	qr/could not open directory/,
diff --git a/src/bin/pg_verifybackup/t/005_bad_manifest.pl b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
index c4ed64b62d5..28c51b6feb0 100644
--- a/src/bin/pg_verifybackup/t/005_bad_manifest.pl
+++ b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
@@ -208,7 +208,7 @@ sub test_bad_manifest
 	print $fh $manifest_contents;
 	close($fh);
 
-	command_fails_like([ 'pg_verifybackup', $tempdir ], $regexp, $test_name);
+	command_fails_like([ 'pg_verifybackup', '-Fp', $tempdir ], $regexp, $test_name);
 	return;
 }
 
diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
index 7a09f3b75b2..9896560adc3 100644
--- a/src/bin/pg_verifybackup/t/008_untar.pl
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -16,6 +16,20 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 
+# Create a couple of directories to use as tablespaces.
+my $TS1_LOCATION = $primary->backup_dir .'/ts1';
+$TS1_LOCATION =~ s/\/\.\//\//g;    # collapse foo/./bar to foo/bar
+mkdir($TS1_LOCATION);
+
+# Create a tablespace with table in it.
+$primary->safe_psql('postgres', qq(
+		CREATE TABLESPACE regress_ts1 LOCATION '$TS1_LOCATION';
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1';
+		CREATE TABLE regress_tbl1(i int) TABLESPACE regress_ts1;
+		INSERT INTO regress_tbl1 VALUES(generate_series(1,5));));
+my $tsoid = $primary->safe_psql('postgres', qq(
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1'));
+
 my $backup_path = $primary->backup_dir . '/server-backup';
 my $extract_path = $primary->backup_dir . '/extracted-backup';
 
@@ -23,39 +37,31 @@ my @test_configuration = (
 	{
 		'compression_method' => 'none',
 		'backup_flags' => [],
-		'backup_archive' => 'base.tar',
+		'backup_archive' => ['base.tar', "$tsoid.tar"],
 		'enabled' => 1
 	},
 	{
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'server-gzip' ],
-		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.gz', "$tsoid.tar.gz" ],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'server-lz4' ],
-		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => [ '-d', '-m' ],
+		'backup_archive' => ['base.tar.lz4', "$tsoid.tar.lz4" ],
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd:level=1,long' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	});
 
@@ -86,47 +92,16 @@ for my $tc (@test_configuration)
 		my $backup_files = join(',',
 			sort grep { $_ ne '.' && $_ ne '..' } slurp_dir($backup_path));
 		my $expected_backup_files =
-		  join(',', sort ('backup_manifest', $tc->{'backup_archive'}));
+		  join(',', sort ('backup_manifest', @{ $tc->{'backup_archive'} }));
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok(['pg_verifybackup', '-n', '-e', $backup_path],
+			"verify backup, compression $method");
 
 		# Cleanup.
-		unlink($backup_path . '/backup_manifest');
-		unlink($backup_path . '/base.tar');
-		unlink($backup_path . '/' . $tc->{'backup_archive'});
+		rmtree($backup_path);
 		rmtree($extract_path);
 	}
 }
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index 8c076d46dee..6b7d7483f6e 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -29,41 +29,30 @@ my @test_configuration = (
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'client-gzip:5' ],
 		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'client-lz4:5' ],
 		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => ['-d'],
-		'output_file' => 'base.tar',
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:5' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:level=1,long' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'parallel zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:workers=3' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1"),
 		'possibly_unsupported' =>
 		  qr/could not set compression worker count to 3: Unsupported parameter/
@@ -118,40 +107,9 @@ for my $tc (@test_configuration)
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			push @decompress, $backup_path . '/' . $tc->{'output_file'}
-			  if $tc->{'output_file'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok( [ 'pg_verifybackup', '-n', '-e', $backup_path ],
+			"verify backup, compression $method");
 
 		# Cleanup.
 		rmtree($extract_path);
-- 
2.18.0

From de07a9a975221e679a8c3d61a0ab9b7ee808539f Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 31 Jul 2024 15:28:12 +0530
Subject: [PATCH v3 07/11] Refactor: split verify_file_checksum() function.

Move the core functionality of verify_file_checksum to a new function
to enable incremental checksum computation.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 102 +++++++++++++++-------
 src/bin/pg_verifybackup/pg_verifybackup.h |   4 +
 2 files changed, 73 insertions(+), 33 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index bb732bf06ca..7e94af387ad 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -771,6 +771,72 @@ verify_backup_checksums(verifier_context *context)
 	progress_report(context, true);
 }
 
+/*
+ * It computes the checksum incrementally for the received bytes, requiring the
+ * caller to pass a properly initialized checksum_ctx parameter. Once the
+ * complete file content is received, which is tracked using the computed_len
+ * parameter, it verifies against the manifest data. If any error occurs, it
+ * returns false; otherwise, it returns true to indicate either the complete
+ * file content is yet to be received or checksum verification is completed
+ * successfully.
+ */
+bool
+verify_content_checksum(verifier_context *context,
+						pg_checksum_context *checksum_ctx,
+						manifest_file *m, uint8 *buffer,
+						int buffer_len, size_t *computed_len)
+{
+	char	   *relpath = m->pathname;
+	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
+	int			checksumlen;
+
+	if (pg_checksum_update(checksum_ctx, buffer, buffer_len) < 0)
+	{
+		report_backup_error(context, "could not update checksum of file \"%s\"",
+							relpath);
+		return false;
+	}
+
+	/* Update the total count of computed checksum bytes. */
+	*computed_len += buffer_len;
+
+	/* Report progress */
+	done_size += buffer_len;
+	progress_report(context, false);
+
+	/* Yet to receive the full content of the file. */
+	if (*computed_len != m->size)
+		return true;
+
+	/* Get the final checksum. */
+	checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
+	if (checksumlen < 0)
+	{
+		report_backup_error(context,
+							"could not finalize checksum of file \"%s\"",
+							relpath);
+		return false;
+	}
+
+	/* And check it against the manifest. */
+	if (checksumlen != m->checksum_length)
+	{
+		report_backup_error(context,
+							"file \"%s\" has checksum of length %d, but expected %d",
+							relpath, m->checksum_length, checksumlen);
+		return false;
+	}
+	else if (memcmp(checksumbuf, m->checksum_payload, checksumlen) != 0)
+	{
+		report_backup_error(context,
+							"checksum mismatch for file \"%s\"",
+							relpath);
+		return false;
+	}
+
+	return true;
+}
+
 /*
  * Verify the checksum of a single file.
  */
@@ -783,8 +849,6 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	int			fd;
 	int			rc;
 	size_t		bytes_read = 0;
-	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
-	int			checksumlen;
 
 	/* Open the target file. */
 	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
@@ -806,19 +870,14 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	/* Read the file chunk by chunk, updating the checksum as we go. */
 	while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
 	{
-		bytes_read += rc;
-		if (pg_checksum_update(&checksum_ctx, buffer, rc) < 0)
+		if (!verify_content_checksum(context, &checksum_ctx, m, buffer, rc,
+									 &bytes_read))
 		{
-			report_backup_error(context, "could not update checksum of file \"%s\"",
-								relpath);
 			close(fd);
 			return;
 		}
-
-		/* Report progress */
-		done_size += rc;
-		progress_report(context, false);
 	}
+
 	if (rc < 0)
 		report_backup_error(context, "could not read file \"%s\": %m",
 							relpath);
@@ -843,32 +902,9 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	 * filesystem misbehavior.
 	 */
 	if (bytes_read != m->size)
-	{
 		report_backup_error(context,
 							"file \"%s\" should contain %zu bytes, but read %zu bytes",
 							relpath, m->size, bytes_read);
-		return;
-	}
-
-	/* Get the final checksum. */
-	checksumlen = pg_checksum_final(&checksum_ctx, checksumbuf);
-	if (checksumlen < 0)
-	{
-		report_backup_error(context,
-							"could not finalize checksum of file \"%s\"",
-							relpath);
-		return;
-	}
-
-	/* And check it against the manifest. */
-	if (checksumlen != m->checksum_length)
-		report_backup_error(context,
-							"file \"%s\" has checksum of length %d, but expected %d",
-							relpath, m->checksum_length, checksumlen);
-	else if (memcmp(checksumbuf, m->checksum_payload, checksumlen) != 0)
-		report_backup_error(context,
-							"checksum mismatch for file \"%s\"",
-							relpath);
 }
 
 /*
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index f4a93d3f137..fe56c63f99b 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -103,6 +103,10 @@ typedef struct verifier_context
 
 extern manifest_file *verify_manifest_entry(verifier_context *context,
 											char *relpath, size_t filesize);
+extern bool verify_content_checksum(verifier_context *context,
+									pg_checksum_context *checksum_ctx,
+									manifest_file *m, uint8 *buf,
+									int buf_len, size_t *computed_len);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From feecd16679b80f8a1b2c98cfd4ac8286e370d796 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 10:26:35 +0530
Subject: [PATCH v3 09/11] pg_verifybackup: Add backup format and compression
 option

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the next patch that implements tar format support.
----
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 143 +++++++++++++++++++++-
 1 file changed, 141 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index ba2a9b44d2d..2a7d4869ce5 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,6 +18,7 @@
 #include <sys/stat.h>
 #include <time.h>
 
+#include "common/compression.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
@@ -56,6 +57,8 @@ static void report_manifest_error(JsonManifestParseContext *context,
 								  const char *fmt,...)
 			pg_attribute_printf(2, 3) pg_attribute_noreturn();
 
+static char find_backup_format(verifier_context *context);
+static pg_compress_algorithm find_backup_compression(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
@@ -78,6 +81,9 @@ static const char *progname;
 static uint64 total_size = 0;
 static uint64 done_size = 0;
 
+char		format = '\0';		/* p(lain)/t(ar) */
+pg_compress_algorithm compress_algorithm = PG_COMPRESSION_NONE;
+
 /*
  * Main entry point.
  */
@@ -88,11 +94,13 @@ main(int argc, char **argv)
 		{"exit-on-error", no_argument, NULL, 'e'},
 		{"ignore", required_argument, NULL, 'i'},
 		{"manifest-path", required_argument, NULL, 'm'},
+		{"format", required_argument, NULL, 'F'},
 		{"no-parse-wal", no_argument, NULL, 'n'},
 		{"progress", no_argument, NULL, 'P'},
 		{"quiet", no_argument, NULL, 'q'},
 		{"skip-checksums", no_argument, NULL, 's'},
 		{"wal-directory", required_argument, NULL, 'w'},
+		{"compress", required_argument, NULL, 'Z'},
 		{NULL, 0, NULL, 0}
 	};
 
@@ -103,6 +111,7 @@ main(int argc, char **argv)
 	bool		quiet = false;
 	char	   *wal_directory = NULL;
 	char	   *pg_waldump_path = NULL;
+	bool		tar_compression_specified = false;
 
 	pg_logging_init(argv[0]);
 	set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_verifybackup"));
@@ -145,7 +154,7 @@ main(int argc, char **argv)
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
 
-	while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1)
+	while ((c = getopt_long(argc, argv, "eF:i:m:nPqsw:Z:", long_options, NULL)) != -1)
 	{
 		switch (c)
 		{
@@ -164,6 +173,15 @@ main(int argc, char **argv)
 				manifest_path = pstrdup(optarg);
 				canonicalize_path(manifest_path);
 				break;
+			case 'F':
+				if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
+					format = 'p';
+				else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
+					format = 't';
+				else
+					pg_fatal("invalid backup format \"%s\", must be \"plain\" or \"tar\"",
+							 optarg);
+				break;
 			case 'n':
 				no_parse_wal = true;
 				break;
@@ -180,6 +198,12 @@ main(int argc, char **argv)
 				wal_directory = pstrdup(optarg);
 				canonicalize_path(wal_directory);
 				break;
+			case 'Z':
+				if (!parse_compress_algorithm(optarg, &compress_algorithm))
+					pg_fatal("unrecognized compression algorithm: \"%s\"",
+							 optarg);
+				tar_compression_specified = true;
+				break;
 			default:
 				/* getopt_long already emitted a complaint */
 				pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -211,11 +235,41 @@ main(int argc, char **argv)
 		pg_fatal("cannot specify both %s and %s",
 				 "-P/--progress", "-q/--quiet");
 
+	/* Complain if compression method specified but the format isn't tar. */
+	if (format != 't' && tar_compression_specified)
+	{
+		pg_log_error("only tar mode backups can be compressed");
+		pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+		exit(1);
+	}
+
+	/* Determine the backup format if it hasn't been specified. */
+	if (format == '\0')
+		format = find_backup_format(&context);
+
+	/*
+	 * Determine the tar backup compression method if it hasn't been
+	 * specified.
+	 */
+	if (format == 't' && !tar_compression_specified)
+		compress_algorithm = find_backup_compression(&context);
+
 	/* Unless --no-parse-wal was specified, we will need pg_waldump. */
 	if (!no_parse_wal)
 	{
 		int			ret;
 
+		/*
+		 * XXX: In the future, we should consider enhancing pg_waldump to read
+		 * WAL files from the tar archive.
+		 */
+		if (format != 'p')
+		{
+			pg_log_error("pg_waldump does not support parsing WAL files from a tar archive.");
+			pg_log_error_hint("Try \"%s --help\" to skip parse WAL files option.", progname);
+			exit(1);
+		}
+
 		pg_waldump_path = pg_malloc(MAXPGPATH);
 		ret = find_other_exec(argv[0], "pg_waldump",
 							  "pg_waldump (PostgreSQL) " PG_VERSION "\n",
@@ -270,8 +324,13 @@ main(int argc, char **argv)
 	/*
 	 * Now do the expensive work of verifying file checksums, unless we were
 	 * told to skip it.
+	 *
+	 * We were only checking the plain backup here. For the tar backup, file
+	 * checksums verification (if requested) will be done immediately when the
+	 * file is accessed, as we don't have random access to the files like we
+	 * do with plain backups.
 	 */
-	if (!context.skip_checksums)
+	if (!context.skip_checksums && format == 'p')
 		verify_backup_checksums(&context);
 
 	/*
@@ -1039,6 +1098,84 @@ progress_report(verifier_context *context, bool finished)
 	fputc((!finished && isatty(fileno(stderr))) ? '\r' : '\n', stderr);
 }
 
+/*
+ * To detect the backup format, it checks for the PG_VERSION file in the backup
+ * directory. If found, it will be considered a plain-format backup; otherwise,
+ * it will be assumed to be a tar-format backup.
+ */
+static char
+find_backup_format(verifier_context *context)
+{
+	char		result;
+	char	   *path;
+	struct stat sb;
+
+	/* Should be here only if the backup format is unknown */
+	Assert(format == '\0');
+
+	/* Check PG_VERSION file. */
+	path = psprintf("%s/%s", context->backup_directory, "PG_VERSION");
+	result = (stat(path, &sb) == 0) ? 'p' : 't';
+	pfree(path);
+
+	return result;
+}
+
+/*
+ * To determine the compression format, we will search for the main data
+ * directory archive and its extension, which starts with base.tar, as
+ * pg_basebackup writes the main data directory to an archive file named
+ * base.tar followed by a compression type extension like .gz, .lz4, or .zst.
+ */
+static pg_compress_algorithm
+find_backup_compression(verifier_context *context)
+{
+	char	   *path;
+	struct stat sb;
+	bool		found;
+
+	/* Should be here only for tar backup */
+	Assert(format == 't');
+
+	/*
+	 * Is this a tar archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_NONE;
+
+	/*
+	 * Is this a .tar.gz archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar.gz");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_GZIP;
+
+	/*
+	 * Is this a .tar.lz4 archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar.lz4");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_LZ4;
+
+	/*
+	 * Is this a .tar.zst archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar.zst");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_ZSTD;
+
+	return PG_COMPRESSION_NONE; /* placate compiler */
+}
+
 /*
  * Print out usage information and exit.
  */
@@ -1051,11 +1188,13 @@ usage(void)
 	printf(_("  -e, --exit-on-error         exit immediately on error\n"));
 	printf(_("  -i, --ignore=RELATIVE_PATH  ignore indicated path\n"));
 	printf(_("  -m, --manifest-path=PATH    use specified path for manifest\n"));
+	printf(_("  -F, --format=p|t            backup format (plain, tar)\n"));
 	printf(_("  -n, --no-parse-wal          do not try to parse WAL files\n"));
 	printf(_("  -P, --progress              show progress information\n"));
 	printf(_("  -q, --quiet                 do not print any output, except for errors\n"));
 	printf(_("  -s, --skip-checksums        skip checksum verification\n"));
 	printf(_("  -w, --wal-directory=PATH    use specified path for WAL files\n"));
+	printf(_("  -Z, --compress=METHOD       compress method (gzip, lz4, zstd, none) \n"));
 	printf(_("  -V, --version               output version information, then exit\n"));
 	printf(_("  -?, --help                  show this help, then exit\n"));
 	printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
-- 
2.18.0

From 91cd914f9acb6ade6744cae6d45c6d56d3e78bc0 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 31 Jul 2024 16:08:43 +0530
Subject: [PATCH v3 08/11] Refactor: split verify_control_file.

Moved verify_control_file doing the control file checks to a separate
function that can be called from other places as well.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 44 +++++++++++------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 14 ++++++++
 2 files changed, 34 insertions(+), 24 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 7e94af387ad..ba2a9b44d2d 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,7 +18,6 @@
 #include <sys/stat.h>
 #include <time.h>
 
-#include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
@@ -61,8 +60,6 @@ static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
 							   char *relpath, char *fullpath);
-static void verify_control_file(const char *controlpath,
-								uint64 manifest_system_identifier);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -622,14 +619,20 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 	/* Check the backup manifest entry for this file. */
 	m = verify_manifest_entry(context, relpath, sb.st_size);
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0 &&
-		m != NULL && m->matched && !m->bad)
-		verify_control_file(fullpath, context->manifest->system_identifier);
+	/* Validate the manifest system identifier */
+	if (m != NULL && should_verify_control_data(context->manifest, m))
+	{
+		ControlFileData *control_file;
+		bool		crc_ok;
+
+		pg_log_debug("reading \"%s\"", fullpath);
+		control_file = get_controlfile_by_exact_path(fullpath, &crc_ok);
+
+		verify_control_data(control_file, fullpath, crc_ok,
+							context->manifest->system_identifier);
+		/* Release memory. */
+		pfree(control_file);
+	}
 }
 
 /*
@@ -678,18 +681,14 @@ verify_manifest_entry(verifier_context *context, char *relpath, size_t filesize)
 }
 
 /*
- * Sanity check control file and validate system identifier against manifest
- * system identifier.
+ * Sanity check control file data and validate system identifier against
+ * manifest system identifier.
  */
-static void
-verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
+void
+verify_control_data(ControlFileData *control_file,
+					const char *controlpath, bool crc_ok,
+					uint64 manifest_system_identifier)
 {
-	ControlFileData *control_file;
-	bool		crc_ok;
-
-	pg_log_debug("reading \"%s\"", controlpath);
-	control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
-
 	/* Control file contents not meaningful if CRC is bad. */
 	if (!crc_ok)
 		report_fatal_error("%s: CRC is incorrect", controlpath);
@@ -705,9 +704,6 @@ verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
 						   controlpath,
 						   (unsigned long long) manifest_system_identifier,
 						   (unsigned long long) control_file->system_identifier);
-
-	/* Release memory. */
-	pfree(control_file);
 }
 
 /*
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index fe56c63f99b..a1a34c4b773 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -16,6 +16,7 @@
 
 #include "common/controldata_utils.h"
 #include "common/hashfn_unstable.h"
+#include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 
@@ -46,6 +47,16 @@ typedef struct manifest_file
 #define should_verify_checksum(m) \
 	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
 
+/*
+ * Validate the manifest system identifier against the control file; this
+ * feature is not available in manifest version 1.  This validation should be
+ * carried out only if the manifest entry validation is completed without any
+ * errors.
+ */
+#define should_verify_control_data(manifest, m) \
+	(((manifest)->version != 1) && \
+	 ((m)->matched) && !((m)->bad) && (strcmp((m)->pathname, "global/pg_control") == 0))
+
 /*
  * Define a hash table which we can use to store information about the files
  * mentioned in the backup manifest.
@@ -107,6 +118,9 @@ extern bool verify_content_checksum(verifier_context *context,
 									pg_checksum_context *checksum_ctx,
 									manifest_file *m, uint8 *buf,
 									int buf_len, size_t *computed_len);
+extern void verify_control_data(ControlFileData *control_file,
+								const char *controlpath, bool crc_ok,
+								uint64 manifest_system_identifier);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From bb7de7434e4fb9b85d805946ca6aa7d2b9cc8e9e Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 31 Jul 2024 16:22:07 +0530
Subject: [PATCH v3 10/11] pg_verifybackup: Read tar files and verify its
 contents

---
 src/bin/pg_verifybackup/Makefile           |   4 +-
 src/bin/pg_verifybackup/astreamer_verify.c | 277 +++++++++++++++++++++
 src/bin/pg_verifybackup/meson.build        |   6 +-
 src/bin/pg_verifybackup/pg_verifybackup.c  | 216 +++++++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h  |  10 +
 src/tools/pgindent/typedefs.list           |   1 +
 6 files changed, 505 insertions(+), 9 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/astreamer_verify.c

diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile
index 7c045f142e8..df7aaabd530 100644
--- a/src/bin/pg_verifybackup/Makefile
+++ b/src/bin/pg_verifybackup/Makefile
@@ -17,11 +17,13 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 # We need libpq only because fe_utils does.
+override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 
 OBJS = \
 	$(WIN32RES) \
-	pg_verifybackup.o
+	pg_verifybackup.o \
+	astreamer_verify.o
 
 all: pg_verifybackup
 
diff --git a/src/bin/pg_verifybackup/astreamer_verify.c b/src/bin/pg_verifybackup/astreamer_verify.c
new file mode 100644
index 00000000000..05119e4b7cb
--- /dev/null
+++ b/src/bin/pg_verifybackup/astreamer_verify.c
@@ -0,0 +1,277 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer_verify.c
+ *
+ * Extend fe_utils/astreamer.h archive streaming facility to verify TAR
+ * backup.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * src/bin/pg_verifybackup/astreamer_verify.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include "common/logging.h"
+#include "fe_utils/astreamer.h"
+#include "pg_verifybackup.h"
+
+typedef struct astreamer_verify
+{
+	astreamer	base;
+	verifier_context *context;
+	char	   *archiveName;
+	Oid			tblspcOid;
+
+	manifest_file *mfile;
+	size_t		receivedBytes;
+	bool		verifyChecksums;
+	bool		verifyControlData;
+	pg_checksum_context *checksum_ctx;
+} astreamer_verify;
+
+static void astreamer_verify_content(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len,
+									 astreamer_archive_context context);
+static void astreamer_verify_finalize(astreamer *streamer);
+static void astreamer_verify_free(astreamer *streamer);
+
+static void verify_checksums(astreamer *streamer,
+							 astreamer_member *member,
+							 const char *data, int len);
+static void verify_controldata(astreamer *streamer,
+							   astreamer_member *member,
+							   const char *data, int len);
+
+static const astreamer_ops astreamer_verify_ops = {
+	.content = astreamer_verify_content,
+	.finalize = astreamer_verify_finalize,
+	.free = astreamer_verify_free
+};
+
+/*
+ * Create a astreamer that can verifies content of a TAR file.
+ */
+astreamer *
+astreamer_verify_content_new(astreamer *next, verifier_context *context,
+							 char *archive_name, Oid tblspc_oid)
+{
+	astreamer_verify *streamer;
+
+	streamer = palloc0(sizeof(astreamer_verify));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_verify_ops;
+
+	streamer->base.bbs_next = next;
+	streamer->context = context;
+	streamer->archiveName = archive_name;
+	streamer->tblspcOid = tblspc_oid;
+	initStringInfo(&streamer->base.bbs_buffer);
+
+	return &streamer->base;
+}
+
+/*
+ * It verifies each TAR member entry against the manifest data and performs
+ * checksum verification if enabled. Additionally, it validates the backup's
+ * system identifier against the backup_manifest.
+ */
+static void
+astreamer_verify_content(astreamer *streamer,
+						 astreamer_member *member, const char *data,
+						 int len, astreamer_archive_context context)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	Assert(context != ASTREAMER_UNKNOWN);
+
+	switch (context)
+	{
+		case ASTREAMER_MEMBER_HEADER:
+			if (!member->is_directory && !member->is_link &&
+				!should_ignore_relpath(mystreamer->context, member->pathname))
+			{
+				manifest_file *m;
+
+				/*
+				 * The backup_manifest stores a relative path to the base
+				 * directory for files belong tablespace, whereas
+				 * <tablespaceoid>.tar doesn't. Prepare the required path,
+				 * otherwise, the manfiest entry verification will fail.
+				 */
+				if (OidIsValid(mystreamer->tblspcOid))
+				{
+					char		temp[MAXPGPATH];
+
+					/* Copy original name at temporary space */
+					memcpy(temp, member->pathname, MAXPGPATH);
+
+					snprintf(member->pathname, MAXPGPATH, "%s/%d/%s",
+							 "pg_tblspc", mystreamer->tblspcOid, temp);
+				}
+
+				/* Check the manifest entry */
+				m = verify_manifest_entry(mystreamer->context, member->pathname,
+										  member->size);
+				mystreamer->mfile = (void *) m;
+
+				/*
+				 * Prepare for checksum and manifest system identifier
+				 * verification.
+				 *
+				 * We could have these checks while receiving contents.
+				 * However, since contents are received in multiple
+				 * iterations, this would result in these lengthy checks being
+				 * performed multiple times. Instead, having a single flag
+				 * would be more efficient.
+				 */
+				if (m != NULL)
+				{
+					mystreamer->verifyChecksums =
+						(!mystreamer->context->skip_checksums &&
+						 should_verify_checksum(m));
+					mystreamer->verifyControlData =
+						should_verify_control_data(mystreamer->context->manifest, m);
+				}
+			}
+			break;
+
+		case ASTREAMER_MEMBER_CONTENTS:
+
+			/*
+			 * Perform checksum verification as the file content becomes
+			 * available, since the TAR format does not have random access to
+			 * files like a normal backup directory, where checksum
+			 * verification occurs at different points.
+			 */
+			if (mystreamer->verifyChecksums)
+				verify_checksums(streamer, member, data, len);
+
+			/* Verify pg_control file information */
+			if (mystreamer->verifyControlData)
+				verify_controldata(streamer, member, data, len);
+			break;
+
+		case ASTREAMER_MEMBER_TRAILER:
+			if (mystreamer->checksum_ctx)
+				pfree(mystreamer->checksum_ctx);
+			mystreamer->checksum_ctx = NULL;
+			mystreamer->mfile = NULL;
+			mystreamer->receivedBytes = 0;
+			mystreamer->verifyChecksums = false;
+			mystreamer->verifyControlData = false;
+			break;
+
+		case ASTREAMER_ARCHIVE_TRAILER:
+			break;
+
+		default:
+			/* Shouldn't happen. */
+			pg_fatal("unexpected state while parsing tar archive");
+	}
+}
+
+/*
+ * End-of-stream processing for a astreamer_verify stream.
+ */
+static void
+astreamer_verify_finalize(astreamer *streamer)
+{
+	Assert(streamer->bbs_next == NULL);
+}
+
+/*
+ * Free memory associated with a astreamer_verify stream.
+ */
+static void
+astreamer_verify_free(astreamer *streamer)
+{
+	pfree(streamer->bbs_buffer.data);
+	pfree(streamer);
+}
+
+/*
+ * Perform checksum verification of the file content, which may be received in
+ * multiple iterations.
+ */
+static void
+verify_checksums(astreamer *streamer, astreamer_member *member,
+				 const char *data, int len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	/* If we were first time for this file */
+	if (!mystreamer->checksum_ctx)
+	{
+		mystreamer->checksum_ctx = pg_malloc(sizeof(pg_checksum_context));
+
+		if (pg_checksum_init(mystreamer->checksum_ctx,
+							 mystreamer->mfile->checksum_type) < 0)
+		{
+			report_backup_error(mystreamer->context,
+								"%s: could not initialize checksum of file \"%s\"",
+								mystreamer->archiveName, member->pathname);
+			mystreamer->verifyChecksums = false;
+			return;
+		}
+	}
+
+	/* Compute and do the checksum validation */
+	mystreamer->verifyChecksums =
+		verify_content_checksum(mystreamer->context,
+								mystreamer->checksum_ctx,
+								mystreamer->mfile,
+								(uint8 *) data, len,
+								&mystreamer->receivedBytes);
+}
+
+/*
+ * Prepare the control data from the received file contents, which are supposed
+ * to be from the pg_control file, including CRC calculation. Then, call the
+ * routines that perform the final verification of the control file information.
+ */
+static void
+verify_controldata(astreamer *streamer, astreamer_member *member,
+				   const char *data, int len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_data *manifest = mystreamer->context->manifest;
+	ControlFileData control_file;
+	pg_crc32c	crc;
+	bool		crc_ok;
+
+	/* Should be here only for control file */
+	Assert(strcmp(member->pathname, "global/pg_control") == 0);
+	Assert(manifest->version != 1);
+
+	/* Should have whole control file data. */
+	if (!astreamer_buffer_until(streamer, &data, &len, sizeof(ControlFileData)))
+		return;
+
+	pg_log_debug("%s: reading \"%s\"", mystreamer->archiveName,
+				 member->pathname);
+
+	if (streamer->bbs_buffer.len != sizeof(ControlFileData))
+		report_fatal_error("%s: could not read control file: read %d of %zu",
+						   mystreamer->archiveName, streamer->bbs_buffer.len,
+						   sizeof(ControlFileData));
+
+	memcpy(&control_file, streamer->bbs_buffer.data,
+		   sizeof(ControlFileData));
+
+	/* Check the CRC. */
+	INIT_CRC32C(crc);
+	COMP_CRC32C(crc,
+				(char *) (&control_file),
+				offsetof(ControlFileData, crc));
+	FIN_CRC32C(crc);
+
+	crc_ok = EQ_CRC32C(crc, control_file.crc);
+
+	/* Do the final control data verification */
+	verify_control_data(&control_file, member->pathname, crc_ok,
+						manifest->system_identifier);
+}
diff --git a/src/bin/pg_verifybackup/meson.build b/src/bin/pg_verifybackup/meson.build
index 7c7d31a0350..1e3fcf7ee5a 100644
--- a/src/bin/pg_verifybackup/meson.build
+++ b/src/bin/pg_verifybackup/meson.build
@@ -1,7 +1,8 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 pg_verifybackup_sources = files(
-  'pg_verifybackup.c'
+  'pg_verifybackup.c',
+  'astreamer_verify.c'
 )
 
 if host_system == 'windows'
@@ -10,9 +11,10 @@ if host_system == 'windows'
     '--FILEDESC', 'pg_verifybackup - verify a backup against using a backup manifest'])
 endif
 
+pg_verifybackup_deps = [frontend_code, libpq, lz4, zlib, zstd]
 pg_verifybackup = executable('pg_verifybackup',
   pg_verifybackup_sources,
-  dependencies: [frontend_code, libpq],
+  dependencies: pg_verifybackup_deps,
   kwargs: default_bin_args,
 )
 bin_targets += pg_verifybackup
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 2a7d4869ce5..ffc7842b350 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -20,9 +20,12 @@
 
 #include "common/compression.h"
 #include "common/parse_manifest.h"
+#include "common/relpath.h"
+#include "fe_utils/astreamer.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
 #include "pg_verifybackup.h"
+#include "pgtar.h"
 #include "pgtime.h"
 
 /*
@@ -39,6 +42,11 @@
  */
 #define ESTIMATED_BYTES_PER_MANIFEST_LINE	100
 
+
+static void (*verify_backup_file_cb) (verifier_context *context,
+									  char *relpath, char *fullpath,
+									  size_t filesize);
+
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -63,6 +71,15 @@ static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
 							   char *relpath, char *fullpath);
+static void verify_plain_file_cb(verifier_context *context,
+								 char *relpath, char *fullpath,
+								 size_t filesize);
+static void verify_tar_file_cb(verifier_context *context,
+							   char *relpath, char *fullpath,
+							   size_t filesize);
+static void verify_tar_content(verifier_context *context,
+							   char *relpath, char *fullpath,
+							   astreamer *streamer);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -71,6 +88,9 @@ static void verify_file_checksum(verifier_context *context,
 static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
+static astreamer *create_archive_verifier(verifier_context *context,
+										  char *archive_name,
+										  Oid tblspc_oid);
 
 static void progress_report(verifier_context *context, bool finished);
 static void usage(void);
@@ -150,6 +170,10 @@ main(int argc, char **argv)
 	 */
 	simple_string_list_append(&context.ignore_list, "backup_manifest");
 	simple_string_list_append(&context.ignore_list, "pg_wal");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.gz");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.lz4");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.zst");
 	simple_string_list_append(&context.ignore_list, "postgresql.auto.conf");
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
@@ -254,6 +278,15 @@ main(int argc, char **argv)
 	if (format == 't' && !tar_compression_specified)
 		compress_algorithm = find_backup_compression(&context);
 
+	/*
+	 * Setup the required callback function to verify plain or tar backup
+	 * files.
+	 */
+	if (format == 'p')
+		verify_backup_file_cb = verify_plain_file_cb;
+	else
+		verify_backup_file_cb = verify_tar_file_cb;
+
 	/* Unless --no-parse-wal was specified, we will need pg_waldump. */
 	if (!no_parse_wal)
 	{
@@ -633,7 +666,8 @@ verify_backup_directory(verifier_context *context, char *relpath,
 }
 
 /*
- * Verify one file (which might actually be a directory or a symlink).
+ * Verify one file (which might actually be a directory, a symlink or a
+ * archive).
  *
  * The arguments to this function have the same meaning as the arguments to
  * verify_backup_directory.
@@ -642,7 +676,6 @@ static void
 verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 {
 	struct stat sb;
-	manifest_file *m;
 
 	if (stat(fullpath, &sb) != 0)
 	{
@@ -675,8 +708,25 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		return;
 	}
 
+	/* Do the further verifications */
+	verify_backup_file_cb(context, relpath, fullpath, sb.st_size);
+}
+
+/*
+ * Verify one plan file or a symlink.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_backup_directory. The additional argument is the file size for
+ * verifying against manifest entry.
+ */
+static void
+verify_plain_file_cb(verifier_context *context, char *relpath,
+					 char *fullpath, size_t filesize)
+{
+	manifest_file *m;
+
 	/* Check the backup manifest entry for this file. */
-	m = verify_manifest_entry(context, relpath, sb.st_size);
+	m = verify_manifest_entry(context, relpath, filesize);
 
 	/* Validate the manifest system identifier */
 	if (m != NULL && should_verify_control_data(context->manifest, m))
@@ -694,6 +744,124 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 	}
 }
 
+/*
+ * Verify one tar file.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_backup_directory. The additional argument is the file size for
+ * verifying against manifest entry.
+ */
+static void
+verify_tar_file_cb(verifier_context *context, char *relpath,
+				   char *fullpath, size_t filesize)
+{
+	astreamer  *streamer;
+	Oid			tblspc_oid = InvalidOid;
+	int			file_name_len;
+	int			file_extn_len = 0;	/* placate compiler */
+	char	   *file_extn = "";
+
+	/* Should be tar backup */
+	Assert(format == 't');
+
+	/* Find the tar file extension. */
+	if (compress_algorithm == PG_COMPRESSION_NONE)
+	{
+		file_extn = ".tar";
+		file_extn_len = 4;
+
+	}
+	else if (compress_algorithm == PG_COMPRESSION_GZIP)
+	{
+		file_extn = ".tar.gz";
+		file_extn_len = 7;
+
+	}
+	else if (compress_algorithm == PG_COMPRESSION_LZ4)
+	{
+		file_extn = ".tar.lz4";
+		file_extn_len = 8;
+	}
+	else if (compress_algorithm == PG_COMPRESSION_ZSTD)
+	{
+		file_extn = ".tar.zst";
+		file_extn_len = 8;
+	}
+
+	/*
+	 * Ensure that we have the correct file type corresponding to the backup
+	 * format.
+	 */
+	file_name_len = strlen(relpath);
+	if (file_name_len < file_extn_len ||
+		strcmp(relpath + file_name_len - file_extn_len, file_extn) != 0)
+	{
+		if (compress_algorithm == PG_COMPRESSION_NONE)
+			report_backup_error(context,
+								"\"%s\" is not a valid file, expecting tar file",
+								relpath);
+		else
+			report_backup_error(context,
+								"\"%s\" is not a valid file, expecting \"%s\" compressed tar file",
+								relpath,
+								get_compress_algorithm_name(compress_algorithm));
+		return;
+	}
+
+	/*
+	 * For the tablespace, pg_basebackup writes the data out to
+	 * <tablespaceoid>.tar.  If a file matches that format, then extract the
+	 * tablespaceoid, which we need to prepare the paths of the files
+	 * belonging to that tablespace relative to the base directory.
+	 */
+	if (strspn(relpath, "0123456789") == (file_name_len - file_extn_len))
+		tblspc_oid = strtoi64(relpath, NULL, 10);
+
+	streamer = create_archive_verifier(context, relpath, tblspc_oid);
+	verify_tar_content(context, relpath, fullpath, streamer);
+
+	/* Cleanup. */
+	astreamer_finalize(streamer);
+	astreamer_free(streamer);
+}
+
+/*
+ * Reads a given tar file in predefined chunks and pass to astreamer.  Which
+ * initiates routines for decompression (if necessary) then verification
+ * of each member within the tar archive.
+ */
+static void
+verify_tar_content(verifier_context *context, char *relpath, char *fullpath,
+				   astreamer *streamer)
+{
+	int			fd;
+	int			rc;
+	char	   *buffer;
+
+	/* Open the target file. */
+	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
+	{
+		report_backup_error(context, "could not open file \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
+
+	/* Perform the reads */
+	while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
+		astreamer_content(streamer, NULL, buffer, rc, ASTREAMER_UNKNOWN);
+
+	if (rc < 0)
+		report_backup_error(context, "could not read file \"%s\": %m",
+							relpath);
+
+	/* Close the file. */
+	if (close(fd) != 0)
+		report_backup_error(context, "could not close file \"%s\": %m",
+							relpath);
+}
+
 /*
  * Verify file and its size entry in the manifest.
  */
@@ -1122,10 +1290,10 @@ find_backup_format(verifier_context *context)
 }
 
 /*
- * To determine the compression format, we will search for the main data
- * directory archive and its extension, which starts with base.tar, as
  * pg_basebackup writes the main data directory to an archive file named
- * base.tar followed by a compression type extension like .gz, .lz4, or .zst.
+ * base.tar, followed by a compression type extension such as .gz, .lz4, or
+ * .zst. To determine the compression format, we need to search for this main
+ * data directory archive file.
  */
 static pg_compress_algorithm
 find_backup_compression(verifier_context *context)
@@ -1176,6 +1344,42 @@ find_backup_compression(verifier_context *context)
 	return PG_COMPRESSION_NONE; /* placate compiler */
 }
 
+/*
+ * Identifies the necessary steps for verifying the contents of the
+ * provided tar file.
+ */
+static astreamer *
+create_archive_verifier(verifier_context *context, char *archive_name,
+						Oid tblspc_oid)
+{
+	astreamer  *streamer = NULL;
+
+	/* Should be here only for tar backup */
+	Assert(format == 't');
+
+	/*
+	 * To verify the contents of the tar file, the initial step is to parse
+	 * its content.
+	 */
+	streamer = astreamer_verify_content_new(streamer, context, archive_name,
+											tblspc_oid);
+	streamer = astreamer_tar_parser_new(streamer);
+
+	/*
+	 * If the tar file is compressed, we must perform the appropriate
+	 * decompression operation before proceeding with the verification of its
+	 * contents.
+	 */
+	if (compress_algorithm == PG_COMPRESSION_GZIP)
+		streamer = astreamer_gzip_decompressor_new(streamer);
+	else if (compress_algorithm == PG_COMPRESSION_LZ4)
+		streamer = astreamer_lz4_decompressor_new(streamer);
+	else if (compress_algorithm == PG_COMPRESSION_ZSTD)
+		streamer = astreamer_zstd_decompressor_new(streamer);
+
+	return streamer;
+}
+
 /*
  * Print out usage information and exit.
  */
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index a1a34c4b773..fc53e069854 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -129,4 +129,14 @@ extern void report_fatal_error(const char *pg_restrict fmt,...)
 			pg_attribute_printf(1, 2) pg_attribute_noreturn();
 extern bool should_ignore_relpath(verifier_context *context, const char *relpath);
 
+/* Forward declarations to avoid fe_utils/astreamer.h include. */
+struct astreamer;
+typedef struct astreamer astreamer;
+
+extern astreamer *astreamer_verify_content_new(astreamer *next,
+											   verifier_context *context,
+											   char *archive_name,
+											   Oid tblspc_oid);
+
+
 #endif /* PG_VERIFYBACKUP_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index f59f7acdb7e..e4f4c64c1a2 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3321,6 +3321,7 @@ astreamer_plain_writer
 astreamer_recovery_injector
 astreamer_tar_archiver
 astreamer_tar_parser
+astreamer_verify
 astreamer_zstd_frame
 bgworker_main_type
 bh_node_type
-- 
2.18.0

From 031e17e9e5bf36e1f6ae5e9b1b3ae15b14f60088 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Mon, 17 Jun 2024 14:17:22 +0530
Subject: [PATCH v3 06/11] Refactor: split verify_backup_file() function.

Separate the manifest entry verification code into a new function.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 37 ++++++++++++++++-------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 134d575d738..bb732bf06ca 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -619,6 +619,27 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		return;
 	}
 
+	/* Check the backup manifest entry for this file. */
+	m = verify_manifest_entry(context, relpath, sb.st_size);
+
+	/*
+	 * Validate the manifest system identifier, not available in manifest
+	 * version 1.
+	 */
+	if (context->manifest->version != 1 &&
+		strcmp(relpath, "global/pg_control") == 0 &&
+		m != NULL && m->matched && !m->bad)
+		verify_control_file(fullpath, context->manifest->system_identifier);
+}
+
+/*
+ * Verify file and its size entry in the manifest.
+ */
+manifest_file *
+verify_manifest_entry(verifier_context *context, char *relpath, size_t filesize)
+{
+	manifest_file *m;
+
 	/* Check whether there's an entry in the manifest hash. */
 	m = manifest_files_lookup(context->manifest->files, relpath);
 	if (m == NULL)
@@ -626,29 +647,21 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		report_backup_error(context,
 							"\"%s\" is present on disk but not in the manifest",
 							relpath);
-		return;
+		return NULL;
 	}
 
 	/* Flag this entry as having been encountered in the filesystem. */
 	m->matched = true;
 
 	/* Check that the size matches. */
-	if (m->size != sb.st_size)
+	if (m->size != filesize)
 	{
 		report_backup_error(context,
 							"\"%s\" has size %lld on disk but size %zu in the manifest",
-							relpath, (long long int) sb.st_size, m->size);
+							relpath, (long long int) filesize, m->size);
 		m->bad = true;
 	}
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0)
-		verify_control_file(fullpath, context->manifest->system_identifier);
-
 	/* Update statistics for progress report, if necessary */
 	if (context->show_progress && !context->skip_checksums &&
 		should_verify_checksum(m))
@@ -660,6 +673,8 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 	 * afterwards verify the checksums. That's because computing checksums may
 	 * take a while, and we'd like to report more obvious problems quickly.
 	 */
+
+	return m;
 }
 
 /*
-- 
2.18.0

From cf2e5f89557cb6716760f49be5e78cda9b066914 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 10:32:11 +0530
Subject: [PATCH v3 05/11] Refactor: move some part of pg_verifybackup.c to
 pg_verifybackup.h

---
 src/bin/pg_verifybackup/pg_verifybackup.c |  97 +-----------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 114 ++++++++++++++++++++++
 2 files changed, 119 insertions(+), 92 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/pg_verifybackup.h

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 74b3a66835a..134d575d738 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,12 +18,11 @@
 #include <sys/stat.h>
 #include <time.h>
 
-#include "common/controldata_utils.h"
-#include "common/hashfn_unstable.h"
 #include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
+#include "pg_verifybackup.h"
 #include "pgtime.h"
 
 /*
@@ -40,85 +39,6 @@
  */
 #define ESTIMATED_BYTES_PER_MANIFEST_LINE	100
 
-/*
- * How many bytes should we try to read from a file at once?
- */
-#define READ_CHUNK_SIZE				(128 * 1024)
-
-/*
- * Each file described by the manifest file is parsed to produce an object
- * like this.
- */
-typedef struct manifest_file
-{
-	uint32		status;			/* hash status */
-	const char *pathname;
-	size_t		size;
-	pg_checksum_type checksum_type;
-	int			checksum_length;
-	uint8	   *checksum_payload;
-	bool		matched;
-	bool		bad;
-} manifest_file;
-
-#define should_verify_checksum(m) \
-	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
-
-/*
- * Define a hash table which we can use to store information about the files
- * mentioned in the backup manifest.
- */
-#define SH_PREFIX		manifest_files
-#define SH_ELEMENT_TYPE	manifest_file
-#define SH_KEY_TYPE		const char *
-#define	SH_KEY			pathname
-#define SH_HASH_KEY(tb, key)	hash_string(key)
-#define SH_EQUAL(tb, a, b)		(strcmp(a, b) == 0)
-#define	SH_SCOPE		static inline
-#define SH_RAW_ALLOCATOR	pg_malloc0
-#define SH_DECLARE
-#define SH_DEFINE
-#include "lib/simplehash.h"
-
-/*
- * Each WAL range described by the manifest file is parsed to produce an
- * object like this.
- */
-typedef struct manifest_wal_range
-{
-	TimeLineID	tli;
-	XLogRecPtr	start_lsn;
-	XLogRecPtr	end_lsn;
-	struct manifest_wal_range *next;
-	struct manifest_wal_range *prev;
-} manifest_wal_range;
-
-/*
- * All the data parsed from a backup_manifest file.
- */
-typedef struct manifest_data
-{
-	int			version;
-	uint64		system_identifier;
-	manifest_files_hash *files;
-	manifest_wal_range *first_wal_range;
-	manifest_wal_range *last_wal_range;
-} manifest_data;
-
-/*
- * All of the context information we need while checking a backup manifest.
- */
-typedef struct verifier_context
-{
-	manifest_data *manifest;
-	char	   *backup_directory;
-	SimpleStringList ignore_list;
-	bool		show_progress;
-	bool		skip_checksums;
-	bool		exit_on_error;
-	bool		saw_any_error;
-} verifier_context;
-
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -152,13 +72,6 @@ static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
 
-static void report_backup_error(verifier_context *context,
-								const char *pg_restrict fmt,...)
-			pg_attribute_printf(2, 3);
-static void report_fatal_error(const char *pg_restrict fmt,...)
-			pg_attribute_printf(1, 2) pg_attribute_noreturn();
-static bool should_ignore_relpath(verifier_context *context, const char *relpath);
-
 static void progress_report(verifier_context *context, bool finished);
 static void usage(void);
 
@@ -554,7 +467,7 @@ verifybackup_per_file_cb(JsonManifestParseContext *context,
 	bool		found;
 
 	/* Make a new entry in the hash table for this file. */
-	m = manifest_files_insert(ht, pathname, &found);
+	m = manifest_files_insert(ht, (char *) pathname, &found);
 	if (found)
 		report_fatal_error("duplicate path name in backup manifest: \"%s\"",
 						   pathname);
@@ -978,7 +891,7 @@ parse_required_wal(verifier_context *context, char *pg_waldump_path,
  * Update the context to indicate that we saw an error, and exit if the
  * context says we should.
  */
-static void
+void
 report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
 {
 	va_list		ap;
@@ -995,7 +908,7 @@ report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
 /*
  * Report a fatal error and exit
  */
-static void
+void
 report_fatal_error(const char *pg_restrict fmt,...)
 {
 	va_list		ap;
@@ -1014,7 +927,7 @@ report_fatal_error(const char *pg_restrict fmt,...)
  * Note that by "prefix" we mean a parent directory; for this purpose,
  * "aa/bb" is not a prefix of "aa/bbb", but it is a prefix of "aa/bb/cc".
  */
-static bool
+bool
 should_ignore_relpath(verifier_context *context, const char *relpath)
 {
 	SimpleStringListCell *cell;
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
new file mode 100644
index 00000000000..f4a93d3f137
--- /dev/null
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -0,0 +1,114 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_verifybackup.h
+ *	  Verify a backup against a backup manifest.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_verifybackup/pg_verifybackup.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef PG_VERIFYBACKUP_H
+#define PG_VERIFYBACKUP_H
+
+#include "common/controldata_utils.h"
+#include "common/hashfn_unstable.h"
+#include "common/parse_manifest.h"
+#include "fe_utils/simple_list.h"
+
+extern bool show_progress;
+extern bool skip_checksums;
+
+/*
+ * How many bytes should we try to read from a file at once?
+ */
+#define READ_CHUNK_SIZE				(128 * 1024)
+
+/*
+ * Each file described by the manifest file is parsed to produce an object
+ * like this.
+ */
+typedef struct manifest_file
+{
+	uint32		status;			/* hash status */
+	char	   *pathname;
+	size_t		size;
+	pg_checksum_type checksum_type;
+	int			checksum_length;
+	uint8	   *checksum_payload;
+	bool		matched;
+	bool		bad;
+} manifest_file;
+
+#define should_verify_checksum(m) \
+	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+
+/*
+ * Define a hash table which we can use to store information about the files
+ * mentioned in the backup manifest.
+ */
+#define SH_PREFIX		manifest_files
+#define SH_ELEMENT_TYPE	manifest_file
+#define SH_KEY_TYPE		char *
+#define	SH_KEY			pathname
+#define SH_HASH_KEY(tb, key)	hash_string(key)
+#define SH_EQUAL(tb, a, b)		(strcmp(a, b) == 0)
+#define	SH_SCOPE		static inline
+#define SH_RAW_ALLOCATOR	pg_malloc0
+#define SH_DECLARE
+#define SH_DEFINE
+#include "lib/simplehash.h"
+
+/*
+ * Each WAL range described by the manifest file is parsed to produce an
+ * object like this.
+ */
+typedef struct manifest_wal_range
+{
+	TimeLineID	tli;
+	XLogRecPtr	start_lsn;
+	XLogRecPtr	end_lsn;
+	struct manifest_wal_range *next;
+	struct manifest_wal_range *prev;
+} manifest_wal_range;
+
+/*
+ * All the data parsed from a backup_manifest file.
+ */
+typedef struct manifest_data
+{
+	int			version;
+	uint64		system_identifier;
+	manifest_files_hash *files;
+	manifest_wal_range *first_wal_range;
+	manifest_wal_range *last_wal_range;
+} manifest_data;
+
+/*
+ * All of the context information we need while checking a backup manifest.
+ */
+typedef struct verifier_context
+{
+	manifest_data *manifest;
+	char	   *backup_directory;
+	SimpleStringList ignore_list;
+	bool		show_progress;
+	bool		skip_checksums;
+	bool		exit_on_error;
+	bool		saw_any_error;
+} verifier_context;
+
+extern manifest_file *verify_manifest_entry(verifier_context *context,
+											char *relpath, size_t filesize);
+
+extern void report_backup_error(verifier_context *context,
+								const char *pg_restrict fmt,...)
+			pg_attribute_printf(2, 3);
+extern void report_fatal_error(const char *pg_restrict fmt,...)
+			pg_attribute_printf(1, 2) pg_attribute_noreturn();
+extern bool should_ignore_relpath(verifier_context *context, const char *relpath);
+
+#endif /* PG_VERIFYBACKUP_H */
-- 
2.18.0

From 67985f9eb69510800cd53333ce456a72363499d0 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 31 Jul 2024 11:43:52 +0530
Subject: [PATCH v3 04/11] Refactor: move show_progress and skip_checksums to
 verifier_context struct

---
 src/bin/pg_verifybackup/pg_verifybackup.c | 29 +++++++++++------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index d77e70fbe38..74b3a66835a 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -113,6 +113,8 @@ typedef struct verifier_context
 	manifest_data *manifest;
 	char	   *backup_directory;
 	SimpleStringList ignore_list;
+	bool		show_progress;
+	bool		skip_checksums;
 	bool		exit_on_error;
 	bool		saw_any_error;
 } verifier_context;
@@ -157,15 +159,11 @@ static void report_fatal_error(const char *pg_restrict fmt,...)
 			pg_attribute_printf(1, 2) pg_attribute_noreturn();
 static bool should_ignore_relpath(verifier_context *context, const char *relpath);
 
-static void progress_report(bool finished);
+static void progress_report(verifier_context *context, bool finished);
 static void usage(void);
 
 static const char *progname;
 
-/* options */
-static bool show_progress = false;
-static bool skip_checksums = false;
-
 /* Progress indicators */
 static uint64 total_size = 0;
 static uint64 done_size = 0;
@@ -260,13 +258,13 @@ main(int argc, char **argv)
 				no_parse_wal = true;
 				break;
 			case 'P':
-				show_progress = true;
+				context.show_progress = true;
 				break;
 			case 'q':
 				quiet = true;
 				break;
 			case 's':
-				skip_checksums = true;
+				context.skip_checksums = true;
 				break;
 			case 'w':
 				wal_directory = pstrdup(optarg);
@@ -299,7 +297,7 @@ main(int argc, char **argv)
 	}
 
 	/* Complain if the specified arguments conflict */
-	if (show_progress && quiet)
+	if (context.show_progress && quiet)
 		pg_fatal("cannot specify both %s and %s",
 				 "-P/--progress", "-q/--quiet");
 
@@ -363,7 +361,7 @@ main(int argc, char **argv)
 	 * Now do the expensive work of verifying file checksums, unless we were
 	 * told to skip it.
 	 */
-	if (!skip_checksums)
+	if (!context.skip_checksums)
 		verify_backup_checksums(&context);
 
 	/*
@@ -739,7 +737,8 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		verify_control_file(fullpath, context->manifest->system_identifier);
 
 	/* Update statistics for progress report, if necessary */
-	if (show_progress && !skip_checksums && should_verify_checksum(m))
+	if (context->show_progress && !context->skip_checksums &&
+		should_verify_checksum(m))
 		total_size += m->size;
 
 	/*
@@ -815,7 +814,7 @@ verify_backup_checksums(verifier_context *context)
 	manifest_file *m;
 	uint8	   *buffer;
 
-	progress_report(false);
+	progress_report(context, false);
 
 	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
 
@@ -841,7 +840,7 @@ verify_backup_checksums(verifier_context *context)
 
 	pfree(buffer);
 
-	progress_report(true);
+	progress_report(context, true);
 }
 
 /*
@@ -890,7 +889,7 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 
 		/* Report progress */
 		done_size += rc;
-		progress_report(false);
+		progress_report(context, false);
 	}
 	if (rc < 0)
 		report_backup_error(context, "could not read file \"%s\": %m",
@@ -1045,7 +1044,7 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
  * is moved to the next line.
  */
 static void
-progress_report(bool finished)
+progress_report(verifier_context *context, bool finished)
 {
 	static pg_time_t last_progress_report = 0;
 	pg_time_t	now;
@@ -1053,7 +1052,7 @@ progress_report(bool finished)
 	char		totalsize_str[32];
 	char		donesize_str[32];
 
-	if (!show_progress)
+	if (!context->show_progress)
 		return;
 
 	now = time(NULL);
-- 
2.18.0

From 6577e6c506cba5d91721ba9f55ea9400fc1a72c0 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 31 Jul 2024 11:20:52 +0530
Subject: [PATCH v3 03/11] Refactor: move astreamer* files to fe_utils to make
 common availability of it.

To make it accessible to other code, we need to move the ASTREAMER
code (previously known as BBSTREAMER) to a common location. The
appropriate place would be src/fe_utils, as it is a frontend
infrastructure intended for shared use.
---
 meson.build                                             | 2 +-
 src/bin/pg_basebackup/Makefile                          | 7 +------
 src/bin/pg_basebackup/astreamer_inject.h                | 2 +-
 src/bin/pg_basebackup/meson.build                       | 5 -----
 src/fe_utils/Makefile                                   | 5 +++++
 src/{bin/pg_basebackup => fe_utils}/astreamer_file.c    | 2 +-
 src/{bin/pg_basebackup => fe_utils}/astreamer_gzip.c    | 2 +-
 src/{bin/pg_basebackup => fe_utils}/astreamer_lz4.c     | 2 +-
 src/{bin/pg_basebackup => fe_utils}/astreamer_tar.c     | 2 +-
 src/{bin/pg_basebackup => fe_utils}/astreamer_zstd.c    | 2 +-
 src/fe_utils/meson.build                                | 5 +++++
 src/{bin/pg_basebackup => include/fe_utils}/astreamer.h | 0
 12 files changed, 18 insertions(+), 18 deletions(-)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_file.c (99%)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_gzip.c (99%)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_lz4.c (99%)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_tar.c (99%)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_zstd.c (99%)
 rename src/{bin/pg_basebackup => include/fe_utils}/astreamer.h (100%)

diff --git a/meson.build b/meson.build
index 7de0371226d..f7a5d2aea9a 100644
--- a/meson.build
+++ b/meson.build
@@ -3027,7 +3027,7 @@ frontend_common_code = declare_dependency(
   compile_args: ['-DFRONTEND'],
   include_directories: [postgres_inc],
   sources: generated_headers,
-  dependencies: [os_deps, zlib, zstd],
+  dependencies: [os_deps, zlib, zstd, lz4],
 )
 
 backend_common_code = declare_dependency(
diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile
index a71af2d48a7..f1e73058b23 100644
--- a/src/bin/pg_basebackup/Makefile
+++ b/src/bin/pg_basebackup/Makefile
@@ -37,12 +37,7 @@ OBJS = \
 
 BBOBJS = \
 	pg_basebackup.o \
-	astreamer_file.o \
-	astreamer_gzip.o \
-	astreamer_inject.o \
-	astreamer_lz4.o \
-	astreamer_tar.o \
-	astreamer_zstd.o
+	astreamer_inject.o
 
 all: pg_basebackup pg_createsubscriber pg_receivewal pg_recvlogical
 
diff --git a/src/bin/pg_basebackup/astreamer_inject.h b/src/bin/pg_basebackup/astreamer_inject.h
index 8504b3f5e0d..aeed533862b 100644
--- a/src/bin/pg_basebackup/astreamer_inject.h
+++ b/src/bin/pg_basebackup/astreamer_inject.h
@@ -12,7 +12,7 @@
 #ifndef ASTREAMER_INJECT_H
 #define ASTREAMER_INJECT_H
 
-#include "astreamer.h"
+#include "fe_utils/astreamer.h"
 #include "pqexpbuffer.h"
 
 extern astreamer *astreamer_recovery_injector_new(astreamer *next,
diff --git a/src/bin/pg_basebackup/meson.build b/src/bin/pg_basebackup/meson.build
index a68dbd7837d..9101fc18438 100644
--- a/src/bin/pg_basebackup/meson.build
+++ b/src/bin/pg_basebackup/meson.build
@@ -1,12 +1,7 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 common_sources = files(
-  'astreamer_file.c',
-  'astreamer_gzip.c',
   'astreamer_inject.c',
-  'astreamer_lz4.c',
-  'astreamer_tar.c',
-  'astreamer_zstd.c',
   'receivelog.c',
   'streamutil.c',
   'walmethods.c',
diff --git a/src/fe_utils/Makefile b/src/fe_utils/Makefile
index 946c05258f0..2694be4b859 100644
--- a/src/fe_utils/Makefile
+++ b/src/fe_utils/Makefile
@@ -21,6 +21,11 @@ override CPPFLAGS := -DFRONTEND -I$(libpq_srcdir) $(CPPFLAGS)
 
 OBJS = \
 	archive.o \
+	astreamer_file.o \
+	astreamer_gzip.o \
+	astreamer_lz4.o \
+	astreamer_tar.o \
+	astreamer_zstd.o \
 	cancel.o \
 	conditional.o \
 	connect_utils.o \
diff --git a/src/bin/pg_basebackup/astreamer_file.c b/src/fe_utils/astreamer_file.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_file.c
rename to src/fe_utils/astreamer_file.c
index 2742385e103..13d1192c6e6 100644
--- a/src/bin/pg_basebackup/astreamer_file.c
+++ b/src/fe_utils/astreamer_file.c
@@ -13,10 +13,10 @@
 
 #include <unistd.h>
 
-#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
+#include "fe_utils/astreamer.h"
 
 typedef struct astreamer_plain_writer
 {
diff --git a/src/bin/pg_basebackup/astreamer_gzip.c b/src/fe_utils/astreamer_gzip.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_gzip.c
rename to src/fe_utils/astreamer_gzip.c
index 6f7c27afbbc..dd28defac7b 100644
--- a/src/bin/pg_basebackup/astreamer_gzip.c
+++ b/src/fe_utils/astreamer_gzip.c
@@ -17,10 +17,10 @@
 #include <zlib.h>
 #endif
 
-#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
+#include "fe_utils/astreamer.h"
 
 #ifdef HAVE_LIBZ
 typedef struct astreamer_gzip_writer
diff --git a/src/bin/pg_basebackup/astreamer_lz4.c b/src/fe_utils/astreamer_lz4.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_lz4.c
rename to src/fe_utils/astreamer_lz4.c
index 1c40d7d8ad5..d8b2a367e47 100644
--- a/src/bin/pg_basebackup/astreamer_lz4.c
+++ b/src/fe_utils/astreamer_lz4.c
@@ -17,10 +17,10 @@
 #include <lz4frame.h>
 #endif
 
-#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
+#include "fe_utils/astreamer.h"
 
 #ifdef USE_LZ4
 typedef struct astreamer_lz4_frame
diff --git a/src/bin/pg_basebackup/astreamer_tar.c b/src/fe_utils/astreamer_tar.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_tar.c
rename to src/fe_utils/astreamer_tar.c
index 673690cd18f..f5d3562d280 100644
--- a/src/bin/pg_basebackup/astreamer_tar.c
+++ b/src/fe_utils/astreamer_tar.c
@@ -23,8 +23,8 @@
 
 #include <time.h>
 
-#include "astreamer.h"
 #include "common/logging.h"
+#include "fe_utils/astreamer.h"
 #include "pgtar.h"
 
 typedef struct astreamer_tar_parser
diff --git a/src/bin/pg_basebackup/astreamer_zstd.c b/src/fe_utils/astreamer_zstd.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_zstd.c
rename to src/fe_utils/astreamer_zstd.c
index 58dc679ef99..45f6cb67363 100644
--- a/src/bin/pg_basebackup/astreamer_zstd.c
+++ b/src/fe_utils/astreamer_zstd.c
@@ -17,8 +17,8 @@
 #include <zstd.h>
 #endif
 
-#include "astreamer.h"
 #include "common/logging.h"
+#include "fe_utils/astreamer.h"
 
 #ifdef USE_ZSTD
 
diff --git a/src/fe_utils/meson.build b/src/fe_utils/meson.build
index 14d0482a2cc..043021d826d 100644
--- a/src/fe_utils/meson.build
+++ b/src/fe_utils/meson.build
@@ -2,6 +2,11 @@
 
 fe_utils_sources = files(
   'archive.c',
+  'astreamer_file.c',
+  'astreamer_gzip.c',
+  'astreamer_lz4.c',
+  'astreamer_tar.c',
+  'astreamer_zstd.c',
   'cancel.c',
   'conditional.c',
   'connect_utils.c',
diff --git a/src/bin/pg_basebackup/astreamer.h b/src/include/fe_utils/astreamer.h
similarity index 100%
rename from src/bin/pg_basebackup/astreamer.h
rename to src/include/fe_utils/astreamer.h
-- 
2.18.0

From 90ea115b1a3d9ffcf2d58f9c4a63663245679d1e Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 17 Jul 2024 14:23:27 +0530
Subject: [PATCH v3 02/11] Refactor: Add astreamer_inject.h and move related
 declarations to it.

---
 src/bin/pg_basebackup/astreamer.h        |  6 ------
 src/bin/pg_basebackup/astreamer_inject.c |  2 +-
 src/bin/pg_basebackup/astreamer_inject.h | 24 ++++++++++++++++++++++++
 src/bin/pg_basebackup/pg_basebackup.c    |  2 +-
 4 files changed, 26 insertions(+), 8 deletions(-)
 create mode 100644 src/bin/pg_basebackup/astreamer_inject.h

diff --git a/src/bin/pg_basebackup/astreamer.h b/src/bin/pg_basebackup/astreamer.h
index 6b0047418bb..9d0a8c4d0c2 100644
--- a/src/bin/pg_basebackup/astreamer.h
+++ b/src/bin/pg_basebackup/astreamer.h
@@ -217,10 +217,4 @@ extern astreamer *astreamer_tar_parser_new(astreamer *next);
 extern astreamer *astreamer_tar_terminator_new(astreamer *next);
 extern astreamer *astreamer_tar_archiver_new(astreamer *next);
 
-extern astreamer *astreamer_recovery_injector_new(astreamer *next,
-												  bool is_recovery_guc_supported,
-												  PQExpBuffer recoveryconfcontents);
-extern void astreamer_inject_file(astreamer *streamer, char *pathname,
-								  char *data, int len);
-
 #endif
diff --git a/src/bin/pg_basebackup/astreamer_inject.c b/src/bin/pg_basebackup/astreamer_inject.c
index 7f1decded8d..4ad8381f102 100644
--- a/src/bin/pg_basebackup/astreamer_inject.c
+++ b/src/bin/pg_basebackup/astreamer_inject.c
@@ -11,7 +11,7 @@
 
 #include "postgres_fe.h"
 
-#include "astreamer.h"
+#include "astreamer_inject.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 
diff --git a/src/bin/pg_basebackup/astreamer_inject.h b/src/bin/pg_basebackup/astreamer_inject.h
new file mode 100644
index 00000000000..8504b3f5e0d
--- /dev/null
+++ b/src/bin/pg_basebackup/astreamer_inject.h
@@ -0,0 +1,24 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer_inject.h
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *		  src/bin/pg_basebackup/astreamer_inject.h
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef ASTREAMER_INJECT_H
+#define ASTREAMER_INJECT_H
+
+#include "astreamer.h"
+#include "pqexpbuffer.h"
+
+extern astreamer *astreamer_recovery_injector_new(astreamer *next,
+												  bool is_recovery_guc_supported,
+												  PQExpBuffer recoveryconfcontents);
+extern void astreamer_inject_file(astreamer *streamer, char *pathname,
+								  char *data, int len);
+
+#endif
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 4179b064cbc..1e753e40c97 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -26,7 +26,7 @@
 #endif
 
 #include "access/xlog_internal.h"
-#include "astreamer.h"
+#include "astreamer_inject.h"
 #include "backup/basebackup.h"
 #include "common/compression.h"
 #include "common/file_perm.h"
-- 
2.18.0

From ea8d3d56a37ded2e24535577e5c97f43a782b04d Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 09:39:32 +0530
Subject: [PATCH v3 01/11] Refactor: Rename all bbstreamer references to
 astreamer.

BBSTREAMER is specific to pg_basebackup; we need a more generalized
name so it can be placed in a common area, making it accessible for
other modules. Renaming it to ASTREAMER, short for ARCHIVE STREAMER,
makes it more general.
---
 src/bin/pg_basebackup/Makefile                |  12 +-
 src/bin/pg_basebackup/astreamer.h             | 226 +++++++++++++
 .../{bbstreamer_file.c => astreamer_file.c}   | 148 ++++----
 .../{bbstreamer_gzip.c => astreamer_gzip.c}   | 154 ++++-----
 ...bbstreamer_inject.c => astreamer_inject.c} | 152 ++++-----
 .../{bbstreamer_lz4.c => astreamer_lz4.c}     | 172 +++++-----
 .../{bbstreamer_tar.c => astreamer_tar.c}     | 316 +++++++++---------
 .../{bbstreamer_zstd.c => astreamer_zstd.c}   | 160 ++++-----
 src/bin/pg_basebackup/bbstreamer.h            | 226 -------------
 src/bin/pg_basebackup/meson.build             |  12 +-
 src/bin/pg_basebackup/nls.mk                  |  12 +-
 src/bin/pg_basebackup/pg_basebackup.c         |  74 ++--
 src/tools/pgindent/typedefs.list              |  26 +-
 13 files changed, 845 insertions(+), 845 deletions(-)
 create mode 100644 src/bin/pg_basebackup/astreamer.h
 rename src/bin/pg_basebackup/{bbstreamer_file.c => astreamer_file.c} (69%)
 rename src/bin/pg_basebackup/{bbstreamer_gzip.c => astreamer_gzip.c} (62%)
 rename src/bin/pg_basebackup/{bbstreamer_inject.c => astreamer_inject.c} (53%)
 rename src/bin/pg_basebackup/{bbstreamer_lz4.c => astreamer_lz4.c} (69%)
 rename src/bin/pg_basebackup/{bbstreamer_tar.c => astreamer_tar.c} (50%)
 rename src/bin/pg_basebackup/{bbstreamer_zstd.c => astreamer_zstd.c} (64%)
 delete mode 100644 src/bin/pg_basebackup/bbstreamer.h

diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile
index 26c53e473f5..a71af2d48a7 100644
--- a/src/bin/pg_basebackup/Makefile
+++ b/src/bin/pg_basebackup/Makefile
@@ -37,12 +37,12 @@ OBJS = \
 
 BBOBJS = \
 	pg_basebackup.o \
-	bbstreamer_file.o \
-	bbstreamer_gzip.o \
-	bbstreamer_inject.o \
-	bbstreamer_lz4.o \
-	bbstreamer_tar.o \
-	bbstreamer_zstd.o
+	astreamer_file.o \
+	astreamer_gzip.o \
+	astreamer_inject.o \
+	astreamer_lz4.o \
+	astreamer_tar.o \
+	astreamer_zstd.o
 
 all: pg_basebackup pg_createsubscriber pg_receivewal pg_recvlogical
 
diff --git a/src/bin/pg_basebackup/astreamer.h b/src/bin/pg_basebackup/astreamer.h
new file mode 100644
index 00000000000..6b0047418bb
--- /dev/null
+++ b/src/bin/pg_basebackup/astreamer.h
@@ -0,0 +1,226 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer.h
+ *
+ * Each tar archive returned by the server is passed to one or more
+ * astreamer objects for further processing. The astreamer may do
+ * something simple, like write the archive to a file, perhaps after
+ * compressing it, but it can also do more complicated things, like
+ * annotating the byte stream to indicate which parts of the data
+ * correspond to tar headers or trailing padding, vs. which parts are
+ * payload data. A subsequent astreamer may use this information to
+ * make further decisions about how to process the data; for example,
+ * it might choose to modify the archive contents.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *		  src/bin/pg_basebackup/astreamer.h
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef ASTREAMER_H
+#define ASTREAMER_H
+
+#include "common/compression.h"
+#include "lib/stringinfo.h"
+#include "pqexpbuffer.h"
+
+struct astreamer;
+struct astreamer_ops;
+typedef struct astreamer astreamer;
+typedef struct astreamer_ops astreamer_ops;
+
+/*
+ * Each chunk of archive data passed to a astreamer is classified into one
+ * of these categories. When data is first received from the remote server,
+ * each chunk will be categorized as ASTREAMER_UNKNOWN, and the chunks will
+ * be of whatever size the remote server chose to send.
+ *
+ * If the archive is parsed (e.g. see astreamer_tar_parser_new()), then all
+ * chunks should be labelled as one of the other types listed here. In
+ * addition, there should be exactly one ASTREAMER_MEMBER_HEADER chunk and
+ * exactly one ASTREAMER_MEMBER_TRAILER chunk per archive member, even if
+ * that means a zero-length call. There can be any number of
+ * ASTREAMER_MEMBER_CONTENTS chunks in between those calls. There
+ * should exactly ASTREAMER_ARCHIVE_TRAILER chunk, and it should follow the
+ * last ASTREAMER_MEMBER_TRAILER chunk.
+ *
+ * In theory, we could need other classifications here, such as a way of
+ * indicating an archive header, but the "tar" format doesn't need anything
+ * else, so for the time being there's no point.
+ */
+typedef enum
+{
+	ASTREAMER_UNKNOWN,
+	ASTREAMER_MEMBER_HEADER,
+	ASTREAMER_MEMBER_CONTENTS,
+	ASTREAMER_MEMBER_TRAILER,
+	ASTREAMER_ARCHIVE_TRAILER,
+} astreamer_archive_context;
+
+/*
+ * Each chunk of data that is classified as ASTREAMER_MEMBER_HEADER,
+ * ASTREAMER_MEMBER_CONTENTS, or ASTREAMER_MEMBER_TRAILER should also
+ * pass a pointer to an instance of this struct. The details are expected
+ * to be present in the archive header and used to fill the struct, after
+ * which all subsequent calls for the same archive member are expected to
+ * pass the same details.
+ */
+typedef struct
+{
+	char		pathname[MAXPGPATH];
+	pgoff_t		size;
+	mode_t		mode;
+	uid_t		uid;
+	gid_t		gid;
+	bool		is_directory;
+	bool		is_link;
+	char		linktarget[MAXPGPATH];
+} astreamer_member;
+
+/*
+ * Generally, each type of astreamer will define its own struct, but the
+ * first element should be 'astreamer base'. A astreamer that does not
+ * require any additional private data could use this structure directly.
+ *
+ * bbs_ops is a pointer to the astreamer_ops object which contains the
+ * function pointers appropriate to this type of astreamer.
+ *
+ * bbs_next is a pointer to the successor astreamer, for those types of
+ * astreamer which forward data to a successor. It need not be used and
+ * should be set to NULL when not relevant.
+ *
+ * bbs_buffer is a buffer for accumulating data for temporary storage. Each
+ * type of astreamer makes its own decisions about whether and how to use
+ * this buffer.
+ */
+struct astreamer
+{
+	const astreamer_ops *bbs_ops;
+	astreamer *bbs_next;
+	StringInfoData bbs_buffer;
+};
+
+/*
+ * There are three callbacks for a astreamer. The 'content' callback is
+ * called repeatedly, as described in the astreamer_archive_context comments.
+ * Then, the 'finalize' callback is called once at the end, to give the
+ * astreamer a chance to perform cleanup such as closing files. Finally,
+ * because this code is running in a frontend environment where, as of this
+ * writing, there are no memory contexts, the 'free' callback is called to
+ * release memory. These callbacks should always be invoked using the static
+ * inline functions defined below.
+ */
+struct astreamer_ops
+{
+	void		(*content) (astreamer *streamer, astreamer_member *member,
+							const char *data, int len,
+							astreamer_archive_context context);
+	void		(*finalize) (astreamer *streamer);
+	void		(*free) (astreamer *streamer);
+};
+
+/* Send some content to a astreamer. */
+static inline void
+astreamer_content(astreamer *streamer, astreamer_member *member,
+				  const char *data, int len,
+				  astreamer_archive_context context)
+{
+	Assert(streamer != NULL);
+	streamer->bbs_ops->content(streamer, member, data, len, context);
+}
+
+/* Finalize a astreamer. */
+static inline void
+astreamer_finalize(astreamer *streamer)
+{
+	Assert(streamer != NULL);
+	streamer->bbs_ops->finalize(streamer);
+}
+
+/* Free a astreamer. */
+static inline void
+astreamer_free(astreamer *streamer)
+{
+	Assert(streamer != NULL);
+	streamer->bbs_ops->free(streamer);
+}
+
+/*
+ * This is a convenience method for use when implementing a astreamer; it is
+ * not for use by outside callers. It adds the amount of data specified by
+ * 'nbytes' to the astreamer's buffer and adjusts '*len' and '*data'
+ * accordingly.
+ */
+static inline void
+astreamer_buffer_bytes(astreamer *streamer, const char **data, int *len,
+					   int nbytes)
+{
+	Assert(nbytes <= *len);
+
+	appendBinaryStringInfo(&streamer->bbs_buffer, *data, nbytes);
+	*len -= nbytes;
+	*data += nbytes;
+}
+
+/*
+ * This is a convenience method for use when implementing a astreamer; it is
+ * not for use by outsider callers. It attempts to add enough data to the
+ * astreamer's buffer to reach a length of target_bytes and adjusts '*len'
+ * and '*data' accordingly. It returns true if the target length has been
+ * reached and false otherwise.
+ */
+static inline bool
+astreamer_buffer_until(astreamer *streamer, const char **data, int *len,
+					   int target_bytes)
+{
+	int			buflen = streamer->bbs_buffer.len;
+
+	if (buflen >= target_bytes)
+	{
+		/* Target length already reached; nothing to do. */
+		return true;
+	}
+
+	if (buflen + *len < target_bytes)
+	{
+		/* Not enough data to reach target length; buffer all of it. */
+		astreamer_buffer_bytes(streamer, data, len, *len);
+		return false;
+	}
+
+	/* Buffer just enough to reach the target length. */
+	astreamer_buffer_bytes(streamer, data, len, target_bytes - buflen);
+	return true;
+}
+
+/*
+ * Functions for creating astreamer objects of various types. See the header
+ * comments for each of these functions for details.
+ */
+extern astreamer *astreamer_plain_writer_new(char *pathname, FILE *file);
+extern astreamer *astreamer_gzip_writer_new(char *pathname, FILE *file,
+											pg_compress_specification *compress);
+extern astreamer *astreamer_extractor_new(const char *basepath,
+										  const char *(*link_map) (const char *),
+										  void (*report_output_file) (const char *));
+
+extern astreamer *astreamer_gzip_decompressor_new(astreamer *next);
+extern astreamer *astreamer_lz4_compressor_new(astreamer *next,
+											   pg_compress_specification *compress);
+extern astreamer *astreamer_lz4_decompressor_new(astreamer *next);
+extern astreamer *astreamer_zstd_compressor_new(astreamer *next,
+												pg_compress_specification *compress);
+extern astreamer *astreamer_zstd_decompressor_new(astreamer *next);
+extern astreamer *astreamer_tar_parser_new(astreamer *next);
+extern astreamer *astreamer_tar_terminator_new(astreamer *next);
+extern astreamer *astreamer_tar_archiver_new(astreamer *next);
+
+extern astreamer *astreamer_recovery_injector_new(astreamer *next,
+												  bool is_recovery_guc_supported,
+												  PQExpBuffer recoveryconfcontents);
+extern void astreamer_inject_file(astreamer *streamer, char *pathname,
+								  char *data, int len);
+
+#endif
diff --git a/src/bin/pg_basebackup/bbstreamer_file.c b/src/bin/pg_basebackup/astreamer_file.c
similarity index 69%
rename from src/bin/pg_basebackup/bbstreamer_file.c
rename to src/bin/pg_basebackup/astreamer_file.c
index bab6cd4a6b1..2742385e103 100644
--- a/src/bin/pg_basebackup/bbstreamer_file.c
+++ b/src/bin/pg_basebackup/astreamer_file.c
@@ -1,11 +1,11 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_file.c
+ * astreamer_file.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_file.c
+ *		  src/bin/pg_basebackup/astreamer_file.c
  *-------------------------------------------------------------------------
  */
 
@@ -13,60 +13,60 @@
 
 #include <unistd.h>
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
 
-typedef struct bbstreamer_plain_writer
+typedef struct astreamer_plain_writer
 {
-	bbstreamer	base;
+	astreamer	base;
 	char	   *pathname;
 	FILE	   *file;
 	bool		should_close_file;
-} bbstreamer_plain_writer;
+} astreamer_plain_writer;
 
-typedef struct bbstreamer_extractor
+typedef struct astreamer_extractor
 {
-	bbstreamer	base;
+	astreamer	base;
 	char	   *basepath;
 	const char *(*link_map) (const char *);
 	void		(*report_output_file) (const char *);
 	char		filename[MAXPGPATH];
 	FILE	   *file;
-} bbstreamer_extractor;
+} astreamer_extractor;
 
-static void bbstreamer_plain_writer_content(bbstreamer *streamer,
-											bbstreamer_member *member,
-											const char *data, int len,
-											bbstreamer_archive_context context);
-static void bbstreamer_plain_writer_finalize(bbstreamer *streamer);
-static void bbstreamer_plain_writer_free(bbstreamer *streamer);
+static void astreamer_plain_writer_content(astreamer *streamer,
+										   astreamer_member *member,
+										   const char *data, int len,
+										   astreamer_archive_context context);
+static void astreamer_plain_writer_finalize(astreamer *streamer);
+static void astreamer_plain_writer_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_plain_writer_ops = {
-	.content = bbstreamer_plain_writer_content,
-	.finalize = bbstreamer_plain_writer_finalize,
-	.free = bbstreamer_plain_writer_free
+static const astreamer_ops astreamer_plain_writer_ops = {
+	.content = astreamer_plain_writer_content,
+	.finalize = astreamer_plain_writer_finalize,
+	.free = astreamer_plain_writer_free
 };
 
-static void bbstreamer_extractor_content(bbstreamer *streamer,
-										 bbstreamer_member *member,
-										 const char *data, int len,
-										 bbstreamer_archive_context context);
-static void bbstreamer_extractor_finalize(bbstreamer *streamer);
-static void bbstreamer_extractor_free(bbstreamer *streamer);
+static void astreamer_extractor_content(astreamer *streamer,
+										astreamer_member *member,
+										const char *data, int len,
+										astreamer_archive_context context);
+static void astreamer_extractor_finalize(astreamer *streamer);
+static void astreamer_extractor_free(astreamer *streamer);
 static void extract_directory(const char *filename, mode_t mode);
 static void extract_link(const char *filename, const char *linktarget);
 static FILE *create_file_for_extract(const char *filename, mode_t mode);
 
-static const bbstreamer_ops bbstreamer_extractor_ops = {
-	.content = bbstreamer_extractor_content,
-	.finalize = bbstreamer_extractor_finalize,
-	.free = bbstreamer_extractor_free
+static const astreamer_ops astreamer_extractor_ops = {
+	.content = astreamer_extractor_content,
+	.finalize = astreamer_extractor_finalize,
+	.free = astreamer_extractor_free
 };
 
 /*
- * Create a bbstreamer that just writes data to a file.
+ * Create a astreamer that just writes data to a file.
  *
  * The caller must specify a pathname and may specify a file. The pathname is
  * used for error-reporting purposes either way. If file is NULL, the pathname
@@ -74,14 +74,14 @@ static const bbstreamer_ops bbstreamer_extractor_ops = {
  * for writing and closed when done. If file is not NULL, the data is written
  * there.
  */
-bbstreamer *
-bbstreamer_plain_writer_new(char *pathname, FILE *file)
+astreamer *
+astreamer_plain_writer_new(char *pathname, FILE *file)
 {
-	bbstreamer_plain_writer *streamer;
+	astreamer_plain_writer *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_plain_writer));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_plain_writer_ops;
+	streamer = palloc0(sizeof(astreamer_plain_writer));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_plain_writer_ops;
 
 	streamer->pathname = pstrdup(pathname);
 	streamer->file = file;
@@ -101,13 +101,13 @@ bbstreamer_plain_writer_new(char *pathname, FILE *file)
  * Write archive content to file.
  */
 static void
-bbstreamer_plain_writer_content(bbstreamer *streamer,
-								bbstreamer_member *member, const char *data,
-								int len, bbstreamer_archive_context context)
+astreamer_plain_writer_content(astreamer *streamer,
+							   astreamer_member *member, const char *data,
+							   int len, astreamer_archive_context context)
 {
-	bbstreamer_plain_writer *mystreamer;
+	astreamer_plain_writer *mystreamer;
 
-	mystreamer = (bbstreamer_plain_writer *) streamer;
+	mystreamer = (astreamer_plain_writer *) streamer;
 
 	if (len == 0)
 		return;
@@ -128,11 +128,11 @@ bbstreamer_plain_writer_content(bbstreamer *streamer,
  * the file if we opened it, but not if the caller provided it.
  */
 static void
-bbstreamer_plain_writer_finalize(bbstreamer *streamer)
+astreamer_plain_writer_finalize(astreamer *streamer)
 {
-	bbstreamer_plain_writer *mystreamer;
+	astreamer_plain_writer *mystreamer;
 
-	mystreamer = (bbstreamer_plain_writer *) streamer;
+	mystreamer = (astreamer_plain_writer *) streamer;
 
 	if (mystreamer->should_close_file && fclose(mystreamer->file) != 0)
 		pg_fatal("could not close file \"%s\": %m",
@@ -143,14 +143,14 @@ bbstreamer_plain_writer_finalize(bbstreamer *streamer)
 }
 
 /*
- * Free memory associated with this bbstreamer.
+ * Free memory associated with this astreamer.
  */
 static void
-bbstreamer_plain_writer_free(bbstreamer *streamer)
+astreamer_plain_writer_free(astreamer *streamer)
 {
-	bbstreamer_plain_writer *mystreamer;
+	astreamer_plain_writer *mystreamer;
 
-	mystreamer = (bbstreamer_plain_writer *) streamer;
+	mystreamer = (astreamer_plain_writer *) streamer;
 
 	Assert(!mystreamer->should_close_file);
 	Assert(mystreamer->base.bbs_next == NULL);
@@ -160,13 +160,13 @@ bbstreamer_plain_writer_free(bbstreamer *streamer)
 }
 
 /*
- * Create a bbstreamer that extracts an archive.
+ * Create a astreamer that extracts an archive.
  *
  * All pathnames in the archive are interpreted relative to basepath.
  *
- * Unlike e.g. bbstreamer_plain_writer_new() we can't do anything useful here
+ * Unlike e.g. astreamer_plain_writer_new() we can't do anything useful here
  * with untyped chunks; we need typed chunks which follow the rules described
- * in bbstreamer.h. Assuming we have that, we don't need to worry about the
+ * in astreamer.h. Assuming we have that, we don't need to worry about the
  * original archive format; it's enough to just look at the member information
  * provided and write to the corresponding file.
  *
@@ -179,16 +179,16 @@ bbstreamer_plain_writer_free(bbstreamer *streamer)
  * new output file. The pathname to that file is passed as an argument. If
  * NULL, the call is skipped.
  */
-bbstreamer *
-bbstreamer_extractor_new(const char *basepath,
-						 const char *(*link_map) (const char *),
-						 void (*report_output_file) (const char *))
+astreamer *
+astreamer_extractor_new(const char *basepath,
+						const char *(*link_map) (const char *),
+						void (*report_output_file) (const char *))
 {
-	bbstreamer_extractor *streamer;
+	astreamer_extractor *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_extractor));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_extractor_ops;
+	streamer = palloc0(sizeof(astreamer_extractor));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_extractor_ops;
 	streamer->basepath = pstrdup(basepath);
 	streamer->link_map = link_map;
 	streamer->report_output_file = report_output_file;
@@ -200,19 +200,19 @@ bbstreamer_extractor_new(const char *basepath,
  * Extract archive contents to the filesystem.
  */
 static void
-bbstreamer_extractor_content(bbstreamer *streamer, bbstreamer_member *member,
-							 const char *data, int len,
-							 bbstreamer_archive_context context)
+astreamer_extractor_content(astreamer *streamer, astreamer_member *member,
+							const char *data, int len,
+							astreamer_archive_context context)
 {
-	bbstreamer_extractor *mystreamer = (bbstreamer_extractor *) streamer;
+	astreamer_extractor *mystreamer = (astreamer_extractor *) streamer;
 	int			fnamelen;
 
-	Assert(member != NULL || context == BBSTREAMER_ARCHIVE_TRAILER);
-	Assert(context != BBSTREAMER_UNKNOWN);
+	Assert(member != NULL || context == ASTREAMER_ARCHIVE_TRAILER);
+	Assert(context != ASTREAMER_UNKNOWN);
 
 	switch (context)
 	{
-		case BBSTREAMER_MEMBER_HEADER:
+		case ASTREAMER_MEMBER_HEADER:
 			Assert(mystreamer->file == NULL);
 
 			/* Prepend basepath. */
@@ -245,7 +245,7 @@ bbstreamer_extractor_content(bbstreamer *streamer, bbstreamer_member *member,
 				mystreamer->report_output_file(mystreamer->filename);
 			break;
 
-		case BBSTREAMER_MEMBER_CONTENTS:
+		case ASTREAMER_MEMBER_CONTENTS:
 			if (mystreamer->file == NULL)
 				break;
 
@@ -260,14 +260,14 @@ bbstreamer_extractor_content(bbstreamer *streamer, bbstreamer_member *member,
 			}
 			break;
 
-		case BBSTREAMER_MEMBER_TRAILER:
+		case ASTREAMER_MEMBER_TRAILER:
 			if (mystreamer->file == NULL)
 				break;
 			fclose(mystreamer->file);
 			mystreamer->file = NULL;
 			break;
 
-		case BBSTREAMER_ARCHIVE_TRAILER:
+		case ASTREAMER_ARCHIVE_TRAILER:
 			break;
 
 		default:
@@ -375,10 +375,10 @@ create_file_for_extract(const char *filename, mode_t mode)
  * There's nothing to do here but sanity checking.
  */
 static void
-bbstreamer_extractor_finalize(bbstreamer *streamer)
+astreamer_extractor_finalize(astreamer *streamer)
 {
-	bbstreamer_extractor *mystreamer PG_USED_FOR_ASSERTS_ONLY
-	= (bbstreamer_extractor *) streamer;
+	astreamer_extractor *mystreamer PG_USED_FOR_ASSERTS_ONLY
+	= (astreamer_extractor *) streamer;
 
 	Assert(mystreamer->file == NULL);
 }
@@ -387,9 +387,9 @@ bbstreamer_extractor_finalize(bbstreamer *streamer)
  * Free memory.
  */
 static void
-bbstreamer_extractor_free(bbstreamer *streamer)
+astreamer_extractor_free(astreamer *streamer)
 {
-	bbstreamer_extractor *mystreamer = (bbstreamer_extractor *) streamer;
+	astreamer_extractor *mystreamer = (astreamer_extractor *) streamer;
 
 	pfree(mystreamer->basepath);
 	pfree(mystreamer);
diff --git a/src/bin/pg_basebackup/bbstreamer_gzip.c b/src/bin/pg_basebackup/astreamer_gzip.c
similarity index 62%
rename from src/bin/pg_basebackup/bbstreamer_gzip.c
rename to src/bin/pg_basebackup/astreamer_gzip.c
index 0417fd9bc2c..6f7c27afbbc 100644
--- a/src/bin/pg_basebackup/bbstreamer_gzip.c
+++ b/src/bin/pg_basebackup/astreamer_gzip.c
@@ -1,11 +1,11 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_gzip.c
+ * astreamer_gzip.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_gzip.c
+ *		  src/bin/pg_basebackup/astreamer_gzip.c
  *-------------------------------------------------------------------------
  */
 
@@ -17,74 +17,74 @@
 #include <zlib.h>
 #endif
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
 
 #ifdef HAVE_LIBZ
-typedef struct bbstreamer_gzip_writer
+typedef struct astreamer_gzip_writer
 {
-	bbstreamer	base;
+	astreamer	base;
 	char	   *pathname;
 	gzFile		gzfile;
-} bbstreamer_gzip_writer;
+} astreamer_gzip_writer;
 
-typedef struct bbstreamer_gzip_decompressor
+typedef struct astreamer_gzip_decompressor
 {
-	bbstreamer	base;
+	astreamer	base;
 	z_stream	zstream;
 	size_t		bytes_written;
-} bbstreamer_gzip_decompressor;
+} astreamer_gzip_decompressor;
 
-static void bbstreamer_gzip_writer_content(bbstreamer *streamer,
-										   bbstreamer_member *member,
-										   const char *data, int len,
-										   bbstreamer_archive_context context);
-static void bbstreamer_gzip_writer_finalize(bbstreamer *streamer);
-static void bbstreamer_gzip_writer_free(bbstreamer *streamer);
+static void astreamer_gzip_writer_content(astreamer *streamer,
+										  astreamer_member *member,
+										  const char *data, int len,
+										  astreamer_archive_context context);
+static void astreamer_gzip_writer_finalize(astreamer *streamer);
+static void astreamer_gzip_writer_free(astreamer *streamer);
 static const char *get_gz_error(gzFile gzf);
 
-static const bbstreamer_ops bbstreamer_gzip_writer_ops = {
-	.content = bbstreamer_gzip_writer_content,
-	.finalize = bbstreamer_gzip_writer_finalize,
-	.free = bbstreamer_gzip_writer_free
+static const astreamer_ops astreamer_gzip_writer_ops = {
+	.content = astreamer_gzip_writer_content,
+	.finalize = astreamer_gzip_writer_finalize,
+	.free = astreamer_gzip_writer_free
 };
 
-static void bbstreamer_gzip_decompressor_content(bbstreamer *streamer,
-												 bbstreamer_member *member,
-												 const char *data, int len,
-												 bbstreamer_archive_context context);
-static void bbstreamer_gzip_decompressor_finalize(bbstreamer *streamer);
-static void bbstreamer_gzip_decompressor_free(bbstreamer *streamer);
+static void astreamer_gzip_decompressor_content(astreamer *streamer,
+												astreamer_member *member,
+												const char *data, int len,
+												astreamer_archive_context context);
+static void astreamer_gzip_decompressor_finalize(astreamer *streamer);
+static void astreamer_gzip_decompressor_free(astreamer *streamer);
 static void *gzip_palloc(void *opaque, unsigned items, unsigned size);
 static void gzip_pfree(void *opaque, void *address);
 
-static const bbstreamer_ops bbstreamer_gzip_decompressor_ops = {
-	.content = bbstreamer_gzip_decompressor_content,
-	.finalize = bbstreamer_gzip_decompressor_finalize,
-	.free = bbstreamer_gzip_decompressor_free
+static const astreamer_ops astreamer_gzip_decompressor_ops = {
+	.content = astreamer_gzip_decompressor_content,
+	.finalize = astreamer_gzip_decompressor_finalize,
+	.free = astreamer_gzip_decompressor_free
 };
 #endif
 
 /*
- * Create a bbstreamer that just compresses data using gzip, and then writes
+ * Create a astreamer that just compresses data using gzip, and then writes
  * it to a file.
  *
- * As in the case of bbstreamer_plain_writer_new, pathname is always used
+ * As in the case of astreamer_plain_writer_new, pathname is always used
  * for error reporting purposes; if file is NULL, it is also the opened and
  * closed so that the data may be written there.
  */
-bbstreamer *
-bbstreamer_gzip_writer_new(char *pathname, FILE *file,
-						   pg_compress_specification *compress)
+astreamer *
+astreamer_gzip_writer_new(char *pathname, FILE *file,
+						  pg_compress_specification *compress)
 {
 #ifdef HAVE_LIBZ
-	bbstreamer_gzip_writer *streamer;
+	astreamer_gzip_writer *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_gzip_writer));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_gzip_writer_ops;
+	streamer = palloc0(sizeof(astreamer_gzip_writer));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_gzip_writer_ops;
 
 	streamer->pathname = pstrdup(pathname);
 
@@ -123,13 +123,13 @@ bbstreamer_gzip_writer_new(char *pathname, FILE *file,
  * Write archive content to gzip file.
  */
 static void
-bbstreamer_gzip_writer_content(bbstreamer *streamer,
-							   bbstreamer_member *member, const char *data,
-							   int len, bbstreamer_archive_context context)
+astreamer_gzip_writer_content(astreamer *streamer,
+							  astreamer_member *member, const char *data,
+							  int len, astreamer_archive_context context)
 {
-	bbstreamer_gzip_writer *mystreamer;
+	astreamer_gzip_writer *mystreamer;
 
-	mystreamer = (bbstreamer_gzip_writer *) streamer;
+	mystreamer = (astreamer_gzip_writer *) streamer;
 
 	if (len == 0)
 		return;
@@ -151,16 +151,16 @@ bbstreamer_gzip_writer_content(bbstreamer *streamer,
  *
  * It makes no difference whether we opened the file or the caller did it,
  * because libz provides no way of avoiding a close on the underlying file
- * handle. Notice, however, that bbstreamer_gzip_writer_new() uses dup() to
+ * handle. Notice, however, that astreamer_gzip_writer_new() uses dup() to
  * work around this issue, so that the behavior from the caller's viewpoint
- * is the same as for bbstreamer_plain_writer.
+ * is the same as for astreamer_plain_writer.
  */
 static void
-bbstreamer_gzip_writer_finalize(bbstreamer *streamer)
+astreamer_gzip_writer_finalize(astreamer *streamer)
 {
-	bbstreamer_gzip_writer *mystreamer;
+	astreamer_gzip_writer *mystreamer;
 
-	mystreamer = (bbstreamer_gzip_writer *) streamer;
+	mystreamer = (astreamer_gzip_writer *) streamer;
 
 	errno = 0;					/* in case gzclose() doesn't set it */
 	if (gzclose(mystreamer->gzfile) != 0)
@@ -171,14 +171,14 @@ bbstreamer_gzip_writer_finalize(bbstreamer *streamer)
 }
 
 /*
- * Free memory associated with this bbstreamer.
+ * Free memory associated with this astreamer.
  */
 static void
-bbstreamer_gzip_writer_free(bbstreamer *streamer)
+astreamer_gzip_writer_free(astreamer *streamer)
 {
-	bbstreamer_gzip_writer *mystreamer;
+	astreamer_gzip_writer *mystreamer;
 
-	mystreamer = (bbstreamer_gzip_writer *) streamer;
+	mystreamer = (astreamer_gzip_writer *) streamer;
 
 	Assert(mystreamer->base.bbs_next == NULL);
 	Assert(mystreamer->gzfile == NULL);
@@ -208,18 +208,18 @@ get_gz_error(gzFile gzf)
  * Create a new base backup streamer that performs decompression of gzip
  * compressed blocks.
  */
-bbstreamer *
-bbstreamer_gzip_decompressor_new(bbstreamer *next)
+astreamer *
+astreamer_gzip_decompressor_new(astreamer *next)
 {
 #ifdef HAVE_LIBZ
-	bbstreamer_gzip_decompressor *streamer;
+	astreamer_gzip_decompressor *streamer;
 	z_stream   *zs;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_gzip_decompressor));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_gzip_decompressor_ops;
+	streamer = palloc0(sizeof(astreamer_gzip_decompressor));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_gzip_decompressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -258,15 +258,15 @@ bbstreamer_gzip_decompressor_new(bbstreamer *next)
  * to the next streamer.
  */
 static void
-bbstreamer_gzip_decompressor_content(bbstreamer *streamer,
-									 bbstreamer_member *member,
-									 const char *data, int len,
-									 bbstreamer_archive_context context)
+astreamer_gzip_decompressor_content(astreamer *streamer,
+									astreamer_member *member,
+									const char *data, int len,
+									astreamer_archive_context context)
 {
-	bbstreamer_gzip_decompressor *mystreamer;
+	astreamer_gzip_decompressor *mystreamer;
 	z_stream   *zs;
 
-	mystreamer = (bbstreamer_gzip_decompressor *) streamer;
+	mystreamer = (astreamer_gzip_decompressor *) streamer;
 
 	zs = &mystreamer->zstream;
 	zs->next_in = (const uint8 *) data;
@@ -301,9 +301,9 @@ bbstreamer_gzip_decompressor_content(bbstreamer *streamer,
 		/* If output buffer is full then pass data to next streamer */
 		if (mystreamer->bytes_written >= mystreamer->base.bbs_buffer.maxlen)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, member,
-							   mystreamer->base.bbs_buffer.data,
-							   mystreamer->base.bbs_buffer.maxlen, context);
+			astreamer_content(mystreamer->base.bbs_next, member,
+							  mystreamer->base.bbs_buffer.data,
+							  mystreamer->base.bbs_buffer.maxlen, context);
 			mystreamer->bytes_written = 0;
 		}
 	}
@@ -313,31 +313,31 @@ bbstreamer_gzip_decompressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_gzip_decompressor_finalize(bbstreamer *streamer)
+astreamer_gzip_decompressor_finalize(astreamer *streamer)
 {
-	bbstreamer_gzip_decompressor *mystreamer;
+	astreamer_gzip_decompressor *mystreamer;
 
-	mystreamer = (bbstreamer_gzip_decompressor *) streamer;
+	mystreamer = (astreamer_gzip_decompressor *) streamer;
 
 	/*
 	 * End of the stream, if there is some pending data in output buffers then
 	 * we must forward it to next streamer.
 	 */
-	bbstreamer_content(mystreamer->base.bbs_next, NULL,
-					   mystreamer->base.bbs_buffer.data,
-					   mystreamer->base.bbs_buffer.maxlen,
-					   BBSTREAMER_UNKNOWN);
+	astreamer_content(mystreamer->base.bbs_next, NULL,
+					  mystreamer->base.bbs_buffer.data,
+					  mystreamer->base.bbs_buffer.maxlen,
+					  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_gzip_decompressor_free(bbstreamer *streamer)
+astreamer_gzip_decompressor_free(astreamer *streamer)
 {
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
 }
diff --git a/src/bin/pg_basebackup/bbstreamer_inject.c b/src/bin/pg_basebackup/astreamer_inject.c
similarity index 53%
rename from src/bin/pg_basebackup/bbstreamer_inject.c
rename to src/bin/pg_basebackup/astreamer_inject.c
index 194026b56e9..7f1decded8d 100644
--- a/src/bin/pg_basebackup/bbstreamer_inject.c
+++ b/src/bin/pg_basebackup/astreamer_inject.c
@@ -1,51 +1,51 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_inject.c
+ * astreamer_inject.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_inject.c
+ *		  src/bin/pg_basebackup/astreamer_inject.c
  *-------------------------------------------------------------------------
  */
 
 #include "postgres_fe.h"
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 
-typedef struct bbstreamer_recovery_injector
+typedef struct astreamer_recovery_injector
 {
-	bbstreamer	base;
+	astreamer	base;
 	bool		skip_file;
 	bool		is_recovery_guc_supported;
 	bool		is_postgresql_auto_conf;
 	bool		found_postgresql_auto_conf;
 	PQExpBuffer recoveryconfcontents;
-	bbstreamer_member member;
-} bbstreamer_recovery_injector;
+	astreamer_member member;
+} astreamer_recovery_injector;
 
-static void bbstreamer_recovery_injector_content(bbstreamer *streamer,
-												 bbstreamer_member *member,
-												 const char *data, int len,
-												 bbstreamer_archive_context context);
-static void bbstreamer_recovery_injector_finalize(bbstreamer *streamer);
-static void bbstreamer_recovery_injector_free(bbstreamer *streamer);
+static void astreamer_recovery_injector_content(astreamer *streamer,
+												astreamer_member *member,
+												const char *data, int len,
+												astreamer_archive_context context);
+static void astreamer_recovery_injector_finalize(astreamer *streamer);
+static void astreamer_recovery_injector_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_recovery_injector_ops = {
-	.content = bbstreamer_recovery_injector_content,
-	.finalize = bbstreamer_recovery_injector_finalize,
-	.free = bbstreamer_recovery_injector_free
+static const astreamer_ops astreamer_recovery_injector_ops = {
+	.content = astreamer_recovery_injector_content,
+	.finalize = astreamer_recovery_injector_finalize,
+	.free = astreamer_recovery_injector_free
 };
 
 /*
- * Create a bbstreamer that can edit recoverydata into an archive stream.
+ * Create a astreamer that can edit recoverydata into an archive stream.
  *
- * The input should be a series of typed chunks (not BBSTREAMER_UNKNOWN) as
- * per the conventions described in bbstreamer.h; the chunks forwarded to
- * the next bbstreamer will be similarly typed, but the
- * BBSTREAMER_MEMBER_HEADER chunks may be zero-length in cases where we've
+ * The input should be a series of typed chunks (not ASTREAMER_UNKNOWN) as
+ * per the conventions described in astreamer.h; the chunks forwarded to
+ * the next astreamer will be similarly typed, but the
+ * ASTREAMER_MEMBER_HEADER chunks may be zero-length in cases where we've
  * edited the archive stream.
  *
  * Our goal is to do one of the following three things with the content passed
@@ -61,16 +61,16 @@ static const bbstreamer_ops bbstreamer_recovery_injector_ops = {
  * zero-length standby.signal file, dropping any file with that name from
  * the archive.
  */
-bbstreamer *
-bbstreamer_recovery_injector_new(bbstreamer *next,
-								 bool is_recovery_guc_supported,
-								 PQExpBuffer recoveryconfcontents)
+astreamer *
+astreamer_recovery_injector_new(astreamer *next,
+								bool is_recovery_guc_supported,
+								PQExpBuffer recoveryconfcontents)
 {
-	bbstreamer_recovery_injector *streamer;
+	astreamer_recovery_injector *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_recovery_injector));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_recovery_injector_ops;
+	streamer = palloc0(sizeof(astreamer_recovery_injector));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_recovery_injector_ops;
 	streamer->base.bbs_next = next;
 	streamer->is_recovery_guc_supported = is_recovery_guc_supported;
 	streamer->recoveryconfcontents = recoveryconfcontents;
@@ -82,21 +82,21 @@ bbstreamer_recovery_injector_new(bbstreamer *next,
  * Handle each chunk of tar content while injecting recovery configuration.
  */
 static void
-bbstreamer_recovery_injector_content(bbstreamer *streamer,
-									 bbstreamer_member *member,
-									 const char *data, int len,
-									 bbstreamer_archive_context context)
+astreamer_recovery_injector_content(astreamer *streamer,
+									astreamer_member *member,
+									const char *data, int len,
+									astreamer_archive_context context)
 {
-	bbstreamer_recovery_injector *mystreamer;
+	astreamer_recovery_injector *mystreamer;
 
-	mystreamer = (bbstreamer_recovery_injector *) streamer;
-	Assert(member != NULL || context == BBSTREAMER_ARCHIVE_TRAILER);
+	mystreamer = (astreamer_recovery_injector *) streamer;
+	Assert(member != NULL || context == ASTREAMER_ARCHIVE_TRAILER);
 
 	switch (context)
 	{
-		case BBSTREAMER_MEMBER_HEADER:
+		case ASTREAMER_MEMBER_HEADER:
 			/* Must copy provided data so we have the option to modify it. */
-			memcpy(&mystreamer->member, member, sizeof(bbstreamer_member));
+			memcpy(&mystreamer->member, member, sizeof(astreamer_member));
 
 			/*
 			 * On v12+, skip standby.signal and edit postgresql.auto.conf; on
@@ -119,8 +119,8 @@ bbstreamer_recovery_injector_content(bbstreamer *streamer,
 
 					/*
 					 * Zap data and len because the archive header is no
-					 * longer valid; some subsequent bbstreamer must
-					 * regenerate it if it's necessary.
+					 * longer valid; some subsequent astreamer must regenerate
+					 * it if it's necessary.
 					 */
 					data = NULL;
 					len = 0;
@@ -135,26 +135,26 @@ bbstreamer_recovery_injector_content(bbstreamer *streamer,
 				return;
 			break;
 
-		case BBSTREAMER_MEMBER_CONTENTS:
+		case ASTREAMER_MEMBER_CONTENTS:
 			/* Do not forward if the file is to be skipped. */
 			if (mystreamer->skip_file)
 				return;
 			break;
 
-		case BBSTREAMER_MEMBER_TRAILER:
+		case ASTREAMER_MEMBER_TRAILER:
 			/* Do not forward it the file is to be skipped. */
 			if (mystreamer->skip_file)
 				return;
 
 			/* Append provided content to whatever we already sent. */
 			if (mystreamer->is_postgresql_auto_conf)
-				bbstreamer_content(mystreamer->base.bbs_next, member,
-								   mystreamer->recoveryconfcontents->data,
-								   mystreamer->recoveryconfcontents->len,
-								   BBSTREAMER_MEMBER_CONTENTS);
+				astreamer_content(mystreamer->base.bbs_next, member,
+								  mystreamer->recoveryconfcontents->data,
+								  mystreamer->recoveryconfcontents->len,
+								  ASTREAMER_MEMBER_CONTENTS);
 			break;
 
-		case BBSTREAMER_ARCHIVE_TRAILER:
+		case ASTREAMER_ARCHIVE_TRAILER:
 			if (mystreamer->is_recovery_guc_supported)
 			{
 				/*
@@ -163,22 +163,22 @@ bbstreamer_recovery_injector_content(bbstreamer *streamer,
 				 * member now.
 				 */
 				if (!mystreamer->found_postgresql_auto_conf)
-					bbstreamer_inject_file(mystreamer->base.bbs_next,
-										   "postgresql.auto.conf",
-										   mystreamer->recoveryconfcontents->data,
-										   mystreamer->recoveryconfcontents->len);
+					astreamer_inject_file(mystreamer->base.bbs_next,
+										  "postgresql.auto.conf",
+										  mystreamer->recoveryconfcontents->data,
+										  mystreamer->recoveryconfcontents->len);
 
 				/* Inject empty standby.signal file. */
-				bbstreamer_inject_file(mystreamer->base.bbs_next,
-									   "standby.signal", "", 0);
+				astreamer_inject_file(mystreamer->base.bbs_next,
+									  "standby.signal", "", 0);
 			}
 			else
 			{
 				/* Inject recovery.conf file with specified contents. */
-				bbstreamer_inject_file(mystreamer->base.bbs_next,
-									   "recovery.conf",
-									   mystreamer->recoveryconfcontents->data,
-									   mystreamer->recoveryconfcontents->len);
+				astreamer_inject_file(mystreamer->base.bbs_next,
+									  "recovery.conf",
+									  mystreamer->recoveryconfcontents->data,
+									  mystreamer->recoveryconfcontents->len);
 			}
 
 			/* Nothing to do here. */
@@ -189,26 +189,26 @@ bbstreamer_recovery_injector_content(bbstreamer *streamer,
 			pg_fatal("unexpected state while injecting recovery settings");
 	}
 
-	bbstreamer_content(mystreamer->base.bbs_next, &mystreamer->member,
-					   data, len, context);
+	astreamer_content(mystreamer->base.bbs_next, &mystreamer->member,
+					  data, len, context);
 }
 
 /*
- * End-of-stream processing for this bbstreamer.
+ * End-of-stream processing for this astreamer.
  */
 static void
-bbstreamer_recovery_injector_finalize(bbstreamer *streamer)
+astreamer_recovery_injector_finalize(astreamer *streamer)
 {
-	bbstreamer_finalize(streamer->bbs_next);
+	astreamer_finalize(streamer->bbs_next);
 }
 
 /*
- * Free memory associated with this bbstreamer.
+ * Free memory associated with this astreamer.
  */
 static void
-bbstreamer_recovery_injector_free(bbstreamer *streamer)
+astreamer_recovery_injector_free(astreamer *streamer)
 {
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	pfree(streamer);
 }
 
@@ -216,10 +216,10 @@ bbstreamer_recovery_injector_free(bbstreamer *streamer)
  * Inject a member into the archive with specified contents.
  */
 void
-bbstreamer_inject_file(bbstreamer *streamer, char *pathname, char *data,
-					   int len)
+astreamer_inject_file(astreamer *streamer, char *pathname, char *data,
+					  int len)
 {
-	bbstreamer_member member;
+	astreamer_member member;
 
 	strlcpy(member.pathname, pathname, MAXPGPATH);
 	member.size = len;
@@ -238,12 +238,12 @@ bbstreamer_inject_file(bbstreamer *streamer, char *pathname, char *data,
 	/*
 	 * We don't know here how to generate valid member headers and trailers
 	 * for the archiving format in use, so if those are needed, some successor
-	 * bbstreamer will have to generate them using the data from 'member'.
+	 * astreamer will have to generate them using the data from 'member'.
 	 */
-	bbstreamer_content(streamer, &member, NULL, 0,
-					   BBSTREAMER_MEMBER_HEADER);
-	bbstreamer_content(streamer, &member, data, len,
-					   BBSTREAMER_MEMBER_CONTENTS);
-	bbstreamer_content(streamer, &member, NULL, 0,
-					   BBSTREAMER_MEMBER_TRAILER);
+	astreamer_content(streamer, &member, NULL, 0,
+					  ASTREAMER_MEMBER_HEADER);
+	astreamer_content(streamer, &member, data, len,
+					  ASTREAMER_MEMBER_CONTENTS);
+	astreamer_content(streamer, &member, NULL, 0,
+					  ASTREAMER_MEMBER_TRAILER);
 }
diff --git a/src/bin/pg_basebackup/bbstreamer_lz4.c b/src/bin/pg_basebackup/astreamer_lz4.c
similarity index 69%
rename from src/bin/pg_basebackup/bbstreamer_lz4.c
rename to src/bin/pg_basebackup/astreamer_lz4.c
index f5c9e68150c..1c40d7d8ad5 100644
--- a/src/bin/pg_basebackup/bbstreamer_lz4.c
+++ b/src/bin/pg_basebackup/astreamer_lz4.c
@@ -1,11 +1,11 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_lz4.c
+ * astreamer_lz4.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_lz4.c
+ *		  src/bin/pg_basebackup/astreamer_lz4.c
  *-------------------------------------------------------------------------
  */
 
@@ -17,15 +17,15 @@
 #include <lz4frame.h>
 #endif
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
 
 #ifdef USE_LZ4
-typedef struct bbstreamer_lz4_frame
+typedef struct astreamer_lz4_frame
 {
-	bbstreamer	base;
+	astreamer	base;
 
 	LZ4F_compressionContext_t cctx;
 	LZ4F_decompressionContext_t dctx;
@@ -33,32 +33,32 @@ typedef struct bbstreamer_lz4_frame
 
 	size_t		bytes_written;
 	bool		header_written;
-} bbstreamer_lz4_frame;
+} astreamer_lz4_frame;
 
-static void bbstreamer_lz4_compressor_content(bbstreamer *streamer,
-											  bbstreamer_member *member,
-											  const char *data, int len,
-											  bbstreamer_archive_context context);
-static void bbstreamer_lz4_compressor_finalize(bbstreamer *streamer);
-static void bbstreamer_lz4_compressor_free(bbstreamer *streamer);
+static void astreamer_lz4_compressor_content(astreamer *streamer,
+											 astreamer_member *member,
+											 const char *data, int len,
+											 astreamer_archive_context context);
+static void astreamer_lz4_compressor_finalize(astreamer *streamer);
+static void astreamer_lz4_compressor_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_lz4_compressor_ops = {
-	.content = bbstreamer_lz4_compressor_content,
-	.finalize = bbstreamer_lz4_compressor_finalize,
-	.free = bbstreamer_lz4_compressor_free
+static const astreamer_ops astreamer_lz4_compressor_ops = {
+	.content = astreamer_lz4_compressor_content,
+	.finalize = astreamer_lz4_compressor_finalize,
+	.free = astreamer_lz4_compressor_free
 };
 
-static void bbstreamer_lz4_decompressor_content(bbstreamer *streamer,
-												bbstreamer_member *member,
-												const char *data, int len,
-												bbstreamer_archive_context context);
-static void bbstreamer_lz4_decompressor_finalize(bbstreamer *streamer);
-static void bbstreamer_lz4_decompressor_free(bbstreamer *streamer);
+static void astreamer_lz4_decompressor_content(astreamer *streamer,
+											   astreamer_member *member,
+											   const char *data, int len,
+											   astreamer_archive_context context);
+static void astreamer_lz4_decompressor_finalize(astreamer *streamer);
+static void astreamer_lz4_decompressor_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_lz4_decompressor_ops = {
-	.content = bbstreamer_lz4_decompressor_content,
-	.finalize = bbstreamer_lz4_decompressor_finalize,
-	.free = bbstreamer_lz4_decompressor_free
+static const astreamer_ops astreamer_lz4_decompressor_ops = {
+	.content = astreamer_lz4_decompressor_content,
+	.finalize = astreamer_lz4_decompressor_finalize,
+	.free = astreamer_lz4_decompressor_free
 };
 #endif
 
@@ -66,19 +66,19 @@ static const bbstreamer_ops bbstreamer_lz4_decompressor_ops = {
  * Create a new base backup streamer that performs lz4 compression of tar
  * blocks.
  */
-bbstreamer *
-bbstreamer_lz4_compressor_new(bbstreamer *next, pg_compress_specification *compress)
+astreamer *
+astreamer_lz4_compressor_new(astreamer *next, pg_compress_specification *compress)
 {
 #ifdef USE_LZ4
-	bbstreamer_lz4_frame *streamer;
+	astreamer_lz4_frame *streamer;
 	LZ4F_errorCode_t ctxError;
 	LZ4F_preferences_t *prefs;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_lz4_frame));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_lz4_compressor_ops;
+	streamer = palloc0(sizeof(astreamer_lz4_frame));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_lz4_compressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -113,19 +113,19 @@ bbstreamer_lz4_compressor_new(bbstreamer *next, pg_compress_specification *compr
  * of output buffer to next streamer and empty the buffer.
  */
 static void
-bbstreamer_lz4_compressor_content(bbstreamer *streamer,
-								  bbstreamer_member *member,
-								  const char *data, int len,
-								  bbstreamer_archive_context context)
+astreamer_lz4_compressor_content(astreamer *streamer,
+								 astreamer_member *member,
+								 const char *data, int len,
+								 astreamer_archive_context context)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 	uint8	   *next_in,
 			   *next_out;
 	size_t		out_bound,
 				compressed_size,
 				avail_out;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
+	mystreamer = (astreamer_lz4_frame *) streamer;
 	next_in = (uint8 *) data;
 
 	/* Write header before processing the first input chunk. */
@@ -159,10 +159,10 @@ bbstreamer_lz4_compressor_content(bbstreamer *streamer,
 	out_bound = LZ4F_compressBound(len, &mystreamer->prefs);
 	if (avail_out < out_bound)
 	{
-		bbstreamer_content(mystreamer->base.bbs_next, member,
-						   mystreamer->base.bbs_buffer.data,
-						   mystreamer->bytes_written,
-						   context);
+		astreamer_content(mystreamer->base.bbs_next, member,
+						  mystreamer->base.bbs_buffer.data,
+						  mystreamer->bytes_written,
+						  context);
 
 		/* Enlarge buffer if it falls short of out bound. */
 		if (mystreamer->base.bbs_buffer.maxlen < out_bound)
@@ -196,25 +196,25 @@ bbstreamer_lz4_compressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_lz4_compressor_finalize(bbstreamer *streamer)
+astreamer_lz4_compressor_finalize(astreamer *streamer)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 	uint8	   *next_out;
 	size_t		footer_bound,
 				compressed_size,
 				avail_out;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
+	mystreamer = (astreamer_lz4_frame *) streamer;
 
 	/* Find out the footer bound and update the output buffer. */
 	footer_bound = LZ4F_compressBound(0, &mystreamer->prefs);
 	if ((mystreamer->base.bbs_buffer.maxlen - mystreamer->bytes_written) <
 		footer_bound)
 	{
-		bbstreamer_content(mystreamer->base.bbs_next, NULL,
-						   mystreamer->base.bbs_buffer.data,
-						   mystreamer->bytes_written,
-						   BBSTREAMER_UNKNOWN);
+		astreamer_content(mystreamer->base.bbs_next, NULL,
+						  mystreamer->base.bbs_buffer.data,
+						  mystreamer->bytes_written,
+						  ASTREAMER_UNKNOWN);
 
 		/* Enlarge buffer if it falls short of footer bound. */
 		if (mystreamer->base.bbs_buffer.maxlen < footer_bound)
@@ -243,24 +243,24 @@ bbstreamer_lz4_compressor_finalize(bbstreamer *streamer)
 
 	mystreamer->bytes_written += compressed_size;
 
-	bbstreamer_content(mystreamer->base.bbs_next, NULL,
-					   mystreamer->base.bbs_buffer.data,
-					   mystreamer->bytes_written,
-					   BBSTREAMER_UNKNOWN);
+	astreamer_content(mystreamer->base.bbs_next, NULL,
+					  mystreamer->base.bbs_buffer.data,
+					  mystreamer->bytes_written,
+					  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_lz4_compressor_free(bbstreamer *streamer)
+astreamer_lz4_compressor_free(astreamer *streamer)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
-	bbstreamer_free(streamer->bbs_next);
+	mystreamer = (astreamer_lz4_frame *) streamer;
+	astreamer_free(streamer->bbs_next);
 	LZ4F_freeCompressionContext(mystreamer->cctx);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
@@ -271,18 +271,18 @@ bbstreamer_lz4_compressor_free(bbstreamer *streamer)
  * Create a new base backup streamer that performs decompression of lz4
  * compressed blocks.
  */
-bbstreamer *
-bbstreamer_lz4_decompressor_new(bbstreamer *next)
+astreamer *
+astreamer_lz4_decompressor_new(astreamer *next)
 {
 #ifdef USE_LZ4
-	bbstreamer_lz4_frame *streamer;
+	astreamer_lz4_frame *streamer;
 	LZ4F_errorCode_t ctxError;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_lz4_frame));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_lz4_decompressor_ops;
+	streamer = palloc0(sizeof(astreamer_lz4_frame));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_lz4_decompressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -307,18 +307,18 @@ bbstreamer_lz4_decompressor_new(bbstreamer *next)
  * to the next streamer.
  */
 static void
-bbstreamer_lz4_decompressor_content(bbstreamer *streamer,
-									bbstreamer_member *member,
-									const char *data, int len,
-									bbstreamer_archive_context context)
+astreamer_lz4_decompressor_content(astreamer *streamer,
+								   astreamer_member *member,
+								   const char *data, int len,
+								   astreamer_archive_context context)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 	uint8	   *next_in,
 			   *next_out;
 	size_t		avail_in,
 				avail_out;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
+	mystreamer = (astreamer_lz4_frame *) streamer;
 	next_in = (uint8 *) data;
 	next_out = (uint8 *) mystreamer->base.bbs_buffer.data;
 	avail_in = len;
@@ -366,10 +366,10 @@ bbstreamer_lz4_decompressor_content(bbstreamer *streamer,
 		 */
 		if (mystreamer->bytes_written >= mystreamer->base.bbs_buffer.maxlen)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, member,
-							   mystreamer->base.bbs_buffer.data,
-							   mystreamer->base.bbs_buffer.maxlen,
-							   context);
+			astreamer_content(mystreamer->base.bbs_next, member,
+							  mystreamer->base.bbs_buffer.data,
+							  mystreamer->base.bbs_buffer.maxlen,
+							  context);
 
 			avail_out = mystreamer->base.bbs_buffer.maxlen;
 			mystreamer->bytes_written = 0;
@@ -387,34 +387,34 @@ bbstreamer_lz4_decompressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_lz4_decompressor_finalize(bbstreamer *streamer)
+astreamer_lz4_decompressor_finalize(astreamer *streamer)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
+	mystreamer = (astreamer_lz4_frame *) streamer;
 
 	/*
 	 * End of the stream, if there is some pending data in output buffers then
 	 * we must forward it to next streamer.
 	 */
-	bbstreamer_content(mystreamer->base.bbs_next, NULL,
-					   mystreamer->base.bbs_buffer.data,
-					   mystreamer->base.bbs_buffer.maxlen,
-					   BBSTREAMER_UNKNOWN);
+	astreamer_content(mystreamer->base.bbs_next, NULL,
+					  mystreamer->base.bbs_buffer.data,
+					  mystreamer->base.bbs_buffer.maxlen,
+					  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_lz4_decompressor_free(bbstreamer *streamer)
+astreamer_lz4_decompressor_free(astreamer *streamer)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
-	bbstreamer_free(streamer->bbs_next);
+	mystreamer = (astreamer_lz4_frame *) streamer;
+	astreamer_free(streamer->bbs_next);
 	LZ4F_freeDecompressionContext(mystreamer->dctx);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
diff --git a/src/bin/pg_basebackup/bbstreamer_tar.c b/src/bin/pg_basebackup/astreamer_tar.c
similarity index 50%
rename from src/bin/pg_basebackup/bbstreamer_tar.c
rename to src/bin/pg_basebackup/astreamer_tar.c
index 9137d17ddc1..673690cd18f 100644
--- a/src/bin/pg_basebackup/bbstreamer_tar.c
+++ b/src/bin/pg_basebackup/astreamer_tar.c
@@ -1,13 +1,13 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_tar.c
+ * astreamer_tar.c
  *
  * This module implements three types of tar processing. A tar parser
- * expects unlabelled chunks of data (e.g. BBSTREAMER_UNKNOWN) and splits
- * it into labelled chunks (any other value of bbstreamer_archive_context).
+ * expects unlabelled chunks of data (e.g. ASTREAMER_UNKNOWN) and splits
+ * it into labelled chunks (any other value of astreamer_archive_context).
  * A tar archiver does the reverse: it takes a bunch of labelled chunks
  * and produces a tarfile, optionally replacing member headers and trailers
- * so that upstream bbstreamer objects can perform surgery on the tarfile
+ * so that upstream astreamer objects can perform surgery on the tarfile
  * contents without knowing the details of the tar format. A tar terminator
  * just adds two blocks of NUL bytes to the end of the file, since older
  * server versions produce files with this terminator omitted.
@@ -15,7 +15,7 @@
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_tar.c
+ *		  src/bin/pg_basebackup/astreamer_tar.c
  *-------------------------------------------------------------------------
  */
 
@@ -23,83 +23,83 @@
 
 #include <time.h>
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/logging.h"
 #include "pgtar.h"
 
-typedef struct bbstreamer_tar_parser
+typedef struct astreamer_tar_parser
 {
-	bbstreamer	base;
-	bbstreamer_archive_context next_context;
-	bbstreamer_member member;
+	astreamer	base;
+	astreamer_archive_context next_context;
+	astreamer_member member;
 	size_t		file_bytes_sent;
 	size_t		pad_bytes_expected;
-} bbstreamer_tar_parser;
+} astreamer_tar_parser;
 
-typedef struct bbstreamer_tar_archiver
+typedef struct astreamer_tar_archiver
 {
-	bbstreamer	base;
+	astreamer	base;
 	bool		rearchive_member;
-} bbstreamer_tar_archiver;
+} astreamer_tar_archiver;
 
-static void bbstreamer_tar_parser_content(bbstreamer *streamer,
-										  bbstreamer_member *member,
-										  const char *data, int len,
-										  bbstreamer_archive_context context);
-static void bbstreamer_tar_parser_finalize(bbstreamer *streamer);
-static void bbstreamer_tar_parser_free(bbstreamer *streamer);
-static bool bbstreamer_tar_header(bbstreamer_tar_parser *mystreamer);
+static void astreamer_tar_parser_content(astreamer *streamer,
+										 astreamer_member *member,
+										 const char *data, int len,
+										 astreamer_archive_context context);
+static void astreamer_tar_parser_finalize(astreamer *streamer);
+static void astreamer_tar_parser_free(astreamer *streamer);
+static bool astreamer_tar_header(astreamer_tar_parser *mystreamer);
 
-static const bbstreamer_ops bbstreamer_tar_parser_ops = {
-	.content = bbstreamer_tar_parser_content,
-	.finalize = bbstreamer_tar_parser_finalize,
-	.free = bbstreamer_tar_parser_free
+static const astreamer_ops astreamer_tar_parser_ops = {
+	.content = astreamer_tar_parser_content,
+	.finalize = astreamer_tar_parser_finalize,
+	.free = astreamer_tar_parser_free
 };
 
-static void bbstreamer_tar_archiver_content(bbstreamer *streamer,
-											bbstreamer_member *member,
-											const char *data, int len,
-											bbstreamer_archive_context context);
-static void bbstreamer_tar_archiver_finalize(bbstreamer *streamer);
-static void bbstreamer_tar_archiver_free(bbstreamer *streamer);
+static void astreamer_tar_archiver_content(astreamer *streamer,
+										   astreamer_member *member,
+										   const char *data, int len,
+										   astreamer_archive_context context);
+static void astreamer_tar_archiver_finalize(astreamer *streamer);
+static void astreamer_tar_archiver_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_tar_archiver_ops = {
-	.content = bbstreamer_tar_archiver_content,
-	.finalize = bbstreamer_tar_archiver_finalize,
-	.free = bbstreamer_tar_archiver_free
+static const astreamer_ops astreamer_tar_archiver_ops = {
+	.content = astreamer_tar_archiver_content,
+	.finalize = astreamer_tar_archiver_finalize,
+	.free = astreamer_tar_archiver_free
 };
 
-static void bbstreamer_tar_terminator_content(bbstreamer *streamer,
-											  bbstreamer_member *member,
-											  const char *data, int len,
-											  bbstreamer_archive_context context);
-static void bbstreamer_tar_terminator_finalize(bbstreamer *streamer);
-static void bbstreamer_tar_terminator_free(bbstreamer *streamer);
+static void astreamer_tar_terminator_content(astreamer *streamer,
+											 astreamer_member *member,
+											 const char *data, int len,
+											 astreamer_archive_context context);
+static void astreamer_tar_terminator_finalize(astreamer *streamer);
+static void astreamer_tar_terminator_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_tar_terminator_ops = {
-	.content = bbstreamer_tar_terminator_content,
-	.finalize = bbstreamer_tar_terminator_finalize,
-	.free = bbstreamer_tar_terminator_free
+static const astreamer_ops astreamer_tar_terminator_ops = {
+	.content = astreamer_tar_terminator_content,
+	.finalize = astreamer_tar_terminator_finalize,
+	.free = astreamer_tar_terminator_free
 };
 
 /*
- * Create a bbstreamer that can parse a stream of content as tar data.
+ * Create a astreamer that can parse a stream of content as tar data.
  *
- * The input should be a series of BBSTREAMER_UNKNOWN chunks; the bbstreamer
+ * The input should be a series of ASTREAMER_UNKNOWN chunks; the astreamer
  * specified by 'next' will receive a series of typed chunks, as per the
- * conventions described in bbstreamer.h.
+ * conventions described in astreamer.h.
  */
-bbstreamer *
-bbstreamer_tar_parser_new(bbstreamer *next)
+astreamer *
+astreamer_tar_parser_new(astreamer *next)
 {
-	bbstreamer_tar_parser *streamer;
+	astreamer_tar_parser *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_tar_parser));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_tar_parser_ops;
+	streamer = palloc0(sizeof(astreamer_tar_parser));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_tar_parser_ops;
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
-	streamer->next_context = BBSTREAMER_MEMBER_HEADER;
+	streamer->next_context = ASTREAMER_MEMBER_HEADER;
 
 	return &streamer->base;
 }
@@ -108,29 +108,29 @@ bbstreamer_tar_parser_new(bbstreamer *next)
  * Parse unknown content as tar data.
  */
 static void
-bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
-							  const char *data, int len,
-							  bbstreamer_archive_context context)
+astreamer_tar_parser_content(astreamer *streamer, astreamer_member *member,
+							 const char *data, int len,
+							 astreamer_archive_context context)
 {
-	bbstreamer_tar_parser *mystreamer = (bbstreamer_tar_parser *) streamer;
+	astreamer_tar_parser *mystreamer = (astreamer_tar_parser *) streamer;
 	size_t		nbytes;
 
 	/* Expect unparsed input. */
 	Assert(member == NULL);
-	Assert(context == BBSTREAMER_UNKNOWN);
+	Assert(context == ASTREAMER_UNKNOWN);
 
 	while (len > 0)
 	{
 		switch (mystreamer->next_context)
 		{
-			case BBSTREAMER_MEMBER_HEADER:
+			case ASTREAMER_MEMBER_HEADER:
 
 				/*
 				 * If we're expecting an archive member header, accumulate a
 				 * full block of data before doing anything further.
 				 */
-				if (!bbstreamer_buffer_until(streamer, &data, &len,
-											 TAR_BLOCK_SIZE))
+				if (!astreamer_buffer_until(streamer, &data, &len,
+											TAR_BLOCK_SIZE))
 					return;
 
 				/*
@@ -139,32 +139,32 @@ bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
 				 * thought was the next file header is actually the start of
 				 * the archive trailer. Switch modes accordingly.
 				 */
-				if (bbstreamer_tar_header(mystreamer))
+				if (astreamer_tar_header(mystreamer))
 				{
 					if (mystreamer->member.size == 0)
 					{
 						/* No content; trailer is zero-length. */
-						bbstreamer_content(mystreamer->base.bbs_next,
-										   &mystreamer->member,
-										   NULL, 0,
-										   BBSTREAMER_MEMBER_TRAILER);
+						astreamer_content(mystreamer->base.bbs_next,
+										  &mystreamer->member,
+										  NULL, 0,
+										  ASTREAMER_MEMBER_TRAILER);
 
 						/* Expect next header. */
-						mystreamer->next_context = BBSTREAMER_MEMBER_HEADER;
+						mystreamer->next_context = ASTREAMER_MEMBER_HEADER;
 					}
 					else
 					{
 						/* Expect contents. */
-						mystreamer->next_context = BBSTREAMER_MEMBER_CONTENTS;
+						mystreamer->next_context = ASTREAMER_MEMBER_CONTENTS;
 					}
 					mystreamer->base.bbs_buffer.len = 0;
 					mystreamer->file_bytes_sent = 0;
 				}
 				else
-					mystreamer->next_context = BBSTREAMER_ARCHIVE_TRAILER;
+					mystreamer->next_context = ASTREAMER_ARCHIVE_TRAILER;
 				break;
 
-			case BBSTREAMER_MEMBER_CONTENTS:
+			case ASTREAMER_MEMBER_CONTENTS:
 
 				/*
 				 * Send as much content as we have, but not more than the
@@ -174,10 +174,10 @@ bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
 				nbytes = mystreamer->member.size - mystreamer->file_bytes_sent;
 				nbytes = Min(nbytes, len);
 				Assert(nbytes > 0);
-				bbstreamer_content(mystreamer->base.bbs_next,
-								   &mystreamer->member,
-								   data, nbytes,
-								   BBSTREAMER_MEMBER_CONTENTS);
+				astreamer_content(mystreamer->base.bbs_next,
+								  &mystreamer->member,
+								  data, nbytes,
+								  ASTREAMER_MEMBER_CONTENTS);
 				mystreamer->file_bytes_sent += nbytes;
 				data += nbytes;
 				len -= nbytes;
@@ -193,53 +193,53 @@ bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
 					if (mystreamer->pad_bytes_expected == 0)
 					{
 						/* Trailer is zero-length. */
-						bbstreamer_content(mystreamer->base.bbs_next,
-										   &mystreamer->member,
-										   NULL, 0,
-										   BBSTREAMER_MEMBER_TRAILER);
+						astreamer_content(mystreamer->base.bbs_next,
+										  &mystreamer->member,
+										  NULL, 0,
+										  ASTREAMER_MEMBER_TRAILER);
 
 						/* Expect next header. */
-						mystreamer->next_context = BBSTREAMER_MEMBER_HEADER;
+						mystreamer->next_context = ASTREAMER_MEMBER_HEADER;
 					}
 					else
 					{
 						/* Trailer is not zero-length. */
-						mystreamer->next_context = BBSTREAMER_MEMBER_TRAILER;
+						mystreamer->next_context = ASTREAMER_MEMBER_TRAILER;
 					}
 					mystreamer->base.bbs_buffer.len = 0;
 				}
 				break;
 
-			case BBSTREAMER_MEMBER_TRAILER:
+			case ASTREAMER_MEMBER_TRAILER:
 
 				/*
 				 * If we're expecting an archive member trailer, accumulate
 				 * the expected number of padding bytes before sending
 				 * anything onward.
 				 */
-				if (!bbstreamer_buffer_until(streamer, &data, &len,
-											 mystreamer->pad_bytes_expected))
+				if (!astreamer_buffer_until(streamer, &data, &len,
+											mystreamer->pad_bytes_expected))
 					return;
 
 				/* OK, now we can send it. */
-				bbstreamer_content(mystreamer->base.bbs_next,
-								   &mystreamer->member,
-								   data, mystreamer->pad_bytes_expected,
-								   BBSTREAMER_MEMBER_TRAILER);
+				astreamer_content(mystreamer->base.bbs_next,
+								  &mystreamer->member,
+								  data, mystreamer->pad_bytes_expected,
+								  ASTREAMER_MEMBER_TRAILER);
 
 				/* Expect next file header. */
-				mystreamer->next_context = BBSTREAMER_MEMBER_HEADER;
+				mystreamer->next_context = ASTREAMER_MEMBER_HEADER;
 				mystreamer->base.bbs_buffer.len = 0;
 				break;
 
-			case BBSTREAMER_ARCHIVE_TRAILER:
+			case ASTREAMER_ARCHIVE_TRAILER:
 
 				/*
 				 * We've seen an end-of-archive indicator, so anything more is
 				 * buffered and sent as part of the archive trailer. But we
 				 * don't expect more than 2 blocks.
 				 */
-				bbstreamer_buffer_bytes(streamer, &data, &len, len);
+				astreamer_buffer_bytes(streamer, &data, &len, len);
 				if (len > 2 * TAR_BLOCK_SIZE)
 					pg_fatal("tar file trailer exceeds 2 blocks");
 				return;
@@ -255,14 +255,14 @@ bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
  * Parse a file header within a tar stream.
  *
  * The return value is true if we found a file header and passed it on to the
- * next bbstreamer; it is false if we have reached the archive trailer.
+ * next astreamer; it is false if we have reached the archive trailer.
  */
 static bool
-bbstreamer_tar_header(bbstreamer_tar_parser *mystreamer)
+astreamer_tar_header(astreamer_tar_parser *mystreamer)
 {
 	bool		has_nonzero_byte = false;
 	int			i;
-	bbstreamer_member *member = &mystreamer->member;
+	astreamer_member *member = &mystreamer->member;
 	char	   *buffer = mystreamer->base.bbs_buffer.data;
 
 	Assert(mystreamer->base.bbs_buffer.len == TAR_BLOCK_SIZE);
@@ -304,10 +304,10 @@ bbstreamer_tar_header(bbstreamer_tar_parser *mystreamer)
 	/* Compute number of padding bytes. */
 	mystreamer->pad_bytes_expected = tarPaddingBytesRequired(member->size);
 
-	/* Forward the entire header to the next bbstreamer. */
-	bbstreamer_content(mystreamer->base.bbs_next, member,
-					   buffer, TAR_BLOCK_SIZE,
-					   BBSTREAMER_MEMBER_HEADER);
+	/* Forward the entire header to the next astreamer. */
+	astreamer_content(mystreamer->base.bbs_next, member,
+					  buffer, TAR_BLOCK_SIZE,
+					  ASTREAMER_MEMBER_HEADER);
 
 	return true;
 }
@@ -316,50 +316,50 @@ bbstreamer_tar_header(bbstreamer_tar_parser *mystreamer)
  * End-of-stream processing for a tar parser.
  */
 static void
-bbstreamer_tar_parser_finalize(bbstreamer *streamer)
+astreamer_tar_parser_finalize(astreamer *streamer)
 {
-	bbstreamer_tar_parser *mystreamer = (bbstreamer_tar_parser *) streamer;
+	astreamer_tar_parser *mystreamer = (astreamer_tar_parser *) streamer;
 
-	if (mystreamer->next_context != BBSTREAMER_ARCHIVE_TRAILER &&
-		(mystreamer->next_context != BBSTREAMER_MEMBER_HEADER ||
+	if (mystreamer->next_context != ASTREAMER_ARCHIVE_TRAILER &&
+		(mystreamer->next_context != ASTREAMER_MEMBER_HEADER ||
 		 mystreamer->base.bbs_buffer.len > 0))
 		pg_fatal("COPY stream ended before last file was finished");
 
 	/* Send the archive trailer, even if empty. */
-	bbstreamer_content(streamer->bbs_next, NULL,
-					   streamer->bbs_buffer.data, streamer->bbs_buffer.len,
-					   BBSTREAMER_ARCHIVE_TRAILER);
+	astreamer_content(streamer->bbs_next, NULL,
+					  streamer->bbs_buffer.data, streamer->bbs_buffer.len,
+					  ASTREAMER_ARCHIVE_TRAILER);
 
 	/* Now finalize successor. */
-	bbstreamer_finalize(streamer->bbs_next);
+	astreamer_finalize(streamer->bbs_next);
 }
 
 /*
  * Free memory associated with a tar parser.
  */
 static void
-bbstreamer_tar_parser_free(bbstreamer *streamer)
+astreamer_tar_parser_free(astreamer *streamer)
 {
 	pfree(streamer->bbs_buffer.data);
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 }
 
 /*
- * Create a bbstreamer that can generate a tar archive.
+ * Create a astreamer that can generate a tar archive.
  *
  * This is intended to be usable either for generating a brand-new tar archive
  * or for modifying one on the fly. The input should be a series of typed
- * chunks (i.e. not BBSTREAMER_UNKNOWN). See also the comments for
- * bbstreamer_tar_parser_content.
+ * chunks (i.e. not ASTREAMER_UNKNOWN). See also the comments for
+ * astreamer_tar_parser_content.
  */
-bbstreamer *
-bbstreamer_tar_archiver_new(bbstreamer *next)
+astreamer *
+astreamer_tar_archiver_new(astreamer *next)
 {
-	bbstreamer_tar_archiver *streamer;
+	astreamer_tar_archiver *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_tar_archiver));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_tar_archiver_ops;
+	streamer = palloc0(sizeof(astreamer_tar_archiver));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_tar_archiver_ops;
 	streamer->base.bbs_next = next;
 
 	return &streamer->base;
@@ -368,36 +368,36 @@ bbstreamer_tar_archiver_new(bbstreamer *next)
 /*
  * Fix up the stream of input chunks to create a valid tar file.
  *
- * If a BBSTREAMER_MEMBER_HEADER chunk is of size 0, it is replaced with a
+ * If a ASTREAMER_MEMBER_HEADER chunk is of size 0, it is replaced with a
  * newly-constructed tar header. If it is of size TAR_BLOCK_SIZE, it is
  * passed through without change. Any other size is a fatal error (and
  * indicates a bug).
  *
- * Whenever a new BBSTREAMER_MEMBER_HEADER chunk is constructed, the
- * corresponding BBSTREAMER_MEMBER_TRAILER chunk is also constructed from
+ * Whenever a new ASTREAMER_MEMBER_HEADER chunk is constructed, the
+ * corresponding ASTREAMER_MEMBER_TRAILER chunk is also constructed from
  * scratch. Specifically, we construct a block of zero bytes sufficient to
  * pad out to a block boundary, as required by the tar format. Other
- * BBSTREAMER_MEMBER_TRAILER chunks are passed through without change.
+ * ASTREAMER_MEMBER_TRAILER chunks are passed through without change.
  *
- * Any BBSTREAMER_MEMBER_CONTENTS chunks are passed through without change.
+ * Any ASTREAMER_MEMBER_CONTENTS chunks are passed through without change.
  *
- * The BBSTREAMER_ARCHIVE_TRAILER chunk is replaced with two
+ * The ASTREAMER_ARCHIVE_TRAILER chunk is replaced with two
  * blocks of zero bytes. Not all tar programs require this, but apparently
  * some do. The server does not supply this trailer. If no archive trailer is
- * present, one will be added by bbstreamer_tar_parser_finalize.
+ * present, one will be added by astreamer_tar_parser_finalize.
  */
 static void
-bbstreamer_tar_archiver_content(bbstreamer *streamer,
-								bbstreamer_member *member,
-								const char *data, int len,
-								bbstreamer_archive_context context)
+astreamer_tar_archiver_content(astreamer *streamer,
+							   astreamer_member *member,
+							   const char *data, int len,
+							   astreamer_archive_context context)
 {
-	bbstreamer_tar_archiver *mystreamer = (bbstreamer_tar_archiver *) streamer;
+	astreamer_tar_archiver *mystreamer = (astreamer_tar_archiver *) streamer;
 	char		buffer[2 * TAR_BLOCK_SIZE];
 
-	Assert(context != BBSTREAMER_UNKNOWN);
+	Assert(context != ASTREAMER_UNKNOWN);
 
-	if (context == BBSTREAMER_MEMBER_HEADER && len != TAR_BLOCK_SIZE)
+	if (context == ASTREAMER_MEMBER_HEADER && len != TAR_BLOCK_SIZE)
 	{
 		Assert(len == 0);
 
@@ -411,7 +411,7 @@ bbstreamer_tar_archiver_content(bbstreamer *streamer,
 		/* Also make a note to replace padding, in case size changed. */
 		mystreamer->rearchive_member = true;
 	}
-	else if (context == BBSTREAMER_MEMBER_TRAILER &&
+	else if (context == ASTREAMER_MEMBER_TRAILER &&
 			 mystreamer->rearchive_member)
 	{
 		int			pad_bytes = tarPaddingBytesRequired(member->size);
@@ -424,7 +424,7 @@ bbstreamer_tar_archiver_content(bbstreamer *streamer,
 		/* Don't do this again unless we replace another header. */
 		mystreamer->rearchive_member = false;
 	}
-	else if (context == BBSTREAMER_ARCHIVE_TRAILER)
+	else if (context == ASTREAMER_ARCHIVE_TRAILER)
 	{
 		/* Trailer should always be two blocks of zero bytes. */
 		memset(buffer, 0, 2 * TAR_BLOCK_SIZE);
@@ -432,40 +432,40 @@ bbstreamer_tar_archiver_content(bbstreamer *streamer,
 		len = 2 * TAR_BLOCK_SIZE;
 	}
 
-	bbstreamer_content(streamer->bbs_next, member, data, len, context);
+	astreamer_content(streamer->bbs_next, member, data, len, context);
 }
 
 /*
  * End-of-stream processing for a tar archiver.
  */
 static void
-bbstreamer_tar_archiver_finalize(bbstreamer *streamer)
+astreamer_tar_archiver_finalize(astreamer *streamer)
 {
-	bbstreamer_finalize(streamer->bbs_next);
+	astreamer_finalize(streamer->bbs_next);
 }
 
 /*
  * Free memory associated with a tar archiver.
  */
 static void
-bbstreamer_tar_archiver_free(bbstreamer *streamer)
+astreamer_tar_archiver_free(astreamer *streamer)
 {
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	pfree(streamer);
 }
 
 /*
- * Create a bbstreamer that blindly adds two blocks of NUL bytes to the
+ * Create a astreamer that blindly adds two blocks of NUL bytes to the
  * end of an incomplete tarfile that the server might send us.
  */
-bbstreamer *
-bbstreamer_tar_terminator_new(bbstreamer *next)
+astreamer *
+astreamer_tar_terminator_new(astreamer *next)
 {
-	bbstreamer *streamer;
+	astreamer  *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer));
-	*((const bbstreamer_ops **) &streamer->bbs_ops) =
-		&bbstreamer_tar_terminator_ops;
+	streamer = palloc0(sizeof(astreamer));
+	*((const astreamer_ops **) &streamer->bbs_ops) =
+		&astreamer_tar_terminator_ops;
 	streamer->bbs_next = next;
 
 	return streamer;
@@ -475,17 +475,17 @@ bbstreamer_tar_terminator_new(bbstreamer *next)
  * Pass all the content through without change.
  */
 static void
-bbstreamer_tar_terminator_content(bbstreamer *streamer,
-								  bbstreamer_member *member,
-								  const char *data, int len,
-								  bbstreamer_archive_context context)
+astreamer_tar_terminator_content(astreamer *streamer,
+								 astreamer_member *member,
+								 const char *data, int len,
+								 astreamer_archive_context context)
 {
 	/* Expect unparsed input. */
 	Assert(member == NULL);
-	Assert(context == BBSTREAMER_UNKNOWN);
+	Assert(context == ASTREAMER_UNKNOWN);
 
 	/* Just forward it. */
-	bbstreamer_content(streamer->bbs_next, member, data, len, context);
+	astreamer_content(streamer->bbs_next, member, data, len, context);
 }
 
 /*
@@ -493,22 +493,22 @@ bbstreamer_tar_terminator_content(bbstreamer *streamer,
  * to supply.
  */
 static void
-bbstreamer_tar_terminator_finalize(bbstreamer *streamer)
+astreamer_tar_terminator_finalize(astreamer *streamer)
 {
 	char		buffer[2 * TAR_BLOCK_SIZE];
 
 	memset(buffer, 0, 2 * TAR_BLOCK_SIZE);
-	bbstreamer_content(streamer->bbs_next, NULL, buffer,
-					   2 * TAR_BLOCK_SIZE, BBSTREAMER_UNKNOWN);
-	bbstreamer_finalize(streamer->bbs_next);
+	astreamer_content(streamer->bbs_next, NULL, buffer,
+					  2 * TAR_BLOCK_SIZE, ASTREAMER_UNKNOWN);
+	astreamer_finalize(streamer->bbs_next);
 }
 
 /*
  * Free memory associated with a tar terminator.
  */
 static void
-bbstreamer_tar_terminator_free(bbstreamer *streamer)
+astreamer_tar_terminator_free(astreamer *streamer)
 {
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	pfree(streamer);
 }
diff --git a/src/bin/pg_basebackup/bbstreamer_zstd.c b/src/bin/pg_basebackup/astreamer_zstd.c
similarity index 64%
rename from src/bin/pg_basebackup/bbstreamer_zstd.c
rename to src/bin/pg_basebackup/astreamer_zstd.c
index 20f11d4450e..58dc679ef99 100644
--- a/src/bin/pg_basebackup/bbstreamer_zstd.c
+++ b/src/bin/pg_basebackup/astreamer_zstd.c
@@ -1,11 +1,11 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_zstd.c
+ * astreamer_zstd.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_zstd.c
+ *		  src/bin/pg_basebackup/astreamer_zstd.c
  *-------------------------------------------------------------------------
  */
 
@@ -17,44 +17,44 @@
 #include <zstd.h>
 #endif
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/logging.h"
 
 #ifdef USE_ZSTD
 
-typedef struct bbstreamer_zstd_frame
+typedef struct astreamer_zstd_frame
 {
-	bbstreamer	base;
+	astreamer	base;
 
 	ZSTD_CCtx  *cctx;
 	ZSTD_DCtx  *dctx;
 	ZSTD_outBuffer zstd_outBuf;
-} bbstreamer_zstd_frame;
+} astreamer_zstd_frame;
 
-static void bbstreamer_zstd_compressor_content(bbstreamer *streamer,
-											   bbstreamer_member *member,
-											   const char *data, int len,
-											   bbstreamer_archive_context context);
-static void bbstreamer_zstd_compressor_finalize(bbstreamer *streamer);
-static void bbstreamer_zstd_compressor_free(bbstreamer *streamer);
+static void astreamer_zstd_compressor_content(astreamer *streamer,
+											  astreamer_member *member,
+											  const char *data, int len,
+											  astreamer_archive_context context);
+static void astreamer_zstd_compressor_finalize(astreamer *streamer);
+static void astreamer_zstd_compressor_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_zstd_compressor_ops = {
-	.content = bbstreamer_zstd_compressor_content,
-	.finalize = bbstreamer_zstd_compressor_finalize,
-	.free = bbstreamer_zstd_compressor_free
+static const astreamer_ops astreamer_zstd_compressor_ops = {
+	.content = astreamer_zstd_compressor_content,
+	.finalize = astreamer_zstd_compressor_finalize,
+	.free = astreamer_zstd_compressor_free
 };
 
-static void bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
-												 bbstreamer_member *member,
-												 const char *data, int len,
-												 bbstreamer_archive_context context);
-static void bbstreamer_zstd_decompressor_finalize(bbstreamer *streamer);
-static void bbstreamer_zstd_decompressor_free(bbstreamer *streamer);
+static void astreamer_zstd_decompressor_content(astreamer *streamer,
+												astreamer_member *member,
+												const char *data, int len,
+												astreamer_archive_context context);
+static void astreamer_zstd_decompressor_finalize(astreamer *streamer);
+static void astreamer_zstd_decompressor_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_zstd_decompressor_ops = {
-	.content = bbstreamer_zstd_decompressor_content,
-	.finalize = bbstreamer_zstd_decompressor_finalize,
-	.free = bbstreamer_zstd_decompressor_free
+static const astreamer_ops astreamer_zstd_decompressor_ops = {
+	.content = astreamer_zstd_decompressor_content,
+	.finalize = astreamer_zstd_decompressor_finalize,
+	.free = astreamer_zstd_decompressor_free
 };
 #endif
 
@@ -62,19 +62,19 @@ static const bbstreamer_ops bbstreamer_zstd_decompressor_ops = {
  * Create a new base backup streamer that performs zstd compression of tar
  * blocks.
  */
-bbstreamer *
-bbstreamer_zstd_compressor_new(bbstreamer *next, pg_compress_specification *compress)
+astreamer *
+astreamer_zstd_compressor_new(astreamer *next, pg_compress_specification *compress)
 {
 #ifdef USE_ZSTD
-	bbstreamer_zstd_frame *streamer;
+	astreamer_zstd_frame *streamer;
 	size_t		ret;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_zstd_frame));
+	streamer = palloc0(sizeof(astreamer_zstd_frame));
 
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_zstd_compressor_ops;
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_zstd_compressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -142,12 +142,12 @@ bbstreamer_zstd_compressor_new(bbstreamer *next, pg_compress_specification *comp
  * of output buffer to next streamer and empty the buffer.
  */
 static void
-bbstreamer_zstd_compressor_content(bbstreamer *streamer,
-								   bbstreamer_member *member,
-								   const char *data, int len,
-								   bbstreamer_archive_context context)
+astreamer_zstd_compressor_content(astreamer *streamer,
+								  astreamer_member *member,
+								  const char *data, int len,
+								  astreamer_archive_context context)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 	ZSTD_inBuffer inBuf = {data, len, 0};
 
 	while (inBuf.pos < inBuf.size)
@@ -162,10 +162,10 @@ bbstreamer_zstd_compressor_content(bbstreamer *streamer,
 		if (mystreamer->zstd_outBuf.size - mystreamer->zstd_outBuf.pos <
 			max_needed)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, member,
-							   mystreamer->zstd_outBuf.dst,
-							   mystreamer->zstd_outBuf.pos,
-							   context);
+			astreamer_content(mystreamer->base.bbs_next, member,
+							  mystreamer->zstd_outBuf.dst,
+							  mystreamer->zstd_outBuf.pos,
+							  context);
 
 			/* Reset the ZSTD output buffer. */
 			mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data;
@@ -187,9 +187,9 @@ bbstreamer_zstd_compressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_zstd_compressor_finalize(bbstreamer *streamer)
+astreamer_zstd_compressor_finalize(astreamer *streamer)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 	size_t		yet_to_flush;
 
 	do
@@ -204,10 +204,10 @@ bbstreamer_zstd_compressor_finalize(bbstreamer *streamer)
 		if (mystreamer->zstd_outBuf.size - mystreamer->zstd_outBuf.pos <
 			max_needed)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, NULL,
-							   mystreamer->zstd_outBuf.dst,
-							   mystreamer->zstd_outBuf.pos,
-							   BBSTREAMER_UNKNOWN);
+			astreamer_content(mystreamer->base.bbs_next, NULL,
+							  mystreamer->zstd_outBuf.dst,
+							  mystreamer->zstd_outBuf.pos,
+							  ASTREAMER_UNKNOWN);
 
 			/* Reset the ZSTD output buffer. */
 			mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data;
@@ -227,23 +227,23 @@ bbstreamer_zstd_compressor_finalize(bbstreamer *streamer)
 
 	/* Make sure to pass any remaining bytes to the next streamer. */
 	if (mystreamer->zstd_outBuf.pos > 0)
-		bbstreamer_content(mystreamer->base.bbs_next, NULL,
-						   mystreamer->zstd_outBuf.dst,
-						   mystreamer->zstd_outBuf.pos,
-						   BBSTREAMER_UNKNOWN);
+		astreamer_content(mystreamer->base.bbs_next, NULL,
+						  mystreamer->zstd_outBuf.dst,
+						  mystreamer->zstd_outBuf.pos,
+						  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_zstd_compressor_free(bbstreamer *streamer)
+astreamer_zstd_compressor_free(astreamer *streamer)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	ZSTD_freeCCtx(mystreamer->cctx);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
@@ -254,17 +254,17 @@ bbstreamer_zstd_compressor_free(bbstreamer *streamer)
  * Create a new base backup streamer that performs decompression of zstd
  * compressed blocks.
  */
-bbstreamer *
-bbstreamer_zstd_decompressor_new(bbstreamer *next)
+astreamer *
+astreamer_zstd_decompressor_new(astreamer *next)
 {
 #ifdef USE_ZSTD
-	bbstreamer_zstd_frame *streamer;
+	astreamer_zstd_frame *streamer;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_zstd_frame));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_zstd_decompressor_ops;
+	streamer = palloc0(sizeof(astreamer_zstd_frame));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_zstd_decompressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -293,12 +293,12 @@ bbstreamer_zstd_decompressor_new(bbstreamer *next)
  * to the next streamer.
  */
 static void
-bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
-									 bbstreamer_member *member,
-									 const char *data, int len,
-									 bbstreamer_archive_context context)
+astreamer_zstd_decompressor_content(astreamer *streamer,
+									astreamer_member *member,
+									const char *data, int len,
+									astreamer_archive_context context)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 	ZSTD_inBuffer inBuf = {data, len, 0};
 
 	while (inBuf.pos < inBuf.size)
@@ -311,10 +311,10 @@ bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
 		 */
 		if (mystreamer->zstd_outBuf.pos >= mystreamer->zstd_outBuf.size)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, member,
-							   mystreamer->zstd_outBuf.dst,
-							   mystreamer->zstd_outBuf.pos,
-							   context);
+			astreamer_content(mystreamer->base.bbs_next, member,
+							  mystreamer->zstd_outBuf.dst,
+							  mystreamer->zstd_outBuf.pos,
+							  context);
 
 			/* Reset the ZSTD output buffer. */
 			mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data;
@@ -335,32 +335,32 @@ bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_zstd_decompressor_finalize(bbstreamer *streamer)
+astreamer_zstd_decompressor_finalize(astreamer *streamer)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 
 	/*
 	 * End of the stream, if there is some pending data in output buffers then
 	 * we must forward it to next streamer.
 	 */
 	if (mystreamer->zstd_outBuf.pos > 0)
-		bbstreamer_content(mystreamer->base.bbs_next, NULL,
-						   mystreamer->base.bbs_buffer.data,
-						   mystreamer->base.bbs_buffer.maxlen,
-						   BBSTREAMER_UNKNOWN);
+		astreamer_content(mystreamer->base.bbs_next, NULL,
+						  mystreamer->base.bbs_buffer.data,
+						  mystreamer->base.bbs_buffer.maxlen,
+						  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_zstd_decompressor_free(bbstreamer *streamer)
+astreamer_zstd_decompressor_free(astreamer *streamer)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	ZSTD_freeDCtx(mystreamer->dctx);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
diff --git a/src/bin/pg_basebackup/bbstreamer.h b/src/bin/pg_basebackup/bbstreamer.h
deleted file mode 100644
index 3b820f13b51..00000000000
--- a/src/bin/pg_basebackup/bbstreamer.h
+++ /dev/null
@@ -1,226 +0,0 @@
-/*-------------------------------------------------------------------------
- *
- * bbstreamer.h
- *
- * Each tar archive returned by the server is passed to one or more
- * bbstreamer objects for further processing. The bbstreamer may do
- * something simple, like write the archive to a file, perhaps after
- * compressing it, but it can also do more complicated things, like
- * annotating the byte stream to indicate which parts of the data
- * correspond to tar headers or trailing padding, vs. which parts are
- * payload data. A subsequent bbstreamer may use this information to
- * make further decisions about how to process the data; for example,
- * it might choose to modify the archive contents.
- *
- * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
- *
- * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer.h
- *-------------------------------------------------------------------------
- */
-
-#ifndef BBSTREAMER_H
-#define BBSTREAMER_H
-
-#include "common/compression.h"
-#include "lib/stringinfo.h"
-#include "pqexpbuffer.h"
-
-struct bbstreamer;
-struct bbstreamer_ops;
-typedef struct bbstreamer bbstreamer;
-typedef struct bbstreamer_ops bbstreamer_ops;
-
-/*
- * Each chunk of archive data passed to a bbstreamer is classified into one
- * of these categories. When data is first received from the remote server,
- * each chunk will be categorized as BBSTREAMER_UNKNOWN, and the chunks will
- * be of whatever size the remote server chose to send.
- *
- * If the archive is parsed (e.g. see bbstreamer_tar_parser_new()), then all
- * chunks should be labelled as one of the other types listed here. In
- * addition, there should be exactly one BBSTREAMER_MEMBER_HEADER chunk and
- * exactly one BBSTREAMER_MEMBER_TRAILER chunk per archive member, even if
- * that means a zero-length call. There can be any number of
- * BBSTREAMER_MEMBER_CONTENTS chunks in between those calls. There
- * should exactly BBSTREAMER_ARCHIVE_TRAILER chunk, and it should follow the
- * last BBSTREAMER_MEMBER_TRAILER chunk.
- *
- * In theory, we could need other classifications here, such as a way of
- * indicating an archive header, but the "tar" format doesn't need anything
- * else, so for the time being there's no point.
- */
-typedef enum
-{
-	BBSTREAMER_UNKNOWN,
-	BBSTREAMER_MEMBER_HEADER,
-	BBSTREAMER_MEMBER_CONTENTS,
-	BBSTREAMER_MEMBER_TRAILER,
-	BBSTREAMER_ARCHIVE_TRAILER,
-} bbstreamer_archive_context;
-
-/*
- * Each chunk of data that is classified as BBSTREAMER_MEMBER_HEADER,
- * BBSTREAMER_MEMBER_CONTENTS, or BBSTREAMER_MEMBER_TRAILER should also
- * pass a pointer to an instance of this struct. The details are expected
- * to be present in the archive header and used to fill the struct, after
- * which all subsequent calls for the same archive member are expected to
- * pass the same details.
- */
-typedef struct
-{
-	char		pathname[MAXPGPATH];
-	pgoff_t		size;
-	mode_t		mode;
-	uid_t		uid;
-	gid_t		gid;
-	bool		is_directory;
-	bool		is_link;
-	char		linktarget[MAXPGPATH];
-} bbstreamer_member;
-
-/*
- * Generally, each type of bbstreamer will define its own struct, but the
- * first element should be 'bbstreamer base'. A bbstreamer that does not
- * require any additional private data could use this structure directly.
- *
- * bbs_ops is a pointer to the bbstreamer_ops object which contains the
- * function pointers appropriate to this type of bbstreamer.
- *
- * bbs_next is a pointer to the successor bbstreamer, for those types of
- * bbstreamer which forward data to a successor. It need not be used and
- * should be set to NULL when not relevant.
- *
- * bbs_buffer is a buffer for accumulating data for temporary storage. Each
- * type of bbstreamer makes its own decisions about whether and how to use
- * this buffer.
- */
-struct bbstreamer
-{
-	const bbstreamer_ops *bbs_ops;
-	bbstreamer *bbs_next;
-	StringInfoData bbs_buffer;
-};
-
-/*
- * There are three callbacks for a bbstreamer. The 'content' callback is
- * called repeatedly, as described in the bbstreamer_archive_context comments.
- * Then, the 'finalize' callback is called once at the end, to give the
- * bbstreamer a chance to perform cleanup such as closing files. Finally,
- * because this code is running in a frontend environment where, as of this
- * writing, there are no memory contexts, the 'free' callback is called to
- * release memory. These callbacks should always be invoked using the static
- * inline functions defined below.
- */
-struct bbstreamer_ops
-{
-	void		(*content) (bbstreamer *streamer, bbstreamer_member *member,
-							const char *data, int len,
-							bbstreamer_archive_context context);
-	void		(*finalize) (bbstreamer *streamer);
-	void		(*free) (bbstreamer *streamer);
-};
-
-/* Send some content to a bbstreamer. */
-static inline void
-bbstreamer_content(bbstreamer *streamer, bbstreamer_member *member,
-				   const char *data, int len,
-				   bbstreamer_archive_context context)
-{
-	Assert(streamer != NULL);
-	streamer->bbs_ops->content(streamer, member, data, len, context);
-}
-
-/* Finalize a bbstreamer. */
-static inline void
-bbstreamer_finalize(bbstreamer *streamer)
-{
-	Assert(streamer != NULL);
-	streamer->bbs_ops->finalize(streamer);
-}
-
-/* Free a bbstreamer. */
-static inline void
-bbstreamer_free(bbstreamer *streamer)
-{
-	Assert(streamer != NULL);
-	streamer->bbs_ops->free(streamer);
-}
-
-/*
- * This is a convenience method for use when implementing a bbstreamer; it is
- * not for use by outside callers. It adds the amount of data specified by
- * 'nbytes' to the bbstreamer's buffer and adjusts '*len' and '*data'
- * accordingly.
- */
-static inline void
-bbstreamer_buffer_bytes(bbstreamer *streamer, const char **data, int *len,
-						int nbytes)
-{
-	Assert(nbytes <= *len);
-
-	appendBinaryStringInfo(&streamer->bbs_buffer, *data, nbytes);
-	*len -= nbytes;
-	*data += nbytes;
-}
-
-/*
- * This is a convenience method for use when implementing a bbstreamer; it is
- * not for use by outsider callers. It attempts to add enough data to the
- * bbstreamer's buffer to reach a length of target_bytes and adjusts '*len'
- * and '*data' accordingly. It returns true if the target length has been
- * reached and false otherwise.
- */
-static inline bool
-bbstreamer_buffer_until(bbstreamer *streamer, const char **data, int *len,
-						int target_bytes)
-{
-	int			buflen = streamer->bbs_buffer.len;
-
-	if (buflen >= target_bytes)
-	{
-		/* Target length already reached; nothing to do. */
-		return true;
-	}
-
-	if (buflen + *len < target_bytes)
-	{
-		/* Not enough data to reach target length; buffer all of it. */
-		bbstreamer_buffer_bytes(streamer, data, len, *len);
-		return false;
-	}
-
-	/* Buffer just enough to reach the target length. */
-	bbstreamer_buffer_bytes(streamer, data, len, target_bytes - buflen);
-	return true;
-}
-
-/*
- * Functions for creating bbstreamer objects of various types. See the header
- * comments for each of these functions for details.
- */
-extern bbstreamer *bbstreamer_plain_writer_new(char *pathname, FILE *file);
-extern bbstreamer *bbstreamer_gzip_writer_new(char *pathname, FILE *file,
-											  pg_compress_specification *compress);
-extern bbstreamer *bbstreamer_extractor_new(const char *basepath,
-											const char *(*link_map) (const char *),
-											void (*report_output_file) (const char *));
-
-extern bbstreamer *bbstreamer_gzip_decompressor_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_lz4_compressor_new(bbstreamer *next,
-												 pg_compress_specification *compress);
-extern bbstreamer *bbstreamer_lz4_decompressor_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_zstd_compressor_new(bbstreamer *next,
-												  pg_compress_specification *compress);
-extern bbstreamer *bbstreamer_zstd_decompressor_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_tar_parser_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_tar_terminator_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_tar_archiver_new(bbstreamer *next);
-
-extern bbstreamer *bbstreamer_recovery_injector_new(bbstreamer *next,
-													bool is_recovery_guc_supported,
-													PQExpBuffer recoveryconfcontents);
-extern void bbstreamer_inject_file(bbstreamer *streamer, char *pathname,
-								   char *data, int len);
-
-#endif
diff --git a/src/bin/pg_basebackup/meson.build b/src/bin/pg_basebackup/meson.build
index c00acd5e118..a68dbd7837d 100644
--- a/src/bin/pg_basebackup/meson.build
+++ b/src/bin/pg_basebackup/meson.build
@@ -1,12 +1,12 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 common_sources = files(
-  'bbstreamer_file.c',
-  'bbstreamer_gzip.c',
-  'bbstreamer_inject.c',
-  'bbstreamer_lz4.c',
-  'bbstreamer_tar.c',
-  'bbstreamer_zstd.c',
+  'astreamer_file.c',
+  'astreamer_gzip.c',
+  'astreamer_inject.c',
+  'astreamer_lz4.c',
+  'astreamer_tar.c',
+  'astreamer_zstd.c',
   'receivelog.c',
   'streamutil.c',
   'walmethods.c',
diff --git a/src/bin/pg_basebackup/nls.mk b/src/bin/pg_basebackup/nls.mk
index 384dbb021e9..950b9797b1e 100644
--- a/src/bin/pg_basebackup/nls.mk
+++ b/src/bin/pg_basebackup/nls.mk
@@ -1,12 +1,12 @@
 # src/bin/pg_basebackup/nls.mk
 CATALOG_NAME     = pg_basebackup
 GETTEXT_FILES    = $(FRONTEND_COMMON_GETTEXT_FILES) \
-                   bbstreamer_file.c \
-                   bbstreamer_gzip.c \
-                   bbstreamer_inject.c \
-                   bbstreamer_lz4.c \
-                   bbstreamer_tar.c \
-                   bbstreamer_zstd.c \
+                   astreamer_file.c \
+                   astreamer_gzip.c \
+                   astreamer_inject.c \
+                   astreamer_lz4.c \
+                   astreamer_tar.c \
+                   astreamer_zstd.c \
                    pg_basebackup.c \
                    pg_createsubscriber.c \
                    pg_receivewal.c \
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 8f3dd04fd22..4179b064cbc 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -26,8 +26,8 @@
 #endif
 
 #include "access/xlog_internal.h"
+#include "astreamer.h"
 #include "backup/basebackup.h"
-#include "bbstreamer.h"
 #include "common/compression.h"
 #include "common/file_perm.h"
 #include "common/file_utils.h"
@@ -57,8 +57,8 @@ typedef struct ArchiveStreamState
 {
 	int			tablespacenum;
 	pg_compress_specification *compress;
-	bbstreamer *streamer;
-	bbstreamer *manifest_inject_streamer;
+	astreamer *streamer;
+	astreamer *manifest_inject_streamer;
 	PQExpBuffer manifest_buffer;
 	char		manifest_filename[MAXPGPATH];
 	FILE	   *manifest_file;
@@ -67,7 +67,7 @@ typedef struct ArchiveStreamState
 typedef struct WriteTarState
 {
 	int			tablespacenum;
-	bbstreamer *streamer;
+	astreamer *streamer;
 } WriteTarState;
 
 typedef struct WriteManifestState
@@ -199,8 +199,8 @@ static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *fo
 static void progress_update_filename(const char *filename);
 static void progress_report(int tablespacenum, bool force, bool finished);
 
-static bbstreamer *CreateBackupStreamer(char *archive_name, char *spclocation,
-										bbstreamer **manifest_inject_streamer_p,
+static astreamer *CreateBackupStreamer(char *archive_name, char *spclocation,
+										astreamer **manifest_inject_streamer_p,
 										bool is_recovery_guc_supported,
 										bool expect_unterminated_tarfile,
 										pg_compress_specification *compress);
@@ -1053,19 +1053,19 @@ ReceiveCopyData(PGconn *conn, WriteDataCallback callback,
  * the options selected by the user.  We may just write the results directly
  * to a file, or we might compress first, or we might extract the tar file
  * and write each member separately. This function doesn't do any of that
- * directly, but it works out what kind of bbstreamer we need to create so
+ * directly, but it works out what kind of astreamer we need to create so
  * that the right stuff happens when, down the road, we actually receive
  * the data.
  */
-static bbstreamer *
+static astreamer *
 CreateBackupStreamer(char *archive_name, char *spclocation,
-					 bbstreamer **manifest_inject_streamer_p,
+					 astreamer **manifest_inject_streamer_p,
 					 bool is_recovery_guc_supported,
 					 bool expect_unterminated_tarfile,
 					 pg_compress_specification *compress)
 {
-	bbstreamer *streamer = NULL;
-	bbstreamer *manifest_inject_streamer = NULL;
+	astreamer *streamer = NULL;
+	astreamer *manifest_inject_streamer = NULL;
 	bool		inject_manifest;
 	bool		is_tar,
 				is_tar_gz,
@@ -1160,7 +1160,7 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 			directory = psprintf("%s/%s", basedir, spclocation);
 		else
 			directory = get_tablespace_mapping(spclocation);
-		streamer = bbstreamer_extractor_new(directory,
+		streamer = astreamer_extractor_new(directory,
 											get_tablespace_mapping,
 											progress_update_filename);
 	}
@@ -1188,27 +1188,27 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 		}
 
 		if (compress->algorithm == PG_COMPRESSION_NONE)
-			streamer = bbstreamer_plain_writer_new(archive_filename,
+			streamer = astreamer_plain_writer_new(archive_filename,
 												   archive_file);
 		else if (compress->algorithm == PG_COMPRESSION_GZIP)
 		{
 			strlcat(archive_filename, ".gz", sizeof(archive_filename));
-			streamer = bbstreamer_gzip_writer_new(archive_filename,
+			streamer = astreamer_gzip_writer_new(archive_filename,
 												  archive_file, compress);
 		}
 		else if (compress->algorithm == PG_COMPRESSION_LZ4)
 		{
 			strlcat(archive_filename, ".lz4", sizeof(archive_filename));
-			streamer = bbstreamer_plain_writer_new(archive_filename,
+			streamer = astreamer_plain_writer_new(archive_filename,
 												   archive_file);
-			streamer = bbstreamer_lz4_compressor_new(streamer, compress);
+			streamer = astreamer_lz4_compressor_new(streamer, compress);
 		}
 		else if (compress->algorithm == PG_COMPRESSION_ZSTD)
 		{
 			strlcat(archive_filename, ".zst", sizeof(archive_filename));
-			streamer = bbstreamer_plain_writer_new(archive_filename,
+			streamer = astreamer_plain_writer_new(archive_filename,
 												   archive_file);
-			streamer = bbstreamer_zstd_compressor_new(streamer, compress);
+			streamer = astreamer_zstd_compressor_new(streamer, compress);
 		}
 		else
 		{
@@ -1222,7 +1222,7 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 		 * into it.
 		 */
 		if (must_parse_archive)
-			streamer = bbstreamer_tar_archiver_new(streamer);
+			streamer = astreamer_tar_archiver_new(streamer);
 		progress_update_filename(archive_filename);
 	}
 
@@ -1241,7 +1241,7 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 	if (spclocation == NULL && writerecoveryconf)
 	{
 		Assert(must_parse_archive);
-		streamer = bbstreamer_recovery_injector_new(streamer,
+		streamer = astreamer_recovery_injector_new(streamer,
 													is_recovery_guc_supported,
 													recoveryconfcontents);
 	}
@@ -1253,9 +1253,9 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 	 * we're talking to such a server we'll need to add the terminator here.
 	 */
 	if (must_parse_archive)
-		streamer = bbstreamer_tar_parser_new(streamer);
+		streamer = astreamer_tar_parser_new(streamer);
 	else if (expect_unterminated_tarfile)
-		streamer = bbstreamer_tar_terminator_new(streamer);
+		streamer = astreamer_tar_terminator_new(streamer);
 
 	/*
 	 * If the user has requested a server compressed archive along with
@@ -1264,11 +1264,11 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 	if (format == 'p')
 	{
 		if (is_tar_gz)
-			streamer = bbstreamer_gzip_decompressor_new(streamer);
+			streamer = astreamer_gzip_decompressor_new(streamer);
 		else if (is_tar_lz4)
-			streamer = bbstreamer_lz4_decompressor_new(streamer);
+			streamer = astreamer_lz4_decompressor_new(streamer);
 		else if (is_tar_zstd)
-			streamer = bbstreamer_zstd_decompressor_new(streamer);
+			streamer = astreamer_zstd_decompressor_new(streamer);
 	}
 
 	/* Return the results. */
@@ -1307,7 +1307,7 @@ ReceiveArchiveStream(PGconn *conn, pg_compress_specification *compress)
 	if (state.manifest_inject_streamer != NULL &&
 		state.manifest_buffer != NULL)
 	{
-		bbstreamer_inject_file(state.manifest_inject_streamer,
+		astreamer_inject_file(state.manifest_inject_streamer,
 							   "backup_manifest",
 							   state.manifest_buffer->data,
 							   state.manifest_buffer->len);
@@ -1318,8 +1318,8 @@ ReceiveArchiveStream(PGconn *conn, pg_compress_specification *compress)
 	/* If there's still an archive in progress, end processing. */
 	if (state.streamer != NULL)
 	{
-		bbstreamer_finalize(state.streamer);
-		bbstreamer_free(state.streamer);
+		astreamer_finalize(state.streamer);
+		astreamer_free(state.streamer);
 		state.streamer = NULL;
 	}
 }
@@ -1383,8 +1383,8 @@ ReceiveArchiveStreamChunk(size_t r, char *copybuf, void *callback_data)
 				/* End processing of any prior archive. */
 				if (state->streamer != NULL)
 				{
-					bbstreamer_finalize(state->streamer);
-					bbstreamer_free(state->streamer);
+					astreamer_finalize(state->streamer);
+					astreamer_free(state->streamer);
 					state->streamer = NULL;
 				}
 
@@ -1437,8 +1437,8 @@ ReceiveArchiveStreamChunk(size_t r, char *copybuf, void *callback_data)
 				else if (state->streamer != NULL)
 				{
 					/* Archive data. */
-					bbstreamer_content(state->streamer, NULL, copybuf + 1,
-									   r - 1, BBSTREAMER_UNKNOWN);
+					astreamer_content(state->streamer, NULL, copybuf + 1,
+									   r - 1, ASTREAMER_UNKNOWN);
 				}
 				else
 					pg_fatal("unexpected payload data");
@@ -1600,7 +1600,7 @@ ReceiveTarFile(PGconn *conn, char *archive_name, char *spclocation,
 			   bool tablespacenum, pg_compress_specification *compress)
 {
 	WriteTarState state;
-	bbstreamer *manifest_inject_streamer;
+	astreamer *manifest_inject_streamer;
 	bool		is_recovery_guc_supported;
 	bool		expect_unterminated_tarfile;
 
@@ -1636,7 +1636,7 @@ ReceiveTarFile(PGconn *conn, char *archive_name, char *spclocation,
 			pg_fatal("out of memory");
 
 		/* Inject it into the output tarfile. */
-		bbstreamer_inject_file(manifest_inject_streamer, "backup_manifest",
+		astreamer_inject_file(manifest_inject_streamer, "backup_manifest",
 							   buf.data, buf.len);
 
 		/* Free memory. */
@@ -1644,8 +1644,8 @@ ReceiveTarFile(PGconn *conn, char *archive_name, char *spclocation,
 	}
 
 	/* Cleanup. */
-	bbstreamer_finalize(state.streamer);
-	bbstreamer_free(state.streamer);
+	astreamer_finalize(state.streamer);
+	astreamer_free(state.streamer);
 
 	progress_report(tablespacenum, true, false);
 
@@ -1663,7 +1663,7 @@ ReceiveTarCopyChunk(size_t r, char *copybuf, void *callback_data)
 {
 	WriteTarState *state = callback_data;
 
-	bbstreamer_content(state->streamer, NULL, copybuf, r, BBSTREAMER_UNKNOWN);
+	astreamer_content(state->streamer, NULL, copybuf, r, ASTREAMER_UNKNOWN);
 
 	totaldone += r;
 	progress_report(state->tablespacenum, false, false);
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 3deb6113b80..f59f7acdb7e 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3309,19 +3309,19 @@ bbsink_shell
 bbsink_state
 bbsink_throttle
 bbsink_zstd
-bbstreamer
-bbstreamer_archive_context
-bbstreamer_extractor
-bbstreamer_gzip_decompressor
-bbstreamer_gzip_writer
-bbstreamer_lz4_frame
-bbstreamer_member
-bbstreamer_ops
-bbstreamer_plain_writer
-bbstreamer_recovery_injector
-bbstreamer_tar_archiver
-bbstreamer_tar_parser
-bbstreamer_zstd_frame
+astreamer
+astreamer_archive_context
+astreamer_extractor
+astreamer_gzip_decompressor
+astreamer_gzip_writer
+astreamer_lz4_frame
+astreamer_member
+astreamer_ops
+astreamer_plain_writer
+astreamer_recovery_injector
+astreamer_tar_archiver
+astreamer_tar_parser
+astreamer_zstd_frame
 bgworker_main_type
 bh_node_type
 binaryheap
-- 
2.18.0

From 335d17b88f26235a71ec4cb3b79cbc7b7b4bf22c Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 15:47:26 +0530
Subject: [PATCH v4 08/11] Refactor: split verify_control_file.

Separated the manifest entry verification code into a new function and
introduced the should_verify_control_data() macro, similar to
should_verify_checksum().

Note that should_verify_checksum() has been slightly modified to
include a NULL check for its argument, maintaining the same code
structure as should_verify_control_data().
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 44 +++++++++++------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 18 +++++++++-
 2 files changed, 37 insertions(+), 25 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 9479b439bd4..57cb037bb76 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,7 +18,6 @@
 #include <sys/stat.h>
 #include <time.h>
 
-#include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
@@ -61,8 +60,6 @@ static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
 							   char *relpath, char *fullpath);
-static void verify_control_file(const char *controlpath,
-								uint64 manifest_system_identifier);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -617,14 +614,20 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 	/* Check the backup manifest entry for this file. */
 	m = verify_manifest_entry(context, relpath, sb.st_size);
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0 &&
-		m != NULL && m->matched && !m->bad)
-		verify_control_file(fullpath, context->manifest->system_identifier);
+	/* Validate the manifest system identifier */
+	if (should_verify_control_data(context->manifest, m))
+	{
+		ControlFileData *control_file;
+		bool		crc_ok;
+
+		pg_log_debug("reading \"%s\"", fullpath);
+		control_file = get_controlfile_by_exact_path(fullpath, &crc_ok);
+
+		verify_control_data(control_file, fullpath, crc_ok,
+							context->manifest->system_identifier);
+		/* Release memory. */
+		pfree(control_file);
+	}
 }
 
 /*
@@ -673,18 +676,14 @@ verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
 }
 
 /*
- * Sanity check control file and validate system identifier against manifest
- * system identifier.
+ * Sanity check control file data and validate system identifier against
+ * manifest system identifier.
  */
-static void
-verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
+void
+verify_control_data(ControlFileData *control_file,
+					const char *controlpath, bool crc_ok,
+					uint64 manifest_system_identifier)
 {
-	ControlFileData *control_file;
-	bool		crc_ok;
-
-	pg_log_debug("reading \"%s\"", controlpath);
-	control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
-
 	/* Control file contents not meaningful if CRC is bad. */
 	if (!crc_ok)
 		report_fatal_error("%s: CRC is incorrect", controlpath);
@@ -700,9 +699,6 @@ verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
 						   controlpath,
 						   (unsigned long long) manifest_system_identifier,
 						   (unsigned long long) control_file->system_identifier);
-
-	/* Release memory. */
-	pfree(control_file);
 }
 
 /*
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 1bc5f7a6b4a..c88f71ff14b 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -16,6 +16,7 @@
 
 #include "common/controldata_utils.h"
 #include "common/hashfn_unstable.h"
+#include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 
@@ -44,7 +45,19 @@ typedef struct manifest_file
 } manifest_file;
 
 #define should_verify_checksum(m) \
-	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+	(((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+
+/*
+ * Validate the manifest system identifier against the control file; this
+ * feature is not available in manifest version 1.  This validation should be
+ * carried out only if the manifest entry validation is completed without any
+ * errors.
+ */
+#define should_verify_control_data(manifest, m) \
+	(((manifest)->version != 1) && \
+	 ((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (strcmp((m)->pathname, "global/pg_control") == 0))
 
 /*
  * Define a hash table which we can use to store information about the files
@@ -110,6 +123,9 @@ extern manifest_file *verify_manifest_entry(verifier_context *context,
 extern void verify_checksum(verifier_context *context, manifest_file *m,
 							pg_checksum_context *checksum_ctx,
 							uint8 *checksumbuf);
+extern void verify_control_data(ControlFileData *control_file,
+								const char *controlpath, bool crc_ok,
+								uint64 manifest_system_identifier);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From 816e29e50120b40729377336229714dc0462ec7c Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 16:45:55 +0530
Subject: [PATCH v4 07/11] Refactor: split verify_file_checksum() function.

Move the core functionality of verify_file_checksum to a new function
to reuse it instead of duplicating the code.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 18 ++++++++++++++++--
 src/bin/pg_verifybackup/pg_verifybackup.h |  3 +++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index b8db83ed2cd..9479b439bd4 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -779,7 +779,6 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	int			rc;
 	size_t		bytes_read = 0;
 	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
-	int			checksumlen;
 
 	/* Open the target file. */
 	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
@@ -845,8 +844,23 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 		return;
 	}
 
+	/* Do the final computation and verification. */
+	verify_checksum(context, m, &checksum_ctx, checksumbuf);
+}
+
+/*
+ * A helper function to finalize checksum computation and verify it against the
+ * backup manifest information.
+ */
+void
+verify_checksum(verifier_context *context, manifest_file *m,
+				pg_checksum_context *checksum_ctx, uint8 *checksumbuf)
+{
+	int			checksumlen;
+	const char *relpath = m->pathname;
+
 	/* Get the final checksum. */
-	checksumlen = pg_checksum_final(&checksum_ctx, checksumbuf);
+	checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
 	if (checksumlen < 0)
 	{
 		report_backup_error(context,
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 98c75916255..1bc5f7a6b4a 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -107,6 +107,9 @@ typedef struct verifier_context
 
 extern manifest_file *verify_manifest_entry(verifier_context *context,
 											char *relpath, int64 filesize);
+extern void verify_checksum(verifier_context *context, manifest_file *m,
+							pg_checksum_context *checksum_ctx,
+							uint8 *checksumbuf);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From 694a28770bfc2dfb90b14442be081bdbc54d4987 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 10:26:35 +0530
Subject: [PATCH v4 09/11] pg_verifybackup: Add backup format and compression
 option

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the next patch that implements tar format support.
----
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 143 +++++++++++++++++++++-
 1 file changed, 141 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 57cb037bb76..2ca6b51741f 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,6 +18,7 @@
 #include <sys/stat.h>
 #include <time.h>
 
+#include "common/compression.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
@@ -56,6 +57,8 @@ static void report_manifest_error(JsonManifestParseContext *context,
 								  const char *fmt,...)
 			pg_attribute_printf(2, 3) pg_attribute_noreturn();
 
+static char find_backup_format(verifier_context *context);
+static pg_compress_algorithm find_backup_compression(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
@@ -73,6 +76,9 @@ static void usage(void);
 
 static const char *progname;
 
+char		format = '\0';		/* p(lain)/t(ar) */
+pg_compress_algorithm compress_algorithm = PG_COMPRESSION_NONE;
+
 /*
  * Main entry point.
  */
@@ -83,11 +89,13 @@ main(int argc, char **argv)
 		{"exit-on-error", no_argument, NULL, 'e'},
 		{"ignore", required_argument, NULL, 'i'},
 		{"manifest-path", required_argument, NULL, 'm'},
+		{"format", required_argument, NULL, 'F'},
 		{"no-parse-wal", no_argument, NULL, 'n'},
 		{"progress", no_argument, NULL, 'P'},
 		{"quiet", no_argument, NULL, 'q'},
 		{"skip-checksums", no_argument, NULL, 's'},
 		{"wal-directory", required_argument, NULL, 'w'},
+		{"compress", required_argument, NULL, 'Z'},
 		{NULL, 0, NULL, 0}
 	};
 
@@ -98,6 +106,7 @@ main(int argc, char **argv)
 	bool		quiet = false;
 	char	   *wal_directory = NULL;
 	char	   *pg_waldump_path = NULL;
+	bool		tar_compression_specified = false;
 
 	pg_logging_init(argv[0]);
 	set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_verifybackup"));
@@ -140,7 +149,7 @@ main(int argc, char **argv)
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
 
-	while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1)
+	while ((c = getopt_long(argc, argv, "eF:i:m:nPqsw:Z:", long_options, NULL)) != -1)
 	{
 		switch (c)
 		{
@@ -159,6 +168,15 @@ main(int argc, char **argv)
 				manifest_path = pstrdup(optarg);
 				canonicalize_path(manifest_path);
 				break;
+			case 'F':
+				if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
+					format = 'p';
+				else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
+					format = 't';
+				else
+					pg_fatal("invalid backup format \"%s\", must be \"plain\" or \"tar\"",
+							 optarg);
+				break;
 			case 'n':
 				no_parse_wal = true;
 				break;
@@ -175,6 +193,12 @@ main(int argc, char **argv)
 				wal_directory = pstrdup(optarg);
 				canonicalize_path(wal_directory);
 				break;
+			case 'Z':
+				if (!parse_compress_algorithm(optarg, &compress_algorithm))
+					pg_fatal("unrecognized compression algorithm: \"%s\"",
+							 optarg);
+				tar_compression_specified = true;
+				break;
 			default:
 				/* getopt_long already emitted a complaint */
 				pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -206,11 +230,41 @@ main(int argc, char **argv)
 		pg_fatal("cannot specify both %s and %s",
 				 "-P/--progress", "-q/--quiet");
 
+	/* Complain if compression method specified but the format isn't tar. */
+	if (format != 't' && tar_compression_specified)
+	{
+		pg_log_error("only tar mode backups can be compressed");
+		pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+		exit(1);
+	}
+
+	/* Determine the backup format if it hasn't been specified. */
+	if (format == '\0')
+		format = find_backup_format(&context);
+
+	/*
+	 * Determine the tar backup compression method if it hasn't been
+	 * specified.
+	 */
+	if (format == 't' && !tar_compression_specified)
+		compress_algorithm = find_backup_compression(&context);
+
 	/* Unless --no-parse-wal was specified, we will need pg_waldump. */
 	if (!no_parse_wal)
 	{
 		int			ret;
 
+		/*
+		 * XXX: In the future, we should consider enhancing pg_waldump to read
+		 * WAL files from the tar archive.
+		 */
+		if (format != 'p')
+		{
+			pg_log_error("pg_waldump does not support parsing WAL files from a tar archive.");
+			pg_log_error_hint("Try \"%s --help\" to skip parse WAL files option.", progname);
+			exit(1);
+		}
+
 		pg_waldump_path = pg_malloc(MAXPGPATH);
 		ret = find_other_exec(argv[0], "pg_waldump",
 							  "pg_waldump (PostgreSQL) " PG_VERSION "\n",
@@ -265,8 +319,13 @@ main(int argc, char **argv)
 	/*
 	 * Now do the expensive work of verifying file checksums, unless we were
 	 * told to skip it.
+	 *
+	 * We were only checking the plain backup here. For the tar backup, file
+	 * checksums verification (if requested) will be done immediately when the
+	 * file is accessed, as we don't have random access to the files like we
+	 * do with plain backups.
 	 */
-	if (!context.skip_checksums)
+	if (!context.skip_checksums && format == 'p')
 		verify_backup_checksums(&context);
 
 	/*
@@ -1013,6 +1072,84 @@ progress_report(verifier_context *context, bool finished)
 	fputc((!finished && isatty(fileno(stderr))) ? '\r' : '\n', stderr);
 }
 
+/*
+ * To detect the backup format, it checks for the PG_VERSION file in the backup
+ * directory. If found, it will be considered a plain-format backup; otherwise,
+ * it will be assumed to be a tar-format backup.
+ */
+static char
+find_backup_format(verifier_context *context)
+{
+	char		result;
+	char	   *path;
+	struct stat sb;
+
+	/* Should be here only if the backup format is unknown */
+	Assert(format == '\0');
+
+	/* Check PG_VERSION file. */
+	path = psprintf("%s/%s", context->backup_directory, "PG_VERSION");
+	result = (stat(path, &sb) == 0) ? 'p' : 't';
+	pfree(path);
+
+	return result;
+}
+
+/*
+ * To determine the compression format, we will search for the main data
+ * directory archive and its extension, which starts with base.tar, as
+ * pg_basebackup writes the main data directory to an archive file named
+ * base.tar followed by a compression type extension like .gz, .lz4, or .zst.
+ */
+static pg_compress_algorithm
+find_backup_compression(verifier_context *context)
+{
+	char	   *path;
+	struct stat sb;
+	bool		found;
+
+	/* Should be here only for tar backup */
+	Assert(format == 't');
+
+	/*
+	 * Is this a tar archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_NONE;
+
+	/*
+	 * Is this a .tar.gz archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar.gz");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_GZIP;
+
+	/*
+	 * Is this a .tar.lz4 archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar.lz4");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_LZ4;
+
+	/*
+	 * Is this a .tar.zst archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar.zst");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_ZSTD;
+
+	return PG_COMPRESSION_NONE; /* placate compiler */
+}
+
 /*
  * Print out usage information and exit.
  */
@@ -1025,11 +1162,13 @@ usage(void)
 	printf(_("  -e, --exit-on-error         exit immediately on error\n"));
 	printf(_("  -i, --ignore=RELATIVE_PATH  ignore indicated path\n"));
 	printf(_("  -m, --manifest-path=PATH    use specified path for manifest\n"));
+	printf(_("  -F, --format=p|t            backup format (plain, tar)\n"));
 	printf(_("  -n, --no-parse-wal          do not try to parse WAL files\n"));
 	printf(_("  -P, --progress              show progress information\n"));
 	printf(_("  -q, --quiet                 do not print any output, except for errors\n"));
 	printf(_("  -s, --skip-checksums        skip checksum verification\n"));
 	printf(_("  -w, --wal-directory=PATH    use specified path for WAL files\n"));
+	printf(_("  -Z, --compress=METHOD       compress method (gzip, lz4, zstd, none) \n"));
 	printf(_("  -V, --version               output version information, then exit\n"));
 	printf(_("  -?, --help                  show this help, then exit\n"));
 	printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
-- 
2.18.0

From f3f176fe4f0879e9295b07546b9449471d7e4030 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 31 Jul 2024 16:22:07 +0530
Subject: [PATCH v4 10/11] pg_verifybackup: Read tar files and verify its
 contents

---
 src/bin/pg_verifybackup/Makefile           |   4 +-
 src/bin/pg_verifybackup/astreamer_verify.c | 342 +++++++++++++++++++++
 src/bin/pg_verifybackup/meson.build        |   6 +-
 src/bin/pg_verifybackup/pg_verifybackup.c  | 216 ++++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h  |   9 +
 src/tools/pgindent/typedefs.list           |   1 +
 6 files changed, 569 insertions(+), 9 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/astreamer_verify.c

diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile
index 7c045f142e8..df7aaabd530 100644
--- a/src/bin/pg_verifybackup/Makefile
+++ b/src/bin/pg_verifybackup/Makefile
@@ -17,11 +17,13 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 # We need libpq only because fe_utils does.
+override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 
 OBJS = \
 	$(WIN32RES) \
-	pg_verifybackup.o
+	pg_verifybackup.o \
+	astreamer_verify.o
 
 all: pg_verifybackup
 
diff --git a/src/bin/pg_verifybackup/astreamer_verify.c b/src/bin/pg_verifybackup/astreamer_verify.c
new file mode 100644
index 00000000000..55d352e1754
--- /dev/null
+++ b/src/bin/pg_verifybackup/astreamer_verify.c
@@ -0,0 +1,342 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer_verify.c
+ *
+ * Extend fe_utils/astreamer.h archive streaming facility to verify TAR
+ * backup.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * src/bin/pg_verifybackup/astreamer_verify.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include "common/logging.h"
+#include "fe_utils/astreamer.h"
+#include "pg_verifybackup.h"
+
+typedef struct astreamer_verify
+{
+	astreamer	base;
+	verifier_context *context;
+	char	   *archiveName;
+	Oid			tblspcOid;
+
+	/*
+	 * Hold information for a member file verification that needs to be reset at
+	 * the end of each file.
+	 */
+	manifest_file *mfile;
+	int64		receivedBytes;
+	bool		verifyChecksums;
+	bool		verifyControlData;
+	pg_checksum_context *checksum_ctx;
+} astreamer_verify;
+
+static void astreamer_verify_content(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len,
+									 astreamer_archive_context context);
+static void astreamer_verify_finalize(astreamer *streamer);
+static void astreamer_verify_free(astreamer *streamer);
+
+static void verify_member_file(astreamer *streamer, astreamer_member *member);
+static void verify_content_checksum(astreamer *streamer,
+									astreamer_member *member,
+									const char *buffer, int buffer_len);
+static void verify_controldata(astreamer *streamer,
+							   astreamer_member *member,
+							   const char *data, int len);
+static void reset_member_info(astreamer *streamer);
+
+static const astreamer_ops astreamer_verify_ops = {
+	.content = astreamer_verify_content,
+	.finalize = astreamer_verify_finalize,
+	.free = astreamer_verify_free
+};
+
+/*
+ * Create a astreamer that can verifies content of a TAR file.
+ */
+astreamer *
+astreamer_verify_content_new(astreamer *next, verifier_context *context,
+							 char *archive_name, Oid tblspc_oid)
+{
+	astreamer_verify *streamer;
+
+	streamer = palloc0(sizeof(astreamer_verify));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_verify_ops;
+
+	streamer->base.bbs_next = next;
+	streamer->context = context;
+	streamer->archiveName = archive_name;
+	streamer->tblspcOid = tblspc_oid;
+	initStringInfo(&streamer->base.bbs_buffer);
+
+	return &streamer->base;
+}
+
+/*
+ * It verifies each TAR member entry against the manifest data and performs
+ * checksum verification if enabled. Additionally, it validates the backup's
+ * system identifier against the backup_manifest.
+ */
+static void
+astreamer_verify_content(astreamer *streamer,
+						 astreamer_member *member, const char *data,
+						 int len, astreamer_archive_context context)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	Assert(context != ASTREAMER_UNKNOWN);
+
+	switch (context)
+	{
+		case ASTREAMER_MEMBER_HEADER:
+			if (!member->is_directory && !member->is_link &&
+				!should_ignore_relpath(mystreamer->context, member->pathname))
+				verify_member_file(streamer, member);
+			break;
+
+		case ASTREAMER_MEMBER_CONTENTS:
+
+			/*
+			 * Perform checksum verification as the file content becomes
+			 * available, since the TAR format does not have random access to
+			 * files like a normal backup directory, where checksum
+			 * verification occurs at different points.
+			 */
+			if (mystreamer->verifyChecksums)
+				verify_content_checksum(streamer, member, data, len);
+
+			/* Verify pg_control file information */
+			if (mystreamer->verifyControlData)
+				verify_controldata(streamer, member, data, len);
+			break;
+
+		case ASTREAMER_MEMBER_TRAILER:
+			reset_member_info(streamer);
+			break;
+
+		case ASTREAMER_ARCHIVE_TRAILER:
+			break;
+
+		default:
+			/* Shouldn't happen. */
+			pg_fatal("unexpected state while parsing tar archive");
+	}
+}
+
+/*
+ * End-of-stream processing for a astreamer_verify stream.
+ */
+static void
+astreamer_verify_finalize(astreamer *streamer)
+{
+	Assert(streamer->bbs_next == NULL);
+}
+
+/*
+ * Free memory associated with a astreamer_verify stream.
+ */
+static void
+astreamer_verify_free(astreamer *streamer)
+{
+	pfree(streamer->bbs_buffer.data);
+	pfree(streamer);
+}
+
+/*
+ * Verifies the member file entry against the backup manifest. If the archive
+ * being processed is a tablespace, the required file path is prepared for
+ * subsequent operations.
+ */
+static void
+verify_member_file(astreamer *streamer, astreamer_member *member)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_file *m;
+
+	/*
+	 * The backup_manifest stores a relative path to the base
+	 * directory for files belong tablespace, whereas
+	 * <tablespaceoid>.tar doesn't. Prepare the required path,
+	 * otherwise, the manfiest entry verification will fail.
+	 */
+	if (OidIsValid(mystreamer->tblspcOid))
+	{
+		char		temp[MAXPGPATH];
+
+		/* Copy original name at temporary space */
+		memcpy(temp, member->pathname, MAXPGPATH);
+
+		snprintf(member->pathname, MAXPGPATH, "%s/%d/%s",
+				 "pg_tblspc", mystreamer->tblspcOid, temp);
+	}
+
+	/* Check the manifest entry */
+	m = verify_manifest_entry(mystreamer->context, member->pathname,
+							  member->size);
+	mystreamer->mfile = (void *) m;
+
+	/*
+	 * Prepare for checksum and manifest system identifier
+	 * verification.
+	 *
+	 * We could have these checks while receiving contents.
+	 * However, since contents are received in multiple
+	 * iterations, this would result in these lengthy checks being
+	 * performed multiple times. Instead, having a single flag
+	 * would be more efficient.
+	 */
+	mystreamer->verifyChecksums =
+		(!mystreamer->context->skip_checksums && should_verify_checksum(m));
+	mystreamer->verifyControlData =
+		should_verify_control_data(mystreamer->context->manifest, m);
+}
+
+/*
+ * Similar to verify_file_checksum() in terms of implementation and code, which
+ * computes the checksum for a single file, this function computes the checksum
+ * incrementally for the received file content. On the first call for a file, it
+ * initializes checksum_ctx, which will be used in subsequent calls. Once the
+ * complete file content is received, tracked using the receivedBytes parameter,
+ * it verifies the checksum against the manifest data.
+ */
+static void
+verify_content_checksum(astreamer *streamer, astreamer_member *member,
+						const char *buffer, int buffer_len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	pg_checksum_context *checksum_ctx = mystreamer->checksum_ctx;
+	verifier_context *context = mystreamer->context;
+	manifest_file *m = mystreamer->mfile;
+	const char *relpath = m->pathname;
+	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
+
+	/*
+	 * Mark it false to avoid unexpected re-entrance for the same file content
+	 * (e.g. returned in error should not be revisited).
+	 */
+	Assert(mystreamer->verifyChecksums);
+	mystreamer->verifyChecksums = false;
+
+	/* Should have came for the right file */
+	Assert(strcmp(member->pathname, relpath) == 0);
+
+	/* If we were first time for this file */
+	if (!checksum_ctx)
+	{
+		checksum_ctx = pg_malloc(sizeof(pg_checksum_context));
+		mystreamer->checksum_ctx = checksum_ctx;
+
+		if (pg_checksum_init(checksum_ctx, m->checksum_type) < 0)
+		{
+			report_backup_error(context,
+								"%s: could not initialize checksum of file \"%s\"",
+								mystreamer->archiveName, relpath);
+			return;
+		}
+	}
+
+	/* Update the total count of computed checksum bytes. */
+	mystreamer->receivedBytes += buffer_len;
+
+	if (pg_checksum_update(checksum_ctx, (uint8 *) buffer, buffer_len) < 0)
+	{
+		report_backup_error(context, "could not update checksum of file \"%s\"",
+							relpath);
+		return;
+	}
+
+	/* Report progress */
+	context->done_size += buffer_len;
+	progress_report(context, false);
+
+	/* Yet to receive the full content of the file. */
+	if (mystreamer->receivedBytes < m->size)
+	{
+		mystreamer->verifyChecksums = true;
+		return;
+	}
+
+	/* Do the final computation and verification. */
+	verify_checksum(context, m, checksum_ctx, checksumbuf);
+}
+
+/*
+ * Prepare the control data from the received file contents, which are supposed
+ * to be from the pg_control file, including CRC calculation. Then, call the
+ * routines that perform the final verification of the control file information.
+ */
+static void
+verify_controldata(astreamer *streamer, astreamer_member *member,
+				   const char *data, int len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_data *manifest = mystreamer->context->manifest;
+	ControlFileData control_file;
+	pg_crc32c	crc;
+	bool		crc_ok;
+
+	/* Should be here only for control file */
+	Assert(strcmp(member->pathname, "global/pg_control") == 0);
+	Assert(manifest->version != 1);
+
+	/* Mark it as false to avoid unexpected re-entrance */
+	Assert(mystreamer->verifyControlData);
+	mystreamer->verifyControlData = false;
+
+	/* Should have whole control file data. */
+	if (!astreamer_buffer_until(streamer, &data, &len, sizeof(ControlFileData)))
+	{
+		mystreamer->verifyControlData = true;
+		return;
+	}
+
+	pg_log_debug("%s: reading \"%s\"", mystreamer->archiveName,
+				 member->pathname);
+
+	if (streamer->bbs_buffer.len != sizeof(ControlFileData))
+		report_fatal_error("%s: could not read control file: read %d of %zu",
+						   mystreamer->archiveName, streamer->bbs_buffer.len,
+						   sizeof(ControlFileData));
+
+	memcpy(&control_file, streamer->bbs_buffer.data,
+		   sizeof(ControlFileData));
+
+	/* Check the CRC. */
+	INIT_CRC32C(crc);
+	COMP_CRC32C(crc,
+				(char *) (&control_file),
+				offsetof(ControlFileData, crc));
+	FIN_CRC32C(crc);
+
+	crc_ok = EQ_CRC32C(crc, control_file.crc);
+
+	/* Do the final control data verification. */
+	verify_control_data(&control_file, member->pathname, crc_ok,
+						manifest->system_identifier);
+}
+
+/*
+ * Reset flags and free memory allocations for member file verification.
+ */
+static void
+reset_member_info(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	mystreamer->mfile = NULL;
+	mystreamer->receivedBytes = 0;
+	mystreamer->verifyChecksums = false;
+	mystreamer->verifyControlData = false;
+
+	if (mystreamer->checksum_ctx)
+		pfree(mystreamer->checksum_ctx);
+	mystreamer->checksum_ctx = NULL;
+}
diff --git a/src/bin/pg_verifybackup/meson.build b/src/bin/pg_verifybackup/meson.build
index 7c7d31a0350..1e3fcf7ee5a 100644
--- a/src/bin/pg_verifybackup/meson.build
+++ b/src/bin/pg_verifybackup/meson.build
@@ -1,7 +1,8 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 pg_verifybackup_sources = files(
-  'pg_verifybackup.c'
+  'pg_verifybackup.c',
+  'astreamer_verify.c'
 )
 
 if host_system == 'windows'
@@ -10,9 +11,10 @@ if host_system == 'windows'
     '--FILEDESC', 'pg_verifybackup - verify a backup against using a backup manifest'])
 endif
 
+pg_verifybackup_deps = [frontend_code, libpq, lz4, zlib, zstd]
 pg_verifybackup = executable('pg_verifybackup',
   pg_verifybackup_sources,
-  dependencies: [frontend_code, libpq],
+  dependencies: pg_verifybackup_deps,
   kwargs: default_bin_args,
 )
 bin_targets += pg_verifybackup
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 2ca6b51741f..39f2e524a3e 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -20,9 +20,12 @@
 
 #include "common/compression.h"
 #include "common/parse_manifest.h"
+#include "common/relpath.h"
+#include "fe_utils/astreamer.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
 #include "pg_verifybackup.h"
+#include "pgtar.h"
 #include "pgtime.h"
 
 /*
@@ -39,6 +42,11 @@
  */
 #define ESTIMATED_BYTES_PER_MANIFEST_LINE	100
 
+
+static void (*verify_backup_file_cb) (verifier_context *context,
+									  char *relpath, char *fullpath,
+									  size_t filesize);
+
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -63,6 +71,15 @@ static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
 							   char *relpath, char *fullpath);
+static void verify_plain_file_cb(verifier_context *context,
+								 char *relpath, char *fullpath,
+								 size_t filesize);
+static void verify_tar_file_cb(verifier_context *context,
+							   char *relpath, char *fullpath,
+							   size_t filesize);
+static void verify_tar_content(verifier_context *context,
+							   char *relpath, char *fullpath,
+							   astreamer *streamer);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -71,6 +88,9 @@ static void verify_file_checksum(verifier_context *context,
 static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
+static astreamer *create_archive_verifier(verifier_context *context,
+										  char *archive_name,
+										  Oid tblspc_oid);
 
 static void usage(void);
 
@@ -145,6 +165,10 @@ main(int argc, char **argv)
 	 */
 	simple_string_list_append(&context.ignore_list, "backup_manifest");
 	simple_string_list_append(&context.ignore_list, "pg_wal");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.gz");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.lz4");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.zst");
 	simple_string_list_append(&context.ignore_list, "postgresql.auto.conf");
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
@@ -249,6 +273,15 @@ main(int argc, char **argv)
 	if (format == 't' && !tar_compression_specified)
 		compress_algorithm = find_backup_compression(&context);
 
+	/*
+	 * Setup the required callback function to verify plain or tar backup
+	 * files.
+	 */
+	if (format == 'p')
+		verify_backup_file_cb = verify_plain_file_cb;
+	else
+		verify_backup_file_cb = verify_tar_file_cb;
+
 	/* Unless --no-parse-wal was specified, we will need pg_waldump. */
 	if (!no_parse_wal)
 	{
@@ -628,7 +661,8 @@ verify_backup_directory(verifier_context *context, char *relpath,
 }
 
 /*
- * Verify one file (which might actually be a directory or a symlink).
+ * Verify one file (which might actually be a directory, a symlink or a
+ * archive).
  *
  * The arguments to this function have the same meaning as the arguments to
  * verify_backup_directory.
@@ -637,7 +671,6 @@ static void
 verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 {
 	struct stat sb;
-	manifest_file *m;
 
 	if (stat(fullpath, &sb) != 0)
 	{
@@ -670,8 +703,25 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		return;
 	}
 
+	/* Do the further verifications */
+	verify_backup_file_cb(context, relpath, fullpath, sb.st_size);
+}
+
+/*
+ * Verify one plan file or a symlink.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_backup_directory. The additional argument is the file size for
+ * verifying against manifest entry.
+ */
+static void
+verify_plain_file_cb(verifier_context *context, char *relpath,
+					 char *fullpath, size_t filesize)
+{
+	manifest_file *m;
+
 	/* Check the backup manifest entry for this file. */
-	m = verify_manifest_entry(context, relpath, sb.st_size);
+	m = verify_manifest_entry(context, relpath, filesize);
 
 	/* Validate the manifest system identifier */
 	if (should_verify_control_data(context->manifest, m))
@@ -689,6 +739,124 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 	}
 }
 
+/*
+ * Verify one tar file.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_backup_directory. The additional argument is the file size for
+ * verifying against manifest entry.
+ */
+static void
+verify_tar_file_cb(verifier_context *context, char *relpath,
+				   char *fullpath, size_t filesize)
+{
+	astreamer  *streamer;
+	Oid			tblspc_oid = InvalidOid;
+	int			file_name_len;
+	int			file_extn_len = 0;	/* placate compiler */
+	char	   *file_extn = "";
+
+	/* Should be tar backup */
+	Assert(format == 't');
+
+	/* Find the tar file extension. */
+	if (compress_algorithm == PG_COMPRESSION_NONE)
+	{
+		file_extn = ".tar";
+		file_extn_len = 4;
+
+	}
+	else if (compress_algorithm == PG_COMPRESSION_GZIP)
+	{
+		file_extn = ".tar.gz";
+		file_extn_len = 7;
+
+	}
+	else if (compress_algorithm == PG_COMPRESSION_LZ4)
+	{
+		file_extn = ".tar.lz4";
+		file_extn_len = 8;
+	}
+	else if (compress_algorithm == PG_COMPRESSION_ZSTD)
+	{
+		file_extn = ".tar.zst";
+		file_extn_len = 8;
+	}
+
+	/*
+	 * Ensure that we have the correct file type corresponding to the backup
+	 * format.
+	 */
+	file_name_len = strlen(relpath);
+	if (file_name_len < file_extn_len ||
+		strcmp(relpath + file_name_len - file_extn_len, file_extn) != 0)
+	{
+		if (compress_algorithm == PG_COMPRESSION_NONE)
+			report_backup_error(context,
+								"\"%s\" is not a valid file, expecting tar file",
+								relpath);
+		else
+			report_backup_error(context,
+								"\"%s\" is not a valid file, expecting \"%s\" compressed tar file",
+								relpath,
+								get_compress_algorithm_name(compress_algorithm));
+		return;
+	}
+
+	/*
+	 * For the tablespace, pg_basebackup writes the data out to
+	 * <tablespaceoid>.tar.  If a file matches that format, then extract the
+	 * tablespaceoid, which we need to prepare the paths of the files
+	 * belonging to that tablespace relative to the base directory.
+	 */
+	if (strspn(relpath, "0123456789") == (file_name_len - file_extn_len))
+		tblspc_oid = strtoi64(relpath, NULL, 10);
+
+	streamer = create_archive_verifier(context, relpath, tblspc_oid);
+	verify_tar_content(context, relpath, fullpath, streamer);
+
+	/* Cleanup. */
+	astreamer_finalize(streamer);
+	astreamer_free(streamer);
+}
+
+/*
+ * Reads a given tar file in predefined chunks and pass to astreamer.  Which
+ * initiates routines for decompression (if necessary) then verification
+ * of each member within the tar archive.
+ */
+static void
+verify_tar_content(verifier_context *context, char *relpath, char *fullpath,
+				   astreamer *streamer)
+{
+	int			fd;
+	int			rc;
+	char	   *buffer;
+
+	/* Open the target file. */
+	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
+	{
+		report_backup_error(context, "could not open file \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
+
+	/* Perform the reads */
+	while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
+		astreamer_content(streamer, NULL, buffer, rc, ASTREAMER_UNKNOWN);
+
+	if (rc < 0)
+		report_backup_error(context, "could not read file \"%s\": %m",
+							relpath);
+
+	/* Close the file. */
+	if (close(fd) != 0)
+		report_backup_error(context, "could not close file \"%s\": %m",
+							relpath);
+}
+
 /*
  * Verify file and its size entry in the manifest.
  */
@@ -1096,10 +1264,10 @@ find_backup_format(verifier_context *context)
 }
 
 /*
- * To determine the compression format, we will search for the main data
- * directory archive and its extension, which starts with base.tar, as
  * pg_basebackup writes the main data directory to an archive file named
- * base.tar followed by a compression type extension like .gz, .lz4, or .zst.
+ * base.tar, followed by a compression type extension such as .gz, .lz4, or
+ * .zst. To determine the compression format, we need to search for this main
+ * data directory archive file.
  */
 static pg_compress_algorithm
 find_backup_compression(verifier_context *context)
@@ -1150,6 +1318,42 @@ find_backup_compression(verifier_context *context)
 	return PG_COMPRESSION_NONE; /* placate compiler */
 }
 
+/*
+ * Identifies the necessary steps for verifying the contents of the
+ * provided tar file.
+ */
+static astreamer *
+create_archive_verifier(verifier_context *context, char *archive_name,
+						Oid tblspc_oid)
+{
+	astreamer  *streamer = NULL;
+
+	/* Should be here only for tar backup */
+	Assert(format == 't');
+
+	/*
+	 * To verify the contents of the tar file, the initial step is to parse
+	 * its content.
+	 */
+	streamer = astreamer_verify_content_new(streamer, context, archive_name,
+											tblspc_oid);
+	streamer = astreamer_tar_parser_new(streamer);
+
+	/*
+	 * If the tar file is compressed, we must perform the appropriate
+	 * decompression operation before proceeding with the verification of its
+	 * contents.
+	 */
+	if (compress_algorithm == PG_COMPRESSION_GZIP)
+		streamer = astreamer_gzip_decompressor_new(streamer);
+	else if (compress_algorithm == PG_COMPRESSION_LZ4)
+		streamer = astreamer_lz4_decompressor_new(streamer);
+	else if (compress_algorithm == PG_COMPRESSION_ZSTD)
+		streamer = astreamer_zstd_decompressor_new(streamer);
+
+	return streamer;
+}
+
 /*
  * Print out usage information and exit.
  */
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index c88f71ff14b..f0a7c8918fb 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -137,4 +137,13 @@ extern bool should_ignore_relpath(verifier_context *context,
 
 extern void progress_report(verifier_context *context, bool finished);
 
+/* Forward declarations to avoid fe_utils/astreamer.h include. */
+struct astreamer;
+typedef struct astreamer astreamer;
+
+extern astreamer *astreamer_verify_content_new(astreamer *next,
+											   verifier_context *context,
+											   char *archive_name,
+											   Oid tblspc_oid);
+
 #endif							/* PG_VERIFYBACKUP_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index ba9e0200b3f..8c708b02cc2 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3323,6 +3323,7 @@ astreamer_plain_writer
 astreamer_recovery_injector
 astreamer_tar_archiver
 astreamer_tar_parser
+astreamer_verify
 astreamer_zstd_frame
 bgworker_main_type
 bh_node_type
-- 
2.18.0

From b531a41a2e53617c5ec539951218ac7f0e45233b Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 17:04:56 +0530
Subject: [PATCH v4 11/11] pg_verifybackup: Tests and document

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the previous patch that implements tar format support.
----
---
 doc/src/sgml/ref/pg_verifybackup.sgml         | 54 +++++++++++++-
 src/bin/pg_verifybackup/t/001_basic.pl        | 18 ++++-
 src/bin/pg_verifybackup/t/004_options.pl      |  2 +-
 src/bin/pg_verifybackup/t/005_bad_manifest.pl |  2 +-
 src/bin/pg_verifybackup/t/008_untar.pl        | 73 ++++++-------------
 src/bin/pg_verifybackup/t/010_client_untar.pl | 48 +-----------
 6 files changed, 96 insertions(+), 101 deletions(-)

diff --git a/doc/src/sgml/ref/pg_verifybackup.sgml b/doc/src/sgml/ref/pg_verifybackup.sgml
index a3f167f9f6e..c743bd89a92 100644
--- a/doc/src/sgml/ref/pg_verifybackup.sgml
+++ b/doc/src/sgml/ref/pg_verifybackup.sgml
@@ -34,8 +34,10 @@ PostgreSQL documentation
    integrity of a database cluster backup taken using
    <command>pg_basebackup</command> against a
    <literal>backup_manifest</literal> generated by the server at the time
-   of the backup.  The backup must be stored in the "plain"
-   format; a "tar" format backup can be checked after extracting it.
+   of the backup.  The backup must be stored in the "plain" or "tar"
+   format.  Verification is supported for <literal>gzip</literal>,
+   <literal>lz4</literal>, and  <literal>zstd</literal> compressed tar backup;
+   any other compressed format backups can be checked after decompressing them.
   </para>
 
   <para>
@@ -168,6 +170,42 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-F <replaceable class="parameter">format</replaceable></option></term>
+      <term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
+      <listitem>
+       <para>
+        Specifies the format of the backup. <replaceable>format</replaceable>
+        can be one of the following:
+
+        <variablelist>
+         <varlistentry>
+          <term><literal>p</literal></term>
+          <term><literal>plain</literal></term>
+          <listitem>
+           <para>
+            Backup consists of plain files with the same layout as the
+            source server's data directory and tablespaces.
+           </para>
+          </listitem>
+         </varlistentry>
+
+         <varlistentry>
+          <term><literal>t</literal></term>
+          <term><literal>tar</literal></term>
+          <listitem>
+           <para>
+            Backup consists of tar files. The main data directory's contents
+            will be written to a file named <filename>base.tar</filename>,
+            and each other tablespace will be written to a separate tar file
+            named after that tablespace's OID.
+           </para>
+           </listitem>
+         </varlistentry>
+        </variablelist></para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-n</option></term>
       <term><option>--no-parse-wal</option></term>
@@ -227,6 +265,18 @@ PostgreSQL documentation
        </para>
       </listitem>
      </varlistentry>
+
+     <varlistentry>
+      <term><option>-Z <replaceable class="parameter">method</replaceable></option></term>
+      <term><option>--compress=<replaceable class="parameter">method</replaceable></option></term>
+      <listitem>
+       <para>
+        The tar backup compression method can be <literal>gzip</literal>,
+        <literal>lz4</literal>, <literal>zstd</literal>, or
+        <literal>none</literal> if no compression.
+       </para>
+      </listitem>
+     </varlistentry>
     </variablelist>
    </para>
 
diff --git a/src/bin/pg_verifybackup/t/001_basic.pl b/src/bin/pg_verifybackup/t/001_basic.pl
index 2f3e52d296f..d47ce1f04fc 100644
--- a/src/bin/pg_verifybackup/t/001_basic.pl
+++ b/src/bin/pg_verifybackup/t/001_basic.pl
@@ -17,13 +17,25 @@ command_fails_like(
 	qr/no backup directory specified/,
 	'target directory must be specified');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir ],
 	qr/could not open file.*\/backup_manifest\"/,
 	'pg_verifybackup requires a manifest');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir, $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir, $tempdir ],
 	qr/too many command-line arguments/,
 	'multiple target directories not allowed');
+command_fails_like(
+	[ 'pg_verifybackup', '-Zgzip', $tempdir ],
+	qr/only tar mode backups can be compressed/,
+	'compression method required format option');
+command_fails_like(
+	[ 'pg_verifybackup', '-Fp', '-Zlz4', $tempdir ],
+	qr/only tar mode backups can be compressed/,
+	'compression method required tar format option');
+command_fails_like(
+	[ 'pg_verifybackup', '-Fp', '-Znon_exist', $tempdir ],
+	qr/unrecognized compression algorithm/,
+	'compression method should be valid');
 
 # create fake manifest file
 open(my $fh, '>', "$tempdir/backup_manifest") || die "open: $!";
@@ -31,7 +43,7 @@ close($fh);
 
 # but then try to use an alternate, nonexisting manifest
 command_fails_like(
-	[ 'pg_verifybackup', '-m', "$tempdir/not_the_manifest", $tempdir ],
+	[ 'pg_verifybackup', '-Fp', '-m', "$tempdir/not_the_manifest", $tempdir ],
 	qr/could not open file.*\/not_the_manifest\"/,
 	'pg_verifybackup respects -m flag');
 
diff --git a/src/bin/pg_verifybackup/t/004_options.pl b/src/bin/pg_verifybackup/t/004_options.pl
index 8ed2214408e..2f197648740 100644
--- a/src/bin/pg_verifybackup/t/004_options.pl
+++ b/src/bin/pg_verifybackup/t/004_options.pl
@@ -108,7 +108,7 @@ unlike(
 # Test valid manifest with nonexistent backup directory.
 command_fails_like(
 	[
-		'pg_verifybackup', '-m',
+		'pg_verifybackup', '-Fp', '-m',
 		"$backup_path/backup_manifest", "$backup_path/fake"
 	],
 	qr/could not open directory/,
diff --git a/src/bin/pg_verifybackup/t/005_bad_manifest.pl b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
index c4ed64b62d5..28c51b6feb0 100644
--- a/src/bin/pg_verifybackup/t/005_bad_manifest.pl
+++ b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
@@ -208,7 +208,7 @@ sub test_bad_manifest
 	print $fh $manifest_contents;
 	close($fh);
 
-	command_fails_like([ 'pg_verifybackup', $tempdir ], $regexp, $test_name);
+	command_fails_like([ 'pg_verifybackup', '-Fp', $tempdir ], $regexp, $test_name);
 	return;
 }
 
diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
index 7a09f3b75b2..9896560adc3 100644
--- a/src/bin/pg_verifybackup/t/008_untar.pl
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -16,6 +16,20 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 
+# Create a couple of directories to use as tablespaces.
+my $TS1_LOCATION = $primary->backup_dir .'/ts1';
+$TS1_LOCATION =~ s/\/\.\//\//g;    # collapse foo/./bar to foo/bar
+mkdir($TS1_LOCATION);
+
+# Create a tablespace with table in it.
+$primary->safe_psql('postgres', qq(
+		CREATE TABLESPACE regress_ts1 LOCATION '$TS1_LOCATION';
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1';
+		CREATE TABLE regress_tbl1(i int) TABLESPACE regress_ts1;
+		INSERT INTO regress_tbl1 VALUES(generate_series(1,5));));
+my $tsoid = $primary->safe_psql('postgres', qq(
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1'));
+
 my $backup_path = $primary->backup_dir . '/server-backup';
 my $extract_path = $primary->backup_dir . '/extracted-backup';
 
@@ -23,39 +37,31 @@ my @test_configuration = (
 	{
 		'compression_method' => 'none',
 		'backup_flags' => [],
-		'backup_archive' => 'base.tar',
+		'backup_archive' => ['base.tar', "$tsoid.tar"],
 		'enabled' => 1
 	},
 	{
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'server-gzip' ],
-		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.gz', "$tsoid.tar.gz" ],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'server-lz4' ],
-		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => [ '-d', '-m' ],
+		'backup_archive' => ['base.tar.lz4', "$tsoid.tar.lz4" ],
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd:level=1,long' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	});
 
@@ -86,47 +92,16 @@ for my $tc (@test_configuration)
 		my $backup_files = join(',',
 			sort grep { $_ ne '.' && $_ ne '..' } slurp_dir($backup_path));
 		my $expected_backup_files =
-		  join(',', sort ('backup_manifest', $tc->{'backup_archive'}));
+		  join(',', sort ('backup_manifest', @{ $tc->{'backup_archive'} }));
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok(['pg_verifybackup', '-n', '-e', $backup_path],
+			"verify backup, compression $method");
 
 		# Cleanup.
-		unlink($backup_path . '/backup_manifest');
-		unlink($backup_path . '/base.tar');
-		unlink($backup_path . '/' . $tc->{'backup_archive'});
+		rmtree($backup_path);
 		rmtree($extract_path);
 	}
 }
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index 8c076d46dee..6b7d7483f6e 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -29,41 +29,30 @@ my @test_configuration = (
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'client-gzip:5' ],
 		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'client-lz4:5' ],
 		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => ['-d'],
-		'output_file' => 'base.tar',
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:5' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:level=1,long' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'parallel zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:workers=3' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1"),
 		'possibly_unsupported' =>
 		  qr/could not set compression worker count to 3: Unsupported parameter/
@@ -118,40 +107,9 @@ for my $tc (@test_configuration)
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			push @decompress, $backup_path . '/' . $tc->{'output_file'}
-			  if $tc->{'output_file'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok( [ 'pg_verifybackup', '-n', '-e', $backup_path ],
+			"verify backup, compression $method");
 
 		# Cleanup.
 		rmtree($extract_path);
-- 
2.18.0

From b5f845c39be20f6c2c820e3a053980fa1ca45919 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 12:15:26 +0530
Subject: [PATCH v4 06/11] Refactor: split verify_backup_file() function.

Separate the manifest entry verification code into a new function.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 39 ++++++++++++++++-------
 src/bin/pg_verifybackup/pg_verifybackup.h |  3 ++
 2 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 4e42757c346..b8db83ed2cd 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -614,6 +614,27 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		return;
 	}
 
+	/* Check the backup manifest entry for this file. */
+	m = verify_manifest_entry(context, relpath, sb.st_size);
+
+	/*
+	 * Validate the manifest system identifier, not available in manifest
+	 * version 1.
+	 */
+	if (context->manifest->version != 1 &&
+		strcmp(relpath, "global/pg_control") == 0 &&
+		m != NULL && m->matched && !m->bad)
+		verify_control_file(fullpath, context->manifest->system_identifier);
+}
+
+/*
+ * Verify file and its size entry in the manifest.
+ */
+manifest_file *
+verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
+{
+	manifest_file *m;
+
 	/* Check whether there's an entry in the manifest hash. */
 	m = manifest_files_lookup(context->manifest->files, relpath);
 	if (m == NULL)
@@ -621,29 +642,21 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		report_backup_error(context,
 							"\"%s\" is present on disk but not in the manifest",
 							relpath);
-		return;
+		return NULL;
 	}
 
 	/* Flag this entry as having been encountered in the filesystem. */
 	m->matched = true;
 
 	/* Check that the size matches. */
-	if (m->size != sb.st_size)
+	if (m->size != filesize)
 	{
 		report_backup_error(context,
 							"\"%s\" has size %lld on disk but size %zu in the manifest",
-							relpath, (long long int) sb.st_size, m->size);
+							relpath, (long long int) filesize, m->size);
 		m->bad = true;
 	}
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0)
-		verify_control_file(fullpath, context->manifest->system_identifier);
-
 	/* Update statistics for progress report, if necessary */
 	if (context->show_progress && !context->skip_checksums &&
 		should_verify_checksum(m))
@@ -655,6 +668,8 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 	 * afterwards verify the checksums. That's because computing checksums may
 	 * take a while, and we'd like to report more obvious problems quickly.
 	 */
+
+	return m;
 }
 
 /*
@@ -817,7 +832,7 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 
 	/*
 	 * Double-check that we read the expected number of bytes from the file.
-	 * Normally, a file size mismatch would be caught in verify_backup_file
+	 * Normally, a file size mismatch would be caught in verify_manifest_entry
 	 * and this check would never be reached, but this provides additional
 	 * safety and clarity in the event of concurrent modifications or
 	 * filesystem misbehavior.
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 90900048547..98c75916255 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -105,6 +105,9 @@ typedef struct verifier_context
 	uint64		done_size;
 } verifier_context;
 
+extern manifest_file *verify_manifest_entry(verifier_context *context,
+											char *relpath, int64 filesize);
+
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
 			pg_attribute_printf(2, 3);
-- 
2.18.0

From d6e791de02b2d5e32059326952ecbb7c440bcf99 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 12:10:34 +0530
Subject: [PATCH v4 05/11] Refactor: move some part of pg_verifybackup.c to
 pg_verifybackup.h

---
 src/bin/pg_verifybackup/pg_verifybackup.c | 102 +------------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 118 ++++++++++++++++++++++
 2 files changed, 123 insertions(+), 97 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/pg_verifybackup.h

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 71585ffc50e..4e42757c346 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,12 +18,11 @@
 #include <sys/stat.h>
 #include <time.h>
 
-#include "common/controldata_utils.h"
-#include "common/hashfn_unstable.h"
 #include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
+#include "pg_verifybackup.h"
 #include "pgtime.h"
 
 /*
@@ -40,89 +39,6 @@
  */
 #define ESTIMATED_BYTES_PER_MANIFEST_LINE	100
 
-/*
- * How many bytes should we try to read from a file at once?
- */
-#define READ_CHUNK_SIZE				(128 * 1024)
-
-/*
- * Each file described by the manifest file is parsed to produce an object
- * like this.
- */
-typedef struct manifest_file
-{
-	uint32		status;			/* hash status */
-	const char *pathname;
-	size_t		size;
-	pg_checksum_type checksum_type;
-	int			checksum_length;
-	uint8	   *checksum_payload;
-	bool		matched;
-	bool		bad;
-} manifest_file;
-
-#define should_verify_checksum(m) \
-	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
-
-/*
- * Define a hash table which we can use to store information about the files
- * mentioned in the backup manifest.
- */
-#define SH_PREFIX		manifest_files
-#define SH_ELEMENT_TYPE	manifest_file
-#define SH_KEY_TYPE		const char *
-#define	SH_KEY			pathname
-#define SH_HASH_KEY(tb, key)	hash_string(key)
-#define SH_EQUAL(tb, a, b)		(strcmp(a, b) == 0)
-#define	SH_SCOPE		static inline
-#define SH_RAW_ALLOCATOR	pg_malloc0
-#define SH_DECLARE
-#define SH_DEFINE
-#include "lib/simplehash.h"
-
-/*
- * Each WAL range described by the manifest file is parsed to produce an
- * object like this.
- */
-typedef struct manifest_wal_range
-{
-	TimeLineID	tli;
-	XLogRecPtr	start_lsn;
-	XLogRecPtr	end_lsn;
-	struct manifest_wal_range *next;
-	struct manifest_wal_range *prev;
-} manifest_wal_range;
-
-/*
- * All the data parsed from a backup_manifest file.
- */
-typedef struct manifest_data
-{
-	int			version;
-	uint64		system_identifier;
-	manifest_files_hash *files;
-	manifest_wal_range *first_wal_range;
-	manifest_wal_range *last_wal_range;
-} manifest_data;
-
-/*
- * All of the context information we need while checking a backup manifest.
- */
-typedef struct verifier_context
-{
-	manifest_data *manifest;
-	char	   *backup_directory;
-	SimpleStringList ignore_list;
-	bool		skip_checksums;
-	bool		exit_on_error;
-	bool		saw_any_error;
-
-	/* Progress indicators */
-	bool		show_progress;
-	uint64		total_size;
-	uint64		done_size;
-} verifier_context;
-
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -156,14 +72,6 @@ static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
 
-static void report_backup_error(verifier_context *context,
-								const char *pg_restrict fmt,...)
-			pg_attribute_printf(2, 3);
-static void report_fatal_error(const char *pg_restrict fmt,...)
-			pg_attribute_printf(1, 2) pg_attribute_noreturn();
-static bool should_ignore_relpath(verifier_context *context, const char *relpath);
-
-static void progress_report(verifier_context *context, bool finished);
 static void usage(void);
 
 static const char *progname;
@@ -978,7 +886,7 @@ parse_required_wal(verifier_context *context, char *pg_waldump_path,
  * Update the context to indicate that we saw an error, and exit if the
  * context says we should.
  */
-static void
+void
 report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
 {
 	va_list		ap;
@@ -995,7 +903,7 @@ report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
 /*
  * Report a fatal error and exit
  */
-static void
+void
 report_fatal_error(const char *pg_restrict fmt,...)
 {
 	va_list		ap;
@@ -1014,7 +922,7 @@ report_fatal_error(const char *pg_restrict fmt,...)
  * Note that by "prefix" we mean a parent directory; for this purpose,
  * "aa/bb" is not a prefix of "aa/bbb", but it is a prefix of "aa/bb/cc".
  */
-static bool
+bool
 should_ignore_relpath(verifier_context *context, const char *relpath)
 {
 	SimpleStringListCell *cell;
@@ -1043,7 +951,7 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
  * If finished is set to true, this is the last progress report. The cursor
  * is moved to the next line.
  */
-static void
+void
 progress_report(verifier_context *context, bool finished)
 {
 	static pg_time_t last_progress_report = 0;
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
new file mode 100644
index 00000000000..90900048547
--- /dev/null
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -0,0 +1,118 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_verifybackup.h
+ *	  Verify a backup against a backup manifest.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_verifybackup/pg_verifybackup.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef PG_VERIFYBACKUP_H
+#define PG_VERIFYBACKUP_H
+
+#include "common/controldata_utils.h"
+#include "common/hashfn_unstable.h"
+#include "common/parse_manifest.h"
+#include "fe_utils/simple_list.h"
+
+extern bool show_progress;
+extern bool skip_checksums;
+
+/*
+ * How many bytes should we try to read from a file at once?
+ */
+#define READ_CHUNK_SIZE				(128 * 1024)
+
+/*
+ * Each file described by the manifest file is parsed to produce an object
+ * like this.
+ */
+typedef struct manifest_file
+{
+	uint32		status;			/* hash status */
+	const char *pathname;
+	size_t		size;
+	pg_checksum_type checksum_type;
+	int			checksum_length;
+	uint8	   *checksum_payload;
+	bool		matched;
+	bool		bad;
+} manifest_file;
+
+#define should_verify_checksum(m) \
+	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+
+/*
+ * Define a hash table which we can use to store information about the files
+ * mentioned in the backup manifest.
+ */
+#define SH_PREFIX		manifest_files
+#define SH_ELEMENT_TYPE	manifest_file
+#define SH_KEY_TYPE		const char *
+#define	SH_KEY			pathname
+#define SH_HASH_KEY(tb, key)	hash_string(key)
+#define SH_EQUAL(tb, a, b)		(strcmp(a, b) == 0)
+#define	SH_SCOPE		static inline
+#define SH_RAW_ALLOCATOR	pg_malloc0
+#define SH_DECLARE
+#define SH_DEFINE
+#include "lib/simplehash.h"
+
+/*
+ * Each WAL range described by the manifest file is parsed to produce an
+ * object like this.
+ */
+typedef struct manifest_wal_range
+{
+	TimeLineID	tli;
+	XLogRecPtr	start_lsn;
+	XLogRecPtr	end_lsn;
+	struct manifest_wal_range *next;
+	struct manifest_wal_range *prev;
+} manifest_wal_range;
+
+/*
+ * All the data parsed from a backup_manifest file.
+ */
+typedef struct manifest_data
+{
+	int			version;
+	uint64		system_identifier;
+	manifest_files_hash *files;
+	manifest_wal_range *first_wal_range;
+	manifest_wal_range *last_wal_range;
+} manifest_data;
+
+/*
+ * All of the context information we need while checking a backup manifest.
+ */
+typedef struct verifier_context
+{
+	manifest_data *manifest;
+	char	   *backup_directory;
+	SimpleStringList ignore_list;
+	bool		skip_checksums;
+	bool		exit_on_error;
+	bool		saw_any_error;
+
+	/* Progress indicators */
+	bool		show_progress;
+	uint64		total_size;
+	uint64		done_size;
+} verifier_context;
+
+extern void report_backup_error(verifier_context *context,
+								const char *pg_restrict fmt,...)
+			pg_attribute_printf(2, 3);
+extern void report_fatal_error(const char *pg_restrict fmt,...)
+			pg_attribute_printf(1, 2) pg_attribute_noreturn();
+extern bool should_ignore_relpath(verifier_context *context,
+								  const char *relpath);
+
+extern void progress_report(verifier_context *context, bool finished);
+
+#endif							/* PG_VERIFYBACKUP_H */
-- 
2.18.0

From 59cbaf1aa862fc0969b1efdd2b1702f7530814c1 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 31 Jul 2024 11:20:52 +0530
Subject: [PATCH v4 03/11] Refactor: move astreamer* files to fe_utils to make
 common availability of it.

To make it accessible to other code, we need to move the ASTREAMER
code (previously known as BBSTREAMER) to a common location. The
appropriate place would be src/fe_utils, as it is a frontend
infrastructure intended for shared use.
---
 meson.build                                             | 2 +-
 src/bin/pg_basebackup/Makefile                          | 7 +------
 src/bin/pg_basebackup/astreamer_inject.h                | 2 +-
 src/bin/pg_basebackup/meson.build                       | 5 -----
 src/fe_utils/Makefile                                   | 5 +++++
 src/{bin/pg_basebackup => fe_utils}/astreamer_file.c    | 2 +-
 src/{bin/pg_basebackup => fe_utils}/astreamer_gzip.c    | 2 +-
 src/{bin/pg_basebackup => fe_utils}/astreamer_lz4.c     | 2 +-
 src/{bin/pg_basebackup => fe_utils}/astreamer_tar.c     | 2 +-
 src/{bin/pg_basebackup => fe_utils}/astreamer_zstd.c    | 2 +-
 src/fe_utils/meson.build                                | 5 +++++
 src/{bin/pg_basebackup => include/fe_utils}/astreamer.h | 0
 12 files changed, 18 insertions(+), 18 deletions(-)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_file.c (99%)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_gzip.c (99%)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_lz4.c (99%)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_tar.c (99%)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_zstd.c (99%)
 rename src/{bin/pg_basebackup => include/fe_utils}/astreamer.h (100%)

diff --git a/meson.build b/meson.build
index 7de0371226d..f7a5d2aea9a 100644
--- a/meson.build
+++ b/meson.build
@@ -3027,7 +3027,7 @@ frontend_common_code = declare_dependency(
   compile_args: ['-DFRONTEND'],
   include_directories: [postgres_inc],
   sources: generated_headers,
-  dependencies: [os_deps, zlib, zstd],
+  dependencies: [os_deps, zlib, zstd, lz4],
 )
 
 backend_common_code = declare_dependency(
diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile
index a71af2d48a7..f1e73058b23 100644
--- a/src/bin/pg_basebackup/Makefile
+++ b/src/bin/pg_basebackup/Makefile
@@ -37,12 +37,7 @@ OBJS = \
 
 BBOBJS = \
 	pg_basebackup.o \
-	astreamer_file.o \
-	astreamer_gzip.o \
-	astreamer_inject.o \
-	astreamer_lz4.o \
-	astreamer_tar.o \
-	astreamer_zstd.o
+	astreamer_inject.o
 
 all: pg_basebackup pg_createsubscriber pg_receivewal pg_recvlogical
 
diff --git a/src/bin/pg_basebackup/astreamer_inject.h b/src/bin/pg_basebackup/astreamer_inject.h
index 8504b3f5e0d..aeed533862b 100644
--- a/src/bin/pg_basebackup/astreamer_inject.h
+++ b/src/bin/pg_basebackup/astreamer_inject.h
@@ -12,7 +12,7 @@
 #ifndef ASTREAMER_INJECT_H
 #define ASTREAMER_INJECT_H
 
-#include "astreamer.h"
+#include "fe_utils/astreamer.h"
 #include "pqexpbuffer.h"
 
 extern astreamer *astreamer_recovery_injector_new(astreamer *next,
diff --git a/src/bin/pg_basebackup/meson.build b/src/bin/pg_basebackup/meson.build
index a68dbd7837d..9101fc18438 100644
--- a/src/bin/pg_basebackup/meson.build
+++ b/src/bin/pg_basebackup/meson.build
@@ -1,12 +1,7 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 common_sources = files(
-  'astreamer_file.c',
-  'astreamer_gzip.c',
   'astreamer_inject.c',
-  'astreamer_lz4.c',
-  'astreamer_tar.c',
-  'astreamer_zstd.c',
   'receivelog.c',
   'streamutil.c',
   'walmethods.c',
diff --git a/src/fe_utils/Makefile b/src/fe_utils/Makefile
index 946c05258f0..2694be4b859 100644
--- a/src/fe_utils/Makefile
+++ b/src/fe_utils/Makefile
@@ -21,6 +21,11 @@ override CPPFLAGS := -DFRONTEND -I$(libpq_srcdir) $(CPPFLAGS)
 
 OBJS = \
 	archive.o \
+	astreamer_file.o \
+	astreamer_gzip.o \
+	astreamer_lz4.o \
+	astreamer_tar.o \
+	astreamer_zstd.o \
 	cancel.o \
 	conditional.o \
 	connect_utils.o \
diff --git a/src/bin/pg_basebackup/astreamer_file.c b/src/fe_utils/astreamer_file.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_file.c
rename to src/fe_utils/astreamer_file.c
index 2742385e103..13d1192c6e6 100644
--- a/src/bin/pg_basebackup/astreamer_file.c
+++ b/src/fe_utils/astreamer_file.c
@@ -13,10 +13,10 @@
 
 #include <unistd.h>
 
-#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
+#include "fe_utils/astreamer.h"
 
 typedef struct astreamer_plain_writer
 {
diff --git a/src/bin/pg_basebackup/astreamer_gzip.c b/src/fe_utils/astreamer_gzip.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_gzip.c
rename to src/fe_utils/astreamer_gzip.c
index 6f7c27afbbc..dd28defac7b 100644
--- a/src/bin/pg_basebackup/astreamer_gzip.c
+++ b/src/fe_utils/astreamer_gzip.c
@@ -17,10 +17,10 @@
 #include <zlib.h>
 #endif
 
-#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
+#include "fe_utils/astreamer.h"
 
 #ifdef HAVE_LIBZ
 typedef struct astreamer_gzip_writer
diff --git a/src/bin/pg_basebackup/astreamer_lz4.c b/src/fe_utils/astreamer_lz4.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_lz4.c
rename to src/fe_utils/astreamer_lz4.c
index 1c40d7d8ad5..d8b2a367e47 100644
--- a/src/bin/pg_basebackup/astreamer_lz4.c
+++ b/src/fe_utils/astreamer_lz4.c
@@ -17,10 +17,10 @@
 #include <lz4frame.h>
 #endif
 
-#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
+#include "fe_utils/astreamer.h"
 
 #ifdef USE_LZ4
 typedef struct astreamer_lz4_frame
diff --git a/src/bin/pg_basebackup/astreamer_tar.c b/src/fe_utils/astreamer_tar.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_tar.c
rename to src/fe_utils/astreamer_tar.c
index 673690cd18f..f5d3562d280 100644
--- a/src/bin/pg_basebackup/astreamer_tar.c
+++ b/src/fe_utils/astreamer_tar.c
@@ -23,8 +23,8 @@
 
 #include <time.h>
 
-#include "astreamer.h"
 #include "common/logging.h"
+#include "fe_utils/astreamer.h"
 #include "pgtar.h"
 
 typedef struct astreamer_tar_parser
diff --git a/src/bin/pg_basebackup/astreamer_zstd.c b/src/fe_utils/astreamer_zstd.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_zstd.c
rename to src/fe_utils/astreamer_zstd.c
index 58dc679ef99..45f6cb67363 100644
--- a/src/bin/pg_basebackup/astreamer_zstd.c
+++ b/src/fe_utils/astreamer_zstd.c
@@ -17,8 +17,8 @@
 #include <zstd.h>
 #endif
 
-#include "astreamer.h"
 #include "common/logging.h"
+#include "fe_utils/astreamer.h"
 
 #ifdef USE_ZSTD
 
diff --git a/src/fe_utils/meson.build b/src/fe_utils/meson.build
index 14d0482a2cc..043021d826d 100644
--- a/src/fe_utils/meson.build
+++ b/src/fe_utils/meson.build
@@ -2,6 +2,11 @@
 
 fe_utils_sources = files(
   'archive.c',
+  'astreamer_file.c',
+  'astreamer_gzip.c',
+  'astreamer_lz4.c',
+  'astreamer_tar.c',
+  'astreamer_zstd.c',
   'cancel.c',
   'conditional.c',
   'connect_utils.c',
diff --git a/src/bin/pg_basebackup/astreamer.h b/src/include/fe_utils/astreamer.h
similarity index 100%
rename from src/bin/pg_basebackup/astreamer.h
rename to src/include/fe_utils/astreamer.h
-- 
2.18.0

From 63c2694c9fe17a15bdec481397acf4ee54f00ab1 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 31 Jul 2024 11:43:52 +0530
Subject: [PATCH v4 04/11] Refactor: move few global variable to
 verifier_context struct

Global variables are:
	1. show_progress
	2. skip_checksums
	3. total_size
	4. done_size
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 50 +++++++++++------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index d77e70fbe38..71585ffc50e 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -113,8 +113,14 @@ typedef struct verifier_context
 	manifest_data *manifest;
 	char	   *backup_directory;
 	SimpleStringList ignore_list;
+	bool		skip_checksums;
 	bool		exit_on_error;
 	bool		saw_any_error;
+
+	/* Progress indicators */
+	bool		show_progress;
+	uint64		total_size;
+	uint64		done_size;
 } verifier_context;
 
 static manifest_data *parse_manifest_file(char *manifest_path);
@@ -157,19 +163,11 @@ static void report_fatal_error(const char *pg_restrict fmt,...)
 			pg_attribute_printf(1, 2) pg_attribute_noreturn();
 static bool should_ignore_relpath(verifier_context *context, const char *relpath);
 
-static void progress_report(bool finished);
+static void progress_report(verifier_context *context, bool finished);
 static void usage(void);
 
 static const char *progname;
 
-/* options */
-static bool show_progress = false;
-static bool skip_checksums = false;
-
-/* Progress indicators */
-static uint64 total_size = 0;
-static uint64 done_size = 0;
-
 /*
  * Main entry point.
  */
@@ -260,13 +258,13 @@ main(int argc, char **argv)
 				no_parse_wal = true;
 				break;
 			case 'P':
-				show_progress = true;
+				context.show_progress = true;
 				break;
 			case 'q':
 				quiet = true;
 				break;
 			case 's':
-				skip_checksums = true;
+				context.skip_checksums = true;
 				break;
 			case 'w':
 				wal_directory = pstrdup(optarg);
@@ -299,7 +297,7 @@ main(int argc, char **argv)
 	}
 
 	/* Complain if the specified arguments conflict */
-	if (show_progress && quiet)
+	if (context.show_progress && quiet)
 		pg_fatal("cannot specify both %s and %s",
 				 "-P/--progress", "-q/--quiet");
 
@@ -363,7 +361,7 @@ main(int argc, char **argv)
 	 * Now do the expensive work of verifying file checksums, unless we were
 	 * told to skip it.
 	 */
-	if (!skip_checksums)
+	if (!context.skip_checksums)
 		verify_backup_checksums(&context);
 
 	/*
@@ -739,8 +737,9 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		verify_control_file(fullpath, context->manifest->system_identifier);
 
 	/* Update statistics for progress report, if necessary */
-	if (show_progress && !skip_checksums && should_verify_checksum(m))
-		total_size += m->size;
+	if (context->show_progress && !context->skip_checksums &&
+		should_verify_checksum(m))
+		context->total_size += m->size;
 
 	/*
 	 * We don't verify checksums at this stage. We first finish verifying that
@@ -815,7 +814,7 @@ verify_backup_checksums(verifier_context *context)
 	manifest_file *m;
 	uint8	   *buffer;
 
-	progress_report(false);
+	progress_report(context, false);
 
 	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
 
@@ -841,7 +840,7 @@ verify_backup_checksums(verifier_context *context)
 
 	pfree(buffer);
 
-	progress_report(true);
+	progress_report(context, true);
 }
 
 /*
@@ -889,8 +888,8 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 		}
 
 		/* Report progress */
-		done_size += rc;
-		progress_report(false);
+		context->done_size += rc;
+		progress_report(context, false);
 	}
 	if (rc < 0)
 		report_backup_error(context, "could not read file \"%s\": %m",
@@ -1036,7 +1035,7 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
 }
 
 /*
- * Print a progress report based on the global variables.
+ * Print a progress report based on the variables in verifier_context.
  *
  * Progress report is written at maximum once per second, unless the finished
  * parameter is set to true.
@@ -1045,7 +1044,7 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
  * is moved to the next line.
  */
 static void
-progress_report(bool finished)
+progress_report(verifier_context *context, bool finished)
 {
 	static pg_time_t last_progress_report = 0;
 	pg_time_t	now;
@@ -1053,7 +1052,7 @@ progress_report(bool finished)
 	char		totalsize_str[32];
 	char		donesize_str[32];
 
-	if (!show_progress)
+	if (!context->show_progress)
 		return;
 
 	now = time(NULL);
@@ -1061,12 +1060,13 @@ progress_report(bool finished)
 		return;					/* Max once per second */
 
 	last_progress_report = now;
-	percent_size = total_size ? (int) ((done_size * 100 / total_size)) : 0;
+	percent_size = context->total_size ?
+		(int) ((context->done_size * 100 / context->total_size)) : 0;
 
 	snprintf(totalsize_str, sizeof(totalsize_str), UINT64_FORMAT,
-			 total_size / 1024);
+			 context->total_size / 1024);
 	snprintf(donesize_str, sizeof(donesize_str), UINT64_FORMAT,
-			 done_size / 1024);
+			 context->done_size / 1024);
 
 	fprintf(stderr,
 			_("%*s/%s kB (%d%%) verified"),
-- 
2.18.0

From 5b0e83ae2b1f444b7bd9277b0c202758d2eb5934 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 17 Jul 2024 14:23:27 +0530
Subject: [PATCH v4 02/11] Refactor: Add astreamer_inject.h and move related
 declarations to it.

---
 src/bin/pg_basebackup/astreamer.h        |  6 ------
 src/bin/pg_basebackup/astreamer_inject.c |  2 +-
 src/bin/pg_basebackup/astreamer_inject.h | 24 ++++++++++++++++++++++++
 src/bin/pg_basebackup/pg_basebackup.c    |  2 +-
 4 files changed, 26 insertions(+), 8 deletions(-)
 create mode 100644 src/bin/pg_basebackup/astreamer_inject.h

diff --git a/src/bin/pg_basebackup/astreamer.h b/src/bin/pg_basebackup/astreamer.h
index 6b0047418bb..9d0a8c4d0c2 100644
--- a/src/bin/pg_basebackup/astreamer.h
+++ b/src/bin/pg_basebackup/astreamer.h
@@ -217,10 +217,4 @@ extern astreamer *astreamer_tar_parser_new(astreamer *next);
 extern astreamer *astreamer_tar_terminator_new(astreamer *next);
 extern astreamer *astreamer_tar_archiver_new(astreamer *next);
 
-extern astreamer *astreamer_recovery_injector_new(astreamer *next,
-												  bool is_recovery_guc_supported,
-												  PQExpBuffer recoveryconfcontents);
-extern void astreamer_inject_file(astreamer *streamer, char *pathname,
-								  char *data, int len);
-
 #endif
diff --git a/src/bin/pg_basebackup/astreamer_inject.c b/src/bin/pg_basebackup/astreamer_inject.c
index 7f1decded8d..4ad8381f102 100644
--- a/src/bin/pg_basebackup/astreamer_inject.c
+++ b/src/bin/pg_basebackup/astreamer_inject.c
@@ -11,7 +11,7 @@
 
 #include "postgres_fe.h"
 
-#include "astreamer.h"
+#include "astreamer_inject.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 
diff --git a/src/bin/pg_basebackup/astreamer_inject.h b/src/bin/pg_basebackup/astreamer_inject.h
new file mode 100644
index 00000000000..8504b3f5e0d
--- /dev/null
+++ b/src/bin/pg_basebackup/astreamer_inject.h
@@ -0,0 +1,24 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer_inject.h
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *		  src/bin/pg_basebackup/astreamer_inject.h
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef ASTREAMER_INJECT_H
+#define ASTREAMER_INJECT_H
+
+#include "astreamer.h"
+#include "pqexpbuffer.h"
+
+extern astreamer *astreamer_recovery_injector_new(astreamer *next,
+												  bool is_recovery_guc_supported,
+												  PQExpBuffer recoveryconfcontents);
+extern void astreamer_inject_file(astreamer *streamer, char *pathname,
+								  char *data, int len);
+
+#endif
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 4179b064cbc..1e753e40c97 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -26,7 +26,7 @@
 #endif
 
 #include "access/xlog_internal.h"
-#include "astreamer.h"
+#include "astreamer_inject.h"
 #include "backup/basebackup.h"
 #include "common/compression.h"
 #include "common/file_perm.h"
-- 
2.18.0

From 412266b417727de9eedd58cfb744cca35f0d2c22 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 09:39:32 +0530
Subject: [PATCH v4 01/11] Refactor: Rename all bbstreamer references to
 astreamer.

BBSTREAMER is specific to pg_basebackup; we need a more generalized
name so it can be placed in a common area, making it accessible for
other modules. Renaming it to ASTREAMER, short for ARCHIVE STREAMER,
makes it more general.
---
 src/bin/pg_basebackup/Makefile                |  12 +-
 src/bin/pg_basebackup/astreamer.h             | 226 +++++++++++++
 .../{bbstreamer_file.c => astreamer_file.c}   | 148 ++++----
 .../{bbstreamer_gzip.c => astreamer_gzip.c}   | 154 ++++-----
 ...bbstreamer_inject.c => astreamer_inject.c} | 152 ++++-----
 .../{bbstreamer_lz4.c => astreamer_lz4.c}     | 172 +++++-----
 .../{bbstreamer_tar.c => astreamer_tar.c}     | 316 +++++++++---------
 .../{bbstreamer_zstd.c => astreamer_zstd.c}   | 160 ++++-----
 src/bin/pg_basebackup/bbstreamer.h            | 226 -------------
 src/bin/pg_basebackup/meson.build             |  12 +-
 src/bin/pg_basebackup/nls.mk                  |  12 +-
 src/bin/pg_basebackup/pg_basebackup.c         |  74 ++--
 src/tools/pgindent/typedefs.list              |  26 +-
 13 files changed, 845 insertions(+), 845 deletions(-)
 create mode 100644 src/bin/pg_basebackup/astreamer.h
 rename src/bin/pg_basebackup/{bbstreamer_file.c => astreamer_file.c} (69%)
 rename src/bin/pg_basebackup/{bbstreamer_gzip.c => astreamer_gzip.c} (62%)
 rename src/bin/pg_basebackup/{bbstreamer_inject.c => astreamer_inject.c} (53%)
 rename src/bin/pg_basebackup/{bbstreamer_lz4.c => astreamer_lz4.c} (69%)
 rename src/bin/pg_basebackup/{bbstreamer_tar.c => astreamer_tar.c} (50%)
 rename src/bin/pg_basebackup/{bbstreamer_zstd.c => astreamer_zstd.c} (64%)
 delete mode 100644 src/bin/pg_basebackup/bbstreamer.h

diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile
index 26c53e473f5..a71af2d48a7 100644
--- a/src/bin/pg_basebackup/Makefile
+++ b/src/bin/pg_basebackup/Makefile
@@ -37,12 +37,12 @@ OBJS = \
 
 BBOBJS = \
 	pg_basebackup.o \
-	bbstreamer_file.o \
-	bbstreamer_gzip.o \
-	bbstreamer_inject.o \
-	bbstreamer_lz4.o \
-	bbstreamer_tar.o \
-	bbstreamer_zstd.o
+	astreamer_file.o \
+	astreamer_gzip.o \
+	astreamer_inject.o \
+	astreamer_lz4.o \
+	astreamer_tar.o \
+	astreamer_zstd.o
 
 all: pg_basebackup pg_createsubscriber pg_receivewal pg_recvlogical
 
diff --git a/src/bin/pg_basebackup/astreamer.h b/src/bin/pg_basebackup/astreamer.h
new file mode 100644
index 00000000000..6b0047418bb
--- /dev/null
+++ b/src/bin/pg_basebackup/astreamer.h
@@ -0,0 +1,226 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer.h
+ *
+ * Each tar archive returned by the server is passed to one or more
+ * astreamer objects for further processing. The astreamer may do
+ * something simple, like write the archive to a file, perhaps after
+ * compressing it, but it can also do more complicated things, like
+ * annotating the byte stream to indicate which parts of the data
+ * correspond to tar headers or trailing padding, vs. which parts are
+ * payload data. A subsequent astreamer may use this information to
+ * make further decisions about how to process the data; for example,
+ * it might choose to modify the archive contents.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *		  src/bin/pg_basebackup/astreamer.h
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef ASTREAMER_H
+#define ASTREAMER_H
+
+#include "common/compression.h"
+#include "lib/stringinfo.h"
+#include "pqexpbuffer.h"
+
+struct astreamer;
+struct astreamer_ops;
+typedef struct astreamer astreamer;
+typedef struct astreamer_ops astreamer_ops;
+
+/*
+ * Each chunk of archive data passed to a astreamer is classified into one
+ * of these categories. When data is first received from the remote server,
+ * each chunk will be categorized as ASTREAMER_UNKNOWN, and the chunks will
+ * be of whatever size the remote server chose to send.
+ *
+ * If the archive is parsed (e.g. see astreamer_tar_parser_new()), then all
+ * chunks should be labelled as one of the other types listed here. In
+ * addition, there should be exactly one ASTREAMER_MEMBER_HEADER chunk and
+ * exactly one ASTREAMER_MEMBER_TRAILER chunk per archive member, even if
+ * that means a zero-length call. There can be any number of
+ * ASTREAMER_MEMBER_CONTENTS chunks in between those calls. There
+ * should exactly ASTREAMER_ARCHIVE_TRAILER chunk, and it should follow the
+ * last ASTREAMER_MEMBER_TRAILER chunk.
+ *
+ * In theory, we could need other classifications here, such as a way of
+ * indicating an archive header, but the "tar" format doesn't need anything
+ * else, so for the time being there's no point.
+ */
+typedef enum
+{
+	ASTREAMER_UNKNOWN,
+	ASTREAMER_MEMBER_HEADER,
+	ASTREAMER_MEMBER_CONTENTS,
+	ASTREAMER_MEMBER_TRAILER,
+	ASTREAMER_ARCHIVE_TRAILER,
+} astreamer_archive_context;
+
+/*
+ * Each chunk of data that is classified as ASTREAMER_MEMBER_HEADER,
+ * ASTREAMER_MEMBER_CONTENTS, or ASTREAMER_MEMBER_TRAILER should also
+ * pass a pointer to an instance of this struct. The details are expected
+ * to be present in the archive header and used to fill the struct, after
+ * which all subsequent calls for the same archive member are expected to
+ * pass the same details.
+ */
+typedef struct
+{
+	char		pathname[MAXPGPATH];
+	pgoff_t		size;
+	mode_t		mode;
+	uid_t		uid;
+	gid_t		gid;
+	bool		is_directory;
+	bool		is_link;
+	char		linktarget[MAXPGPATH];
+} astreamer_member;
+
+/*
+ * Generally, each type of astreamer will define its own struct, but the
+ * first element should be 'astreamer base'. A astreamer that does not
+ * require any additional private data could use this structure directly.
+ *
+ * bbs_ops is a pointer to the astreamer_ops object which contains the
+ * function pointers appropriate to this type of astreamer.
+ *
+ * bbs_next is a pointer to the successor astreamer, for those types of
+ * astreamer which forward data to a successor. It need not be used and
+ * should be set to NULL when not relevant.
+ *
+ * bbs_buffer is a buffer for accumulating data for temporary storage. Each
+ * type of astreamer makes its own decisions about whether and how to use
+ * this buffer.
+ */
+struct astreamer
+{
+	const astreamer_ops *bbs_ops;
+	astreamer *bbs_next;
+	StringInfoData bbs_buffer;
+};
+
+/*
+ * There are three callbacks for a astreamer. The 'content' callback is
+ * called repeatedly, as described in the astreamer_archive_context comments.
+ * Then, the 'finalize' callback is called once at the end, to give the
+ * astreamer a chance to perform cleanup such as closing files. Finally,
+ * because this code is running in a frontend environment where, as of this
+ * writing, there are no memory contexts, the 'free' callback is called to
+ * release memory. These callbacks should always be invoked using the static
+ * inline functions defined below.
+ */
+struct astreamer_ops
+{
+	void		(*content) (astreamer *streamer, astreamer_member *member,
+							const char *data, int len,
+							astreamer_archive_context context);
+	void		(*finalize) (astreamer *streamer);
+	void		(*free) (astreamer *streamer);
+};
+
+/* Send some content to a astreamer. */
+static inline void
+astreamer_content(astreamer *streamer, astreamer_member *member,
+				  const char *data, int len,
+				  astreamer_archive_context context)
+{
+	Assert(streamer != NULL);
+	streamer->bbs_ops->content(streamer, member, data, len, context);
+}
+
+/* Finalize a astreamer. */
+static inline void
+astreamer_finalize(astreamer *streamer)
+{
+	Assert(streamer != NULL);
+	streamer->bbs_ops->finalize(streamer);
+}
+
+/* Free a astreamer. */
+static inline void
+astreamer_free(astreamer *streamer)
+{
+	Assert(streamer != NULL);
+	streamer->bbs_ops->free(streamer);
+}
+
+/*
+ * This is a convenience method for use when implementing a astreamer; it is
+ * not for use by outside callers. It adds the amount of data specified by
+ * 'nbytes' to the astreamer's buffer and adjusts '*len' and '*data'
+ * accordingly.
+ */
+static inline void
+astreamer_buffer_bytes(astreamer *streamer, const char **data, int *len,
+					   int nbytes)
+{
+	Assert(nbytes <= *len);
+
+	appendBinaryStringInfo(&streamer->bbs_buffer, *data, nbytes);
+	*len -= nbytes;
+	*data += nbytes;
+}
+
+/*
+ * This is a convenience method for use when implementing a astreamer; it is
+ * not for use by outsider callers. It attempts to add enough data to the
+ * astreamer's buffer to reach a length of target_bytes and adjusts '*len'
+ * and '*data' accordingly. It returns true if the target length has been
+ * reached and false otherwise.
+ */
+static inline bool
+astreamer_buffer_until(astreamer *streamer, const char **data, int *len,
+					   int target_bytes)
+{
+	int			buflen = streamer->bbs_buffer.len;
+
+	if (buflen >= target_bytes)
+	{
+		/* Target length already reached; nothing to do. */
+		return true;
+	}
+
+	if (buflen + *len < target_bytes)
+	{
+		/* Not enough data to reach target length; buffer all of it. */
+		astreamer_buffer_bytes(streamer, data, len, *len);
+		return false;
+	}
+
+	/* Buffer just enough to reach the target length. */
+	astreamer_buffer_bytes(streamer, data, len, target_bytes - buflen);
+	return true;
+}
+
+/*
+ * Functions for creating astreamer objects of various types. See the header
+ * comments for each of these functions for details.
+ */
+extern astreamer *astreamer_plain_writer_new(char *pathname, FILE *file);
+extern astreamer *astreamer_gzip_writer_new(char *pathname, FILE *file,
+											pg_compress_specification *compress);
+extern astreamer *astreamer_extractor_new(const char *basepath,
+										  const char *(*link_map) (const char *),
+										  void (*report_output_file) (const char *));
+
+extern astreamer *astreamer_gzip_decompressor_new(astreamer *next);
+extern astreamer *astreamer_lz4_compressor_new(astreamer *next,
+											   pg_compress_specification *compress);
+extern astreamer *astreamer_lz4_decompressor_new(astreamer *next);
+extern astreamer *astreamer_zstd_compressor_new(astreamer *next,
+												pg_compress_specification *compress);
+extern astreamer *astreamer_zstd_decompressor_new(astreamer *next);
+extern astreamer *astreamer_tar_parser_new(astreamer *next);
+extern astreamer *astreamer_tar_terminator_new(astreamer *next);
+extern astreamer *astreamer_tar_archiver_new(astreamer *next);
+
+extern astreamer *astreamer_recovery_injector_new(astreamer *next,
+												  bool is_recovery_guc_supported,
+												  PQExpBuffer recoveryconfcontents);
+extern void astreamer_inject_file(astreamer *streamer, char *pathname,
+								  char *data, int len);
+
+#endif
diff --git a/src/bin/pg_basebackup/bbstreamer_file.c b/src/bin/pg_basebackup/astreamer_file.c
similarity index 69%
rename from src/bin/pg_basebackup/bbstreamer_file.c
rename to src/bin/pg_basebackup/astreamer_file.c
index bab6cd4a6b1..2742385e103 100644
--- a/src/bin/pg_basebackup/bbstreamer_file.c
+++ b/src/bin/pg_basebackup/astreamer_file.c
@@ -1,11 +1,11 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_file.c
+ * astreamer_file.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_file.c
+ *		  src/bin/pg_basebackup/astreamer_file.c
  *-------------------------------------------------------------------------
  */
 
@@ -13,60 +13,60 @@
 
 #include <unistd.h>
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
 
-typedef struct bbstreamer_plain_writer
+typedef struct astreamer_plain_writer
 {
-	bbstreamer	base;
+	astreamer	base;
 	char	   *pathname;
 	FILE	   *file;
 	bool		should_close_file;
-} bbstreamer_plain_writer;
+} astreamer_plain_writer;
 
-typedef struct bbstreamer_extractor
+typedef struct astreamer_extractor
 {
-	bbstreamer	base;
+	astreamer	base;
 	char	   *basepath;
 	const char *(*link_map) (const char *);
 	void		(*report_output_file) (const char *);
 	char		filename[MAXPGPATH];
 	FILE	   *file;
-} bbstreamer_extractor;
+} astreamer_extractor;
 
-static void bbstreamer_plain_writer_content(bbstreamer *streamer,
-											bbstreamer_member *member,
-											const char *data, int len,
-											bbstreamer_archive_context context);
-static void bbstreamer_plain_writer_finalize(bbstreamer *streamer);
-static void bbstreamer_plain_writer_free(bbstreamer *streamer);
+static void astreamer_plain_writer_content(astreamer *streamer,
+										   astreamer_member *member,
+										   const char *data, int len,
+										   astreamer_archive_context context);
+static void astreamer_plain_writer_finalize(astreamer *streamer);
+static void astreamer_plain_writer_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_plain_writer_ops = {
-	.content = bbstreamer_plain_writer_content,
-	.finalize = bbstreamer_plain_writer_finalize,
-	.free = bbstreamer_plain_writer_free
+static const astreamer_ops astreamer_plain_writer_ops = {
+	.content = astreamer_plain_writer_content,
+	.finalize = astreamer_plain_writer_finalize,
+	.free = astreamer_plain_writer_free
 };
 
-static void bbstreamer_extractor_content(bbstreamer *streamer,
-										 bbstreamer_member *member,
-										 const char *data, int len,
-										 bbstreamer_archive_context context);
-static void bbstreamer_extractor_finalize(bbstreamer *streamer);
-static void bbstreamer_extractor_free(bbstreamer *streamer);
+static void astreamer_extractor_content(astreamer *streamer,
+										astreamer_member *member,
+										const char *data, int len,
+										astreamer_archive_context context);
+static void astreamer_extractor_finalize(astreamer *streamer);
+static void astreamer_extractor_free(astreamer *streamer);
 static void extract_directory(const char *filename, mode_t mode);
 static void extract_link(const char *filename, const char *linktarget);
 static FILE *create_file_for_extract(const char *filename, mode_t mode);
 
-static const bbstreamer_ops bbstreamer_extractor_ops = {
-	.content = bbstreamer_extractor_content,
-	.finalize = bbstreamer_extractor_finalize,
-	.free = bbstreamer_extractor_free
+static const astreamer_ops astreamer_extractor_ops = {
+	.content = astreamer_extractor_content,
+	.finalize = astreamer_extractor_finalize,
+	.free = astreamer_extractor_free
 };
 
 /*
- * Create a bbstreamer that just writes data to a file.
+ * Create a astreamer that just writes data to a file.
  *
  * The caller must specify a pathname and may specify a file. The pathname is
  * used for error-reporting purposes either way. If file is NULL, the pathname
@@ -74,14 +74,14 @@ static const bbstreamer_ops bbstreamer_extractor_ops = {
  * for writing and closed when done. If file is not NULL, the data is written
  * there.
  */
-bbstreamer *
-bbstreamer_plain_writer_new(char *pathname, FILE *file)
+astreamer *
+astreamer_plain_writer_new(char *pathname, FILE *file)
 {
-	bbstreamer_plain_writer *streamer;
+	astreamer_plain_writer *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_plain_writer));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_plain_writer_ops;
+	streamer = palloc0(sizeof(astreamer_plain_writer));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_plain_writer_ops;
 
 	streamer->pathname = pstrdup(pathname);
 	streamer->file = file;
@@ -101,13 +101,13 @@ bbstreamer_plain_writer_new(char *pathname, FILE *file)
  * Write archive content to file.
  */
 static void
-bbstreamer_plain_writer_content(bbstreamer *streamer,
-								bbstreamer_member *member, const char *data,
-								int len, bbstreamer_archive_context context)
+astreamer_plain_writer_content(astreamer *streamer,
+							   astreamer_member *member, const char *data,
+							   int len, astreamer_archive_context context)
 {
-	bbstreamer_plain_writer *mystreamer;
+	astreamer_plain_writer *mystreamer;
 
-	mystreamer = (bbstreamer_plain_writer *) streamer;
+	mystreamer = (astreamer_plain_writer *) streamer;
 
 	if (len == 0)
 		return;
@@ -128,11 +128,11 @@ bbstreamer_plain_writer_content(bbstreamer *streamer,
  * the file if we opened it, but not if the caller provided it.
  */
 static void
-bbstreamer_plain_writer_finalize(bbstreamer *streamer)
+astreamer_plain_writer_finalize(astreamer *streamer)
 {
-	bbstreamer_plain_writer *mystreamer;
+	astreamer_plain_writer *mystreamer;
 
-	mystreamer = (bbstreamer_plain_writer *) streamer;
+	mystreamer = (astreamer_plain_writer *) streamer;
 
 	if (mystreamer->should_close_file && fclose(mystreamer->file) != 0)
 		pg_fatal("could not close file \"%s\": %m",
@@ -143,14 +143,14 @@ bbstreamer_plain_writer_finalize(bbstreamer *streamer)
 }
 
 /*
- * Free memory associated with this bbstreamer.
+ * Free memory associated with this astreamer.
  */
 static void
-bbstreamer_plain_writer_free(bbstreamer *streamer)
+astreamer_plain_writer_free(astreamer *streamer)
 {
-	bbstreamer_plain_writer *mystreamer;
+	astreamer_plain_writer *mystreamer;
 
-	mystreamer = (bbstreamer_plain_writer *) streamer;
+	mystreamer = (astreamer_plain_writer *) streamer;
 
 	Assert(!mystreamer->should_close_file);
 	Assert(mystreamer->base.bbs_next == NULL);
@@ -160,13 +160,13 @@ bbstreamer_plain_writer_free(bbstreamer *streamer)
 }
 
 /*
- * Create a bbstreamer that extracts an archive.
+ * Create a astreamer that extracts an archive.
  *
  * All pathnames in the archive are interpreted relative to basepath.
  *
- * Unlike e.g. bbstreamer_plain_writer_new() we can't do anything useful here
+ * Unlike e.g. astreamer_plain_writer_new() we can't do anything useful here
  * with untyped chunks; we need typed chunks which follow the rules described
- * in bbstreamer.h. Assuming we have that, we don't need to worry about the
+ * in astreamer.h. Assuming we have that, we don't need to worry about the
  * original archive format; it's enough to just look at the member information
  * provided and write to the corresponding file.
  *
@@ -179,16 +179,16 @@ bbstreamer_plain_writer_free(bbstreamer *streamer)
  * new output file. The pathname to that file is passed as an argument. If
  * NULL, the call is skipped.
  */
-bbstreamer *
-bbstreamer_extractor_new(const char *basepath,
-						 const char *(*link_map) (const char *),
-						 void (*report_output_file) (const char *))
+astreamer *
+astreamer_extractor_new(const char *basepath,
+						const char *(*link_map) (const char *),
+						void (*report_output_file) (const char *))
 {
-	bbstreamer_extractor *streamer;
+	astreamer_extractor *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_extractor));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_extractor_ops;
+	streamer = palloc0(sizeof(astreamer_extractor));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_extractor_ops;
 	streamer->basepath = pstrdup(basepath);
 	streamer->link_map = link_map;
 	streamer->report_output_file = report_output_file;
@@ -200,19 +200,19 @@ bbstreamer_extractor_new(const char *basepath,
  * Extract archive contents to the filesystem.
  */
 static void
-bbstreamer_extractor_content(bbstreamer *streamer, bbstreamer_member *member,
-							 const char *data, int len,
-							 bbstreamer_archive_context context)
+astreamer_extractor_content(astreamer *streamer, astreamer_member *member,
+							const char *data, int len,
+							astreamer_archive_context context)
 {
-	bbstreamer_extractor *mystreamer = (bbstreamer_extractor *) streamer;
+	astreamer_extractor *mystreamer = (astreamer_extractor *) streamer;
 	int			fnamelen;
 
-	Assert(member != NULL || context == BBSTREAMER_ARCHIVE_TRAILER);
-	Assert(context != BBSTREAMER_UNKNOWN);
+	Assert(member != NULL || context == ASTREAMER_ARCHIVE_TRAILER);
+	Assert(context != ASTREAMER_UNKNOWN);
 
 	switch (context)
 	{
-		case BBSTREAMER_MEMBER_HEADER:
+		case ASTREAMER_MEMBER_HEADER:
 			Assert(mystreamer->file == NULL);
 
 			/* Prepend basepath. */
@@ -245,7 +245,7 @@ bbstreamer_extractor_content(bbstreamer *streamer, bbstreamer_member *member,
 				mystreamer->report_output_file(mystreamer->filename);
 			break;
 
-		case BBSTREAMER_MEMBER_CONTENTS:
+		case ASTREAMER_MEMBER_CONTENTS:
 			if (mystreamer->file == NULL)
 				break;
 
@@ -260,14 +260,14 @@ bbstreamer_extractor_content(bbstreamer *streamer, bbstreamer_member *member,
 			}
 			break;
 
-		case BBSTREAMER_MEMBER_TRAILER:
+		case ASTREAMER_MEMBER_TRAILER:
 			if (mystreamer->file == NULL)
 				break;
 			fclose(mystreamer->file);
 			mystreamer->file = NULL;
 			break;
 
-		case BBSTREAMER_ARCHIVE_TRAILER:
+		case ASTREAMER_ARCHIVE_TRAILER:
 			break;
 
 		default:
@@ -375,10 +375,10 @@ create_file_for_extract(const char *filename, mode_t mode)
  * There's nothing to do here but sanity checking.
  */
 static void
-bbstreamer_extractor_finalize(bbstreamer *streamer)
+astreamer_extractor_finalize(astreamer *streamer)
 {
-	bbstreamer_extractor *mystreamer PG_USED_FOR_ASSERTS_ONLY
-	= (bbstreamer_extractor *) streamer;
+	astreamer_extractor *mystreamer PG_USED_FOR_ASSERTS_ONLY
+	= (astreamer_extractor *) streamer;
 
 	Assert(mystreamer->file == NULL);
 }
@@ -387,9 +387,9 @@ bbstreamer_extractor_finalize(bbstreamer *streamer)
  * Free memory.
  */
 static void
-bbstreamer_extractor_free(bbstreamer *streamer)
+astreamer_extractor_free(astreamer *streamer)
 {
-	bbstreamer_extractor *mystreamer = (bbstreamer_extractor *) streamer;
+	astreamer_extractor *mystreamer = (astreamer_extractor *) streamer;
 
 	pfree(mystreamer->basepath);
 	pfree(mystreamer);
diff --git a/src/bin/pg_basebackup/bbstreamer_gzip.c b/src/bin/pg_basebackup/astreamer_gzip.c
similarity index 62%
rename from src/bin/pg_basebackup/bbstreamer_gzip.c
rename to src/bin/pg_basebackup/astreamer_gzip.c
index 0417fd9bc2c..6f7c27afbbc 100644
--- a/src/bin/pg_basebackup/bbstreamer_gzip.c
+++ b/src/bin/pg_basebackup/astreamer_gzip.c
@@ -1,11 +1,11 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_gzip.c
+ * astreamer_gzip.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_gzip.c
+ *		  src/bin/pg_basebackup/astreamer_gzip.c
  *-------------------------------------------------------------------------
  */
 
@@ -17,74 +17,74 @@
 #include <zlib.h>
 #endif
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
 
 #ifdef HAVE_LIBZ
-typedef struct bbstreamer_gzip_writer
+typedef struct astreamer_gzip_writer
 {
-	bbstreamer	base;
+	astreamer	base;
 	char	   *pathname;
 	gzFile		gzfile;
-} bbstreamer_gzip_writer;
+} astreamer_gzip_writer;
 
-typedef struct bbstreamer_gzip_decompressor
+typedef struct astreamer_gzip_decompressor
 {
-	bbstreamer	base;
+	astreamer	base;
 	z_stream	zstream;
 	size_t		bytes_written;
-} bbstreamer_gzip_decompressor;
+} astreamer_gzip_decompressor;
 
-static void bbstreamer_gzip_writer_content(bbstreamer *streamer,
-										   bbstreamer_member *member,
-										   const char *data, int len,
-										   bbstreamer_archive_context context);
-static void bbstreamer_gzip_writer_finalize(bbstreamer *streamer);
-static void bbstreamer_gzip_writer_free(bbstreamer *streamer);
+static void astreamer_gzip_writer_content(astreamer *streamer,
+										  astreamer_member *member,
+										  const char *data, int len,
+										  astreamer_archive_context context);
+static void astreamer_gzip_writer_finalize(astreamer *streamer);
+static void astreamer_gzip_writer_free(astreamer *streamer);
 static const char *get_gz_error(gzFile gzf);
 
-static const bbstreamer_ops bbstreamer_gzip_writer_ops = {
-	.content = bbstreamer_gzip_writer_content,
-	.finalize = bbstreamer_gzip_writer_finalize,
-	.free = bbstreamer_gzip_writer_free
+static const astreamer_ops astreamer_gzip_writer_ops = {
+	.content = astreamer_gzip_writer_content,
+	.finalize = astreamer_gzip_writer_finalize,
+	.free = astreamer_gzip_writer_free
 };
 
-static void bbstreamer_gzip_decompressor_content(bbstreamer *streamer,
-												 bbstreamer_member *member,
-												 const char *data, int len,
-												 bbstreamer_archive_context context);
-static void bbstreamer_gzip_decompressor_finalize(bbstreamer *streamer);
-static void bbstreamer_gzip_decompressor_free(bbstreamer *streamer);
+static void astreamer_gzip_decompressor_content(astreamer *streamer,
+												astreamer_member *member,
+												const char *data, int len,
+												astreamer_archive_context context);
+static void astreamer_gzip_decompressor_finalize(astreamer *streamer);
+static void astreamer_gzip_decompressor_free(astreamer *streamer);
 static void *gzip_palloc(void *opaque, unsigned items, unsigned size);
 static void gzip_pfree(void *opaque, void *address);
 
-static const bbstreamer_ops bbstreamer_gzip_decompressor_ops = {
-	.content = bbstreamer_gzip_decompressor_content,
-	.finalize = bbstreamer_gzip_decompressor_finalize,
-	.free = bbstreamer_gzip_decompressor_free
+static const astreamer_ops astreamer_gzip_decompressor_ops = {
+	.content = astreamer_gzip_decompressor_content,
+	.finalize = astreamer_gzip_decompressor_finalize,
+	.free = astreamer_gzip_decompressor_free
 };
 #endif
 
 /*
- * Create a bbstreamer that just compresses data using gzip, and then writes
+ * Create a astreamer that just compresses data using gzip, and then writes
  * it to a file.
  *
- * As in the case of bbstreamer_plain_writer_new, pathname is always used
+ * As in the case of astreamer_plain_writer_new, pathname is always used
  * for error reporting purposes; if file is NULL, it is also the opened and
  * closed so that the data may be written there.
  */
-bbstreamer *
-bbstreamer_gzip_writer_new(char *pathname, FILE *file,
-						   pg_compress_specification *compress)
+astreamer *
+astreamer_gzip_writer_new(char *pathname, FILE *file,
+						  pg_compress_specification *compress)
 {
 #ifdef HAVE_LIBZ
-	bbstreamer_gzip_writer *streamer;
+	astreamer_gzip_writer *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_gzip_writer));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_gzip_writer_ops;
+	streamer = palloc0(sizeof(astreamer_gzip_writer));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_gzip_writer_ops;
 
 	streamer->pathname = pstrdup(pathname);
 
@@ -123,13 +123,13 @@ bbstreamer_gzip_writer_new(char *pathname, FILE *file,
  * Write archive content to gzip file.
  */
 static void
-bbstreamer_gzip_writer_content(bbstreamer *streamer,
-							   bbstreamer_member *member, const char *data,
-							   int len, bbstreamer_archive_context context)
+astreamer_gzip_writer_content(astreamer *streamer,
+							  astreamer_member *member, const char *data,
+							  int len, astreamer_archive_context context)
 {
-	bbstreamer_gzip_writer *mystreamer;
+	astreamer_gzip_writer *mystreamer;
 
-	mystreamer = (bbstreamer_gzip_writer *) streamer;
+	mystreamer = (astreamer_gzip_writer *) streamer;
 
 	if (len == 0)
 		return;
@@ -151,16 +151,16 @@ bbstreamer_gzip_writer_content(bbstreamer *streamer,
  *
  * It makes no difference whether we opened the file or the caller did it,
  * because libz provides no way of avoiding a close on the underlying file
- * handle. Notice, however, that bbstreamer_gzip_writer_new() uses dup() to
+ * handle. Notice, however, that astreamer_gzip_writer_new() uses dup() to
  * work around this issue, so that the behavior from the caller's viewpoint
- * is the same as for bbstreamer_plain_writer.
+ * is the same as for astreamer_plain_writer.
  */
 static void
-bbstreamer_gzip_writer_finalize(bbstreamer *streamer)
+astreamer_gzip_writer_finalize(astreamer *streamer)
 {
-	bbstreamer_gzip_writer *mystreamer;
+	astreamer_gzip_writer *mystreamer;
 
-	mystreamer = (bbstreamer_gzip_writer *) streamer;
+	mystreamer = (astreamer_gzip_writer *) streamer;
 
 	errno = 0;					/* in case gzclose() doesn't set it */
 	if (gzclose(mystreamer->gzfile) != 0)
@@ -171,14 +171,14 @@ bbstreamer_gzip_writer_finalize(bbstreamer *streamer)
 }
 
 /*
- * Free memory associated with this bbstreamer.
+ * Free memory associated with this astreamer.
  */
 static void
-bbstreamer_gzip_writer_free(bbstreamer *streamer)
+astreamer_gzip_writer_free(astreamer *streamer)
 {
-	bbstreamer_gzip_writer *mystreamer;
+	astreamer_gzip_writer *mystreamer;
 
-	mystreamer = (bbstreamer_gzip_writer *) streamer;
+	mystreamer = (astreamer_gzip_writer *) streamer;
 
 	Assert(mystreamer->base.bbs_next == NULL);
 	Assert(mystreamer->gzfile == NULL);
@@ -208,18 +208,18 @@ get_gz_error(gzFile gzf)
  * Create a new base backup streamer that performs decompression of gzip
  * compressed blocks.
  */
-bbstreamer *
-bbstreamer_gzip_decompressor_new(bbstreamer *next)
+astreamer *
+astreamer_gzip_decompressor_new(astreamer *next)
 {
 #ifdef HAVE_LIBZ
-	bbstreamer_gzip_decompressor *streamer;
+	astreamer_gzip_decompressor *streamer;
 	z_stream   *zs;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_gzip_decompressor));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_gzip_decompressor_ops;
+	streamer = palloc0(sizeof(astreamer_gzip_decompressor));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_gzip_decompressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -258,15 +258,15 @@ bbstreamer_gzip_decompressor_new(bbstreamer *next)
  * to the next streamer.
  */
 static void
-bbstreamer_gzip_decompressor_content(bbstreamer *streamer,
-									 bbstreamer_member *member,
-									 const char *data, int len,
-									 bbstreamer_archive_context context)
+astreamer_gzip_decompressor_content(astreamer *streamer,
+									astreamer_member *member,
+									const char *data, int len,
+									astreamer_archive_context context)
 {
-	bbstreamer_gzip_decompressor *mystreamer;
+	astreamer_gzip_decompressor *mystreamer;
 	z_stream   *zs;
 
-	mystreamer = (bbstreamer_gzip_decompressor *) streamer;
+	mystreamer = (astreamer_gzip_decompressor *) streamer;
 
 	zs = &mystreamer->zstream;
 	zs->next_in = (const uint8 *) data;
@@ -301,9 +301,9 @@ bbstreamer_gzip_decompressor_content(bbstreamer *streamer,
 		/* If output buffer is full then pass data to next streamer */
 		if (mystreamer->bytes_written >= mystreamer->base.bbs_buffer.maxlen)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, member,
-							   mystreamer->base.bbs_buffer.data,
-							   mystreamer->base.bbs_buffer.maxlen, context);
+			astreamer_content(mystreamer->base.bbs_next, member,
+							  mystreamer->base.bbs_buffer.data,
+							  mystreamer->base.bbs_buffer.maxlen, context);
 			mystreamer->bytes_written = 0;
 		}
 	}
@@ -313,31 +313,31 @@ bbstreamer_gzip_decompressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_gzip_decompressor_finalize(bbstreamer *streamer)
+astreamer_gzip_decompressor_finalize(astreamer *streamer)
 {
-	bbstreamer_gzip_decompressor *mystreamer;
+	astreamer_gzip_decompressor *mystreamer;
 
-	mystreamer = (bbstreamer_gzip_decompressor *) streamer;
+	mystreamer = (astreamer_gzip_decompressor *) streamer;
 
 	/*
 	 * End of the stream, if there is some pending data in output buffers then
 	 * we must forward it to next streamer.
 	 */
-	bbstreamer_content(mystreamer->base.bbs_next, NULL,
-					   mystreamer->base.bbs_buffer.data,
-					   mystreamer->base.bbs_buffer.maxlen,
-					   BBSTREAMER_UNKNOWN);
+	astreamer_content(mystreamer->base.bbs_next, NULL,
+					  mystreamer->base.bbs_buffer.data,
+					  mystreamer->base.bbs_buffer.maxlen,
+					  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_gzip_decompressor_free(bbstreamer *streamer)
+astreamer_gzip_decompressor_free(astreamer *streamer)
 {
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
 }
diff --git a/src/bin/pg_basebackup/bbstreamer_inject.c b/src/bin/pg_basebackup/astreamer_inject.c
similarity index 53%
rename from src/bin/pg_basebackup/bbstreamer_inject.c
rename to src/bin/pg_basebackup/astreamer_inject.c
index 194026b56e9..7f1decded8d 100644
--- a/src/bin/pg_basebackup/bbstreamer_inject.c
+++ b/src/bin/pg_basebackup/astreamer_inject.c
@@ -1,51 +1,51 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_inject.c
+ * astreamer_inject.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_inject.c
+ *		  src/bin/pg_basebackup/astreamer_inject.c
  *-------------------------------------------------------------------------
  */
 
 #include "postgres_fe.h"
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 
-typedef struct bbstreamer_recovery_injector
+typedef struct astreamer_recovery_injector
 {
-	bbstreamer	base;
+	astreamer	base;
 	bool		skip_file;
 	bool		is_recovery_guc_supported;
 	bool		is_postgresql_auto_conf;
 	bool		found_postgresql_auto_conf;
 	PQExpBuffer recoveryconfcontents;
-	bbstreamer_member member;
-} bbstreamer_recovery_injector;
+	astreamer_member member;
+} astreamer_recovery_injector;
 
-static void bbstreamer_recovery_injector_content(bbstreamer *streamer,
-												 bbstreamer_member *member,
-												 const char *data, int len,
-												 bbstreamer_archive_context context);
-static void bbstreamer_recovery_injector_finalize(bbstreamer *streamer);
-static void bbstreamer_recovery_injector_free(bbstreamer *streamer);
+static void astreamer_recovery_injector_content(astreamer *streamer,
+												astreamer_member *member,
+												const char *data, int len,
+												astreamer_archive_context context);
+static void astreamer_recovery_injector_finalize(astreamer *streamer);
+static void astreamer_recovery_injector_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_recovery_injector_ops = {
-	.content = bbstreamer_recovery_injector_content,
-	.finalize = bbstreamer_recovery_injector_finalize,
-	.free = bbstreamer_recovery_injector_free
+static const astreamer_ops astreamer_recovery_injector_ops = {
+	.content = astreamer_recovery_injector_content,
+	.finalize = astreamer_recovery_injector_finalize,
+	.free = astreamer_recovery_injector_free
 };
 
 /*
- * Create a bbstreamer that can edit recoverydata into an archive stream.
+ * Create a astreamer that can edit recoverydata into an archive stream.
  *
- * The input should be a series of typed chunks (not BBSTREAMER_UNKNOWN) as
- * per the conventions described in bbstreamer.h; the chunks forwarded to
- * the next bbstreamer will be similarly typed, but the
- * BBSTREAMER_MEMBER_HEADER chunks may be zero-length in cases where we've
+ * The input should be a series of typed chunks (not ASTREAMER_UNKNOWN) as
+ * per the conventions described in astreamer.h; the chunks forwarded to
+ * the next astreamer will be similarly typed, but the
+ * ASTREAMER_MEMBER_HEADER chunks may be zero-length in cases where we've
  * edited the archive stream.
  *
  * Our goal is to do one of the following three things with the content passed
@@ -61,16 +61,16 @@ static const bbstreamer_ops bbstreamer_recovery_injector_ops = {
  * zero-length standby.signal file, dropping any file with that name from
  * the archive.
  */
-bbstreamer *
-bbstreamer_recovery_injector_new(bbstreamer *next,
-								 bool is_recovery_guc_supported,
-								 PQExpBuffer recoveryconfcontents)
+astreamer *
+astreamer_recovery_injector_new(astreamer *next,
+								bool is_recovery_guc_supported,
+								PQExpBuffer recoveryconfcontents)
 {
-	bbstreamer_recovery_injector *streamer;
+	astreamer_recovery_injector *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_recovery_injector));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_recovery_injector_ops;
+	streamer = palloc0(sizeof(astreamer_recovery_injector));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_recovery_injector_ops;
 	streamer->base.bbs_next = next;
 	streamer->is_recovery_guc_supported = is_recovery_guc_supported;
 	streamer->recoveryconfcontents = recoveryconfcontents;
@@ -82,21 +82,21 @@ bbstreamer_recovery_injector_new(bbstreamer *next,
  * Handle each chunk of tar content while injecting recovery configuration.
  */
 static void
-bbstreamer_recovery_injector_content(bbstreamer *streamer,
-									 bbstreamer_member *member,
-									 const char *data, int len,
-									 bbstreamer_archive_context context)
+astreamer_recovery_injector_content(astreamer *streamer,
+									astreamer_member *member,
+									const char *data, int len,
+									astreamer_archive_context context)
 {
-	bbstreamer_recovery_injector *mystreamer;
+	astreamer_recovery_injector *mystreamer;
 
-	mystreamer = (bbstreamer_recovery_injector *) streamer;
-	Assert(member != NULL || context == BBSTREAMER_ARCHIVE_TRAILER);
+	mystreamer = (astreamer_recovery_injector *) streamer;
+	Assert(member != NULL || context == ASTREAMER_ARCHIVE_TRAILER);
 
 	switch (context)
 	{
-		case BBSTREAMER_MEMBER_HEADER:
+		case ASTREAMER_MEMBER_HEADER:
 			/* Must copy provided data so we have the option to modify it. */
-			memcpy(&mystreamer->member, member, sizeof(bbstreamer_member));
+			memcpy(&mystreamer->member, member, sizeof(astreamer_member));
 
 			/*
 			 * On v12+, skip standby.signal and edit postgresql.auto.conf; on
@@ -119,8 +119,8 @@ bbstreamer_recovery_injector_content(bbstreamer *streamer,
 
 					/*
 					 * Zap data and len because the archive header is no
-					 * longer valid; some subsequent bbstreamer must
-					 * regenerate it if it's necessary.
+					 * longer valid; some subsequent astreamer must regenerate
+					 * it if it's necessary.
 					 */
 					data = NULL;
 					len = 0;
@@ -135,26 +135,26 @@ bbstreamer_recovery_injector_content(bbstreamer *streamer,
 				return;
 			break;
 
-		case BBSTREAMER_MEMBER_CONTENTS:
+		case ASTREAMER_MEMBER_CONTENTS:
 			/* Do not forward if the file is to be skipped. */
 			if (mystreamer->skip_file)
 				return;
 			break;
 
-		case BBSTREAMER_MEMBER_TRAILER:
+		case ASTREAMER_MEMBER_TRAILER:
 			/* Do not forward it the file is to be skipped. */
 			if (mystreamer->skip_file)
 				return;
 
 			/* Append provided content to whatever we already sent. */
 			if (mystreamer->is_postgresql_auto_conf)
-				bbstreamer_content(mystreamer->base.bbs_next, member,
-								   mystreamer->recoveryconfcontents->data,
-								   mystreamer->recoveryconfcontents->len,
-								   BBSTREAMER_MEMBER_CONTENTS);
+				astreamer_content(mystreamer->base.bbs_next, member,
+								  mystreamer->recoveryconfcontents->data,
+								  mystreamer->recoveryconfcontents->len,
+								  ASTREAMER_MEMBER_CONTENTS);
 			break;
 
-		case BBSTREAMER_ARCHIVE_TRAILER:
+		case ASTREAMER_ARCHIVE_TRAILER:
 			if (mystreamer->is_recovery_guc_supported)
 			{
 				/*
@@ -163,22 +163,22 @@ bbstreamer_recovery_injector_content(bbstreamer *streamer,
 				 * member now.
 				 */
 				if (!mystreamer->found_postgresql_auto_conf)
-					bbstreamer_inject_file(mystreamer->base.bbs_next,
-										   "postgresql.auto.conf",
-										   mystreamer->recoveryconfcontents->data,
-										   mystreamer->recoveryconfcontents->len);
+					astreamer_inject_file(mystreamer->base.bbs_next,
+										  "postgresql.auto.conf",
+										  mystreamer->recoveryconfcontents->data,
+										  mystreamer->recoveryconfcontents->len);
 
 				/* Inject empty standby.signal file. */
-				bbstreamer_inject_file(mystreamer->base.bbs_next,
-									   "standby.signal", "", 0);
+				astreamer_inject_file(mystreamer->base.bbs_next,
+									  "standby.signal", "", 0);
 			}
 			else
 			{
 				/* Inject recovery.conf file with specified contents. */
-				bbstreamer_inject_file(mystreamer->base.bbs_next,
-									   "recovery.conf",
-									   mystreamer->recoveryconfcontents->data,
-									   mystreamer->recoveryconfcontents->len);
+				astreamer_inject_file(mystreamer->base.bbs_next,
+									  "recovery.conf",
+									  mystreamer->recoveryconfcontents->data,
+									  mystreamer->recoveryconfcontents->len);
 			}
 
 			/* Nothing to do here. */
@@ -189,26 +189,26 @@ bbstreamer_recovery_injector_content(bbstreamer *streamer,
 			pg_fatal("unexpected state while injecting recovery settings");
 	}
 
-	bbstreamer_content(mystreamer->base.bbs_next, &mystreamer->member,
-					   data, len, context);
+	astreamer_content(mystreamer->base.bbs_next, &mystreamer->member,
+					  data, len, context);
 }
 
 /*
- * End-of-stream processing for this bbstreamer.
+ * End-of-stream processing for this astreamer.
  */
 static void
-bbstreamer_recovery_injector_finalize(bbstreamer *streamer)
+astreamer_recovery_injector_finalize(astreamer *streamer)
 {
-	bbstreamer_finalize(streamer->bbs_next);
+	astreamer_finalize(streamer->bbs_next);
 }
 
 /*
- * Free memory associated with this bbstreamer.
+ * Free memory associated with this astreamer.
  */
 static void
-bbstreamer_recovery_injector_free(bbstreamer *streamer)
+astreamer_recovery_injector_free(astreamer *streamer)
 {
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	pfree(streamer);
 }
 
@@ -216,10 +216,10 @@ bbstreamer_recovery_injector_free(bbstreamer *streamer)
  * Inject a member into the archive with specified contents.
  */
 void
-bbstreamer_inject_file(bbstreamer *streamer, char *pathname, char *data,
-					   int len)
+astreamer_inject_file(astreamer *streamer, char *pathname, char *data,
+					  int len)
 {
-	bbstreamer_member member;
+	astreamer_member member;
 
 	strlcpy(member.pathname, pathname, MAXPGPATH);
 	member.size = len;
@@ -238,12 +238,12 @@ bbstreamer_inject_file(bbstreamer *streamer, char *pathname, char *data,
 	/*
 	 * We don't know here how to generate valid member headers and trailers
 	 * for the archiving format in use, so if those are needed, some successor
-	 * bbstreamer will have to generate them using the data from 'member'.
+	 * astreamer will have to generate them using the data from 'member'.
 	 */
-	bbstreamer_content(streamer, &member, NULL, 0,
-					   BBSTREAMER_MEMBER_HEADER);
-	bbstreamer_content(streamer, &member, data, len,
-					   BBSTREAMER_MEMBER_CONTENTS);
-	bbstreamer_content(streamer, &member, NULL, 0,
-					   BBSTREAMER_MEMBER_TRAILER);
+	astreamer_content(streamer, &member, NULL, 0,
+					  ASTREAMER_MEMBER_HEADER);
+	astreamer_content(streamer, &member, data, len,
+					  ASTREAMER_MEMBER_CONTENTS);
+	astreamer_content(streamer, &member, NULL, 0,
+					  ASTREAMER_MEMBER_TRAILER);
 }
diff --git a/src/bin/pg_basebackup/bbstreamer_lz4.c b/src/bin/pg_basebackup/astreamer_lz4.c
similarity index 69%
rename from src/bin/pg_basebackup/bbstreamer_lz4.c
rename to src/bin/pg_basebackup/astreamer_lz4.c
index f5c9e68150c..1c40d7d8ad5 100644
--- a/src/bin/pg_basebackup/bbstreamer_lz4.c
+++ b/src/bin/pg_basebackup/astreamer_lz4.c
@@ -1,11 +1,11 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_lz4.c
+ * astreamer_lz4.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_lz4.c
+ *		  src/bin/pg_basebackup/astreamer_lz4.c
  *-------------------------------------------------------------------------
  */
 
@@ -17,15 +17,15 @@
 #include <lz4frame.h>
 #endif
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
 
 #ifdef USE_LZ4
-typedef struct bbstreamer_lz4_frame
+typedef struct astreamer_lz4_frame
 {
-	bbstreamer	base;
+	astreamer	base;
 
 	LZ4F_compressionContext_t cctx;
 	LZ4F_decompressionContext_t dctx;
@@ -33,32 +33,32 @@ typedef struct bbstreamer_lz4_frame
 
 	size_t		bytes_written;
 	bool		header_written;
-} bbstreamer_lz4_frame;
+} astreamer_lz4_frame;
 
-static void bbstreamer_lz4_compressor_content(bbstreamer *streamer,
-											  bbstreamer_member *member,
-											  const char *data, int len,
-											  bbstreamer_archive_context context);
-static void bbstreamer_lz4_compressor_finalize(bbstreamer *streamer);
-static void bbstreamer_lz4_compressor_free(bbstreamer *streamer);
+static void astreamer_lz4_compressor_content(astreamer *streamer,
+											 astreamer_member *member,
+											 const char *data, int len,
+											 astreamer_archive_context context);
+static void astreamer_lz4_compressor_finalize(astreamer *streamer);
+static void astreamer_lz4_compressor_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_lz4_compressor_ops = {
-	.content = bbstreamer_lz4_compressor_content,
-	.finalize = bbstreamer_lz4_compressor_finalize,
-	.free = bbstreamer_lz4_compressor_free
+static const astreamer_ops astreamer_lz4_compressor_ops = {
+	.content = astreamer_lz4_compressor_content,
+	.finalize = astreamer_lz4_compressor_finalize,
+	.free = astreamer_lz4_compressor_free
 };
 
-static void bbstreamer_lz4_decompressor_content(bbstreamer *streamer,
-												bbstreamer_member *member,
-												const char *data, int len,
-												bbstreamer_archive_context context);
-static void bbstreamer_lz4_decompressor_finalize(bbstreamer *streamer);
-static void bbstreamer_lz4_decompressor_free(bbstreamer *streamer);
+static void astreamer_lz4_decompressor_content(astreamer *streamer,
+											   astreamer_member *member,
+											   const char *data, int len,
+											   astreamer_archive_context context);
+static void astreamer_lz4_decompressor_finalize(astreamer *streamer);
+static void astreamer_lz4_decompressor_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_lz4_decompressor_ops = {
-	.content = bbstreamer_lz4_decompressor_content,
-	.finalize = bbstreamer_lz4_decompressor_finalize,
-	.free = bbstreamer_lz4_decompressor_free
+static const astreamer_ops astreamer_lz4_decompressor_ops = {
+	.content = astreamer_lz4_decompressor_content,
+	.finalize = astreamer_lz4_decompressor_finalize,
+	.free = astreamer_lz4_decompressor_free
 };
 #endif
 
@@ -66,19 +66,19 @@ static const bbstreamer_ops bbstreamer_lz4_decompressor_ops = {
  * Create a new base backup streamer that performs lz4 compression of tar
  * blocks.
  */
-bbstreamer *
-bbstreamer_lz4_compressor_new(bbstreamer *next, pg_compress_specification *compress)
+astreamer *
+astreamer_lz4_compressor_new(astreamer *next, pg_compress_specification *compress)
 {
 #ifdef USE_LZ4
-	bbstreamer_lz4_frame *streamer;
+	astreamer_lz4_frame *streamer;
 	LZ4F_errorCode_t ctxError;
 	LZ4F_preferences_t *prefs;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_lz4_frame));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_lz4_compressor_ops;
+	streamer = palloc0(sizeof(astreamer_lz4_frame));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_lz4_compressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -113,19 +113,19 @@ bbstreamer_lz4_compressor_new(bbstreamer *next, pg_compress_specification *compr
  * of output buffer to next streamer and empty the buffer.
  */
 static void
-bbstreamer_lz4_compressor_content(bbstreamer *streamer,
-								  bbstreamer_member *member,
-								  const char *data, int len,
-								  bbstreamer_archive_context context)
+astreamer_lz4_compressor_content(astreamer *streamer,
+								 astreamer_member *member,
+								 const char *data, int len,
+								 astreamer_archive_context context)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 	uint8	   *next_in,
 			   *next_out;
 	size_t		out_bound,
 				compressed_size,
 				avail_out;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
+	mystreamer = (astreamer_lz4_frame *) streamer;
 	next_in = (uint8 *) data;
 
 	/* Write header before processing the first input chunk. */
@@ -159,10 +159,10 @@ bbstreamer_lz4_compressor_content(bbstreamer *streamer,
 	out_bound = LZ4F_compressBound(len, &mystreamer->prefs);
 	if (avail_out < out_bound)
 	{
-		bbstreamer_content(mystreamer->base.bbs_next, member,
-						   mystreamer->base.bbs_buffer.data,
-						   mystreamer->bytes_written,
-						   context);
+		astreamer_content(mystreamer->base.bbs_next, member,
+						  mystreamer->base.bbs_buffer.data,
+						  mystreamer->bytes_written,
+						  context);
 
 		/* Enlarge buffer if it falls short of out bound. */
 		if (mystreamer->base.bbs_buffer.maxlen < out_bound)
@@ -196,25 +196,25 @@ bbstreamer_lz4_compressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_lz4_compressor_finalize(bbstreamer *streamer)
+astreamer_lz4_compressor_finalize(astreamer *streamer)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 	uint8	   *next_out;
 	size_t		footer_bound,
 				compressed_size,
 				avail_out;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
+	mystreamer = (astreamer_lz4_frame *) streamer;
 
 	/* Find out the footer bound and update the output buffer. */
 	footer_bound = LZ4F_compressBound(0, &mystreamer->prefs);
 	if ((mystreamer->base.bbs_buffer.maxlen - mystreamer->bytes_written) <
 		footer_bound)
 	{
-		bbstreamer_content(mystreamer->base.bbs_next, NULL,
-						   mystreamer->base.bbs_buffer.data,
-						   mystreamer->bytes_written,
-						   BBSTREAMER_UNKNOWN);
+		astreamer_content(mystreamer->base.bbs_next, NULL,
+						  mystreamer->base.bbs_buffer.data,
+						  mystreamer->bytes_written,
+						  ASTREAMER_UNKNOWN);
 
 		/* Enlarge buffer if it falls short of footer bound. */
 		if (mystreamer->base.bbs_buffer.maxlen < footer_bound)
@@ -243,24 +243,24 @@ bbstreamer_lz4_compressor_finalize(bbstreamer *streamer)
 
 	mystreamer->bytes_written += compressed_size;
 
-	bbstreamer_content(mystreamer->base.bbs_next, NULL,
-					   mystreamer->base.bbs_buffer.data,
-					   mystreamer->bytes_written,
-					   BBSTREAMER_UNKNOWN);
+	astreamer_content(mystreamer->base.bbs_next, NULL,
+					  mystreamer->base.bbs_buffer.data,
+					  mystreamer->bytes_written,
+					  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_lz4_compressor_free(bbstreamer *streamer)
+astreamer_lz4_compressor_free(astreamer *streamer)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
-	bbstreamer_free(streamer->bbs_next);
+	mystreamer = (astreamer_lz4_frame *) streamer;
+	astreamer_free(streamer->bbs_next);
 	LZ4F_freeCompressionContext(mystreamer->cctx);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
@@ -271,18 +271,18 @@ bbstreamer_lz4_compressor_free(bbstreamer *streamer)
  * Create a new base backup streamer that performs decompression of lz4
  * compressed blocks.
  */
-bbstreamer *
-bbstreamer_lz4_decompressor_new(bbstreamer *next)
+astreamer *
+astreamer_lz4_decompressor_new(astreamer *next)
 {
 #ifdef USE_LZ4
-	bbstreamer_lz4_frame *streamer;
+	astreamer_lz4_frame *streamer;
 	LZ4F_errorCode_t ctxError;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_lz4_frame));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_lz4_decompressor_ops;
+	streamer = palloc0(sizeof(astreamer_lz4_frame));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_lz4_decompressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -307,18 +307,18 @@ bbstreamer_lz4_decompressor_new(bbstreamer *next)
  * to the next streamer.
  */
 static void
-bbstreamer_lz4_decompressor_content(bbstreamer *streamer,
-									bbstreamer_member *member,
-									const char *data, int len,
-									bbstreamer_archive_context context)
+astreamer_lz4_decompressor_content(astreamer *streamer,
+								   astreamer_member *member,
+								   const char *data, int len,
+								   astreamer_archive_context context)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 	uint8	   *next_in,
 			   *next_out;
 	size_t		avail_in,
 				avail_out;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
+	mystreamer = (astreamer_lz4_frame *) streamer;
 	next_in = (uint8 *) data;
 	next_out = (uint8 *) mystreamer->base.bbs_buffer.data;
 	avail_in = len;
@@ -366,10 +366,10 @@ bbstreamer_lz4_decompressor_content(bbstreamer *streamer,
 		 */
 		if (mystreamer->bytes_written >= mystreamer->base.bbs_buffer.maxlen)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, member,
-							   mystreamer->base.bbs_buffer.data,
-							   mystreamer->base.bbs_buffer.maxlen,
-							   context);
+			astreamer_content(mystreamer->base.bbs_next, member,
+							  mystreamer->base.bbs_buffer.data,
+							  mystreamer->base.bbs_buffer.maxlen,
+							  context);
 
 			avail_out = mystreamer->base.bbs_buffer.maxlen;
 			mystreamer->bytes_written = 0;
@@ -387,34 +387,34 @@ bbstreamer_lz4_decompressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_lz4_decompressor_finalize(bbstreamer *streamer)
+astreamer_lz4_decompressor_finalize(astreamer *streamer)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
+	mystreamer = (astreamer_lz4_frame *) streamer;
 
 	/*
 	 * End of the stream, if there is some pending data in output buffers then
 	 * we must forward it to next streamer.
 	 */
-	bbstreamer_content(mystreamer->base.bbs_next, NULL,
-					   mystreamer->base.bbs_buffer.data,
-					   mystreamer->base.bbs_buffer.maxlen,
-					   BBSTREAMER_UNKNOWN);
+	astreamer_content(mystreamer->base.bbs_next, NULL,
+					  mystreamer->base.bbs_buffer.data,
+					  mystreamer->base.bbs_buffer.maxlen,
+					  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_lz4_decompressor_free(bbstreamer *streamer)
+astreamer_lz4_decompressor_free(astreamer *streamer)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
-	bbstreamer_free(streamer->bbs_next);
+	mystreamer = (astreamer_lz4_frame *) streamer;
+	astreamer_free(streamer->bbs_next);
 	LZ4F_freeDecompressionContext(mystreamer->dctx);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
diff --git a/src/bin/pg_basebackup/bbstreamer_tar.c b/src/bin/pg_basebackup/astreamer_tar.c
similarity index 50%
rename from src/bin/pg_basebackup/bbstreamer_tar.c
rename to src/bin/pg_basebackup/astreamer_tar.c
index 9137d17ddc1..673690cd18f 100644
--- a/src/bin/pg_basebackup/bbstreamer_tar.c
+++ b/src/bin/pg_basebackup/astreamer_tar.c
@@ -1,13 +1,13 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_tar.c
+ * astreamer_tar.c
  *
  * This module implements three types of tar processing. A tar parser
- * expects unlabelled chunks of data (e.g. BBSTREAMER_UNKNOWN) and splits
- * it into labelled chunks (any other value of bbstreamer_archive_context).
+ * expects unlabelled chunks of data (e.g. ASTREAMER_UNKNOWN) and splits
+ * it into labelled chunks (any other value of astreamer_archive_context).
  * A tar archiver does the reverse: it takes a bunch of labelled chunks
  * and produces a tarfile, optionally replacing member headers and trailers
- * so that upstream bbstreamer objects can perform surgery on the tarfile
+ * so that upstream astreamer objects can perform surgery on the tarfile
  * contents without knowing the details of the tar format. A tar terminator
  * just adds two blocks of NUL bytes to the end of the file, since older
  * server versions produce files with this terminator omitted.
@@ -15,7 +15,7 @@
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_tar.c
+ *		  src/bin/pg_basebackup/astreamer_tar.c
  *-------------------------------------------------------------------------
  */
 
@@ -23,83 +23,83 @@
 
 #include <time.h>
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/logging.h"
 #include "pgtar.h"
 
-typedef struct bbstreamer_tar_parser
+typedef struct astreamer_tar_parser
 {
-	bbstreamer	base;
-	bbstreamer_archive_context next_context;
-	bbstreamer_member member;
+	astreamer	base;
+	astreamer_archive_context next_context;
+	astreamer_member member;
 	size_t		file_bytes_sent;
 	size_t		pad_bytes_expected;
-} bbstreamer_tar_parser;
+} astreamer_tar_parser;
 
-typedef struct bbstreamer_tar_archiver
+typedef struct astreamer_tar_archiver
 {
-	bbstreamer	base;
+	astreamer	base;
 	bool		rearchive_member;
-} bbstreamer_tar_archiver;
+} astreamer_tar_archiver;
 
-static void bbstreamer_tar_parser_content(bbstreamer *streamer,
-										  bbstreamer_member *member,
-										  const char *data, int len,
-										  bbstreamer_archive_context context);
-static void bbstreamer_tar_parser_finalize(bbstreamer *streamer);
-static void bbstreamer_tar_parser_free(bbstreamer *streamer);
-static bool bbstreamer_tar_header(bbstreamer_tar_parser *mystreamer);
+static void astreamer_tar_parser_content(astreamer *streamer,
+										 astreamer_member *member,
+										 const char *data, int len,
+										 astreamer_archive_context context);
+static void astreamer_tar_parser_finalize(astreamer *streamer);
+static void astreamer_tar_parser_free(astreamer *streamer);
+static bool astreamer_tar_header(astreamer_tar_parser *mystreamer);
 
-static const bbstreamer_ops bbstreamer_tar_parser_ops = {
-	.content = bbstreamer_tar_parser_content,
-	.finalize = bbstreamer_tar_parser_finalize,
-	.free = bbstreamer_tar_parser_free
+static const astreamer_ops astreamer_tar_parser_ops = {
+	.content = astreamer_tar_parser_content,
+	.finalize = astreamer_tar_parser_finalize,
+	.free = astreamer_tar_parser_free
 };
 
-static void bbstreamer_tar_archiver_content(bbstreamer *streamer,
-											bbstreamer_member *member,
-											const char *data, int len,
-											bbstreamer_archive_context context);
-static void bbstreamer_tar_archiver_finalize(bbstreamer *streamer);
-static void bbstreamer_tar_archiver_free(bbstreamer *streamer);
+static void astreamer_tar_archiver_content(astreamer *streamer,
+										   astreamer_member *member,
+										   const char *data, int len,
+										   astreamer_archive_context context);
+static void astreamer_tar_archiver_finalize(astreamer *streamer);
+static void astreamer_tar_archiver_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_tar_archiver_ops = {
-	.content = bbstreamer_tar_archiver_content,
-	.finalize = bbstreamer_tar_archiver_finalize,
-	.free = bbstreamer_tar_archiver_free
+static const astreamer_ops astreamer_tar_archiver_ops = {
+	.content = astreamer_tar_archiver_content,
+	.finalize = astreamer_tar_archiver_finalize,
+	.free = astreamer_tar_archiver_free
 };
 
-static void bbstreamer_tar_terminator_content(bbstreamer *streamer,
-											  bbstreamer_member *member,
-											  const char *data, int len,
-											  bbstreamer_archive_context context);
-static void bbstreamer_tar_terminator_finalize(bbstreamer *streamer);
-static void bbstreamer_tar_terminator_free(bbstreamer *streamer);
+static void astreamer_tar_terminator_content(astreamer *streamer,
+											 astreamer_member *member,
+											 const char *data, int len,
+											 astreamer_archive_context context);
+static void astreamer_tar_terminator_finalize(astreamer *streamer);
+static void astreamer_tar_terminator_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_tar_terminator_ops = {
-	.content = bbstreamer_tar_terminator_content,
-	.finalize = bbstreamer_tar_terminator_finalize,
-	.free = bbstreamer_tar_terminator_free
+static const astreamer_ops astreamer_tar_terminator_ops = {
+	.content = astreamer_tar_terminator_content,
+	.finalize = astreamer_tar_terminator_finalize,
+	.free = astreamer_tar_terminator_free
 };
 
 /*
- * Create a bbstreamer that can parse a stream of content as tar data.
+ * Create a astreamer that can parse a stream of content as tar data.
  *
- * The input should be a series of BBSTREAMER_UNKNOWN chunks; the bbstreamer
+ * The input should be a series of ASTREAMER_UNKNOWN chunks; the astreamer
  * specified by 'next' will receive a series of typed chunks, as per the
- * conventions described in bbstreamer.h.
+ * conventions described in astreamer.h.
  */
-bbstreamer *
-bbstreamer_tar_parser_new(bbstreamer *next)
+astreamer *
+astreamer_tar_parser_new(astreamer *next)
 {
-	bbstreamer_tar_parser *streamer;
+	astreamer_tar_parser *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_tar_parser));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_tar_parser_ops;
+	streamer = palloc0(sizeof(astreamer_tar_parser));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_tar_parser_ops;
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
-	streamer->next_context = BBSTREAMER_MEMBER_HEADER;
+	streamer->next_context = ASTREAMER_MEMBER_HEADER;
 
 	return &streamer->base;
 }
@@ -108,29 +108,29 @@ bbstreamer_tar_parser_new(bbstreamer *next)
  * Parse unknown content as tar data.
  */
 static void
-bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
-							  const char *data, int len,
-							  bbstreamer_archive_context context)
+astreamer_tar_parser_content(astreamer *streamer, astreamer_member *member,
+							 const char *data, int len,
+							 astreamer_archive_context context)
 {
-	bbstreamer_tar_parser *mystreamer = (bbstreamer_tar_parser *) streamer;
+	astreamer_tar_parser *mystreamer = (astreamer_tar_parser *) streamer;
 	size_t		nbytes;
 
 	/* Expect unparsed input. */
 	Assert(member == NULL);
-	Assert(context == BBSTREAMER_UNKNOWN);
+	Assert(context == ASTREAMER_UNKNOWN);
 
 	while (len > 0)
 	{
 		switch (mystreamer->next_context)
 		{
-			case BBSTREAMER_MEMBER_HEADER:
+			case ASTREAMER_MEMBER_HEADER:
 
 				/*
 				 * If we're expecting an archive member header, accumulate a
 				 * full block of data before doing anything further.
 				 */
-				if (!bbstreamer_buffer_until(streamer, &data, &len,
-											 TAR_BLOCK_SIZE))
+				if (!astreamer_buffer_until(streamer, &data, &len,
+											TAR_BLOCK_SIZE))
 					return;
 
 				/*
@@ -139,32 +139,32 @@ bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
 				 * thought was the next file header is actually the start of
 				 * the archive trailer. Switch modes accordingly.
 				 */
-				if (bbstreamer_tar_header(mystreamer))
+				if (astreamer_tar_header(mystreamer))
 				{
 					if (mystreamer->member.size == 0)
 					{
 						/* No content; trailer is zero-length. */
-						bbstreamer_content(mystreamer->base.bbs_next,
-										   &mystreamer->member,
-										   NULL, 0,
-										   BBSTREAMER_MEMBER_TRAILER);
+						astreamer_content(mystreamer->base.bbs_next,
+										  &mystreamer->member,
+										  NULL, 0,
+										  ASTREAMER_MEMBER_TRAILER);
 
 						/* Expect next header. */
-						mystreamer->next_context = BBSTREAMER_MEMBER_HEADER;
+						mystreamer->next_context = ASTREAMER_MEMBER_HEADER;
 					}
 					else
 					{
 						/* Expect contents. */
-						mystreamer->next_context = BBSTREAMER_MEMBER_CONTENTS;
+						mystreamer->next_context = ASTREAMER_MEMBER_CONTENTS;
 					}
 					mystreamer->base.bbs_buffer.len = 0;
 					mystreamer->file_bytes_sent = 0;
 				}
 				else
-					mystreamer->next_context = BBSTREAMER_ARCHIVE_TRAILER;
+					mystreamer->next_context = ASTREAMER_ARCHIVE_TRAILER;
 				break;
 
-			case BBSTREAMER_MEMBER_CONTENTS:
+			case ASTREAMER_MEMBER_CONTENTS:
 
 				/*
 				 * Send as much content as we have, but not more than the
@@ -174,10 +174,10 @@ bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
 				nbytes = mystreamer->member.size - mystreamer->file_bytes_sent;
 				nbytes = Min(nbytes, len);
 				Assert(nbytes > 0);
-				bbstreamer_content(mystreamer->base.bbs_next,
-								   &mystreamer->member,
-								   data, nbytes,
-								   BBSTREAMER_MEMBER_CONTENTS);
+				astreamer_content(mystreamer->base.bbs_next,
+								  &mystreamer->member,
+								  data, nbytes,
+								  ASTREAMER_MEMBER_CONTENTS);
 				mystreamer->file_bytes_sent += nbytes;
 				data += nbytes;
 				len -= nbytes;
@@ -193,53 +193,53 @@ bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
 					if (mystreamer->pad_bytes_expected == 0)
 					{
 						/* Trailer is zero-length. */
-						bbstreamer_content(mystreamer->base.bbs_next,
-										   &mystreamer->member,
-										   NULL, 0,
-										   BBSTREAMER_MEMBER_TRAILER);
+						astreamer_content(mystreamer->base.bbs_next,
+										  &mystreamer->member,
+										  NULL, 0,
+										  ASTREAMER_MEMBER_TRAILER);
 
 						/* Expect next header. */
-						mystreamer->next_context = BBSTREAMER_MEMBER_HEADER;
+						mystreamer->next_context = ASTREAMER_MEMBER_HEADER;
 					}
 					else
 					{
 						/* Trailer is not zero-length. */
-						mystreamer->next_context = BBSTREAMER_MEMBER_TRAILER;
+						mystreamer->next_context = ASTREAMER_MEMBER_TRAILER;
 					}
 					mystreamer->base.bbs_buffer.len = 0;
 				}
 				break;
 
-			case BBSTREAMER_MEMBER_TRAILER:
+			case ASTREAMER_MEMBER_TRAILER:
 
 				/*
 				 * If we're expecting an archive member trailer, accumulate
 				 * the expected number of padding bytes before sending
 				 * anything onward.
 				 */
-				if (!bbstreamer_buffer_until(streamer, &data, &len,
-											 mystreamer->pad_bytes_expected))
+				if (!astreamer_buffer_until(streamer, &data, &len,
+											mystreamer->pad_bytes_expected))
 					return;
 
 				/* OK, now we can send it. */
-				bbstreamer_content(mystreamer->base.bbs_next,
-								   &mystreamer->member,
-								   data, mystreamer->pad_bytes_expected,
-								   BBSTREAMER_MEMBER_TRAILER);
+				astreamer_content(mystreamer->base.bbs_next,
+								  &mystreamer->member,
+								  data, mystreamer->pad_bytes_expected,
+								  ASTREAMER_MEMBER_TRAILER);
 
 				/* Expect next file header. */
-				mystreamer->next_context = BBSTREAMER_MEMBER_HEADER;
+				mystreamer->next_context = ASTREAMER_MEMBER_HEADER;
 				mystreamer->base.bbs_buffer.len = 0;
 				break;
 
-			case BBSTREAMER_ARCHIVE_TRAILER:
+			case ASTREAMER_ARCHIVE_TRAILER:
 
 				/*
 				 * We've seen an end-of-archive indicator, so anything more is
 				 * buffered and sent as part of the archive trailer. But we
 				 * don't expect more than 2 blocks.
 				 */
-				bbstreamer_buffer_bytes(streamer, &data, &len, len);
+				astreamer_buffer_bytes(streamer, &data, &len, len);
 				if (len > 2 * TAR_BLOCK_SIZE)
 					pg_fatal("tar file trailer exceeds 2 blocks");
 				return;
@@ -255,14 +255,14 @@ bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
  * Parse a file header within a tar stream.
  *
  * The return value is true if we found a file header and passed it on to the
- * next bbstreamer; it is false if we have reached the archive trailer.
+ * next astreamer; it is false if we have reached the archive trailer.
  */
 static bool
-bbstreamer_tar_header(bbstreamer_tar_parser *mystreamer)
+astreamer_tar_header(astreamer_tar_parser *mystreamer)
 {
 	bool		has_nonzero_byte = false;
 	int			i;
-	bbstreamer_member *member = &mystreamer->member;
+	astreamer_member *member = &mystreamer->member;
 	char	   *buffer = mystreamer->base.bbs_buffer.data;
 
 	Assert(mystreamer->base.bbs_buffer.len == TAR_BLOCK_SIZE);
@@ -304,10 +304,10 @@ bbstreamer_tar_header(bbstreamer_tar_parser *mystreamer)
 	/* Compute number of padding bytes. */
 	mystreamer->pad_bytes_expected = tarPaddingBytesRequired(member->size);
 
-	/* Forward the entire header to the next bbstreamer. */
-	bbstreamer_content(mystreamer->base.bbs_next, member,
-					   buffer, TAR_BLOCK_SIZE,
-					   BBSTREAMER_MEMBER_HEADER);
+	/* Forward the entire header to the next astreamer. */
+	astreamer_content(mystreamer->base.bbs_next, member,
+					  buffer, TAR_BLOCK_SIZE,
+					  ASTREAMER_MEMBER_HEADER);
 
 	return true;
 }
@@ -316,50 +316,50 @@ bbstreamer_tar_header(bbstreamer_tar_parser *mystreamer)
  * End-of-stream processing for a tar parser.
  */
 static void
-bbstreamer_tar_parser_finalize(bbstreamer *streamer)
+astreamer_tar_parser_finalize(astreamer *streamer)
 {
-	bbstreamer_tar_parser *mystreamer = (bbstreamer_tar_parser *) streamer;
+	astreamer_tar_parser *mystreamer = (astreamer_tar_parser *) streamer;
 
-	if (mystreamer->next_context != BBSTREAMER_ARCHIVE_TRAILER &&
-		(mystreamer->next_context != BBSTREAMER_MEMBER_HEADER ||
+	if (mystreamer->next_context != ASTREAMER_ARCHIVE_TRAILER &&
+		(mystreamer->next_context != ASTREAMER_MEMBER_HEADER ||
 		 mystreamer->base.bbs_buffer.len > 0))
 		pg_fatal("COPY stream ended before last file was finished");
 
 	/* Send the archive trailer, even if empty. */
-	bbstreamer_content(streamer->bbs_next, NULL,
-					   streamer->bbs_buffer.data, streamer->bbs_buffer.len,
-					   BBSTREAMER_ARCHIVE_TRAILER);
+	astreamer_content(streamer->bbs_next, NULL,
+					  streamer->bbs_buffer.data, streamer->bbs_buffer.len,
+					  ASTREAMER_ARCHIVE_TRAILER);
 
 	/* Now finalize successor. */
-	bbstreamer_finalize(streamer->bbs_next);
+	astreamer_finalize(streamer->bbs_next);
 }
 
 /*
  * Free memory associated with a tar parser.
  */
 static void
-bbstreamer_tar_parser_free(bbstreamer *streamer)
+astreamer_tar_parser_free(astreamer *streamer)
 {
 	pfree(streamer->bbs_buffer.data);
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 }
 
 /*
- * Create a bbstreamer that can generate a tar archive.
+ * Create a astreamer that can generate a tar archive.
  *
  * This is intended to be usable either for generating a brand-new tar archive
  * or for modifying one on the fly. The input should be a series of typed
- * chunks (i.e. not BBSTREAMER_UNKNOWN). See also the comments for
- * bbstreamer_tar_parser_content.
+ * chunks (i.e. not ASTREAMER_UNKNOWN). See also the comments for
+ * astreamer_tar_parser_content.
  */
-bbstreamer *
-bbstreamer_tar_archiver_new(bbstreamer *next)
+astreamer *
+astreamer_tar_archiver_new(astreamer *next)
 {
-	bbstreamer_tar_archiver *streamer;
+	astreamer_tar_archiver *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_tar_archiver));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_tar_archiver_ops;
+	streamer = palloc0(sizeof(astreamer_tar_archiver));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_tar_archiver_ops;
 	streamer->base.bbs_next = next;
 
 	return &streamer->base;
@@ -368,36 +368,36 @@ bbstreamer_tar_archiver_new(bbstreamer *next)
 /*
  * Fix up the stream of input chunks to create a valid tar file.
  *
- * If a BBSTREAMER_MEMBER_HEADER chunk is of size 0, it is replaced with a
+ * If a ASTREAMER_MEMBER_HEADER chunk is of size 0, it is replaced with a
  * newly-constructed tar header. If it is of size TAR_BLOCK_SIZE, it is
  * passed through without change. Any other size is a fatal error (and
  * indicates a bug).
  *
- * Whenever a new BBSTREAMER_MEMBER_HEADER chunk is constructed, the
- * corresponding BBSTREAMER_MEMBER_TRAILER chunk is also constructed from
+ * Whenever a new ASTREAMER_MEMBER_HEADER chunk is constructed, the
+ * corresponding ASTREAMER_MEMBER_TRAILER chunk is also constructed from
  * scratch. Specifically, we construct a block of zero bytes sufficient to
  * pad out to a block boundary, as required by the tar format. Other
- * BBSTREAMER_MEMBER_TRAILER chunks are passed through without change.
+ * ASTREAMER_MEMBER_TRAILER chunks are passed through without change.
  *
- * Any BBSTREAMER_MEMBER_CONTENTS chunks are passed through without change.
+ * Any ASTREAMER_MEMBER_CONTENTS chunks are passed through without change.
  *
- * The BBSTREAMER_ARCHIVE_TRAILER chunk is replaced with two
+ * The ASTREAMER_ARCHIVE_TRAILER chunk is replaced with two
  * blocks of zero bytes. Not all tar programs require this, but apparently
  * some do. The server does not supply this trailer. If no archive trailer is
- * present, one will be added by bbstreamer_tar_parser_finalize.
+ * present, one will be added by astreamer_tar_parser_finalize.
  */
 static void
-bbstreamer_tar_archiver_content(bbstreamer *streamer,
-								bbstreamer_member *member,
-								const char *data, int len,
-								bbstreamer_archive_context context)
+astreamer_tar_archiver_content(astreamer *streamer,
+							   astreamer_member *member,
+							   const char *data, int len,
+							   astreamer_archive_context context)
 {
-	bbstreamer_tar_archiver *mystreamer = (bbstreamer_tar_archiver *) streamer;
+	astreamer_tar_archiver *mystreamer = (astreamer_tar_archiver *) streamer;
 	char		buffer[2 * TAR_BLOCK_SIZE];
 
-	Assert(context != BBSTREAMER_UNKNOWN);
+	Assert(context != ASTREAMER_UNKNOWN);
 
-	if (context == BBSTREAMER_MEMBER_HEADER && len != TAR_BLOCK_SIZE)
+	if (context == ASTREAMER_MEMBER_HEADER && len != TAR_BLOCK_SIZE)
 	{
 		Assert(len == 0);
 
@@ -411,7 +411,7 @@ bbstreamer_tar_archiver_content(bbstreamer *streamer,
 		/* Also make a note to replace padding, in case size changed. */
 		mystreamer->rearchive_member = true;
 	}
-	else if (context == BBSTREAMER_MEMBER_TRAILER &&
+	else if (context == ASTREAMER_MEMBER_TRAILER &&
 			 mystreamer->rearchive_member)
 	{
 		int			pad_bytes = tarPaddingBytesRequired(member->size);
@@ -424,7 +424,7 @@ bbstreamer_tar_archiver_content(bbstreamer *streamer,
 		/* Don't do this again unless we replace another header. */
 		mystreamer->rearchive_member = false;
 	}
-	else if (context == BBSTREAMER_ARCHIVE_TRAILER)
+	else if (context == ASTREAMER_ARCHIVE_TRAILER)
 	{
 		/* Trailer should always be two blocks of zero bytes. */
 		memset(buffer, 0, 2 * TAR_BLOCK_SIZE);
@@ -432,40 +432,40 @@ bbstreamer_tar_archiver_content(bbstreamer *streamer,
 		len = 2 * TAR_BLOCK_SIZE;
 	}
 
-	bbstreamer_content(streamer->bbs_next, member, data, len, context);
+	astreamer_content(streamer->bbs_next, member, data, len, context);
 }
 
 /*
  * End-of-stream processing for a tar archiver.
  */
 static void
-bbstreamer_tar_archiver_finalize(bbstreamer *streamer)
+astreamer_tar_archiver_finalize(astreamer *streamer)
 {
-	bbstreamer_finalize(streamer->bbs_next);
+	astreamer_finalize(streamer->bbs_next);
 }
 
 /*
  * Free memory associated with a tar archiver.
  */
 static void
-bbstreamer_tar_archiver_free(bbstreamer *streamer)
+astreamer_tar_archiver_free(astreamer *streamer)
 {
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	pfree(streamer);
 }
 
 /*
- * Create a bbstreamer that blindly adds two blocks of NUL bytes to the
+ * Create a astreamer that blindly adds two blocks of NUL bytes to the
  * end of an incomplete tarfile that the server might send us.
  */
-bbstreamer *
-bbstreamer_tar_terminator_new(bbstreamer *next)
+astreamer *
+astreamer_tar_terminator_new(astreamer *next)
 {
-	bbstreamer *streamer;
+	astreamer  *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer));
-	*((const bbstreamer_ops **) &streamer->bbs_ops) =
-		&bbstreamer_tar_terminator_ops;
+	streamer = palloc0(sizeof(astreamer));
+	*((const astreamer_ops **) &streamer->bbs_ops) =
+		&astreamer_tar_terminator_ops;
 	streamer->bbs_next = next;
 
 	return streamer;
@@ -475,17 +475,17 @@ bbstreamer_tar_terminator_new(bbstreamer *next)
  * Pass all the content through without change.
  */
 static void
-bbstreamer_tar_terminator_content(bbstreamer *streamer,
-								  bbstreamer_member *member,
-								  const char *data, int len,
-								  bbstreamer_archive_context context)
+astreamer_tar_terminator_content(astreamer *streamer,
+								 astreamer_member *member,
+								 const char *data, int len,
+								 astreamer_archive_context context)
 {
 	/* Expect unparsed input. */
 	Assert(member == NULL);
-	Assert(context == BBSTREAMER_UNKNOWN);
+	Assert(context == ASTREAMER_UNKNOWN);
 
 	/* Just forward it. */
-	bbstreamer_content(streamer->bbs_next, member, data, len, context);
+	astreamer_content(streamer->bbs_next, member, data, len, context);
 }
 
 /*
@@ -493,22 +493,22 @@ bbstreamer_tar_terminator_content(bbstreamer *streamer,
  * to supply.
  */
 static void
-bbstreamer_tar_terminator_finalize(bbstreamer *streamer)
+astreamer_tar_terminator_finalize(astreamer *streamer)
 {
 	char		buffer[2 * TAR_BLOCK_SIZE];
 
 	memset(buffer, 0, 2 * TAR_BLOCK_SIZE);
-	bbstreamer_content(streamer->bbs_next, NULL, buffer,
-					   2 * TAR_BLOCK_SIZE, BBSTREAMER_UNKNOWN);
-	bbstreamer_finalize(streamer->bbs_next);
+	astreamer_content(streamer->bbs_next, NULL, buffer,
+					  2 * TAR_BLOCK_SIZE, ASTREAMER_UNKNOWN);
+	astreamer_finalize(streamer->bbs_next);
 }
 
 /*
  * Free memory associated with a tar terminator.
  */
 static void
-bbstreamer_tar_terminator_free(bbstreamer *streamer)
+astreamer_tar_terminator_free(astreamer *streamer)
 {
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	pfree(streamer);
 }
diff --git a/src/bin/pg_basebackup/bbstreamer_zstd.c b/src/bin/pg_basebackup/astreamer_zstd.c
similarity index 64%
rename from src/bin/pg_basebackup/bbstreamer_zstd.c
rename to src/bin/pg_basebackup/astreamer_zstd.c
index 20f11d4450e..58dc679ef99 100644
--- a/src/bin/pg_basebackup/bbstreamer_zstd.c
+++ b/src/bin/pg_basebackup/astreamer_zstd.c
@@ -1,11 +1,11 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_zstd.c
+ * astreamer_zstd.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_zstd.c
+ *		  src/bin/pg_basebackup/astreamer_zstd.c
  *-------------------------------------------------------------------------
  */
 
@@ -17,44 +17,44 @@
 #include <zstd.h>
 #endif
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/logging.h"
 
 #ifdef USE_ZSTD
 
-typedef struct bbstreamer_zstd_frame
+typedef struct astreamer_zstd_frame
 {
-	bbstreamer	base;
+	astreamer	base;
 
 	ZSTD_CCtx  *cctx;
 	ZSTD_DCtx  *dctx;
 	ZSTD_outBuffer zstd_outBuf;
-} bbstreamer_zstd_frame;
+} astreamer_zstd_frame;
 
-static void bbstreamer_zstd_compressor_content(bbstreamer *streamer,
-											   bbstreamer_member *member,
-											   const char *data, int len,
-											   bbstreamer_archive_context context);
-static void bbstreamer_zstd_compressor_finalize(bbstreamer *streamer);
-static void bbstreamer_zstd_compressor_free(bbstreamer *streamer);
+static void astreamer_zstd_compressor_content(astreamer *streamer,
+											  astreamer_member *member,
+											  const char *data, int len,
+											  astreamer_archive_context context);
+static void astreamer_zstd_compressor_finalize(astreamer *streamer);
+static void astreamer_zstd_compressor_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_zstd_compressor_ops = {
-	.content = bbstreamer_zstd_compressor_content,
-	.finalize = bbstreamer_zstd_compressor_finalize,
-	.free = bbstreamer_zstd_compressor_free
+static const astreamer_ops astreamer_zstd_compressor_ops = {
+	.content = astreamer_zstd_compressor_content,
+	.finalize = astreamer_zstd_compressor_finalize,
+	.free = astreamer_zstd_compressor_free
 };
 
-static void bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
-												 bbstreamer_member *member,
-												 const char *data, int len,
-												 bbstreamer_archive_context context);
-static void bbstreamer_zstd_decompressor_finalize(bbstreamer *streamer);
-static void bbstreamer_zstd_decompressor_free(bbstreamer *streamer);
+static void astreamer_zstd_decompressor_content(astreamer *streamer,
+												astreamer_member *member,
+												const char *data, int len,
+												astreamer_archive_context context);
+static void astreamer_zstd_decompressor_finalize(astreamer *streamer);
+static void astreamer_zstd_decompressor_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_zstd_decompressor_ops = {
-	.content = bbstreamer_zstd_decompressor_content,
-	.finalize = bbstreamer_zstd_decompressor_finalize,
-	.free = bbstreamer_zstd_decompressor_free
+static const astreamer_ops astreamer_zstd_decompressor_ops = {
+	.content = astreamer_zstd_decompressor_content,
+	.finalize = astreamer_zstd_decompressor_finalize,
+	.free = astreamer_zstd_decompressor_free
 };
 #endif
 
@@ -62,19 +62,19 @@ static const bbstreamer_ops bbstreamer_zstd_decompressor_ops = {
  * Create a new base backup streamer that performs zstd compression of tar
  * blocks.
  */
-bbstreamer *
-bbstreamer_zstd_compressor_new(bbstreamer *next, pg_compress_specification *compress)
+astreamer *
+astreamer_zstd_compressor_new(astreamer *next, pg_compress_specification *compress)
 {
 #ifdef USE_ZSTD
-	bbstreamer_zstd_frame *streamer;
+	astreamer_zstd_frame *streamer;
 	size_t		ret;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_zstd_frame));
+	streamer = palloc0(sizeof(astreamer_zstd_frame));
 
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_zstd_compressor_ops;
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_zstd_compressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -142,12 +142,12 @@ bbstreamer_zstd_compressor_new(bbstreamer *next, pg_compress_specification *comp
  * of output buffer to next streamer and empty the buffer.
  */
 static void
-bbstreamer_zstd_compressor_content(bbstreamer *streamer,
-								   bbstreamer_member *member,
-								   const char *data, int len,
-								   bbstreamer_archive_context context)
+astreamer_zstd_compressor_content(astreamer *streamer,
+								  astreamer_member *member,
+								  const char *data, int len,
+								  astreamer_archive_context context)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 	ZSTD_inBuffer inBuf = {data, len, 0};
 
 	while (inBuf.pos < inBuf.size)
@@ -162,10 +162,10 @@ bbstreamer_zstd_compressor_content(bbstreamer *streamer,
 		if (mystreamer->zstd_outBuf.size - mystreamer->zstd_outBuf.pos <
 			max_needed)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, member,
-							   mystreamer->zstd_outBuf.dst,
-							   mystreamer->zstd_outBuf.pos,
-							   context);
+			astreamer_content(mystreamer->base.bbs_next, member,
+							  mystreamer->zstd_outBuf.dst,
+							  mystreamer->zstd_outBuf.pos,
+							  context);
 
 			/* Reset the ZSTD output buffer. */
 			mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data;
@@ -187,9 +187,9 @@ bbstreamer_zstd_compressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_zstd_compressor_finalize(bbstreamer *streamer)
+astreamer_zstd_compressor_finalize(astreamer *streamer)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 	size_t		yet_to_flush;
 
 	do
@@ -204,10 +204,10 @@ bbstreamer_zstd_compressor_finalize(bbstreamer *streamer)
 		if (mystreamer->zstd_outBuf.size - mystreamer->zstd_outBuf.pos <
 			max_needed)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, NULL,
-							   mystreamer->zstd_outBuf.dst,
-							   mystreamer->zstd_outBuf.pos,
-							   BBSTREAMER_UNKNOWN);
+			astreamer_content(mystreamer->base.bbs_next, NULL,
+							  mystreamer->zstd_outBuf.dst,
+							  mystreamer->zstd_outBuf.pos,
+							  ASTREAMER_UNKNOWN);
 
 			/* Reset the ZSTD output buffer. */
 			mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data;
@@ -227,23 +227,23 @@ bbstreamer_zstd_compressor_finalize(bbstreamer *streamer)
 
 	/* Make sure to pass any remaining bytes to the next streamer. */
 	if (mystreamer->zstd_outBuf.pos > 0)
-		bbstreamer_content(mystreamer->base.bbs_next, NULL,
-						   mystreamer->zstd_outBuf.dst,
-						   mystreamer->zstd_outBuf.pos,
-						   BBSTREAMER_UNKNOWN);
+		astreamer_content(mystreamer->base.bbs_next, NULL,
+						  mystreamer->zstd_outBuf.dst,
+						  mystreamer->zstd_outBuf.pos,
+						  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_zstd_compressor_free(bbstreamer *streamer)
+astreamer_zstd_compressor_free(astreamer *streamer)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	ZSTD_freeCCtx(mystreamer->cctx);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
@@ -254,17 +254,17 @@ bbstreamer_zstd_compressor_free(bbstreamer *streamer)
  * Create a new base backup streamer that performs decompression of zstd
  * compressed blocks.
  */
-bbstreamer *
-bbstreamer_zstd_decompressor_new(bbstreamer *next)
+astreamer *
+astreamer_zstd_decompressor_new(astreamer *next)
 {
 #ifdef USE_ZSTD
-	bbstreamer_zstd_frame *streamer;
+	astreamer_zstd_frame *streamer;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_zstd_frame));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_zstd_decompressor_ops;
+	streamer = palloc0(sizeof(astreamer_zstd_frame));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_zstd_decompressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -293,12 +293,12 @@ bbstreamer_zstd_decompressor_new(bbstreamer *next)
  * to the next streamer.
  */
 static void
-bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
-									 bbstreamer_member *member,
-									 const char *data, int len,
-									 bbstreamer_archive_context context)
+astreamer_zstd_decompressor_content(astreamer *streamer,
+									astreamer_member *member,
+									const char *data, int len,
+									astreamer_archive_context context)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 	ZSTD_inBuffer inBuf = {data, len, 0};
 
 	while (inBuf.pos < inBuf.size)
@@ -311,10 +311,10 @@ bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
 		 */
 		if (mystreamer->zstd_outBuf.pos >= mystreamer->zstd_outBuf.size)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, member,
-							   mystreamer->zstd_outBuf.dst,
-							   mystreamer->zstd_outBuf.pos,
-							   context);
+			astreamer_content(mystreamer->base.bbs_next, member,
+							  mystreamer->zstd_outBuf.dst,
+							  mystreamer->zstd_outBuf.pos,
+							  context);
 
 			/* Reset the ZSTD output buffer. */
 			mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data;
@@ -335,32 +335,32 @@ bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_zstd_decompressor_finalize(bbstreamer *streamer)
+astreamer_zstd_decompressor_finalize(astreamer *streamer)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 
 	/*
 	 * End of the stream, if there is some pending data in output buffers then
 	 * we must forward it to next streamer.
 	 */
 	if (mystreamer->zstd_outBuf.pos > 0)
-		bbstreamer_content(mystreamer->base.bbs_next, NULL,
-						   mystreamer->base.bbs_buffer.data,
-						   mystreamer->base.bbs_buffer.maxlen,
-						   BBSTREAMER_UNKNOWN);
+		astreamer_content(mystreamer->base.bbs_next, NULL,
+						  mystreamer->base.bbs_buffer.data,
+						  mystreamer->base.bbs_buffer.maxlen,
+						  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_zstd_decompressor_free(bbstreamer *streamer)
+astreamer_zstd_decompressor_free(astreamer *streamer)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	ZSTD_freeDCtx(mystreamer->dctx);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
diff --git a/src/bin/pg_basebackup/bbstreamer.h b/src/bin/pg_basebackup/bbstreamer.h
deleted file mode 100644
index 3b820f13b51..00000000000
--- a/src/bin/pg_basebackup/bbstreamer.h
+++ /dev/null
@@ -1,226 +0,0 @@
-/*-------------------------------------------------------------------------
- *
- * bbstreamer.h
- *
- * Each tar archive returned by the server is passed to one or more
- * bbstreamer objects for further processing. The bbstreamer may do
- * something simple, like write the archive to a file, perhaps after
- * compressing it, but it can also do more complicated things, like
- * annotating the byte stream to indicate which parts of the data
- * correspond to tar headers or trailing padding, vs. which parts are
- * payload data. A subsequent bbstreamer may use this information to
- * make further decisions about how to process the data; for example,
- * it might choose to modify the archive contents.
- *
- * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
- *
- * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer.h
- *-------------------------------------------------------------------------
- */
-
-#ifndef BBSTREAMER_H
-#define BBSTREAMER_H
-
-#include "common/compression.h"
-#include "lib/stringinfo.h"
-#include "pqexpbuffer.h"
-
-struct bbstreamer;
-struct bbstreamer_ops;
-typedef struct bbstreamer bbstreamer;
-typedef struct bbstreamer_ops bbstreamer_ops;
-
-/*
- * Each chunk of archive data passed to a bbstreamer is classified into one
- * of these categories. When data is first received from the remote server,
- * each chunk will be categorized as BBSTREAMER_UNKNOWN, and the chunks will
- * be of whatever size the remote server chose to send.
- *
- * If the archive is parsed (e.g. see bbstreamer_tar_parser_new()), then all
- * chunks should be labelled as one of the other types listed here. In
- * addition, there should be exactly one BBSTREAMER_MEMBER_HEADER chunk and
- * exactly one BBSTREAMER_MEMBER_TRAILER chunk per archive member, even if
- * that means a zero-length call. There can be any number of
- * BBSTREAMER_MEMBER_CONTENTS chunks in between those calls. There
- * should exactly BBSTREAMER_ARCHIVE_TRAILER chunk, and it should follow the
- * last BBSTREAMER_MEMBER_TRAILER chunk.
- *
- * In theory, we could need other classifications here, such as a way of
- * indicating an archive header, but the "tar" format doesn't need anything
- * else, so for the time being there's no point.
- */
-typedef enum
-{
-	BBSTREAMER_UNKNOWN,
-	BBSTREAMER_MEMBER_HEADER,
-	BBSTREAMER_MEMBER_CONTENTS,
-	BBSTREAMER_MEMBER_TRAILER,
-	BBSTREAMER_ARCHIVE_TRAILER,
-} bbstreamer_archive_context;
-
-/*
- * Each chunk of data that is classified as BBSTREAMER_MEMBER_HEADER,
- * BBSTREAMER_MEMBER_CONTENTS, or BBSTREAMER_MEMBER_TRAILER should also
- * pass a pointer to an instance of this struct. The details are expected
- * to be present in the archive header and used to fill the struct, after
- * which all subsequent calls for the same archive member are expected to
- * pass the same details.
- */
-typedef struct
-{
-	char		pathname[MAXPGPATH];
-	pgoff_t		size;
-	mode_t		mode;
-	uid_t		uid;
-	gid_t		gid;
-	bool		is_directory;
-	bool		is_link;
-	char		linktarget[MAXPGPATH];
-} bbstreamer_member;
-
-/*
- * Generally, each type of bbstreamer will define its own struct, but the
- * first element should be 'bbstreamer base'. A bbstreamer that does not
- * require any additional private data could use this structure directly.
- *
- * bbs_ops is a pointer to the bbstreamer_ops object which contains the
- * function pointers appropriate to this type of bbstreamer.
- *
- * bbs_next is a pointer to the successor bbstreamer, for those types of
- * bbstreamer which forward data to a successor. It need not be used and
- * should be set to NULL when not relevant.
- *
- * bbs_buffer is a buffer for accumulating data for temporary storage. Each
- * type of bbstreamer makes its own decisions about whether and how to use
- * this buffer.
- */
-struct bbstreamer
-{
-	const bbstreamer_ops *bbs_ops;
-	bbstreamer *bbs_next;
-	StringInfoData bbs_buffer;
-};
-
-/*
- * There are three callbacks for a bbstreamer. The 'content' callback is
- * called repeatedly, as described in the bbstreamer_archive_context comments.
- * Then, the 'finalize' callback is called once at the end, to give the
- * bbstreamer a chance to perform cleanup such as closing files. Finally,
- * because this code is running in a frontend environment where, as of this
- * writing, there are no memory contexts, the 'free' callback is called to
- * release memory. These callbacks should always be invoked using the static
- * inline functions defined below.
- */
-struct bbstreamer_ops
-{
-	void		(*content) (bbstreamer *streamer, bbstreamer_member *member,
-							const char *data, int len,
-							bbstreamer_archive_context context);
-	void		(*finalize) (bbstreamer *streamer);
-	void		(*free) (bbstreamer *streamer);
-};
-
-/* Send some content to a bbstreamer. */
-static inline void
-bbstreamer_content(bbstreamer *streamer, bbstreamer_member *member,
-				   const char *data, int len,
-				   bbstreamer_archive_context context)
-{
-	Assert(streamer != NULL);
-	streamer->bbs_ops->content(streamer, member, data, len, context);
-}
-
-/* Finalize a bbstreamer. */
-static inline void
-bbstreamer_finalize(bbstreamer *streamer)
-{
-	Assert(streamer != NULL);
-	streamer->bbs_ops->finalize(streamer);
-}
-
-/* Free a bbstreamer. */
-static inline void
-bbstreamer_free(bbstreamer *streamer)
-{
-	Assert(streamer != NULL);
-	streamer->bbs_ops->free(streamer);
-}
-
-/*
- * This is a convenience method for use when implementing a bbstreamer; it is
- * not for use by outside callers. It adds the amount of data specified by
- * 'nbytes' to the bbstreamer's buffer and adjusts '*len' and '*data'
- * accordingly.
- */
-static inline void
-bbstreamer_buffer_bytes(bbstreamer *streamer, const char **data, int *len,
-						int nbytes)
-{
-	Assert(nbytes <= *len);
-
-	appendBinaryStringInfo(&streamer->bbs_buffer, *data, nbytes);
-	*len -= nbytes;
-	*data += nbytes;
-}
-
-/*
- * This is a convenience method for use when implementing a bbstreamer; it is
- * not for use by outsider callers. It attempts to add enough data to the
- * bbstreamer's buffer to reach a length of target_bytes and adjusts '*len'
- * and '*data' accordingly. It returns true if the target length has been
- * reached and false otherwise.
- */
-static inline bool
-bbstreamer_buffer_until(bbstreamer *streamer, const char **data, int *len,
-						int target_bytes)
-{
-	int			buflen = streamer->bbs_buffer.len;
-
-	if (buflen >= target_bytes)
-	{
-		/* Target length already reached; nothing to do. */
-		return true;
-	}
-
-	if (buflen + *len < target_bytes)
-	{
-		/* Not enough data to reach target length; buffer all of it. */
-		bbstreamer_buffer_bytes(streamer, data, len, *len);
-		return false;
-	}
-
-	/* Buffer just enough to reach the target length. */
-	bbstreamer_buffer_bytes(streamer, data, len, target_bytes - buflen);
-	return true;
-}
-
-/*
- * Functions for creating bbstreamer objects of various types. See the header
- * comments for each of these functions for details.
- */
-extern bbstreamer *bbstreamer_plain_writer_new(char *pathname, FILE *file);
-extern bbstreamer *bbstreamer_gzip_writer_new(char *pathname, FILE *file,
-											  pg_compress_specification *compress);
-extern bbstreamer *bbstreamer_extractor_new(const char *basepath,
-											const char *(*link_map) (const char *),
-											void (*report_output_file) (const char *));
-
-extern bbstreamer *bbstreamer_gzip_decompressor_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_lz4_compressor_new(bbstreamer *next,
-												 pg_compress_specification *compress);
-extern bbstreamer *bbstreamer_lz4_decompressor_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_zstd_compressor_new(bbstreamer *next,
-												  pg_compress_specification *compress);
-extern bbstreamer *bbstreamer_zstd_decompressor_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_tar_parser_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_tar_terminator_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_tar_archiver_new(bbstreamer *next);
-
-extern bbstreamer *bbstreamer_recovery_injector_new(bbstreamer *next,
-													bool is_recovery_guc_supported,
-													PQExpBuffer recoveryconfcontents);
-extern void bbstreamer_inject_file(bbstreamer *streamer, char *pathname,
-								   char *data, int len);
-
-#endif
diff --git a/src/bin/pg_basebackup/meson.build b/src/bin/pg_basebackup/meson.build
index c00acd5e118..a68dbd7837d 100644
--- a/src/bin/pg_basebackup/meson.build
+++ b/src/bin/pg_basebackup/meson.build
@@ -1,12 +1,12 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 common_sources = files(
-  'bbstreamer_file.c',
-  'bbstreamer_gzip.c',
-  'bbstreamer_inject.c',
-  'bbstreamer_lz4.c',
-  'bbstreamer_tar.c',
-  'bbstreamer_zstd.c',
+  'astreamer_file.c',
+  'astreamer_gzip.c',
+  'astreamer_inject.c',
+  'astreamer_lz4.c',
+  'astreamer_tar.c',
+  'astreamer_zstd.c',
   'receivelog.c',
   'streamutil.c',
   'walmethods.c',
diff --git a/src/bin/pg_basebackup/nls.mk b/src/bin/pg_basebackup/nls.mk
index 384dbb021e9..950b9797b1e 100644
--- a/src/bin/pg_basebackup/nls.mk
+++ b/src/bin/pg_basebackup/nls.mk
@@ -1,12 +1,12 @@
 # src/bin/pg_basebackup/nls.mk
 CATALOG_NAME     = pg_basebackup
 GETTEXT_FILES    = $(FRONTEND_COMMON_GETTEXT_FILES) \
-                   bbstreamer_file.c \
-                   bbstreamer_gzip.c \
-                   bbstreamer_inject.c \
-                   bbstreamer_lz4.c \
-                   bbstreamer_tar.c \
-                   bbstreamer_zstd.c \
+                   astreamer_file.c \
+                   astreamer_gzip.c \
+                   astreamer_inject.c \
+                   astreamer_lz4.c \
+                   astreamer_tar.c \
+                   astreamer_zstd.c \
                    pg_basebackup.c \
                    pg_createsubscriber.c \
                    pg_receivewal.c \
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 8f3dd04fd22..4179b064cbc 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -26,8 +26,8 @@
 #endif
 
 #include "access/xlog_internal.h"
+#include "astreamer.h"
 #include "backup/basebackup.h"
-#include "bbstreamer.h"
 #include "common/compression.h"
 #include "common/file_perm.h"
 #include "common/file_utils.h"
@@ -57,8 +57,8 @@ typedef struct ArchiveStreamState
 {
 	int			tablespacenum;
 	pg_compress_specification *compress;
-	bbstreamer *streamer;
-	bbstreamer *manifest_inject_streamer;
+	astreamer *streamer;
+	astreamer *manifest_inject_streamer;
 	PQExpBuffer manifest_buffer;
 	char		manifest_filename[MAXPGPATH];
 	FILE	   *manifest_file;
@@ -67,7 +67,7 @@ typedef struct ArchiveStreamState
 typedef struct WriteTarState
 {
 	int			tablespacenum;
-	bbstreamer *streamer;
+	astreamer *streamer;
 } WriteTarState;
 
 typedef struct WriteManifestState
@@ -199,8 +199,8 @@ static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *fo
 static void progress_update_filename(const char *filename);
 static void progress_report(int tablespacenum, bool force, bool finished);
 
-static bbstreamer *CreateBackupStreamer(char *archive_name, char *spclocation,
-										bbstreamer **manifest_inject_streamer_p,
+static astreamer *CreateBackupStreamer(char *archive_name, char *spclocation,
+										astreamer **manifest_inject_streamer_p,
 										bool is_recovery_guc_supported,
 										bool expect_unterminated_tarfile,
 										pg_compress_specification *compress);
@@ -1053,19 +1053,19 @@ ReceiveCopyData(PGconn *conn, WriteDataCallback callback,
  * the options selected by the user.  We may just write the results directly
  * to a file, or we might compress first, or we might extract the tar file
  * and write each member separately. This function doesn't do any of that
- * directly, but it works out what kind of bbstreamer we need to create so
+ * directly, but it works out what kind of astreamer we need to create so
  * that the right stuff happens when, down the road, we actually receive
  * the data.
  */
-static bbstreamer *
+static astreamer *
 CreateBackupStreamer(char *archive_name, char *spclocation,
-					 bbstreamer **manifest_inject_streamer_p,
+					 astreamer **manifest_inject_streamer_p,
 					 bool is_recovery_guc_supported,
 					 bool expect_unterminated_tarfile,
 					 pg_compress_specification *compress)
 {
-	bbstreamer *streamer = NULL;
-	bbstreamer *manifest_inject_streamer = NULL;
+	astreamer *streamer = NULL;
+	astreamer *manifest_inject_streamer = NULL;
 	bool		inject_manifest;
 	bool		is_tar,
 				is_tar_gz,
@@ -1160,7 +1160,7 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 			directory = psprintf("%s/%s", basedir, spclocation);
 		else
 			directory = get_tablespace_mapping(spclocation);
-		streamer = bbstreamer_extractor_new(directory,
+		streamer = astreamer_extractor_new(directory,
 											get_tablespace_mapping,
 											progress_update_filename);
 	}
@@ -1188,27 +1188,27 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 		}
 
 		if (compress->algorithm == PG_COMPRESSION_NONE)
-			streamer = bbstreamer_plain_writer_new(archive_filename,
+			streamer = astreamer_plain_writer_new(archive_filename,
 												   archive_file);
 		else if (compress->algorithm == PG_COMPRESSION_GZIP)
 		{
 			strlcat(archive_filename, ".gz", sizeof(archive_filename));
-			streamer = bbstreamer_gzip_writer_new(archive_filename,
+			streamer = astreamer_gzip_writer_new(archive_filename,
 												  archive_file, compress);
 		}
 		else if (compress->algorithm == PG_COMPRESSION_LZ4)
 		{
 			strlcat(archive_filename, ".lz4", sizeof(archive_filename));
-			streamer = bbstreamer_plain_writer_new(archive_filename,
+			streamer = astreamer_plain_writer_new(archive_filename,
 												   archive_file);
-			streamer = bbstreamer_lz4_compressor_new(streamer, compress);
+			streamer = astreamer_lz4_compressor_new(streamer, compress);
 		}
 		else if (compress->algorithm == PG_COMPRESSION_ZSTD)
 		{
 			strlcat(archive_filename, ".zst", sizeof(archive_filename));
-			streamer = bbstreamer_plain_writer_new(archive_filename,
+			streamer = astreamer_plain_writer_new(archive_filename,
 												   archive_file);
-			streamer = bbstreamer_zstd_compressor_new(streamer, compress);
+			streamer = astreamer_zstd_compressor_new(streamer, compress);
 		}
 		else
 		{
@@ -1222,7 +1222,7 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 		 * into it.
 		 */
 		if (must_parse_archive)
-			streamer = bbstreamer_tar_archiver_new(streamer);
+			streamer = astreamer_tar_archiver_new(streamer);
 		progress_update_filename(archive_filename);
 	}
 
@@ -1241,7 +1241,7 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 	if (spclocation == NULL && writerecoveryconf)
 	{
 		Assert(must_parse_archive);
-		streamer = bbstreamer_recovery_injector_new(streamer,
+		streamer = astreamer_recovery_injector_new(streamer,
 													is_recovery_guc_supported,
 													recoveryconfcontents);
 	}
@@ -1253,9 +1253,9 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 	 * we're talking to such a server we'll need to add the terminator here.
 	 */
 	if (must_parse_archive)
-		streamer = bbstreamer_tar_parser_new(streamer);
+		streamer = astreamer_tar_parser_new(streamer);
 	else if (expect_unterminated_tarfile)
-		streamer = bbstreamer_tar_terminator_new(streamer);
+		streamer = astreamer_tar_terminator_new(streamer);
 
 	/*
 	 * If the user has requested a server compressed archive along with
@@ -1264,11 +1264,11 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 	if (format == 'p')
 	{
 		if (is_tar_gz)
-			streamer = bbstreamer_gzip_decompressor_new(streamer);
+			streamer = astreamer_gzip_decompressor_new(streamer);
 		else if (is_tar_lz4)
-			streamer = bbstreamer_lz4_decompressor_new(streamer);
+			streamer = astreamer_lz4_decompressor_new(streamer);
 		else if (is_tar_zstd)
-			streamer = bbstreamer_zstd_decompressor_new(streamer);
+			streamer = astreamer_zstd_decompressor_new(streamer);
 	}
 
 	/* Return the results. */
@@ -1307,7 +1307,7 @@ ReceiveArchiveStream(PGconn *conn, pg_compress_specification *compress)
 	if (state.manifest_inject_streamer != NULL &&
 		state.manifest_buffer != NULL)
 	{
-		bbstreamer_inject_file(state.manifest_inject_streamer,
+		astreamer_inject_file(state.manifest_inject_streamer,
 							   "backup_manifest",
 							   state.manifest_buffer->data,
 							   state.manifest_buffer->len);
@@ -1318,8 +1318,8 @@ ReceiveArchiveStream(PGconn *conn, pg_compress_specification *compress)
 	/* If there's still an archive in progress, end processing. */
 	if (state.streamer != NULL)
 	{
-		bbstreamer_finalize(state.streamer);
-		bbstreamer_free(state.streamer);
+		astreamer_finalize(state.streamer);
+		astreamer_free(state.streamer);
 		state.streamer = NULL;
 	}
 }
@@ -1383,8 +1383,8 @@ ReceiveArchiveStreamChunk(size_t r, char *copybuf, void *callback_data)
 				/* End processing of any prior archive. */
 				if (state->streamer != NULL)
 				{
-					bbstreamer_finalize(state->streamer);
-					bbstreamer_free(state->streamer);
+					astreamer_finalize(state->streamer);
+					astreamer_free(state->streamer);
 					state->streamer = NULL;
 				}
 
@@ -1437,8 +1437,8 @@ ReceiveArchiveStreamChunk(size_t r, char *copybuf, void *callback_data)
 				else if (state->streamer != NULL)
 				{
 					/* Archive data. */
-					bbstreamer_content(state->streamer, NULL, copybuf + 1,
-									   r - 1, BBSTREAMER_UNKNOWN);
+					astreamer_content(state->streamer, NULL, copybuf + 1,
+									   r - 1, ASTREAMER_UNKNOWN);
 				}
 				else
 					pg_fatal("unexpected payload data");
@@ -1600,7 +1600,7 @@ ReceiveTarFile(PGconn *conn, char *archive_name, char *spclocation,
 			   bool tablespacenum, pg_compress_specification *compress)
 {
 	WriteTarState state;
-	bbstreamer *manifest_inject_streamer;
+	astreamer *manifest_inject_streamer;
 	bool		is_recovery_guc_supported;
 	bool		expect_unterminated_tarfile;
 
@@ -1636,7 +1636,7 @@ ReceiveTarFile(PGconn *conn, char *archive_name, char *spclocation,
 			pg_fatal("out of memory");
 
 		/* Inject it into the output tarfile. */
-		bbstreamer_inject_file(manifest_inject_streamer, "backup_manifest",
+		astreamer_inject_file(manifest_inject_streamer, "backup_manifest",
 							   buf.data, buf.len);
 
 		/* Free memory. */
@@ -1644,8 +1644,8 @@ ReceiveTarFile(PGconn *conn, char *archive_name, char *spclocation,
 	}
 
 	/* Cleanup. */
-	bbstreamer_finalize(state.streamer);
-	bbstreamer_free(state.streamer);
+	astreamer_finalize(state.streamer);
+	astreamer_free(state.streamer);
 
 	progress_report(tablespacenum, true, false);
 
@@ -1663,7 +1663,7 @@ ReceiveTarCopyChunk(size_t r, char *copybuf, void *callback_data)
 {
 	WriteTarState *state = callback_data;
 
-	bbstreamer_content(state->streamer, NULL, copybuf, r, BBSTREAMER_UNKNOWN);
+	astreamer_content(state->streamer, NULL, copybuf, r, ASTREAMER_UNKNOWN);
 
 	totaldone += r;
 	progress_report(state->tablespacenum, false, false);
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 8de9978ad8d..ba9e0200b3f 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3311,19 +3311,19 @@ bbsink_shell
 bbsink_state
 bbsink_throttle
 bbsink_zstd
-bbstreamer
-bbstreamer_archive_context
-bbstreamer_extractor
-bbstreamer_gzip_decompressor
-bbstreamer_gzip_writer
-bbstreamer_lz4_frame
-bbstreamer_member
-bbstreamer_ops
-bbstreamer_plain_writer
-bbstreamer_recovery_injector
-bbstreamer_tar_archiver
-bbstreamer_tar_parser
-bbstreamer_zstd_frame
+astreamer
+astreamer_archive_context
+astreamer_extractor
+astreamer_gzip_decompressor
+astreamer_gzip_writer
+astreamer_lz4_frame
+astreamer_member
+astreamer_ops
+astreamer_plain_writer
+astreamer_recovery_injector
+astreamer_tar_archiver
+astreamer_tar_parser
+astreamer_zstd_frame
 bgworker_main_type
 bh_node_type
 binaryheap
-- 
2.18.0

From 4b3af1ab14a61419e2ef3258b7fadc7ef6230da2 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 10:26:35 +0530
Subject: [PATCH v5 10/12] pg_verifybackup: Add backup format and compression
 option

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the next patch that implements tar format support.
----
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 146 +++++++++++++++++++++-
 1 file changed, 144 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 801e13886c2..2738bb30d83 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,6 +18,7 @@
 #include <sys/stat.h>
 #include <time.h>
 
+#include "common/compression.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
@@ -56,6 +57,8 @@ static void report_manifest_error(JsonManifestParseContext *context,
 								  const char *fmt,...)
 			pg_attribute_printf(2, 3) pg_attribute_noreturn();
 
+static char find_backup_format(verifier_context *context);
+static pg_compress_algorithm find_backup_compression(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
@@ -74,6 +77,9 @@ static void usage(void);
 
 static const char *progname;
 
+char		format = '\0';		/* p(lain)/t(ar) */
+pg_compress_algorithm compress_algorithm = PG_COMPRESSION_NONE;
+
 /*
  * Main entry point.
  */
@@ -84,11 +90,13 @@ main(int argc, char **argv)
 		{"exit-on-error", no_argument, NULL, 'e'},
 		{"ignore", required_argument, NULL, 'i'},
 		{"manifest-path", required_argument, NULL, 'm'},
+		{"format", required_argument, NULL, 'F'},
 		{"no-parse-wal", no_argument, NULL, 'n'},
 		{"progress", no_argument, NULL, 'P'},
 		{"quiet", no_argument, NULL, 'q'},
 		{"skip-checksums", no_argument, NULL, 's'},
 		{"wal-directory", required_argument, NULL, 'w'},
+		{"compress", required_argument, NULL, 'Z'},
 		{NULL, 0, NULL, 0}
 	};
 
@@ -99,6 +107,7 @@ main(int argc, char **argv)
 	bool		quiet = false;
 	char	   *wal_directory = NULL;
 	char	   *pg_waldump_path = NULL;
+	bool		tar_compression_specified = false;
 
 	pg_logging_init(argv[0]);
 	set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_verifybackup"));
@@ -141,7 +150,7 @@ main(int argc, char **argv)
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
 
-	while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1)
+	while ((c = getopt_long(argc, argv, "eF:i:m:nPqsw:Z:", long_options, NULL)) != -1)
 	{
 		switch (c)
 		{
@@ -160,6 +169,15 @@ main(int argc, char **argv)
 				manifest_path = pstrdup(optarg);
 				canonicalize_path(manifest_path);
 				break;
+			case 'F':
+				if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
+					format = 'p';
+				else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
+					format = 't';
+				else
+					pg_fatal("invalid backup format \"%s\", must be \"plain\" or \"tar\"",
+							 optarg);
+				break;
 			case 'n':
 				no_parse_wal = true;
 				break;
@@ -176,6 +194,12 @@ main(int argc, char **argv)
 				wal_directory = pstrdup(optarg);
 				canonicalize_path(wal_directory);
 				break;
+			case 'Z':
+				if (!parse_compress_algorithm(optarg, &compress_algorithm))
+					pg_fatal("unrecognized compression algorithm: \"%s\"",
+							 optarg);
+				tar_compression_specified = true;
+				break;
 			default:
 				/* getopt_long already emitted a complaint */
 				pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -207,11 +231,41 @@ main(int argc, char **argv)
 		pg_fatal("cannot specify both %s and %s",
 				 "-P/--progress", "-q/--quiet");
 
+	/* Complain if compression method specified but the format isn't tar. */
+	if (format != 't' && tar_compression_specified)
+	{
+		pg_log_error("only tar mode backups can be compressed");
+		pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+		exit(1);
+	}
+
+	/* Determine the backup format if it hasn't been specified. */
+	if (format == '\0')
+		format = find_backup_format(&context);
+
+	/*
+	 * Determine the tar backup compression method if it hasn't been
+	 * specified.
+	 */
+	if (format == 't' && !tar_compression_specified)
+		compress_algorithm = find_backup_compression(&context);
+
 	/* Unless --no-parse-wal was specified, we will need pg_waldump. */
 	if (!no_parse_wal)
 	{
 		int			ret;
 
+		/*
+		 * XXX: In the future, we should consider enhancing pg_waldump to read
+		 * WAL files from the tar archive.
+		 */
+		if (format != 'p')
+		{
+			pg_log_error("pg_waldump does not support parsing WAL files from a tar archive.");
+			pg_log_error_hint("Try \"%s --help\" to skip parse WAL files option.", progname);
+			exit(1);
+		}
+
 		pg_waldump_path = pg_malloc(MAXPGPATH);
 		ret = find_other_exec(argv[0], "pg_waldump",
 							  "pg_waldump (PostgreSQL) " PG_VERSION "\n",
@@ -279,7 +333,15 @@ main(int argc, char **argv)
 	 */
 	if (!context.skip_checksums)
 	{
-		verify_backup_checksums(&context);
+		/*
+		 * We were only checking the plain backup here. For the tar backup,
+		 * file checksums verification (if requested) will be done immediately
+		 * when the file is accessed, as we don't have random access to the
+		 * files like we do with plain backups.
+		 */
+		if (format == 'p')
+			verify_backup_checksums(&context);
+
 		progress_report(&context, true);
 	}
 
@@ -972,6 +1034,84 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
 	return false;
 }
 
+/*
+ * To detect the backup format, it checks for the PG_VERSION file in the backup
+ * directory. If found, it will be considered a plain-format backup; otherwise,
+ * it will be assumed to be a tar-format backup.
+ */
+static char
+find_backup_format(verifier_context *context)
+{
+	char		result;
+	char	   *path;
+	struct stat sb;
+
+	/* Should be here only if the backup format is unknown */
+	Assert(format == '\0');
+
+	/* Check PG_VERSION file. */
+	path = psprintf("%s/%s", context->backup_directory, "PG_VERSION");
+	result = (stat(path, &sb) == 0) ? 'p' : 't';
+	pfree(path);
+
+	return result;
+}
+
+/*
+ * To determine the compression format, we will search for the main data
+ * directory archive and its extension, which starts with base.tar, as
+ * pg_basebackup writes the main data directory to an archive file named
+ * base.tar followed by a compression type extension like .gz, .lz4, or .zst.
+ */
+static pg_compress_algorithm
+find_backup_compression(verifier_context *context)
+{
+	char	   *path;
+	struct stat sb;
+	bool		found;
+
+	/* Should be here only for tar backup */
+	Assert(format == 't');
+
+	/*
+	 * Is this a tar archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_NONE;
+
+	/*
+	 * Is this a .tar.gz archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar.gz");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_GZIP;
+
+	/*
+	 * Is this a .tar.lz4 archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar.lz4");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_LZ4;
+
+	/*
+	 * Is this a .tar.zst archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar.zst");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_ZSTD;
+
+	return PG_COMPRESSION_NONE; /* placate compiler */
+}
+
 /*
  * Print a progress report based on the variables in verifier_context.
  *
@@ -1054,11 +1194,13 @@ usage(void)
 	printf(_("  -e, --exit-on-error         exit immediately on error\n"));
 	printf(_("  -i, --ignore=RELATIVE_PATH  ignore indicated path\n"));
 	printf(_("  -m, --manifest-path=PATH    use specified path for manifest\n"));
+	printf(_("  -F, --format=p|t            backup format (plain, tar)\n"));
 	printf(_("  -n, --no-parse-wal          do not try to parse WAL files\n"));
 	printf(_("  -P, --progress              show progress information\n"));
 	printf(_("  -q, --quiet                 do not print any output, except for errors\n"));
 	printf(_("  -s, --skip-checksums        skip checksum verification\n"));
 	printf(_("  -w, --wal-directory=PATH    use specified path for WAL files\n"));
+	printf(_("  -Z, --compress=METHOD       compress method (gzip, lz4, zstd, none) \n"));
 	printf(_("  -V, --version               output version information, then exit\n"));
 	printf(_("  -?, --help                  show this help, then exit\n"));
 	printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
-- 
2.18.0

From f34279f9f934fdcad6d3b3dbfb0721bd7959bfdf Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 31 Jul 2024 16:22:07 +0530
Subject: [PATCH v5 11/12] pg_verifybackup: Read tar files and verify its
 contents

---
 src/bin/pg_verifybackup/Makefile           |   4 +-
 src/bin/pg_verifybackup/astreamer_verify.c | 367 +++++++++++++++++++++
 src/bin/pg_verifybackup/meson.build        |   6 +-
 src/bin/pg_verifybackup/pg_verifybackup.c  | 216 +++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h  |   9 +
 src/tools/pgindent/typedefs.list           |   1 +
 6 files changed, 594 insertions(+), 9 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/astreamer_verify.c

diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile
index 7c045f142e8..df7aaabd530 100644
--- a/src/bin/pg_verifybackup/Makefile
+++ b/src/bin/pg_verifybackup/Makefile
@@ -17,11 +17,13 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 # We need libpq only because fe_utils does.
+override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 
 OBJS = \
 	$(WIN32RES) \
-	pg_verifybackup.o
+	pg_verifybackup.o \
+	astreamer_verify.o
 
 all: pg_verifybackup
 
diff --git a/src/bin/pg_verifybackup/astreamer_verify.c b/src/bin/pg_verifybackup/astreamer_verify.c
new file mode 100644
index 00000000000..be40922c042
--- /dev/null
+++ b/src/bin/pg_verifybackup/astreamer_verify.c
@@ -0,0 +1,367 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer_verify.c
+ *
+ * Extend fe_utils/astreamer.h archive streaming facility to verify TAR
+ * backup.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * src/bin/pg_verifybackup/astreamer_verify.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include "common/logging.h"
+#include "fe_utils/astreamer.h"
+#include "pg_verifybackup.h"
+
+typedef struct astreamer_verify
+{
+	astreamer	base;
+	verifier_context *context;
+	char	   *archiveName;
+	Oid			tblspcOid;
+
+	/* Hold information for a member file verification */
+	manifest_file *mfile;
+	int64		receivedBytes;
+	bool		verifyChecksums;
+	bool		verifyControlData;
+	pg_checksum_context *checksum_ctx;
+} astreamer_verify;
+
+static void astreamer_verify_content(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len,
+									 astreamer_archive_context context);
+static void astreamer_verify_finalize(astreamer *streamer);
+static void astreamer_verify_free(astreamer *streamer);
+
+static void verify_member_header(astreamer *streamer, astreamer_member *member);
+static void verify_member_contents(astreamer *streamer,
+								   astreamer_member *member,
+								   const char *data, int len);
+static void verify_content_checksum(astreamer *streamer,
+									astreamer_member *member,
+									const char *buffer, int buffer_len);
+static void verify_controldata(astreamer *streamer,
+							   astreamer_member *member,
+							   const char *data, int len);
+static void reset_member_info(astreamer *streamer);
+
+static const astreamer_ops astreamer_verify_ops = {
+	.content = astreamer_verify_content,
+	.finalize = astreamer_verify_finalize,
+	.free = astreamer_verify_free
+};
+
+/*
+ * Create a astreamer that can verifies content of a TAR file.
+ */
+astreamer *
+astreamer_verify_content_new(astreamer *next, verifier_context *context,
+							 char *archive_name, Oid tblspc_oid)
+{
+	astreamer_verify *streamer;
+
+	streamer = palloc0(sizeof(astreamer_verify));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_verify_ops;
+
+	streamer->base.bbs_next = next;
+	streamer->context = context;
+	streamer->archiveName = archive_name;
+	streamer->tblspcOid = tblspc_oid;
+	initStringInfo(&streamer->base.bbs_buffer);
+
+	return &streamer->base;
+}
+
+/*
+ * It verifies each TAR member entry against the manifest data and performs
+ * checksum verification if enabled. Additionally, it validates the backup's
+ * system identifier against the backup_manifest.
+ */
+static void
+astreamer_verify_content(astreamer *streamer,
+						 astreamer_member *member, const char *data,
+						 int len, astreamer_archive_context context)
+{
+	Assert(context != ASTREAMER_UNKNOWN);
+
+	switch (context)
+	{
+		case ASTREAMER_MEMBER_HEADER:
+
+			/*
+			 * Perform the initial check and setup for the verification.
+			 */
+			verify_member_header(streamer, member);
+			break;
+
+		case ASTREAMER_MEMBER_CONTENTS:
+
+			/*
+			 * Peform the required contants verification.
+			 */
+			verify_member_contents(streamer, member, data, len);
+			break;
+
+		case ASTREAMER_MEMBER_TRAILER:
+
+			/*
+			 * Reset the temporary information stored for a verification.
+			 */
+			reset_member_info(streamer);
+			break;
+
+		case ASTREAMER_ARCHIVE_TRAILER:
+			break;
+
+		default:
+			/* Shouldn't happen. */
+			pg_fatal("unexpected state while parsing tar archive");
+	}
+}
+
+/*
+ * End-of-stream processing for a astreamer_verify stream.
+ */
+static void
+astreamer_verify_finalize(astreamer *streamer)
+{
+	Assert(streamer->bbs_next == NULL);
+}
+
+/*
+ * Free memory associated with a astreamer_verify stream.
+ */
+static void
+astreamer_verify_free(astreamer *streamer)
+{
+	pfree(streamer->bbs_buffer.data);
+	pfree(streamer);
+}
+
+/*
+ * Verify the entry if it is a file in the backup manifest. If the archive being
+ * processed is a tablespace, prepare the required file path for subsequent
+ * operations. Finally, check if it needs to perform checksum verification and
+ * control data verification during file content processing.
+ */
+static void
+verify_member_header(astreamer *streamer, astreamer_member *member)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_file *m;
+
+	/* We are only interested in files that are not in the ignore list. */
+	if (member->is_directory || member->is_link ||
+		should_ignore_relpath(mystreamer->context, member->pathname))
+		return;
+
+	/*
+	 * The backup_manifest stores a relative path to the base directory for
+	 * files belong tablespace, whereas <tablespaceoid>.tar doesn't. Prepare
+	 * the required path, otherwise, the manfiest entry verification will
+	 * fail.
+	 */
+	if (OidIsValid(mystreamer->tblspcOid))
+	{
+		char		temp[MAXPGPATH];
+
+		/* Copy original name at temporary space */
+		memcpy(temp, member->pathname, MAXPGPATH);
+
+		snprintf(member->pathname, MAXPGPATH, "%s/%d/%s",
+				 "pg_tblspc", mystreamer->tblspcOid, temp);
+	}
+
+	/* Check the manifest entry */
+	m = verify_manifest_entry(mystreamer->context, member->pathname,
+							  member->size);
+	mystreamer->mfile = (void *) m;
+
+	/*
+	 * Prepare for checksum and manifest system identifier verification.
+	 *
+	 * We could have these checks while receiving contents. However, since
+	 * contents are received in multiple iterations, this would result in
+	 * these lengthy checks being performed multiple times. Instead, having a
+	 * single flag would be more efficient.
+	 */
+	mystreamer->verifyChecksums =
+		(!mystreamer->context->skip_checksums && should_verify_checksum(m));
+	mystreamer->verifyControlData =
+		should_verify_control_data(mystreamer->context->manifest, m);
+}
+
+/*
+ * Process the member content according to the flags set by the member header
+ * processing routine for checksum and control data verification.
+ */
+static void
+verify_member_contents(astreamer *streamer, astreamer_member *member,
+					   const char *data, int len)
+
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	/* Verify the checksums */
+	if (mystreamer->verifyChecksums)
+		verify_content_checksum(streamer, member, data, len);
+
+	/* Verify pg_control information */
+	if (mystreamer->verifyControlData)
+		verify_controldata(streamer, member, data, len);
+}
+
+/*
+ * Similar to verify_file_checksum() but this function computes the checksum
+ * incrementally for the received file content. Unlike a normal backup
+ * directory, TAR format files do not allow random access, so checksum
+ * verification occurs progressively. Additionally, the function calls the
+ * routine for control data verification if the flags indicate that it is
+ * required.
+ *
+ * On the first visit, the function initializes checksum_ctx, which will be used
+ * for incremental checksum calculation. Once the complete file content is
+ * received (tracked using the receivedBytes), the routine that performs the
+ * final checksum verification is called
+ */
+static void
+verify_content_checksum(astreamer *streamer, astreamer_member *member,
+						const char *buffer, int buffer_len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	pg_checksum_context *checksum_ctx = mystreamer->checksum_ctx;
+	verifier_context *context = mystreamer->context;
+	manifest_file *m = mystreamer->mfile;
+	const char *relpath = m->pathname;
+	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
+
+	/*
+	 * Mark it false to avoid unexpected re-entrance for the same file content
+	 * (e.g. returned in error should not be revisited).
+	 */
+	Assert(mystreamer->verifyChecksums);
+	mystreamer->verifyChecksums = false;
+
+	/* Should have came for the right file */
+	Assert(strcmp(member->pathname, relpath) == 0);
+
+	/* If we were first time for this file */
+	if (!checksum_ctx)
+	{
+		checksum_ctx = pg_malloc(sizeof(pg_checksum_context));
+		mystreamer->checksum_ctx = checksum_ctx;
+
+		if (pg_checksum_init(checksum_ctx, m->checksum_type) < 0)
+		{
+			report_backup_error(context,
+								"%s: could not initialize checksum of file \"%s\"",
+								mystreamer->archiveName, relpath);
+			return;
+		}
+	}
+
+	/* Update the total count of computed checksum bytes. */
+	mystreamer->receivedBytes += buffer_len;
+
+	if (pg_checksum_update(checksum_ctx, (uint8 *) buffer, buffer_len) < 0)
+	{
+		report_backup_error(context, "could not update checksum of file \"%s\"",
+							relpath);
+		return;
+	}
+
+	/* Report progress */
+	context->done_size += buffer_len;
+	progress_report(context, false);
+
+	/* Yet to receive the full content of the file. */
+	if (mystreamer->receivedBytes < m->size)
+	{
+		mystreamer->verifyChecksums = true;
+		return;
+	}
+
+	/* Do the final computation and verification. */
+	verify_checksum(context, m, checksum_ctx, checksumbuf);
+}
+
+/*
+ * Prepare the control data from the received file contents, which are supposed
+ * to be from the pg_control file, including CRC calculation. Then, call the
+ * routines that perform the final verification of the control file information.
+ */
+static void
+verify_controldata(astreamer *streamer, astreamer_member *member,
+				   const char *data, int len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_data *manifest = mystreamer->context->manifest;
+	ControlFileData control_file;
+	pg_crc32c	crc;
+	bool		crc_ok;
+
+	/* Should be here only for control file */
+	Assert(strcmp(member->pathname, "global/pg_control") == 0);
+	Assert(manifest->version != 1);
+
+	/* Mark it as false to avoid unexpected re-entrance */
+	Assert(mystreamer->verifyControlData);
+	mystreamer->verifyControlData = false;
+
+	/* Should have whole control file data. */
+	if (!astreamer_buffer_until(streamer, &data, &len, sizeof(ControlFileData)))
+	{
+		mystreamer->verifyControlData = true;
+		return;
+	}
+
+	pg_log_debug("%s: reading \"%s\"", mystreamer->archiveName,
+				 member->pathname);
+
+	if (streamer->bbs_buffer.len != sizeof(ControlFileData))
+		report_fatal_error("%s: could not read control file: read %d of %zu",
+						   mystreamer->archiveName, streamer->bbs_buffer.len,
+						   sizeof(ControlFileData));
+
+	memcpy(&control_file, streamer->bbs_buffer.data,
+		   sizeof(ControlFileData));
+
+	/* Check the CRC. */
+	INIT_CRC32C(crc);
+	COMP_CRC32C(crc,
+				(char *) (&control_file),
+				offsetof(ControlFileData, crc));
+	FIN_CRC32C(crc);
+
+	crc_ok = EQ_CRC32C(crc, control_file.crc);
+
+	/* Do the final control data verification. */
+	verify_control_data(&control_file, member->pathname, crc_ok,
+						manifest->system_identifier);
+}
+
+/*
+ * Reset flags and free memory allocations for member file verification.
+ */
+static void
+reset_member_info(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	mystreamer->mfile = NULL;
+	mystreamer->receivedBytes = 0;
+	mystreamer->verifyChecksums = false;
+	mystreamer->verifyControlData = false;
+
+	if (mystreamer->checksum_ctx)
+		pfree(mystreamer->checksum_ctx);
+	mystreamer->checksum_ctx = NULL;
+}
diff --git a/src/bin/pg_verifybackup/meson.build b/src/bin/pg_verifybackup/meson.build
index 7c7d31a0350..1e3fcf7ee5a 100644
--- a/src/bin/pg_verifybackup/meson.build
+++ b/src/bin/pg_verifybackup/meson.build
@@ -1,7 +1,8 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 pg_verifybackup_sources = files(
-  'pg_verifybackup.c'
+  'pg_verifybackup.c',
+  'astreamer_verify.c'
 )
 
 if host_system == 'windows'
@@ -10,9 +11,10 @@ if host_system == 'windows'
     '--FILEDESC', 'pg_verifybackup - verify a backup against using a backup manifest'])
 endif
 
+pg_verifybackup_deps = [frontend_code, libpq, lz4, zlib, zstd]
 pg_verifybackup = executable('pg_verifybackup',
   pg_verifybackup_sources,
-  dependencies: [frontend_code, libpq],
+  dependencies: pg_verifybackup_deps,
   kwargs: default_bin_args,
 )
 bin_targets += pg_verifybackup
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 2738bb30d83..d8ab9a3a3dc 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -20,9 +20,12 @@
 
 #include "common/compression.h"
 #include "common/parse_manifest.h"
+#include "common/relpath.h"
+#include "fe_utils/astreamer.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
 #include "pg_verifybackup.h"
+#include "pgtar.h"
 #include "pgtime.h"
 
 /*
@@ -39,6 +42,11 @@
  */
 #define ESTIMATED_BYTES_PER_MANIFEST_LINE	100
 
+
+static void (*verify_backup_file_cb) (verifier_context *context,
+									  char *relpath, char *fullpath,
+									  size_t filesize);
+
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -63,6 +71,15 @@ static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
 							   char *relpath, char *fullpath);
+static void verify_plain_file_cb(verifier_context *context,
+								 char *relpath, char *fullpath,
+								 size_t filesize);
+static void verify_tar_file_cb(verifier_context *context,
+							   char *relpath, char *fullpath,
+							   size_t filesize);
+static void verify_tar_content(verifier_context *context,
+							   char *relpath, char *fullpath,
+							   astreamer *streamer);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -71,6 +88,9 @@ static void verify_file_checksum(verifier_context *context,
 static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
+static astreamer *create_archive_verifier(verifier_context *context,
+										  char *archive_name,
+										  Oid tblspc_oid);
 
 static void compute_total_size(verifier_context *context);
 static void usage(void);
@@ -146,6 +166,10 @@ main(int argc, char **argv)
 	 */
 	simple_string_list_append(&context.ignore_list, "backup_manifest");
 	simple_string_list_append(&context.ignore_list, "pg_wal");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.gz");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.lz4");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.zst");
 	simple_string_list_append(&context.ignore_list, "postgresql.auto.conf");
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
@@ -250,6 +274,15 @@ main(int argc, char **argv)
 	if (format == 't' && !tar_compression_specified)
 		compress_algorithm = find_backup_compression(&context);
 
+	/*
+	 * Setup the required callback function to verify plain or tar backup
+	 * files.
+	 */
+	if (format == 'p')
+		verify_backup_file_cb = verify_plain_file_cb;
+	else
+		verify_backup_file_cb = verify_tar_file_cb;
+
 	/* Unless --no-parse-wal was specified, we will need pg_waldump. */
 	if (!no_parse_wal)
 	{
@@ -645,7 +678,8 @@ verify_backup_directory(verifier_context *context, char *relpath,
 }
 
 /*
- * Verify one file (which might actually be a directory or a symlink).
+ * Verify one file (which might actually be a directory, a symlink or a
+ * archive).
  *
  * The arguments to this function have the same meaning as the arguments to
  * verify_backup_directory.
@@ -654,7 +688,6 @@ static void
 verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 {
 	struct stat sb;
-	manifest_file *m;
 
 	if (stat(fullpath, &sb) != 0)
 	{
@@ -687,8 +720,25 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		return;
 	}
 
+	/* Do the further verifications */
+	verify_backup_file_cb(context, relpath, fullpath, sb.st_size);
+}
+
+/*
+ * Verify one plan file or a symlink.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_backup_directory. The additional argument is the file size for
+ * verifying against manifest entry.
+ */
+static void
+verify_plain_file_cb(verifier_context *context, char *relpath,
+					 char *fullpath, size_t filesize)
+{
+	manifest_file *m;
+
 	/* Check the backup manifest entry for this file. */
-	m = verify_manifest_entry(context, relpath, sb.st_size);
+	m = verify_manifest_entry(context, relpath, filesize);
 
 	/* Validate the manifest system identifier */
 	if (should_verify_control_data(context->manifest, m))
@@ -706,6 +756,124 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 	}
 }
 
+/*
+ * Verify one tar file.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_backup_directory. The additional argument is the file size for
+ * verifying against manifest entry.
+ */
+static void
+verify_tar_file_cb(verifier_context *context, char *relpath,
+				   char *fullpath, size_t filesize)
+{
+	astreamer  *streamer;
+	Oid			tblspc_oid = InvalidOid;
+	int			file_name_len;
+	int			file_extn_len = 0;	/* placate compiler */
+	char	   *file_extn = "";
+
+	/* Should be tar backup */
+	Assert(format == 't');
+
+	/* Find the tar file extension. */
+	if (compress_algorithm == PG_COMPRESSION_NONE)
+	{
+		file_extn = ".tar";
+		file_extn_len = 4;
+
+	}
+	else if (compress_algorithm == PG_COMPRESSION_GZIP)
+	{
+		file_extn = ".tar.gz";
+		file_extn_len = 7;
+
+	}
+	else if (compress_algorithm == PG_COMPRESSION_LZ4)
+	{
+		file_extn = ".tar.lz4";
+		file_extn_len = 8;
+	}
+	else if (compress_algorithm == PG_COMPRESSION_ZSTD)
+	{
+		file_extn = ".tar.zst";
+		file_extn_len = 8;
+	}
+
+	/*
+	 * Ensure that we have the correct file type corresponding to the backup
+	 * format.
+	 */
+	file_name_len = strlen(relpath);
+	if (file_name_len < file_extn_len ||
+		strcmp(relpath + file_name_len - file_extn_len, file_extn) != 0)
+	{
+		if (compress_algorithm == PG_COMPRESSION_NONE)
+			report_backup_error(context,
+								"\"%s\" is not a valid file, expecting tar file",
+								relpath);
+		else
+			report_backup_error(context,
+								"\"%s\" is not a valid file, expecting \"%s\" compressed tar file",
+								relpath,
+								get_compress_algorithm_name(compress_algorithm));
+		return;
+	}
+
+	/*
+	 * For the tablespace, pg_basebackup writes the data out to
+	 * <tablespaceoid>.tar.  If a file matches that format, then extract the
+	 * tablespaceoid, which we need to prepare the paths of the files
+	 * belonging to that tablespace relative to the base directory.
+	 */
+	if (strspn(relpath, "0123456789") == (file_name_len - file_extn_len))
+		tblspc_oid = strtoi64(relpath, NULL, 10);
+
+	streamer = create_archive_verifier(context, relpath, tblspc_oid);
+	verify_tar_content(context, relpath, fullpath, streamer);
+
+	/* Cleanup. */
+	astreamer_finalize(streamer);
+	astreamer_free(streamer);
+}
+
+/*
+ * Reads a given tar file in predefined chunks and pass to astreamer.  Which
+ * initiates routines for decompression (if necessary) then verification
+ * of each member within the tar archive.
+ */
+static void
+verify_tar_content(verifier_context *context, char *relpath, char *fullpath,
+				   astreamer *streamer)
+{
+	int			fd;
+	int			rc;
+	char	   *buffer;
+
+	/* Open the target file. */
+	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
+	{
+		report_backup_error(context, "could not open file \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
+
+	/* Perform the reads */
+	while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
+		astreamer_content(streamer, NULL, buffer, rc, ASTREAMER_UNKNOWN);
+
+	if (rc < 0)
+		report_backup_error(context, "could not read file \"%s\": %m",
+							relpath);
+
+	/* Close the file. */
+	if (close(fd) != 0)
+		report_backup_error(context, "could not close file \"%s\": %m",
+							relpath);
+}
+
 /*
  * Verify file and its size entry in the manifest.
  */
@@ -1058,10 +1226,10 @@ find_backup_format(verifier_context *context)
 }
 
 /*
- * To determine the compression format, we will search for the main data
- * directory archive and its extension, which starts with base.tar, as
  * pg_basebackup writes the main data directory to an archive file named
- * base.tar followed by a compression type extension like .gz, .lz4, or .zst.
+ * base.tar, followed by a compression type extension such as .gz, .lz4, or
+ * .zst. To determine the compression format, we need to search for this main
+ * data directory archive file.
  */
 static pg_compress_algorithm
 find_backup_compression(verifier_context *context)
@@ -1112,6 +1280,42 @@ find_backup_compression(verifier_context *context)
 	return PG_COMPRESSION_NONE; /* placate compiler */
 }
 
+/*
+ * Identifies the necessary steps for verifying the contents of the
+ * provided tar file.
+ */
+static astreamer *
+create_archive_verifier(verifier_context *context, char *archive_name,
+						Oid tblspc_oid)
+{
+	astreamer  *streamer = NULL;
+
+	/* Should be here only for tar backup */
+	Assert(format == 't');
+
+	/*
+	 * To verify the contents of the tar file, the initial step is to parse
+	 * its content.
+	 */
+	streamer = astreamer_verify_content_new(streamer, context, archive_name,
+											tblspc_oid);
+	streamer = astreamer_tar_parser_new(streamer);
+
+	/*
+	 * If the tar file is compressed, we must perform the appropriate
+	 * decompression operation before proceeding with the verification of its
+	 * contents.
+	 */
+	if (compress_algorithm == PG_COMPRESSION_GZIP)
+		streamer = astreamer_gzip_decompressor_new(streamer);
+	else if (compress_algorithm == PG_COMPRESSION_LZ4)
+		streamer = astreamer_lz4_decompressor_new(streamer);
+	else if (compress_algorithm == PG_COMPRESSION_ZSTD)
+		streamer = astreamer_zstd_decompressor_new(streamer);
+
+	return streamer;
+}
+
 /*
  * Print a progress report based on the variables in verifier_context.
  *
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index c88f71ff14b..f0a7c8918fb 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -137,4 +137,13 @@ extern bool should_ignore_relpath(verifier_context *context,
 
 extern void progress_report(verifier_context *context, bool finished);
 
+/* Forward declarations to avoid fe_utils/astreamer.h include. */
+struct astreamer;
+typedef struct astreamer astreamer;
+
+extern astreamer *astreamer_verify_content_new(astreamer *next,
+											   verifier_context *context,
+											   char *archive_name,
+											   Oid tblspc_oid);
+
 #endif							/* PG_VERIFYBACKUP_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index ba9e0200b3f..8c708b02cc2 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3323,6 +3323,7 @@ astreamer_plain_writer
 astreamer_recovery_injector
 astreamer_tar_archiver
 astreamer_tar_parser
+astreamer_verify
 astreamer_zstd_frame
 bgworker_main_type
 bh_node_type
-- 
2.18.0

From beff1feb10a75336369d3157650117d9597899b7 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 17:04:56 +0530
Subject: [PATCH v5 12/12] pg_verifybackup: Tests and document

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the previous patch that implements tar format support.
----
---
 doc/src/sgml/ref/pg_verifybackup.sgml         | 54 +++++++++++++-
 src/bin/pg_verifybackup/t/001_basic.pl        | 18 ++++-
 src/bin/pg_verifybackup/t/004_options.pl      |  2 +-
 src/bin/pg_verifybackup/t/005_bad_manifest.pl |  2 +-
 src/bin/pg_verifybackup/t/008_untar.pl        | 73 ++++++-------------
 src/bin/pg_verifybackup/t/010_client_untar.pl | 48 +-----------
 6 files changed, 96 insertions(+), 101 deletions(-)

diff --git a/doc/src/sgml/ref/pg_verifybackup.sgml b/doc/src/sgml/ref/pg_verifybackup.sgml
index a3f167f9f6e..c743bd89a92 100644
--- a/doc/src/sgml/ref/pg_verifybackup.sgml
+++ b/doc/src/sgml/ref/pg_verifybackup.sgml
@@ -34,8 +34,10 @@ PostgreSQL documentation
    integrity of a database cluster backup taken using
    <command>pg_basebackup</command> against a
    <literal>backup_manifest</literal> generated by the server at the time
-   of the backup.  The backup must be stored in the "plain"
-   format; a "tar" format backup can be checked after extracting it.
+   of the backup.  The backup must be stored in the "plain" or "tar"
+   format.  Verification is supported for <literal>gzip</literal>,
+   <literal>lz4</literal>, and  <literal>zstd</literal> compressed tar backup;
+   any other compressed format backups can be checked after decompressing them.
   </para>
 
   <para>
@@ -168,6 +170,42 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-F <replaceable class="parameter">format</replaceable></option></term>
+      <term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
+      <listitem>
+       <para>
+        Specifies the format of the backup. <replaceable>format</replaceable>
+        can be one of the following:
+
+        <variablelist>
+         <varlistentry>
+          <term><literal>p</literal></term>
+          <term><literal>plain</literal></term>
+          <listitem>
+           <para>
+            Backup consists of plain files with the same layout as the
+            source server's data directory and tablespaces.
+           </para>
+          </listitem>
+         </varlistentry>
+
+         <varlistentry>
+          <term><literal>t</literal></term>
+          <term><literal>tar</literal></term>
+          <listitem>
+           <para>
+            Backup consists of tar files. The main data directory's contents
+            will be written to a file named <filename>base.tar</filename>,
+            and each other tablespace will be written to a separate tar file
+            named after that tablespace's OID.
+           </para>
+           </listitem>
+         </varlistentry>
+        </variablelist></para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-n</option></term>
       <term><option>--no-parse-wal</option></term>
@@ -227,6 +265,18 @@ PostgreSQL documentation
        </para>
       </listitem>
      </varlistentry>
+
+     <varlistentry>
+      <term><option>-Z <replaceable class="parameter">method</replaceable></option></term>
+      <term><option>--compress=<replaceable class="parameter">method</replaceable></option></term>
+      <listitem>
+       <para>
+        The tar backup compression method can be <literal>gzip</literal>,
+        <literal>lz4</literal>, <literal>zstd</literal>, or
+        <literal>none</literal> if no compression.
+       </para>
+      </listitem>
+     </varlistentry>
     </variablelist>
    </para>
 
diff --git a/src/bin/pg_verifybackup/t/001_basic.pl b/src/bin/pg_verifybackup/t/001_basic.pl
index 2f3e52d296f..d47ce1f04fc 100644
--- a/src/bin/pg_verifybackup/t/001_basic.pl
+++ b/src/bin/pg_verifybackup/t/001_basic.pl
@@ -17,13 +17,25 @@ command_fails_like(
 	qr/no backup directory specified/,
 	'target directory must be specified');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir ],
 	qr/could not open file.*\/backup_manifest\"/,
 	'pg_verifybackup requires a manifest');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir, $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir, $tempdir ],
 	qr/too many command-line arguments/,
 	'multiple target directories not allowed');
+command_fails_like(
+	[ 'pg_verifybackup', '-Zgzip', $tempdir ],
+	qr/only tar mode backups can be compressed/,
+	'compression method required format option');
+command_fails_like(
+	[ 'pg_verifybackup', '-Fp', '-Zlz4', $tempdir ],
+	qr/only tar mode backups can be compressed/,
+	'compression method required tar format option');
+command_fails_like(
+	[ 'pg_verifybackup', '-Fp', '-Znon_exist', $tempdir ],
+	qr/unrecognized compression algorithm/,
+	'compression method should be valid');
 
 # create fake manifest file
 open(my $fh, '>', "$tempdir/backup_manifest") || die "open: $!";
@@ -31,7 +43,7 @@ close($fh);
 
 # but then try to use an alternate, nonexisting manifest
 command_fails_like(
-	[ 'pg_verifybackup', '-m', "$tempdir/not_the_manifest", $tempdir ],
+	[ 'pg_verifybackup', '-Fp', '-m', "$tempdir/not_the_manifest", $tempdir ],
 	qr/could not open file.*\/not_the_manifest\"/,
 	'pg_verifybackup respects -m flag');
 
diff --git a/src/bin/pg_verifybackup/t/004_options.pl b/src/bin/pg_verifybackup/t/004_options.pl
index 8ed2214408e..2f197648740 100644
--- a/src/bin/pg_verifybackup/t/004_options.pl
+++ b/src/bin/pg_verifybackup/t/004_options.pl
@@ -108,7 +108,7 @@ unlike(
 # Test valid manifest with nonexistent backup directory.
 command_fails_like(
 	[
-		'pg_verifybackup', '-m',
+		'pg_verifybackup', '-Fp', '-m',
 		"$backup_path/backup_manifest", "$backup_path/fake"
 	],
 	qr/could not open directory/,
diff --git a/src/bin/pg_verifybackup/t/005_bad_manifest.pl b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
index c4ed64b62d5..28c51b6feb0 100644
--- a/src/bin/pg_verifybackup/t/005_bad_manifest.pl
+++ b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
@@ -208,7 +208,7 @@ sub test_bad_manifest
 	print $fh $manifest_contents;
 	close($fh);
 
-	command_fails_like([ 'pg_verifybackup', $tempdir ], $regexp, $test_name);
+	command_fails_like([ 'pg_verifybackup', '-Fp', $tempdir ], $regexp, $test_name);
 	return;
 }
 
diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
index 7a09f3b75b2..9896560adc3 100644
--- a/src/bin/pg_verifybackup/t/008_untar.pl
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -16,6 +16,20 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 
+# Create a couple of directories to use as tablespaces.
+my $TS1_LOCATION = $primary->backup_dir .'/ts1';
+$TS1_LOCATION =~ s/\/\.\//\//g;    # collapse foo/./bar to foo/bar
+mkdir($TS1_LOCATION);
+
+# Create a tablespace with table in it.
+$primary->safe_psql('postgres', qq(
+		CREATE TABLESPACE regress_ts1 LOCATION '$TS1_LOCATION';
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1';
+		CREATE TABLE regress_tbl1(i int) TABLESPACE regress_ts1;
+		INSERT INTO regress_tbl1 VALUES(generate_series(1,5));));
+my $tsoid = $primary->safe_psql('postgres', qq(
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1'));
+
 my $backup_path = $primary->backup_dir . '/server-backup';
 my $extract_path = $primary->backup_dir . '/extracted-backup';
 
@@ -23,39 +37,31 @@ my @test_configuration = (
 	{
 		'compression_method' => 'none',
 		'backup_flags' => [],
-		'backup_archive' => 'base.tar',
+		'backup_archive' => ['base.tar', "$tsoid.tar"],
 		'enabled' => 1
 	},
 	{
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'server-gzip' ],
-		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.gz', "$tsoid.tar.gz" ],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'server-lz4' ],
-		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => [ '-d', '-m' ],
+		'backup_archive' => ['base.tar.lz4', "$tsoid.tar.lz4" ],
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd:level=1,long' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	});
 
@@ -86,47 +92,16 @@ for my $tc (@test_configuration)
 		my $backup_files = join(',',
 			sort grep { $_ ne '.' && $_ ne '..' } slurp_dir($backup_path));
 		my $expected_backup_files =
-		  join(',', sort ('backup_manifest', $tc->{'backup_archive'}));
+		  join(',', sort ('backup_manifest', @{ $tc->{'backup_archive'} }));
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok(['pg_verifybackup', '-n', '-e', $backup_path],
+			"verify backup, compression $method");
 
 		# Cleanup.
-		unlink($backup_path . '/backup_manifest');
-		unlink($backup_path . '/base.tar');
-		unlink($backup_path . '/' . $tc->{'backup_archive'});
+		rmtree($backup_path);
 		rmtree($extract_path);
 	}
 }
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index 8c076d46dee..6b7d7483f6e 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -29,41 +29,30 @@ my @test_configuration = (
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'client-gzip:5' ],
 		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'client-lz4:5' ],
 		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => ['-d'],
-		'output_file' => 'base.tar',
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:5' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:level=1,long' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'parallel zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:workers=3' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1"),
 		'possibly_unsupported' =>
 		  qr/could not set compression worker count to 3: Unsupported parameter/
@@ -118,40 +107,9 @@ for my $tc (@test_configuration)
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			push @decompress, $backup_path . '/' . $tc->{'output_file'}
-			  if $tc->{'output_file'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok( [ 'pg_verifybackup', '-n', '-e', $backup_path ],
+			"verify backup, compression $method");
 
 		# Cleanup.
 		rmtree($extract_path);
-- 
2.18.0

From 498ee467c0123d816d14717c20746e55b26d974f Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Fri, 2 Aug 2024 16:37:38 +0530
Subject: [PATCH v5 09/12] Refactor: move first and last progress_report call
 to Main.

---
 src/bin/pg_verifybackup/pg_verifybackup.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 5f055a23a63..801e13886c2 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -253,7 +253,10 @@ main(int argc, char **argv)
 	 * read, which occurs only when checksum verification is enabled.
 	 */
 	if (!context.skip_checksums)
+	{
 		compute_total_size(&context);
+		progress_report(&context, false);
+	}
 
 	/*
 	 * Now scan the files in the backup directory. At this stage, we verify
@@ -275,7 +278,10 @@ main(int argc, char **argv)
 	 * told to skip it.
 	 */
 	if (!context.skip_checksums)
+	{
 		verify_backup_checksums(&context);
+		progress_report(&context, true);
+	}
 
 	/*
 	 * Try to parse the required ranges of WAL records, unless we were told
@@ -736,8 +742,6 @@ verify_backup_checksums(verifier_context *context)
 	manifest_file *m;
 	uint8	   *buffer;
 
-	progress_report(context, false);
-
 	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
 
 	manifest_files_start_iterate(manifest->files, &it);
@@ -761,8 +765,6 @@ verify_backup_checksums(verifier_context *context)
 	}
 
 	pfree(buffer);
-
-	progress_report(context, true);
 }
 
 /*
-- 
2.18.0

From 07e402b82c7f9af43703aa5759da5ee7c8fcc3df Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 15:47:26 +0530
Subject: [PATCH v5 08/12] Refactor: split verify_control_file.

Separated the manifest entry verification code into a new function and
introduced the should_verify_control_data() macro, similar to
should_verify_checksum().

Note that should_verify_checksum() has been slightly modified to
include a NULL check for its argument, maintaining the same code
structure as should_verify_control_data().
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 44 +++++++++++------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 18 +++++++++-
 2 files changed, 37 insertions(+), 25 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 3eddaa2468e..5f055a23a63 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,7 +18,6 @@
 #include <sys/stat.h>
 #include <time.h>
 
-#include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
@@ -61,8 +60,6 @@ static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
 							   char *relpath, char *fullpath);
-static void verify_control_file(const char *controlpath,
-								uint64 manifest_system_identifier);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -625,14 +622,20 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 	/* Check the backup manifest entry for this file. */
 	m = verify_manifest_entry(context, relpath, sb.st_size);
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0 &&
-		m != NULL && m->matched && !m->bad)
-		verify_control_file(fullpath, context->manifest->system_identifier);
+	/* Validate the manifest system identifier */
+	if (should_verify_control_data(context->manifest, m))
+	{
+		ControlFileData *control_file;
+		bool		crc_ok;
+
+		pg_log_debug("reading \"%s\"", fullpath);
+		control_file = get_controlfile_by_exact_path(fullpath, &crc_ok);
+
+		verify_control_data(control_file, fullpath, crc_ok,
+							context->manifest->system_identifier);
+		/* Release memory. */
+		pfree(control_file);
+	}
 }
 
 /*
@@ -676,18 +679,14 @@ verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
 }
 
 /*
- * Sanity check control file and validate system identifier against manifest
- * system identifier.
+ * Sanity check control file data and validate system identifier against
+ * manifest system identifier.
  */
-static void
-verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
+void
+verify_control_data(ControlFileData *control_file,
+					const char *controlpath, bool crc_ok,
+					uint64 manifest_system_identifier)
 {
-	ControlFileData *control_file;
-	bool		crc_ok;
-
-	pg_log_debug("reading \"%s\"", controlpath);
-	control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
-
 	/* Control file contents not meaningful if CRC is bad. */
 	if (!crc_ok)
 		report_fatal_error("%s: CRC is incorrect", controlpath);
@@ -703,9 +702,6 @@ verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
 						   controlpath,
 						   (unsigned long long) manifest_system_identifier,
 						   (unsigned long long) control_file->system_identifier);
-
-	/* Release memory. */
-	pfree(control_file);
 }
 
 /*
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 1bc5f7a6b4a..c88f71ff14b 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -16,6 +16,7 @@
 
 #include "common/controldata_utils.h"
 #include "common/hashfn_unstable.h"
+#include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 
@@ -44,7 +45,19 @@ typedef struct manifest_file
 } manifest_file;
 
 #define should_verify_checksum(m) \
-	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+	(((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+
+/*
+ * Validate the manifest system identifier against the control file; this
+ * feature is not available in manifest version 1.  This validation should be
+ * carried out only if the manifest entry validation is completed without any
+ * errors.
+ */
+#define should_verify_control_data(manifest, m) \
+	(((manifest)->version != 1) && \
+	 ((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (strcmp((m)->pathname, "global/pg_control") == 0))
 
 /*
  * Define a hash table which we can use to store information about the files
@@ -110,6 +123,9 @@ extern manifest_file *verify_manifest_entry(verifier_context *context,
 extern void verify_checksum(verifier_context *context, manifest_file *m,
 							pg_checksum_context *checksum_ctx,
 							uint8 *checksumbuf);
+extern void verify_control_data(ControlFileData *control_file,
+								const char *controlpath, bool crc_ok,
+								uint64 manifest_system_identifier);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From dc90660cbc524ff04b88aadb027be22b81c94ff7 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 16:45:55 +0530
Subject: [PATCH v5 07/12] Refactor: split verify_file_checksum() function.

Move the core functionality of verify_file_checksum to a new function
to reuse it instead of duplicating the code.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 18 ++++++++++++++++--
 src/bin/pg_verifybackup/pg_verifybackup.h |  3 +++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index ab6bda8c9dc..3eddaa2468e 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -782,7 +782,6 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	int			rc;
 	size_t		bytes_read = 0;
 	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
-	int			checksumlen;
 
 	/* Open the target file. */
 	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
@@ -848,8 +847,23 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 		return;
 	}
 
+	/* Do the final computation and verification. */
+	verify_checksum(context, m, &checksum_ctx, checksumbuf);
+}
+
+/*
+ * A helper function to finalize checksum computation and verify it against the
+ * backup manifest information.
+ */
+void
+verify_checksum(verifier_context *context, manifest_file *m,
+				pg_checksum_context *checksum_ctx, uint8 *checksumbuf)
+{
+	int			checksumlen;
+	const char *relpath = m->pathname;
+
 	/* Get the final checksum. */
-	checksumlen = pg_checksum_final(&checksum_ctx, checksumbuf);
+	checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
 	if (checksumlen < 0)
 	{
 		report_backup_error(context,
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 98c75916255..1bc5f7a6b4a 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -107,6 +107,9 @@ typedef struct verifier_context
 
 extern manifest_file *verify_manifest_entry(verifier_context *context,
 											char *relpath, int64 filesize);
+extern void verify_checksum(verifier_context *context, manifest_file *m,
+							pg_checksum_context *checksum_ctx,
+							uint8 *checksumbuf);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From 8befb1d9759e3a23a2a4b951d7a6ac800dc8e72e Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 12:10:34 +0530
Subject: [PATCH v5 05/12] Refactor: move some part of pg_verifybackup.c to
 pg_verifybackup.h

---
 src/bin/pg_verifybackup/pg_verifybackup.c | 102 +------------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 118 ++++++++++++++++++++++
 2 files changed, 123 insertions(+), 97 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/pg_verifybackup.h

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 71585ffc50e..4e42757c346 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,12 +18,11 @@
 #include <sys/stat.h>
 #include <time.h>
 
-#include "common/controldata_utils.h"
-#include "common/hashfn_unstable.h"
 #include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
+#include "pg_verifybackup.h"
 #include "pgtime.h"
 
 /*
@@ -40,89 +39,6 @@
  */
 #define ESTIMATED_BYTES_PER_MANIFEST_LINE	100
 
-/*
- * How many bytes should we try to read from a file at once?
- */
-#define READ_CHUNK_SIZE				(128 * 1024)
-
-/*
- * Each file described by the manifest file is parsed to produce an object
- * like this.
- */
-typedef struct manifest_file
-{
-	uint32		status;			/* hash status */
-	const char *pathname;
-	size_t		size;
-	pg_checksum_type checksum_type;
-	int			checksum_length;
-	uint8	   *checksum_payload;
-	bool		matched;
-	bool		bad;
-} manifest_file;
-
-#define should_verify_checksum(m) \
-	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
-
-/*
- * Define a hash table which we can use to store information about the files
- * mentioned in the backup manifest.
- */
-#define SH_PREFIX		manifest_files
-#define SH_ELEMENT_TYPE	manifest_file
-#define SH_KEY_TYPE		const char *
-#define	SH_KEY			pathname
-#define SH_HASH_KEY(tb, key)	hash_string(key)
-#define SH_EQUAL(tb, a, b)		(strcmp(a, b) == 0)
-#define	SH_SCOPE		static inline
-#define SH_RAW_ALLOCATOR	pg_malloc0
-#define SH_DECLARE
-#define SH_DEFINE
-#include "lib/simplehash.h"
-
-/*
- * Each WAL range described by the manifest file is parsed to produce an
- * object like this.
- */
-typedef struct manifest_wal_range
-{
-	TimeLineID	tli;
-	XLogRecPtr	start_lsn;
-	XLogRecPtr	end_lsn;
-	struct manifest_wal_range *next;
-	struct manifest_wal_range *prev;
-} manifest_wal_range;
-
-/*
- * All the data parsed from a backup_manifest file.
- */
-typedef struct manifest_data
-{
-	int			version;
-	uint64		system_identifier;
-	manifest_files_hash *files;
-	manifest_wal_range *first_wal_range;
-	manifest_wal_range *last_wal_range;
-} manifest_data;
-
-/*
- * All of the context information we need while checking a backup manifest.
- */
-typedef struct verifier_context
-{
-	manifest_data *manifest;
-	char	   *backup_directory;
-	SimpleStringList ignore_list;
-	bool		skip_checksums;
-	bool		exit_on_error;
-	bool		saw_any_error;
-
-	/* Progress indicators */
-	bool		show_progress;
-	uint64		total_size;
-	uint64		done_size;
-} verifier_context;
-
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -156,14 +72,6 @@ static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
 
-static void report_backup_error(verifier_context *context,
-								const char *pg_restrict fmt,...)
-			pg_attribute_printf(2, 3);
-static void report_fatal_error(const char *pg_restrict fmt,...)
-			pg_attribute_printf(1, 2) pg_attribute_noreturn();
-static bool should_ignore_relpath(verifier_context *context, const char *relpath);
-
-static void progress_report(verifier_context *context, bool finished);
 static void usage(void);
 
 static const char *progname;
@@ -978,7 +886,7 @@ parse_required_wal(verifier_context *context, char *pg_waldump_path,
  * Update the context to indicate that we saw an error, and exit if the
  * context says we should.
  */
-static void
+void
 report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
 {
 	va_list		ap;
@@ -995,7 +903,7 @@ report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
 /*
  * Report a fatal error and exit
  */
-static void
+void
 report_fatal_error(const char *pg_restrict fmt,...)
 {
 	va_list		ap;
@@ -1014,7 +922,7 @@ report_fatal_error(const char *pg_restrict fmt,...)
  * Note that by "prefix" we mean a parent directory; for this purpose,
  * "aa/bb" is not a prefix of "aa/bbb", but it is a prefix of "aa/bb/cc".
  */
-static bool
+bool
 should_ignore_relpath(verifier_context *context, const char *relpath)
 {
 	SimpleStringListCell *cell;
@@ -1043,7 +951,7 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
  * If finished is set to true, this is the last progress report. The cursor
  * is moved to the next line.
  */
-static void
+void
 progress_report(verifier_context *context, bool finished)
 {
 	static pg_time_t last_progress_report = 0;
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
new file mode 100644
index 00000000000..90900048547
--- /dev/null
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -0,0 +1,118 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_verifybackup.h
+ *	  Verify a backup against a backup manifest.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_verifybackup/pg_verifybackup.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef PG_VERIFYBACKUP_H
+#define PG_VERIFYBACKUP_H
+
+#include "common/controldata_utils.h"
+#include "common/hashfn_unstable.h"
+#include "common/parse_manifest.h"
+#include "fe_utils/simple_list.h"
+
+extern bool show_progress;
+extern bool skip_checksums;
+
+/*
+ * How many bytes should we try to read from a file at once?
+ */
+#define READ_CHUNK_SIZE				(128 * 1024)
+
+/*
+ * Each file described by the manifest file is parsed to produce an object
+ * like this.
+ */
+typedef struct manifest_file
+{
+	uint32		status;			/* hash status */
+	const char *pathname;
+	size_t		size;
+	pg_checksum_type checksum_type;
+	int			checksum_length;
+	uint8	   *checksum_payload;
+	bool		matched;
+	bool		bad;
+} manifest_file;
+
+#define should_verify_checksum(m) \
+	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+
+/*
+ * Define a hash table which we can use to store information about the files
+ * mentioned in the backup manifest.
+ */
+#define SH_PREFIX		manifest_files
+#define SH_ELEMENT_TYPE	manifest_file
+#define SH_KEY_TYPE		const char *
+#define	SH_KEY			pathname
+#define SH_HASH_KEY(tb, key)	hash_string(key)
+#define SH_EQUAL(tb, a, b)		(strcmp(a, b) == 0)
+#define	SH_SCOPE		static inline
+#define SH_RAW_ALLOCATOR	pg_malloc0
+#define SH_DECLARE
+#define SH_DEFINE
+#include "lib/simplehash.h"
+
+/*
+ * Each WAL range described by the manifest file is parsed to produce an
+ * object like this.
+ */
+typedef struct manifest_wal_range
+{
+	TimeLineID	tli;
+	XLogRecPtr	start_lsn;
+	XLogRecPtr	end_lsn;
+	struct manifest_wal_range *next;
+	struct manifest_wal_range *prev;
+} manifest_wal_range;
+
+/*
+ * All the data parsed from a backup_manifest file.
+ */
+typedef struct manifest_data
+{
+	int			version;
+	uint64		system_identifier;
+	manifest_files_hash *files;
+	manifest_wal_range *first_wal_range;
+	manifest_wal_range *last_wal_range;
+} manifest_data;
+
+/*
+ * All of the context information we need while checking a backup manifest.
+ */
+typedef struct verifier_context
+{
+	manifest_data *manifest;
+	char	   *backup_directory;
+	SimpleStringList ignore_list;
+	bool		skip_checksums;
+	bool		exit_on_error;
+	bool		saw_any_error;
+
+	/* Progress indicators */
+	bool		show_progress;
+	uint64		total_size;
+	uint64		done_size;
+} verifier_context;
+
+extern void report_backup_error(verifier_context *context,
+								const char *pg_restrict fmt,...)
+			pg_attribute_printf(2, 3);
+extern void report_fatal_error(const char *pg_restrict fmt,...)
+			pg_attribute_printf(1, 2) pg_attribute_noreturn();
+extern bool should_ignore_relpath(verifier_context *context,
+								  const char *relpath);
+
+extern void progress_report(verifier_context *context, bool finished);
+
+#endif							/* PG_VERIFYBACKUP_H */
-- 
2.18.0

From 7720bebed0b8456fec741f9083d3fd71278dea42 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 31 Jul 2024 11:43:52 +0530
Subject: [PATCH v5 04/12] Refactor: move few global variable to
 verifier_context struct

Global variables are:
	1. show_progress
	2. skip_checksums
	3. total_size
	4. done_size
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 50 +++++++++++------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index d77e70fbe38..71585ffc50e 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -113,8 +113,14 @@ typedef struct verifier_context
 	manifest_data *manifest;
 	char	   *backup_directory;
 	SimpleStringList ignore_list;
+	bool		skip_checksums;
 	bool		exit_on_error;
 	bool		saw_any_error;
+
+	/* Progress indicators */
+	bool		show_progress;
+	uint64		total_size;
+	uint64		done_size;
 } verifier_context;
 
 static manifest_data *parse_manifest_file(char *manifest_path);
@@ -157,19 +163,11 @@ static void report_fatal_error(const char *pg_restrict fmt,...)
 			pg_attribute_printf(1, 2) pg_attribute_noreturn();
 static bool should_ignore_relpath(verifier_context *context, const char *relpath);
 
-static void progress_report(bool finished);
+static void progress_report(verifier_context *context, bool finished);
 static void usage(void);
 
 static const char *progname;
 
-/* options */
-static bool show_progress = false;
-static bool skip_checksums = false;
-
-/* Progress indicators */
-static uint64 total_size = 0;
-static uint64 done_size = 0;
-
 /*
  * Main entry point.
  */
@@ -260,13 +258,13 @@ main(int argc, char **argv)
 				no_parse_wal = true;
 				break;
 			case 'P':
-				show_progress = true;
+				context.show_progress = true;
 				break;
 			case 'q':
 				quiet = true;
 				break;
 			case 's':
-				skip_checksums = true;
+				context.skip_checksums = true;
 				break;
 			case 'w':
 				wal_directory = pstrdup(optarg);
@@ -299,7 +297,7 @@ main(int argc, char **argv)
 	}
 
 	/* Complain if the specified arguments conflict */
-	if (show_progress && quiet)
+	if (context.show_progress && quiet)
 		pg_fatal("cannot specify both %s and %s",
 				 "-P/--progress", "-q/--quiet");
 
@@ -363,7 +361,7 @@ main(int argc, char **argv)
 	 * Now do the expensive work of verifying file checksums, unless we were
 	 * told to skip it.
 	 */
-	if (!skip_checksums)
+	if (!context.skip_checksums)
 		verify_backup_checksums(&context);
 
 	/*
@@ -739,8 +737,9 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		verify_control_file(fullpath, context->manifest->system_identifier);
 
 	/* Update statistics for progress report, if necessary */
-	if (show_progress && !skip_checksums && should_verify_checksum(m))
-		total_size += m->size;
+	if (context->show_progress && !context->skip_checksums &&
+		should_verify_checksum(m))
+		context->total_size += m->size;
 
 	/*
 	 * We don't verify checksums at this stage. We first finish verifying that
@@ -815,7 +814,7 @@ verify_backup_checksums(verifier_context *context)
 	manifest_file *m;
 	uint8	   *buffer;
 
-	progress_report(false);
+	progress_report(context, false);
 
 	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
 
@@ -841,7 +840,7 @@ verify_backup_checksums(verifier_context *context)
 
 	pfree(buffer);
 
-	progress_report(true);
+	progress_report(context, true);
 }
 
 /*
@@ -889,8 +888,8 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 		}
 
 		/* Report progress */
-		done_size += rc;
-		progress_report(false);
+		context->done_size += rc;
+		progress_report(context, false);
 	}
 	if (rc < 0)
 		report_backup_error(context, "could not read file \"%s\": %m",
@@ -1036,7 +1035,7 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
 }
 
 /*
- * Print a progress report based on the global variables.
+ * Print a progress report based on the variables in verifier_context.
  *
  * Progress report is written at maximum once per second, unless the finished
  * parameter is set to true.
@@ -1045,7 +1044,7 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
  * is moved to the next line.
  */
 static void
-progress_report(bool finished)
+progress_report(verifier_context *context, bool finished)
 {
 	static pg_time_t last_progress_report = 0;
 	pg_time_t	now;
@@ -1053,7 +1052,7 @@ progress_report(bool finished)
 	char		totalsize_str[32];
 	char		donesize_str[32];
 
-	if (!show_progress)
+	if (!context->show_progress)
 		return;
 
 	now = time(NULL);
@@ -1061,12 +1060,13 @@ progress_report(bool finished)
 		return;					/* Max once per second */
 
 	last_progress_report = now;
-	percent_size = total_size ? (int) ((done_size * 100 / total_size)) : 0;
+	percent_size = context->total_size ?
+		(int) ((context->done_size * 100 / context->total_size)) : 0;
 
 	snprintf(totalsize_str, sizeof(totalsize_str), UINT64_FORMAT,
-			 total_size / 1024);
+			 context->total_size / 1024);
 	snprintf(donesize_str, sizeof(donesize_str), UINT64_FORMAT,
-			 done_size / 1024);
+			 context->done_size / 1024);
 
 	fprintf(stderr,
 			_("%*s/%s kB (%d%%) verified"),
-- 
2.18.0

From 2206efa24af8882877ec216e0001dee8538e30c1 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 12:15:26 +0530
Subject: [PATCH v5 06/12] Refactor: split verify_backup_file() function.

Move the manifest entry verification code into a new function as
verify_manifest_entry(). And the total size computation code into
another new function, compute_total_size(), which is called from the
main.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 76 ++++++++++++++++++-----
 src/bin/pg_verifybackup/pg_verifybackup.h |  3 +
 2 files changed, 62 insertions(+), 17 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 4e42757c346..ab6bda8c9dc 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -72,6 +72,7 @@ static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
 
+static void compute_total_size(verifier_context *context);
 static void usage(void);
 
 static const char *progname;
@@ -250,6 +251,13 @@ main(int argc, char **argv)
 	 */
 	context.manifest = parse_manifest_file(manifest_path);
 
+	/*
+	 * For the progress report, compute the total size of the files to be
+	 * read, which occurs only when checksum verification is enabled.
+	 */
+	if (!context.skip_checksums)
+		compute_total_size(&context);
+
 	/*
 	 * Now scan the files in the backup directory. At this stage, we verify
 	 * that every file on disk is present in the manifest and that the sizes
@@ -614,6 +622,27 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		return;
 	}
 
+	/* Check the backup manifest entry for this file. */
+	m = verify_manifest_entry(context, relpath, sb.st_size);
+
+	/*
+	 * Validate the manifest system identifier, not available in manifest
+	 * version 1.
+	 */
+	if (context->manifest->version != 1 &&
+		strcmp(relpath, "global/pg_control") == 0 &&
+		m != NULL && m->matched && !m->bad)
+		verify_control_file(fullpath, context->manifest->system_identifier);
+}
+
+/*
+ * Verify file and its size entry in the manifest.
+ */
+manifest_file *
+verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
+{
+	manifest_file *m;
+
 	/* Check whether there's an entry in the manifest hash. */
 	m = manifest_files_lookup(context->manifest->files, relpath);
 	if (m == NULL)
@@ -621,40 +650,29 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		report_backup_error(context,
 							"\"%s\" is present on disk but not in the manifest",
 							relpath);
-		return;
+		return NULL;
 	}
 
 	/* Flag this entry as having been encountered in the filesystem. */
 	m->matched = true;
 
 	/* Check that the size matches. */
-	if (m->size != sb.st_size)
+	if (m->size != filesize)
 	{
 		report_backup_error(context,
 							"\"%s\" has size %lld on disk but size %zu in the manifest",
-							relpath, (long long int) sb.st_size, m->size);
+							relpath, (long long int) filesize, m->size);
 		m->bad = true;
 	}
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0)
-		verify_control_file(fullpath, context->manifest->system_identifier);
-
-	/* Update statistics for progress report, if necessary */
-	if (context->show_progress && !context->skip_checksums &&
-		should_verify_checksum(m))
-		context->total_size += m->size;
-
 	/*
 	 * We don't verify checksums at this stage. We first finish verifying that
 	 * we have the expected set of files with the expected sizes, and only
 	 * afterwards verify the checksums. That's because computing checksums may
 	 * take a while, and we'd like to report more obvious problems quickly.
 	 */
+
+	return m;
 }
 
 /*
@@ -817,7 +835,7 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 
 	/*
 	 * Double-check that we read the expected number of bytes from the file.
-	 * Normally, a file size mismatch would be caught in verify_backup_file
+	 * Normally, a file size mismatch would be caught in verify_manifest_entry
 	 * and this check would never be reached, but this provides additional
 	 * safety and clarity in the event of concurrent modifications or
 	 * filesystem misbehavior.
@@ -988,6 +1006,30 @@ progress_report(verifier_context *context, bool finished)
 	fputc((!finished && isatty(fileno(stderr))) ? '\r' : '\n', stderr);
 }
 
+/*
+ * Compute the total size of backup files for progress reporting.
+ */
+static void
+compute_total_size(verifier_context *context)
+{
+	manifest_data *manifest = context->manifest;
+	manifest_files_iterator it;
+	manifest_file *m;
+	uint64		total_size = 0;
+
+	if (!context->show_progress)
+		return;
+
+	manifest_files_start_iterate(manifest->files, &it);
+	while ((m = manifest_files_iterate(manifest->files, &it)) != NULL)
+	{
+		if (!should_ignore_relpath(context, m->pathname))
+			total_size += m->size;
+	}
+
+	context->total_size = total_size;
+}
+
 /*
  * Print out usage information and exit.
  */
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 90900048547..98c75916255 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -105,6 +105,9 @@ typedef struct verifier_context
 	uint64		done_size;
 } verifier_context;
 
+extern manifest_file *verify_manifest_entry(verifier_context *context,
+											char *relpath, int64 filesize);
+
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
 			pg_attribute_printf(2, 3);
-- 
2.18.0

From ecc3a97ee176b00810a57c21bc84e612ddd94fe0 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 31 Jul 2024 11:20:52 +0530
Subject: [PATCH v5 03/12] Refactor: move astreamer* files to fe_utils to make
 common availability of it.

To make it accessible to other code, we need to move the ASTREAMER
code (previously known as BBSTREAMER) to a common location. The
appropriate place would be src/fe_utils, as it is a frontend
infrastructure intended for shared use.
---
 meson.build                                             | 2 +-
 src/bin/pg_basebackup/Makefile                          | 7 +------
 src/bin/pg_basebackup/astreamer_inject.h                | 2 +-
 src/bin/pg_basebackup/meson.build                       | 5 -----
 src/fe_utils/Makefile                                   | 5 +++++
 src/{bin/pg_basebackup => fe_utils}/astreamer_file.c    | 2 +-
 src/{bin/pg_basebackup => fe_utils}/astreamer_gzip.c    | 2 +-
 src/{bin/pg_basebackup => fe_utils}/astreamer_lz4.c     | 2 +-
 src/{bin/pg_basebackup => fe_utils}/astreamer_tar.c     | 2 +-
 src/{bin/pg_basebackup => fe_utils}/astreamer_zstd.c    | 2 +-
 src/fe_utils/meson.build                                | 5 +++++
 src/{bin/pg_basebackup => include/fe_utils}/astreamer.h | 0
 12 files changed, 18 insertions(+), 18 deletions(-)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_file.c (99%)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_gzip.c (99%)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_lz4.c (99%)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_tar.c (99%)
 rename src/{bin/pg_basebackup => fe_utils}/astreamer_zstd.c (99%)
 rename src/{bin/pg_basebackup => include/fe_utils}/astreamer.h (100%)

diff --git a/meson.build b/meson.build
index 7de0371226d..f7a5d2aea9a 100644
--- a/meson.build
+++ b/meson.build
@@ -3027,7 +3027,7 @@ frontend_common_code = declare_dependency(
   compile_args: ['-DFRONTEND'],
   include_directories: [postgres_inc],
   sources: generated_headers,
-  dependencies: [os_deps, zlib, zstd],
+  dependencies: [os_deps, zlib, zstd, lz4],
 )
 
 backend_common_code = declare_dependency(
diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile
index a71af2d48a7..f1e73058b23 100644
--- a/src/bin/pg_basebackup/Makefile
+++ b/src/bin/pg_basebackup/Makefile
@@ -37,12 +37,7 @@ OBJS = \
 
 BBOBJS = \
 	pg_basebackup.o \
-	astreamer_file.o \
-	astreamer_gzip.o \
-	astreamer_inject.o \
-	astreamer_lz4.o \
-	astreamer_tar.o \
-	astreamer_zstd.o
+	astreamer_inject.o
 
 all: pg_basebackup pg_createsubscriber pg_receivewal pg_recvlogical
 
diff --git a/src/bin/pg_basebackup/astreamer_inject.h b/src/bin/pg_basebackup/astreamer_inject.h
index 8504b3f5e0d..aeed533862b 100644
--- a/src/bin/pg_basebackup/astreamer_inject.h
+++ b/src/bin/pg_basebackup/astreamer_inject.h
@@ -12,7 +12,7 @@
 #ifndef ASTREAMER_INJECT_H
 #define ASTREAMER_INJECT_H
 
-#include "astreamer.h"
+#include "fe_utils/astreamer.h"
 #include "pqexpbuffer.h"
 
 extern astreamer *astreamer_recovery_injector_new(astreamer *next,
diff --git a/src/bin/pg_basebackup/meson.build b/src/bin/pg_basebackup/meson.build
index a68dbd7837d..9101fc18438 100644
--- a/src/bin/pg_basebackup/meson.build
+++ b/src/bin/pg_basebackup/meson.build
@@ -1,12 +1,7 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 common_sources = files(
-  'astreamer_file.c',
-  'astreamer_gzip.c',
   'astreamer_inject.c',
-  'astreamer_lz4.c',
-  'astreamer_tar.c',
-  'astreamer_zstd.c',
   'receivelog.c',
   'streamutil.c',
   'walmethods.c',
diff --git a/src/fe_utils/Makefile b/src/fe_utils/Makefile
index 946c05258f0..2694be4b859 100644
--- a/src/fe_utils/Makefile
+++ b/src/fe_utils/Makefile
@@ -21,6 +21,11 @@ override CPPFLAGS := -DFRONTEND -I$(libpq_srcdir) $(CPPFLAGS)
 
 OBJS = \
 	archive.o \
+	astreamer_file.o \
+	astreamer_gzip.o \
+	astreamer_lz4.o \
+	astreamer_tar.o \
+	astreamer_zstd.o \
 	cancel.o \
 	conditional.o \
 	connect_utils.o \
diff --git a/src/bin/pg_basebackup/astreamer_file.c b/src/fe_utils/astreamer_file.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_file.c
rename to src/fe_utils/astreamer_file.c
index 2742385e103..13d1192c6e6 100644
--- a/src/bin/pg_basebackup/astreamer_file.c
+++ b/src/fe_utils/astreamer_file.c
@@ -13,10 +13,10 @@
 
 #include <unistd.h>
 
-#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
+#include "fe_utils/astreamer.h"
 
 typedef struct astreamer_plain_writer
 {
diff --git a/src/bin/pg_basebackup/astreamer_gzip.c b/src/fe_utils/astreamer_gzip.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_gzip.c
rename to src/fe_utils/astreamer_gzip.c
index 6f7c27afbbc..dd28defac7b 100644
--- a/src/bin/pg_basebackup/astreamer_gzip.c
+++ b/src/fe_utils/astreamer_gzip.c
@@ -17,10 +17,10 @@
 #include <zlib.h>
 #endif
 
-#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
+#include "fe_utils/astreamer.h"
 
 #ifdef HAVE_LIBZ
 typedef struct astreamer_gzip_writer
diff --git a/src/bin/pg_basebackup/astreamer_lz4.c b/src/fe_utils/astreamer_lz4.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_lz4.c
rename to src/fe_utils/astreamer_lz4.c
index 1c40d7d8ad5..d8b2a367e47 100644
--- a/src/bin/pg_basebackup/astreamer_lz4.c
+++ b/src/fe_utils/astreamer_lz4.c
@@ -17,10 +17,10 @@
 #include <lz4frame.h>
 #endif
 
-#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
+#include "fe_utils/astreamer.h"
 
 #ifdef USE_LZ4
 typedef struct astreamer_lz4_frame
diff --git a/src/bin/pg_basebackup/astreamer_tar.c b/src/fe_utils/astreamer_tar.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_tar.c
rename to src/fe_utils/astreamer_tar.c
index 673690cd18f..f5d3562d280 100644
--- a/src/bin/pg_basebackup/astreamer_tar.c
+++ b/src/fe_utils/astreamer_tar.c
@@ -23,8 +23,8 @@
 
 #include <time.h>
 
-#include "astreamer.h"
 #include "common/logging.h"
+#include "fe_utils/astreamer.h"
 #include "pgtar.h"
 
 typedef struct astreamer_tar_parser
diff --git a/src/bin/pg_basebackup/astreamer_zstd.c b/src/fe_utils/astreamer_zstd.c
similarity index 99%
rename from src/bin/pg_basebackup/astreamer_zstd.c
rename to src/fe_utils/astreamer_zstd.c
index 58dc679ef99..45f6cb67363 100644
--- a/src/bin/pg_basebackup/astreamer_zstd.c
+++ b/src/fe_utils/astreamer_zstd.c
@@ -17,8 +17,8 @@
 #include <zstd.h>
 #endif
 
-#include "astreamer.h"
 #include "common/logging.h"
+#include "fe_utils/astreamer.h"
 
 #ifdef USE_ZSTD
 
diff --git a/src/fe_utils/meson.build b/src/fe_utils/meson.build
index 14d0482a2cc..043021d826d 100644
--- a/src/fe_utils/meson.build
+++ b/src/fe_utils/meson.build
@@ -2,6 +2,11 @@
 
 fe_utils_sources = files(
   'archive.c',
+  'astreamer_file.c',
+  'astreamer_gzip.c',
+  'astreamer_lz4.c',
+  'astreamer_tar.c',
+  'astreamer_zstd.c',
   'cancel.c',
   'conditional.c',
   'connect_utils.c',
diff --git a/src/bin/pg_basebackup/astreamer.h b/src/include/fe_utils/astreamer.h
similarity index 100%
rename from src/bin/pg_basebackup/astreamer.h
rename to src/include/fe_utils/astreamer.h
-- 
2.18.0

From 8614be90305258751b760e746a44cbbe7ba0ad6c Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 17 Jul 2024 14:23:27 +0530
Subject: [PATCH v5 02/12] Refactor: Add astreamer_inject.h and move related
 declarations to it.

---
 src/bin/pg_basebackup/astreamer.h        |  6 ------
 src/bin/pg_basebackup/astreamer_inject.c |  2 +-
 src/bin/pg_basebackup/astreamer_inject.h | 24 ++++++++++++++++++++++++
 src/bin/pg_basebackup/pg_basebackup.c    |  2 +-
 4 files changed, 26 insertions(+), 8 deletions(-)
 create mode 100644 src/bin/pg_basebackup/astreamer_inject.h

diff --git a/src/bin/pg_basebackup/astreamer.h b/src/bin/pg_basebackup/astreamer.h
index 6b0047418bb..9d0a8c4d0c2 100644
--- a/src/bin/pg_basebackup/astreamer.h
+++ b/src/bin/pg_basebackup/astreamer.h
@@ -217,10 +217,4 @@ extern astreamer *astreamer_tar_parser_new(astreamer *next);
 extern astreamer *astreamer_tar_terminator_new(astreamer *next);
 extern astreamer *astreamer_tar_archiver_new(astreamer *next);
 
-extern astreamer *astreamer_recovery_injector_new(astreamer *next,
-												  bool is_recovery_guc_supported,
-												  PQExpBuffer recoveryconfcontents);
-extern void astreamer_inject_file(astreamer *streamer, char *pathname,
-								  char *data, int len);
-
 #endif
diff --git a/src/bin/pg_basebackup/astreamer_inject.c b/src/bin/pg_basebackup/astreamer_inject.c
index 7f1decded8d..4ad8381f102 100644
--- a/src/bin/pg_basebackup/astreamer_inject.c
+++ b/src/bin/pg_basebackup/astreamer_inject.c
@@ -11,7 +11,7 @@
 
 #include "postgres_fe.h"
 
-#include "astreamer.h"
+#include "astreamer_inject.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 
diff --git a/src/bin/pg_basebackup/astreamer_inject.h b/src/bin/pg_basebackup/astreamer_inject.h
new file mode 100644
index 00000000000..8504b3f5e0d
--- /dev/null
+++ b/src/bin/pg_basebackup/astreamer_inject.h
@@ -0,0 +1,24 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer_inject.h
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *		  src/bin/pg_basebackup/astreamer_inject.h
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef ASTREAMER_INJECT_H
+#define ASTREAMER_INJECT_H
+
+#include "astreamer.h"
+#include "pqexpbuffer.h"
+
+extern astreamer *astreamer_recovery_injector_new(astreamer *next,
+												  bool is_recovery_guc_supported,
+												  PQExpBuffer recoveryconfcontents);
+extern void astreamer_inject_file(astreamer *streamer, char *pathname,
+								  char *data, int len);
+
+#endif
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 4179b064cbc..1e753e40c97 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -26,7 +26,7 @@
 #endif
 
 #include "access/xlog_internal.h"
-#include "astreamer.h"
+#include "astreamer_inject.h"
 #include "backup/basebackup.h"
 #include "common/compression.h"
 #include "common/file_perm.h"
-- 
2.18.0

From 1107087beed27e1e242a058fba30e4fe62b4e620 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 09:39:32 +0530
Subject: [PATCH v5 01/12] Refactor: Rename all bbstreamer references to
 astreamer.

BBSTREAMER is specific to pg_basebackup; we need a more generalized
name so it can be placed in a common area, making it accessible for
other modules. Renaming it to ASTREAMER, short for ARCHIVE STREAMER,
makes it more general.
---
 src/bin/pg_basebackup/Makefile                |  12 +-
 src/bin/pg_basebackup/astreamer.h             | 226 +++++++++++++
 .../{bbstreamer_file.c => astreamer_file.c}   | 148 ++++----
 .../{bbstreamer_gzip.c => astreamer_gzip.c}   | 154 ++++-----
 ...bbstreamer_inject.c => astreamer_inject.c} | 152 ++++-----
 .../{bbstreamer_lz4.c => astreamer_lz4.c}     | 172 +++++-----
 .../{bbstreamer_tar.c => astreamer_tar.c}     | 316 +++++++++---------
 .../{bbstreamer_zstd.c => astreamer_zstd.c}   | 160 ++++-----
 src/bin/pg_basebackup/bbstreamer.h            | 226 -------------
 src/bin/pg_basebackup/meson.build             |  12 +-
 src/bin/pg_basebackup/nls.mk                  |  12 +-
 src/bin/pg_basebackup/pg_basebackup.c         |  74 ++--
 src/tools/pgindent/typedefs.list              |  26 +-
 13 files changed, 845 insertions(+), 845 deletions(-)
 create mode 100644 src/bin/pg_basebackup/astreamer.h
 rename src/bin/pg_basebackup/{bbstreamer_file.c => astreamer_file.c} (69%)
 rename src/bin/pg_basebackup/{bbstreamer_gzip.c => astreamer_gzip.c} (62%)
 rename src/bin/pg_basebackup/{bbstreamer_inject.c => astreamer_inject.c} (53%)
 rename src/bin/pg_basebackup/{bbstreamer_lz4.c => astreamer_lz4.c} (69%)
 rename src/bin/pg_basebackup/{bbstreamer_tar.c => astreamer_tar.c} (50%)
 rename src/bin/pg_basebackup/{bbstreamer_zstd.c => astreamer_zstd.c} (64%)
 delete mode 100644 src/bin/pg_basebackup/bbstreamer.h

diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile
index 26c53e473f5..a71af2d48a7 100644
--- a/src/bin/pg_basebackup/Makefile
+++ b/src/bin/pg_basebackup/Makefile
@@ -37,12 +37,12 @@ OBJS = \
 
 BBOBJS = \
 	pg_basebackup.o \
-	bbstreamer_file.o \
-	bbstreamer_gzip.o \
-	bbstreamer_inject.o \
-	bbstreamer_lz4.o \
-	bbstreamer_tar.o \
-	bbstreamer_zstd.o
+	astreamer_file.o \
+	astreamer_gzip.o \
+	astreamer_inject.o \
+	astreamer_lz4.o \
+	astreamer_tar.o \
+	astreamer_zstd.o
 
 all: pg_basebackup pg_createsubscriber pg_receivewal pg_recvlogical
 
diff --git a/src/bin/pg_basebackup/astreamer.h b/src/bin/pg_basebackup/astreamer.h
new file mode 100644
index 00000000000..6b0047418bb
--- /dev/null
+++ b/src/bin/pg_basebackup/astreamer.h
@@ -0,0 +1,226 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer.h
+ *
+ * Each tar archive returned by the server is passed to one or more
+ * astreamer objects for further processing. The astreamer may do
+ * something simple, like write the archive to a file, perhaps after
+ * compressing it, but it can also do more complicated things, like
+ * annotating the byte stream to indicate which parts of the data
+ * correspond to tar headers or trailing padding, vs. which parts are
+ * payload data. A subsequent astreamer may use this information to
+ * make further decisions about how to process the data; for example,
+ * it might choose to modify the archive contents.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *		  src/bin/pg_basebackup/astreamer.h
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef ASTREAMER_H
+#define ASTREAMER_H
+
+#include "common/compression.h"
+#include "lib/stringinfo.h"
+#include "pqexpbuffer.h"
+
+struct astreamer;
+struct astreamer_ops;
+typedef struct astreamer astreamer;
+typedef struct astreamer_ops astreamer_ops;
+
+/*
+ * Each chunk of archive data passed to a astreamer is classified into one
+ * of these categories. When data is first received from the remote server,
+ * each chunk will be categorized as ASTREAMER_UNKNOWN, and the chunks will
+ * be of whatever size the remote server chose to send.
+ *
+ * If the archive is parsed (e.g. see astreamer_tar_parser_new()), then all
+ * chunks should be labelled as one of the other types listed here. In
+ * addition, there should be exactly one ASTREAMER_MEMBER_HEADER chunk and
+ * exactly one ASTREAMER_MEMBER_TRAILER chunk per archive member, even if
+ * that means a zero-length call. There can be any number of
+ * ASTREAMER_MEMBER_CONTENTS chunks in between those calls. There
+ * should exactly ASTREAMER_ARCHIVE_TRAILER chunk, and it should follow the
+ * last ASTREAMER_MEMBER_TRAILER chunk.
+ *
+ * In theory, we could need other classifications here, such as a way of
+ * indicating an archive header, but the "tar" format doesn't need anything
+ * else, so for the time being there's no point.
+ */
+typedef enum
+{
+	ASTREAMER_UNKNOWN,
+	ASTREAMER_MEMBER_HEADER,
+	ASTREAMER_MEMBER_CONTENTS,
+	ASTREAMER_MEMBER_TRAILER,
+	ASTREAMER_ARCHIVE_TRAILER,
+} astreamer_archive_context;
+
+/*
+ * Each chunk of data that is classified as ASTREAMER_MEMBER_HEADER,
+ * ASTREAMER_MEMBER_CONTENTS, or ASTREAMER_MEMBER_TRAILER should also
+ * pass a pointer to an instance of this struct. The details are expected
+ * to be present in the archive header and used to fill the struct, after
+ * which all subsequent calls for the same archive member are expected to
+ * pass the same details.
+ */
+typedef struct
+{
+	char		pathname[MAXPGPATH];
+	pgoff_t		size;
+	mode_t		mode;
+	uid_t		uid;
+	gid_t		gid;
+	bool		is_directory;
+	bool		is_link;
+	char		linktarget[MAXPGPATH];
+} astreamer_member;
+
+/*
+ * Generally, each type of astreamer will define its own struct, but the
+ * first element should be 'astreamer base'. A astreamer that does not
+ * require any additional private data could use this structure directly.
+ *
+ * bbs_ops is a pointer to the astreamer_ops object which contains the
+ * function pointers appropriate to this type of astreamer.
+ *
+ * bbs_next is a pointer to the successor astreamer, for those types of
+ * astreamer which forward data to a successor. It need not be used and
+ * should be set to NULL when not relevant.
+ *
+ * bbs_buffer is a buffer for accumulating data for temporary storage. Each
+ * type of astreamer makes its own decisions about whether and how to use
+ * this buffer.
+ */
+struct astreamer
+{
+	const astreamer_ops *bbs_ops;
+	astreamer *bbs_next;
+	StringInfoData bbs_buffer;
+};
+
+/*
+ * There are three callbacks for a astreamer. The 'content' callback is
+ * called repeatedly, as described in the astreamer_archive_context comments.
+ * Then, the 'finalize' callback is called once at the end, to give the
+ * astreamer a chance to perform cleanup such as closing files. Finally,
+ * because this code is running in a frontend environment where, as of this
+ * writing, there are no memory contexts, the 'free' callback is called to
+ * release memory. These callbacks should always be invoked using the static
+ * inline functions defined below.
+ */
+struct astreamer_ops
+{
+	void		(*content) (astreamer *streamer, astreamer_member *member,
+							const char *data, int len,
+							astreamer_archive_context context);
+	void		(*finalize) (astreamer *streamer);
+	void		(*free) (astreamer *streamer);
+};
+
+/* Send some content to a astreamer. */
+static inline void
+astreamer_content(astreamer *streamer, astreamer_member *member,
+				  const char *data, int len,
+				  astreamer_archive_context context)
+{
+	Assert(streamer != NULL);
+	streamer->bbs_ops->content(streamer, member, data, len, context);
+}
+
+/* Finalize a astreamer. */
+static inline void
+astreamer_finalize(astreamer *streamer)
+{
+	Assert(streamer != NULL);
+	streamer->bbs_ops->finalize(streamer);
+}
+
+/* Free a astreamer. */
+static inline void
+astreamer_free(astreamer *streamer)
+{
+	Assert(streamer != NULL);
+	streamer->bbs_ops->free(streamer);
+}
+
+/*
+ * This is a convenience method for use when implementing a astreamer; it is
+ * not for use by outside callers. It adds the amount of data specified by
+ * 'nbytes' to the astreamer's buffer and adjusts '*len' and '*data'
+ * accordingly.
+ */
+static inline void
+astreamer_buffer_bytes(astreamer *streamer, const char **data, int *len,
+					   int nbytes)
+{
+	Assert(nbytes <= *len);
+
+	appendBinaryStringInfo(&streamer->bbs_buffer, *data, nbytes);
+	*len -= nbytes;
+	*data += nbytes;
+}
+
+/*
+ * This is a convenience method for use when implementing a astreamer; it is
+ * not for use by outsider callers. It attempts to add enough data to the
+ * astreamer's buffer to reach a length of target_bytes and adjusts '*len'
+ * and '*data' accordingly. It returns true if the target length has been
+ * reached and false otherwise.
+ */
+static inline bool
+astreamer_buffer_until(astreamer *streamer, const char **data, int *len,
+					   int target_bytes)
+{
+	int			buflen = streamer->bbs_buffer.len;
+
+	if (buflen >= target_bytes)
+	{
+		/* Target length already reached; nothing to do. */
+		return true;
+	}
+
+	if (buflen + *len < target_bytes)
+	{
+		/* Not enough data to reach target length; buffer all of it. */
+		astreamer_buffer_bytes(streamer, data, len, *len);
+		return false;
+	}
+
+	/* Buffer just enough to reach the target length. */
+	astreamer_buffer_bytes(streamer, data, len, target_bytes - buflen);
+	return true;
+}
+
+/*
+ * Functions for creating astreamer objects of various types. See the header
+ * comments for each of these functions for details.
+ */
+extern astreamer *astreamer_plain_writer_new(char *pathname, FILE *file);
+extern astreamer *astreamer_gzip_writer_new(char *pathname, FILE *file,
+											pg_compress_specification *compress);
+extern astreamer *astreamer_extractor_new(const char *basepath,
+										  const char *(*link_map) (const char *),
+										  void (*report_output_file) (const char *));
+
+extern astreamer *astreamer_gzip_decompressor_new(astreamer *next);
+extern astreamer *astreamer_lz4_compressor_new(astreamer *next,
+											   pg_compress_specification *compress);
+extern astreamer *astreamer_lz4_decompressor_new(astreamer *next);
+extern astreamer *astreamer_zstd_compressor_new(astreamer *next,
+												pg_compress_specification *compress);
+extern astreamer *astreamer_zstd_decompressor_new(astreamer *next);
+extern astreamer *astreamer_tar_parser_new(astreamer *next);
+extern astreamer *astreamer_tar_terminator_new(astreamer *next);
+extern astreamer *astreamer_tar_archiver_new(astreamer *next);
+
+extern astreamer *astreamer_recovery_injector_new(astreamer *next,
+												  bool is_recovery_guc_supported,
+												  PQExpBuffer recoveryconfcontents);
+extern void astreamer_inject_file(astreamer *streamer, char *pathname,
+								  char *data, int len);
+
+#endif
diff --git a/src/bin/pg_basebackup/bbstreamer_file.c b/src/bin/pg_basebackup/astreamer_file.c
similarity index 69%
rename from src/bin/pg_basebackup/bbstreamer_file.c
rename to src/bin/pg_basebackup/astreamer_file.c
index bab6cd4a6b1..2742385e103 100644
--- a/src/bin/pg_basebackup/bbstreamer_file.c
+++ b/src/bin/pg_basebackup/astreamer_file.c
@@ -1,11 +1,11 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_file.c
+ * astreamer_file.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_file.c
+ *		  src/bin/pg_basebackup/astreamer_file.c
  *-------------------------------------------------------------------------
  */
 
@@ -13,60 +13,60 @@
 
 #include <unistd.h>
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
 
-typedef struct bbstreamer_plain_writer
+typedef struct astreamer_plain_writer
 {
-	bbstreamer	base;
+	astreamer	base;
 	char	   *pathname;
 	FILE	   *file;
 	bool		should_close_file;
-} bbstreamer_plain_writer;
+} astreamer_plain_writer;
 
-typedef struct bbstreamer_extractor
+typedef struct astreamer_extractor
 {
-	bbstreamer	base;
+	astreamer	base;
 	char	   *basepath;
 	const char *(*link_map) (const char *);
 	void		(*report_output_file) (const char *);
 	char		filename[MAXPGPATH];
 	FILE	   *file;
-} bbstreamer_extractor;
+} astreamer_extractor;
 
-static void bbstreamer_plain_writer_content(bbstreamer *streamer,
-											bbstreamer_member *member,
-											const char *data, int len,
-											bbstreamer_archive_context context);
-static void bbstreamer_plain_writer_finalize(bbstreamer *streamer);
-static void bbstreamer_plain_writer_free(bbstreamer *streamer);
+static void astreamer_plain_writer_content(astreamer *streamer,
+										   astreamer_member *member,
+										   const char *data, int len,
+										   astreamer_archive_context context);
+static void astreamer_plain_writer_finalize(astreamer *streamer);
+static void astreamer_plain_writer_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_plain_writer_ops = {
-	.content = bbstreamer_plain_writer_content,
-	.finalize = bbstreamer_plain_writer_finalize,
-	.free = bbstreamer_plain_writer_free
+static const astreamer_ops astreamer_plain_writer_ops = {
+	.content = astreamer_plain_writer_content,
+	.finalize = astreamer_plain_writer_finalize,
+	.free = astreamer_plain_writer_free
 };
 
-static void bbstreamer_extractor_content(bbstreamer *streamer,
-										 bbstreamer_member *member,
-										 const char *data, int len,
-										 bbstreamer_archive_context context);
-static void bbstreamer_extractor_finalize(bbstreamer *streamer);
-static void bbstreamer_extractor_free(bbstreamer *streamer);
+static void astreamer_extractor_content(astreamer *streamer,
+										astreamer_member *member,
+										const char *data, int len,
+										astreamer_archive_context context);
+static void astreamer_extractor_finalize(astreamer *streamer);
+static void astreamer_extractor_free(astreamer *streamer);
 static void extract_directory(const char *filename, mode_t mode);
 static void extract_link(const char *filename, const char *linktarget);
 static FILE *create_file_for_extract(const char *filename, mode_t mode);
 
-static const bbstreamer_ops bbstreamer_extractor_ops = {
-	.content = bbstreamer_extractor_content,
-	.finalize = bbstreamer_extractor_finalize,
-	.free = bbstreamer_extractor_free
+static const astreamer_ops astreamer_extractor_ops = {
+	.content = astreamer_extractor_content,
+	.finalize = astreamer_extractor_finalize,
+	.free = astreamer_extractor_free
 };
 
 /*
- * Create a bbstreamer that just writes data to a file.
+ * Create a astreamer that just writes data to a file.
  *
  * The caller must specify a pathname and may specify a file. The pathname is
  * used for error-reporting purposes either way. If file is NULL, the pathname
@@ -74,14 +74,14 @@ static const bbstreamer_ops bbstreamer_extractor_ops = {
  * for writing and closed when done. If file is not NULL, the data is written
  * there.
  */
-bbstreamer *
-bbstreamer_plain_writer_new(char *pathname, FILE *file)
+astreamer *
+astreamer_plain_writer_new(char *pathname, FILE *file)
 {
-	bbstreamer_plain_writer *streamer;
+	astreamer_plain_writer *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_plain_writer));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_plain_writer_ops;
+	streamer = palloc0(sizeof(astreamer_plain_writer));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_plain_writer_ops;
 
 	streamer->pathname = pstrdup(pathname);
 	streamer->file = file;
@@ -101,13 +101,13 @@ bbstreamer_plain_writer_new(char *pathname, FILE *file)
  * Write archive content to file.
  */
 static void
-bbstreamer_plain_writer_content(bbstreamer *streamer,
-								bbstreamer_member *member, const char *data,
-								int len, bbstreamer_archive_context context)
+astreamer_plain_writer_content(astreamer *streamer,
+							   astreamer_member *member, const char *data,
+							   int len, astreamer_archive_context context)
 {
-	bbstreamer_plain_writer *mystreamer;
+	astreamer_plain_writer *mystreamer;
 
-	mystreamer = (bbstreamer_plain_writer *) streamer;
+	mystreamer = (astreamer_plain_writer *) streamer;
 
 	if (len == 0)
 		return;
@@ -128,11 +128,11 @@ bbstreamer_plain_writer_content(bbstreamer *streamer,
  * the file if we opened it, but not if the caller provided it.
  */
 static void
-bbstreamer_plain_writer_finalize(bbstreamer *streamer)
+astreamer_plain_writer_finalize(astreamer *streamer)
 {
-	bbstreamer_plain_writer *mystreamer;
+	astreamer_plain_writer *mystreamer;
 
-	mystreamer = (bbstreamer_plain_writer *) streamer;
+	mystreamer = (astreamer_plain_writer *) streamer;
 
 	if (mystreamer->should_close_file && fclose(mystreamer->file) != 0)
 		pg_fatal("could not close file \"%s\": %m",
@@ -143,14 +143,14 @@ bbstreamer_plain_writer_finalize(bbstreamer *streamer)
 }
 
 /*
- * Free memory associated with this bbstreamer.
+ * Free memory associated with this astreamer.
  */
 static void
-bbstreamer_plain_writer_free(bbstreamer *streamer)
+astreamer_plain_writer_free(astreamer *streamer)
 {
-	bbstreamer_plain_writer *mystreamer;
+	astreamer_plain_writer *mystreamer;
 
-	mystreamer = (bbstreamer_plain_writer *) streamer;
+	mystreamer = (astreamer_plain_writer *) streamer;
 
 	Assert(!mystreamer->should_close_file);
 	Assert(mystreamer->base.bbs_next == NULL);
@@ -160,13 +160,13 @@ bbstreamer_plain_writer_free(bbstreamer *streamer)
 }
 
 /*
- * Create a bbstreamer that extracts an archive.
+ * Create a astreamer that extracts an archive.
  *
  * All pathnames in the archive are interpreted relative to basepath.
  *
- * Unlike e.g. bbstreamer_plain_writer_new() we can't do anything useful here
+ * Unlike e.g. astreamer_plain_writer_new() we can't do anything useful here
  * with untyped chunks; we need typed chunks which follow the rules described
- * in bbstreamer.h. Assuming we have that, we don't need to worry about the
+ * in astreamer.h. Assuming we have that, we don't need to worry about the
  * original archive format; it's enough to just look at the member information
  * provided and write to the corresponding file.
  *
@@ -179,16 +179,16 @@ bbstreamer_plain_writer_free(bbstreamer *streamer)
  * new output file. The pathname to that file is passed as an argument. If
  * NULL, the call is skipped.
  */
-bbstreamer *
-bbstreamer_extractor_new(const char *basepath,
-						 const char *(*link_map) (const char *),
-						 void (*report_output_file) (const char *))
+astreamer *
+astreamer_extractor_new(const char *basepath,
+						const char *(*link_map) (const char *),
+						void (*report_output_file) (const char *))
 {
-	bbstreamer_extractor *streamer;
+	astreamer_extractor *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_extractor));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_extractor_ops;
+	streamer = palloc0(sizeof(astreamer_extractor));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_extractor_ops;
 	streamer->basepath = pstrdup(basepath);
 	streamer->link_map = link_map;
 	streamer->report_output_file = report_output_file;
@@ -200,19 +200,19 @@ bbstreamer_extractor_new(const char *basepath,
  * Extract archive contents to the filesystem.
  */
 static void
-bbstreamer_extractor_content(bbstreamer *streamer, bbstreamer_member *member,
-							 const char *data, int len,
-							 bbstreamer_archive_context context)
+astreamer_extractor_content(astreamer *streamer, astreamer_member *member,
+							const char *data, int len,
+							astreamer_archive_context context)
 {
-	bbstreamer_extractor *mystreamer = (bbstreamer_extractor *) streamer;
+	astreamer_extractor *mystreamer = (astreamer_extractor *) streamer;
 	int			fnamelen;
 
-	Assert(member != NULL || context == BBSTREAMER_ARCHIVE_TRAILER);
-	Assert(context != BBSTREAMER_UNKNOWN);
+	Assert(member != NULL || context == ASTREAMER_ARCHIVE_TRAILER);
+	Assert(context != ASTREAMER_UNKNOWN);
 
 	switch (context)
 	{
-		case BBSTREAMER_MEMBER_HEADER:
+		case ASTREAMER_MEMBER_HEADER:
 			Assert(mystreamer->file == NULL);
 
 			/* Prepend basepath. */
@@ -245,7 +245,7 @@ bbstreamer_extractor_content(bbstreamer *streamer, bbstreamer_member *member,
 				mystreamer->report_output_file(mystreamer->filename);
 			break;
 
-		case BBSTREAMER_MEMBER_CONTENTS:
+		case ASTREAMER_MEMBER_CONTENTS:
 			if (mystreamer->file == NULL)
 				break;
 
@@ -260,14 +260,14 @@ bbstreamer_extractor_content(bbstreamer *streamer, bbstreamer_member *member,
 			}
 			break;
 
-		case BBSTREAMER_MEMBER_TRAILER:
+		case ASTREAMER_MEMBER_TRAILER:
 			if (mystreamer->file == NULL)
 				break;
 			fclose(mystreamer->file);
 			mystreamer->file = NULL;
 			break;
 
-		case BBSTREAMER_ARCHIVE_TRAILER:
+		case ASTREAMER_ARCHIVE_TRAILER:
 			break;
 
 		default:
@@ -375,10 +375,10 @@ create_file_for_extract(const char *filename, mode_t mode)
  * There's nothing to do here but sanity checking.
  */
 static void
-bbstreamer_extractor_finalize(bbstreamer *streamer)
+astreamer_extractor_finalize(astreamer *streamer)
 {
-	bbstreamer_extractor *mystreamer PG_USED_FOR_ASSERTS_ONLY
-	= (bbstreamer_extractor *) streamer;
+	astreamer_extractor *mystreamer PG_USED_FOR_ASSERTS_ONLY
+	= (astreamer_extractor *) streamer;
 
 	Assert(mystreamer->file == NULL);
 }
@@ -387,9 +387,9 @@ bbstreamer_extractor_finalize(bbstreamer *streamer)
  * Free memory.
  */
 static void
-bbstreamer_extractor_free(bbstreamer *streamer)
+astreamer_extractor_free(astreamer *streamer)
 {
-	bbstreamer_extractor *mystreamer = (bbstreamer_extractor *) streamer;
+	astreamer_extractor *mystreamer = (astreamer_extractor *) streamer;
 
 	pfree(mystreamer->basepath);
 	pfree(mystreamer);
diff --git a/src/bin/pg_basebackup/bbstreamer_gzip.c b/src/bin/pg_basebackup/astreamer_gzip.c
similarity index 62%
rename from src/bin/pg_basebackup/bbstreamer_gzip.c
rename to src/bin/pg_basebackup/astreamer_gzip.c
index 0417fd9bc2c..6f7c27afbbc 100644
--- a/src/bin/pg_basebackup/bbstreamer_gzip.c
+++ b/src/bin/pg_basebackup/astreamer_gzip.c
@@ -1,11 +1,11 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_gzip.c
+ * astreamer_gzip.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_gzip.c
+ *		  src/bin/pg_basebackup/astreamer_gzip.c
  *-------------------------------------------------------------------------
  */
 
@@ -17,74 +17,74 @@
 #include <zlib.h>
 #endif
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
 
 #ifdef HAVE_LIBZ
-typedef struct bbstreamer_gzip_writer
+typedef struct astreamer_gzip_writer
 {
-	bbstreamer	base;
+	astreamer	base;
 	char	   *pathname;
 	gzFile		gzfile;
-} bbstreamer_gzip_writer;
+} astreamer_gzip_writer;
 
-typedef struct bbstreamer_gzip_decompressor
+typedef struct astreamer_gzip_decompressor
 {
-	bbstreamer	base;
+	astreamer	base;
 	z_stream	zstream;
 	size_t		bytes_written;
-} bbstreamer_gzip_decompressor;
+} astreamer_gzip_decompressor;
 
-static void bbstreamer_gzip_writer_content(bbstreamer *streamer,
-										   bbstreamer_member *member,
-										   const char *data, int len,
-										   bbstreamer_archive_context context);
-static void bbstreamer_gzip_writer_finalize(bbstreamer *streamer);
-static void bbstreamer_gzip_writer_free(bbstreamer *streamer);
+static void astreamer_gzip_writer_content(astreamer *streamer,
+										  astreamer_member *member,
+										  const char *data, int len,
+										  astreamer_archive_context context);
+static void astreamer_gzip_writer_finalize(astreamer *streamer);
+static void astreamer_gzip_writer_free(astreamer *streamer);
 static const char *get_gz_error(gzFile gzf);
 
-static const bbstreamer_ops bbstreamer_gzip_writer_ops = {
-	.content = bbstreamer_gzip_writer_content,
-	.finalize = bbstreamer_gzip_writer_finalize,
-	.free = bbstreamer_gzip_writer_free
+static const astreamer_ops astreamer_gzip_writer_ops = {
+	.content = astreamer_gzip_writer_content,
+	.finalize = astreamer_gzip_writer_finalize,
+	.free = astreamer_gzip_writer_free
 };
 
-static void bbstreamer_gzip_decompressor_content(bbstreamer *streamer,
-												 bbstreamer_member *member,
-												 const char *data, int len,
-												 bbstreamer_archive_context context);
-static void bbstreamer_gzip_decompressor_finalize(bbstreamer *streamer);
-static void bbstreamer_gzip_decompressor_free(bbstreamer *streamer);
+static void astreamer_gzip_decompressor_content(astreamer *streamer,
+												astreamer_member *member,
+												const char *data, int len,
+												astreamer_archive_context context);
+static void astreamer_gzip_decompressor_finalize(astreamer *streamer);
+static void astreamer_gzip_decompressor_free(astreamer *streamer);
 static void *gzip_palloc(void *opaque, unsigned items, unsigned size);
 static void gzip_pfree(void *opaque, void *address);
 
-static const bbstreamer_ops bbstreamer_gzip_decompressor_ops = {
-	.content = bbstreamer_gzip_decompressor_content,
-	.finalize = bbstreamer_gzip_decompressor_finalize,
-	.free = bbstreamer_gzip_decompressor_free
+static const astreamer_ops astreamer_gzip_decompressor_ops = {
+	.content = astreamer_gzip_decompressor_content,
+	.finalize = astreamer_gzip_decompressor_finalize,
+	.free = astreamer_gzip_decompressor_free
 };
 #endif
 
 /*
- * Create a bbstreamer that just compresses data using gzip, and then writes
+ * Create a astreamer that just compresses data using gzip, and then writes
  * it to a file.
  *
- * As in the case of bbstreamer_plain_writer_new, pathname is always used
+ * As in the case of astreamer_plain_writer_new, pathname is always used
  * for error reporting purposes; if file is NULL, it is also the opened and
  * closed so that the data may be written there.
  */
-bbstreamer *
-bbstreamer_gzip_writer_new(char *pathname, FILE *file,
-						   pg_compress_specification *compress)
+astreamer *
+astreamer_gzip_writer_new(char *pathname, FILE *file,
+						  pg_compress_specification *compress)
 {
 #ifdef HAVE_LIBZ
-	bbstreamer_gzip_writer *streamer;
+	astreamer_gzip_writer *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_gzip_writer));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_gzip_writer_ops;
+	streamer = palloc0(sizeof(astreamer_gzip_writer));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_gzip_writer_ops;
 
 	streamer->pathname = pstrdup(pathname);
 
@@ -123,13 +123,13 @@ bbstreamer_gzip_writer_new(char *pathname, FILE *file,
  * Write archive content to gzip file.
  */
 static void
-bbstreamer_gzip_writer_content(bbstreamer *streamer,
-							   bbstreamer_member *member, const char *data,
-							   int len, bbstreamer_archive_context context)
+astreamer_gzip_writer_content(astreamer *streamer,
+							  astreamer_member *member, const char *data,
+							  int len, astreamer_archive_context context)
 {
-	bbstreamer_gzip_writer *mystreamer;
+	astreamer_gzip_writer *mystreamer;
 
-	mystreamer = (bbstreamer_gzip_writer *) streamer;
+	mystreamer = (astreamer_gzip_writer *) streamer;
 
 	if (len == 0)
 		return;
@@ -151,16 +151,16 @@ bbstreamer_gzip_writer_content(bbstreamer *streamer,
  *
  * It makes no difference whether we opened the file or the caller did it,
  * because libz provides no way of avoiding a close on the underlying file
- * handle. Notice, however, that bbstreamer_gzip_writer_new() uses dup() to
+ * handle. Notice, however, that astreamer_gzip_writer_new() uses dup() to
  * work around this issue, so that the behavior from the caller's viewpoint
- * is the same as for bbstreamer_plain_writer.
+ * is the same as for astreamer_plain_writer.
  */
 static void
-bbstreamer_gzip_writer_finalize(bbstreamer *streamer)
+astreamer_gzip_writer_finalize(astreamer *streamer)
 {
-	bbstreamer_gzip_writer *mystreamer;
+	astreamer_gzip_writer *mystreamer;
 
-	mystreamer = (bbstreamer_gzip_writer *) streamer;
+	mystreamer = (astreamer_gzip_writer *) streamer;
 
 	errno = 0;					/* in case gzclose() doesn't set it */
 	if (gzclose(mystreamer->gzfile) != 0)
@@ -171,14 +171,14 @@ bbstreamer_gzip_writer_finalize(bbstreamer *streamer)
 }
 
 /*
- * Free memory associated with this bbstreamer.
+ * Free memory associated with this astreamer.
  */
 static void
-bbstreamer_gzip_writer_free(bbstreamer *streamer)
+astreamer_gzip_writer_free(astreamer *streamer)
 {
-	bbstreamer_gzip_writer *mystreamer;
+	astreamer_gzip_writer *mystreamer;
 
-	mystreamer = (bbstreamer_gzip_writer *) streamer;
+	mystreamer = (astreamer_gzip_writer *) streamer;
 
 	Assert(mystreamer->base.bbs_next == NULL);
 	Assert(mystreamer->gzfile == NULL);
@@ -208,18 +208,18 @@ get_gz_error(gzFile gzf)
  * Create a new base backup streamer that performs decompression of gzip
  * compressed blocks.
  */
-bbstreamer *
-bbstreamer_gzip_decompressor_new(bbstreamer *next)
+astreamer *
+astreamer_gzip_decompressor_new(astreamer *next)
 {
 #ifdef HAVE_LIBZ
-	bbstreamer_gzip_decompressor *streamer;
+	astreamer_gzip_decompressor *streamer;
 	z_stream   *zs;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_gzip_decompressor));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_gzip_decompressor_ops;
+	streamer = palloc0(sizeof(astreamer_gzip_decompressor));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_gzip_decompressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -258,15 +258,15 @@ bbstreamer_gzip_decompressor_new(bbstreamer *next)
  * to the next streamer.
  */
 static void
-bbstreamer_gzip_decompressor_content(bbstreamer *streamer,
-									 bbstreamer_member *member,
-									 const char *data, int len,
-									 bbstreamer_archive_context context)
+astreamer_gzip_decompressor_content(astreamer *streamer,
+									astreamer_member *member,
+									const char *data, int len,
+									astreamer_archive_context context)
 {
-	bbstreamer_gzip_decompressor *mystreamer;
+	astreamer_gzip_decompressor *mystreamer;
 	z_stream   *zs;
 
-	mystreamer = (bbstreamer_gzip_decompressor *) streamer;
+	mystreamer = (astreamer_gzip_decompressor *) streamer;
 
 	zs = &mystreamer->zstream;
 	zs->next_in = (const uint8 *) data;
@@ -301,9 +301,9 @@ bbstreamer_gzip_decompressor_content(bbstreamer *streamer,
 		/* If output buffer is full then pass data to next streamer */
 		if (mystreamer->bytes_written >= mystreamer->base.bbs_buffer.maxlen)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, member,
-							   mystreamer->base.bbs_buffer.data,
-							   mystreamer->base.bbs_buffer.maxlen, context);
+			astreamer_content(mystreamer->base.bbs_next, member,
+							  mystreamer->base.bbs_buffer.data,
+							  mystreamer->base.bbs_buffer.maxlen, context);
 			mystreamer->bytes_written = 0;
 		}
 	}
@@ -313,31 +313,31 @@ bbstreamer_gzip_decompressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_gzip_decompressor_finalize(bbstreamer *streamer)
+astreamer_gzip_decompressor_finalize(astreamer *streamer)
 {
-	bbstreamer_gzip_decompressor *mystreamer;
+	astreamer_gzip_decompressor *mystreamer;
 
-	mystreamer = (bbstreamer_gzip_decompressor *) streamer;
+	mystreamer = (astreamer_gzip_decompressor *) streamer;
 
 	/*
 	 * End of the stream, if there is some pending data in output buffers then
 	 * we must forward it to next streamer.
 	 */
-	bbstreamer_content(mystreamer->base.bbs_next, NULL,
-					   mystreamer->base.bbs_buffer.data,
-					   mystreamer->base.bbs_buffer.maxlen,
-					   BBSTREAMER_UNKNOWN);
+	astreamer_content(mystreamer->base.bbs_next, NULL,
+					  mystreamer->base.bbs_buffer.data,
+					  mystreamer->base.bbs_buffer.maxlen,
+					  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_gzip_decompressor_free(bbstreamer *streamer)
+astreamer_gzip_decompressor_free(astreamer *streamer)
 {
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
 }
diff --git a/src/bin/pg_basebackup/bbstreamer_inject.c b/src/bin/pg_basebackup/astreamer_inject.c
similarity index 53%
rename from src/bin/pg_basebackup/bbstreamer_inject.c
rename to src/bin/pg_basebackup/astreamer_inject.c
index 194026b56e9..7f1decded8d 100644
--- a/src/bin/pg_basebackup/bbstreamer_inject.c
+++ b/src/bin/pg_basebackup/astreamer_inject.c
@@ -1,51 +1,51 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_inject.c
+ * astreamer_inject.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_inject.c
+ *		  src/bin/pg_basebackup/astreamer_inject.c
  *-------------------------------------------------------------------------
  */
 
 #include "postgres_fe.h"
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 
-typedef struct bbstreamer_recovery_injector
+typedef struct astreamer_recovery_injector
 {
-	bbstreamer	base;
+	astreamer	base;
 	bool		skip_file;
 	bool		is_recovery_guc_supported;
 	bool		is_postgresql_auto_conf;
 	bool		found_postgresql_auto_conf;
 	PQExpBuffer recoveryconfcontents;
-	bbstreamer_member member;
-} bbstreamer_recovery_injector;
+	astreamer_member member;
+} astreamer_recovery_injector;
 
-static void bbstreamer_recovery_injector_content(bbstreamer *streamer,
-												 bbstreamer_member *member,
-												 const char *data, int len,
-												 bbstreamer_archive_context context);
-static void bbstreamer_recovery_injector_finalize(bbstreamer *streamer);
-static void bbstreamer_recovery_injector_free(bbstreamer *streamer);
+static void astreamer_recovery_injector_content(astreamer *streamer,
+												astreamer_member *member,
+												const char *data, int len,
+												astreamer_archive_context context);
+static void astreamer_recovery_injector_finalize(astreamer *streamer);
+static void astreamer_recovery_injector_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_recovery_injector_ops = {
-	.content = bbstreamer_recovery_injector_content,
-	.finalize = bbstreamer_recovery_injector_finalize,
-	.free = bbstreamer_recovery_injector_free
+static const astreamer_ops astreamer_recovery_injector_ops = {
+	.content = astreamer_recovery_injector_content,
+	.finalize = astreamer_recovery_injector_finalize,
+	.free = astreamer_recovery_injector_free
 };
 
 /*
- * Create a bbstreamer that can edit recoverydata into an archive stream.
+ * Create a astreamer that can edit recoverydata into an archive stream.
  *
- * The input should be a series of typed chunks (not BBSTREAMER_UNKNOWN) as
- * per the conventions described in bbstreamer.h; the chunks forwarded to
- * the next bbstreamer will be similarly typed, but the
- * BBSTREAMER_MEMBER_HEADER chunks may be zero-length in cases where we've
+ * The input should be a series of typed chunks (not ASTREAMER_UNKNOWN) as
+ * per the conventions described in astreamer.h; the chunks forwarded to
+ * the next astreamer will be similarly typed, but the
+ * ASTREAMER_MEMBER_HEADER chunks may be zero-length in cases where we've
  * edited the archive stream.
  *
  * Our goal is to do one of the following three things with the content passed
@@ -61,16 +61,16 @@ static const bbstreamer_ops bbstreamer_recovery_injector_ops = {
  * zero-length standby.signal file, dropping any file with that name from
  * the archive.
  */
-bbstreamer *
-bbstreamer_recovery_injector_new(bbstreamer *next,
-								 bool is_recovery_guc_supported,
-								 PQExpBuffer recoveryconfcontents)
+astreamer *
+astreamer_recovery_injector_new(astreamer *next,
+								bool is_recovery_guc_supported,
+								PQExpBuffer recoveryconfcontents)
 {
-	bbstreamer_recovery_injector *streamer;
+	astreamer_recovery_injector *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_recovery_injector));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_recovery_injector_ops;
+	streamer = palloc0(sizeof(astreamer_recovery_injector));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_recovery_injector_ops;
 	streamer->base.bbs_next = next;
 	streamer->is_recovery_guc_supported = is_recovery_guc_supported;
 	streamer->recoveryconfcontents = recoveryconfcontents;
@@ -82,21 +82,21 @@ bbstreamer_recovery_injector_new(bbstreamer *next,
  * Handle each chunk of tar content while injecting recovery configuration.
  */
 static void
-bbstreamer_recovery_injector_content(bbstreamer *streamer,
-									 bbstreamer_member *member,
-									 const char *data, int len,
-									 bbstreamer_archive_context context)
+astreamer_recovery_injector_content(astreamer *streamer,
+									astreamer_member *member,
+									const char *data, int len,
+									astreamer_archive_context context)
 {
-	bbstreamer_recovery_injector *mystreamer;
+	astreamer_recovery_injector *mystreamer;
 
-	mystreamer = (bbstreamer_recovery_injector *) streamer;
-	Assert(member != NULL || context == BBSTREAMER_ARCHIVE_TRAILER);
+	mystreamer = (astreamer_recovery_injector *) streamer;
+	Assert(member != NULL || context == ASTREAMER_ARCHIVE_TRAILER);
 
 	switch (context)
 	{
-		case BBSTREAMER_MEMBER_HEADER:
+		case ASTREAMER_MEMBER_HEADER:
 			/* Must copy provided data so we have the option to modify it. */
-			memcpy(&mystreamer->member, member, sizeof(bbstreamer_member));
+			memcpy(&mystreamer->member, member, sizeof(astreamer_member));
 
 			/*
 			 * On v12+, skip standby.signal and edit postgresql.auto.conf; on
@@ -119,8 +119,8 @@ bbstreamer_recovery_injector_content(bbstreamer *streamer,
 
 					/*
 					 * Zap data and len because the archive header is no
-					 * longer valid; some subsequent bbstreamer must
-					 * regenerate it if it's necessary.
+					 * longer valid; some subsequent astreamer must regenerate
+					 * it if it's necessary.
 					 */
 					data = NULL;
 					len = 0;
@@ -135,26 +135,26 @@ bbstreamer_recovery_injector_content(bbstreamer *streamer,
 				return;
 			break;
 
-		case BBSTREAMER_MEMBER_CONTENTS:
+		case ASTREAMER_MEMBER_CONTENTS:
 			/* Do not forward if the file is to be skipped. */
 			if (mystreamer->skip_file)
 				return;
 			break;
 
-		case BBSTREAMER_MEMBER_TRAILER:
+		case ASTREAMER_MEMBER_TRAILER:
 			/* Do not forward it the file is to be skipped. */
 			if (mystreamer->skip_file)
 				return;
 
 			/* Append provided content to whatever we already sent. */
 			if (mystreamer->is_postgresql_auto_conf)
-				bbstreamer_content(mystreamer->base.bbs_next, member,
-								   mystreamer->recoveryconfcontents->data,
-								   mystreamer->recoveryconfcontents->len,
-								   BBSTREAMER_MEMBER_CONTENTS);
+				astreamer_content(mystreamer->base.bbs_next, member,
+								  mystreamer->recoveryconfcontents->data,
+								  mystreamer->recoveryconfcontents->len,
+								  ASTREAMER_MEMBER_CONTENTS);
 			break;
 
-		case BBSTREAMER_ARCHIVE_TRAILER:
+		case ASTREAMER_ARCHIVE_TRAILER:
 			if (mystreamer->is_recovery_guc_supported)
 			{
 				/*
@@ -163,22 +163,22 @@ bbstreamer_recovery_injector_content(bbstreamer *streamer,
 				 * member now.
 				 */
 				if (!mystreamer->found_postgresql_auto_conf)
-					bbstreamer_inject_file(mystreamer->base.bbs_next,
-										   "postgresql.auto.conf",
-										   mystreamer->recoveryconfcontents->data,
-										   mystreamer->recoveryconfcontents->len);
+					astreamer_inject_file(mystreamer->base.bbs_next,
+										  "postgresql.auto.conf",
+										  mystreamer->recoveryconfcontents->data,
+										  mystreamer->recoveryconfcontents->len);
 
 				/* Inject empty standby.signal file. */
-				bbstreamer_inject_file(mystreamer->base.bbs_next,
-									   "standby.signal", "", 0);
+				astreamer_inject_file(mystreamer->base.bbs_next,
+									  "standby.signal", "", 0);
 			}
 			else
 			{
 				/* Inject recovery.conf file with specified contents. */
-				bbstreamer_inject_file(mystreamer->base.bbs_next,
-									   "recovery.conf",
-									   mystreamer->recoveryconfcontents->data,
-									   mystreamer->recoveryconfcontents->len);
+				astreamer_inject_file(mystreamer->base.bbs_next,
+									  "recovery.conf",
+									  mystreamer->recoveryconfcontents->data,
+									  mystreamer->recoveryconfcontents->len);
 			}
 
 			/* Nothing to do here. */
@@ -189,26 +189,26 @@ bbstreamer_recovery_injector_content(bbstreamer *streamer,
 			pg_fatal("unexpected state while injecting recovery settings");
 	}
 
-	bbstreamer_content(mystreamer->base.bbs_next, &mystreamer->member,
-					   data, len, context);
+	astreamer_content(mystreamer->base.bbs_next, &mystreamer->member,
+					  data, len, context);
 }
 
 /*
- * End-of-stream processing for this bbstreamer.
+ * End-of-stream processing for this astreamer.
  */
 static void
-bbstreamer_recovery_injector_finalize(bbstreamer *streamer)
+astreamer_recovery_injector_finalize(astreamer *streamer)
 {
-	bbstreamer_finalize(streamer->bbs_next);
+	astreamer_finalize(streamer->bbs_next);
 }
 
 /*
- * Free memory associated with this bbstreamer.
+ * Free memory associated with this astreamer.
  */
 static void
-bbstreamer_recovery_injector_free(bbstreamer *streamer)
+astreamer_recovery_injector_free(astreamer *streamer)
 {
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	pfree(streamer);
 }
 
@@ -216,10 +216,10 @@ bbstreamer_recovery_injector_free(bbstreamer *streamer)
  * Inject a member into the archive with specified contents.
  */
 void
-bbstreamer_inject_file(bbstreamer *streamer, char *pathname, char *data,
-					   int len)
+astreamer_inject_file(astreamer *streamer, char *pathname, char *data,
+					  int len)
 {
-	bbstreamer_member member;
+	astreamer_member member;
 
 	strlcpy(member.pathname, pathname, MAXPGPATH);
 	member.size = len;
@@ -238,12 +238,12 @@ bbstreamer_inject_file(bbstreamer *streamer, char *pathname, char *data,
 	/*
 	 * We don't know here how to generate valid member headers and trailers
 	 * for the archiving format in use, so if those are needed, some successor
-	 * bbstreamer will have to generate them using the data from 'member'.
+	 * astreamer will have to generate them using the data from 'member'.
 	 */
-	bbstreamer_content(streamer, &member, NULL, 0,
-					   BBSTREAMER_MEMBER_HEADER);
-	bbstreamer_content(streamer, &member, data, len,
-					   BBSTREAMER_MEMBER_CONTENTS);
-	bbstreamer_content(streamer, &member, NULL, 0,
-					   BBSTREAMER_MEMBER_TRAILER);
+	astreamer_content(streamer, &member, NULL, 0,
+					  ASTREAMER_MEMBER_HEADER);
+	astreamer_content(streamer, &member, data, len,
+					  ASTREAMER_MEMBER_CONTENTS);
+	astreamer_content(streamer, &member, NULL, 0,
+					  ASTREAMER_MEMBER_TRAILER);
 }
diff --git a/src/bin/pg_basebackup/bbstreamer_lz4.c b/src/bin/pg_basebackup/astreamer_lz4.c
similarity index 69%
rename from src/bin/pg_basebackup/bbstreamer_lz4.c
rename to src/bin/pg_basebackup/astreamer_lz4.c
index f5c9e68150c..1c40d7d8ad5 100644
--- a/src/bin/pg_basebackup/bbstreamer_lz4.c
+++ b/src/bin/pg_basebackup/astreamer_lz4.c
@@ -1,11 +1,11 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_lz4.c
+ * astreamer_lz4.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_lz4.c
+ *		  src/bin/pg_basebackup/astreamer_lz4.c
  *-------------------------------------------------------------------------
  */
 
@@ -17,15 +17,15 @@
 #include <lz4frame.h>
 #endif
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/file_perm.h"
 #include "common/logging.h"
 #include "common/string.h"
 
 #ifdef USE_LZ4
-typedef struct bbstreamer_lz4_frame
+typedef struct astreamer_lz4_frame
 {
-	bbstreamer	base;
+	astreamer	base;
 
 	LZ4F_compressionContext_t cctx;
 	LZ4F_decompressionContext_t dctx;
@@ -33,32 +33,32 @@ typedef struct bbstreamer_lz4_frame
 
 	size_t		bytes_written;
 	bool		header_written;
-} bbstreamer_lz4_frame;
+} astreamer_lz4_frame;
 
-static void bbstreamer_lz4_compressor_content(bbstreamer *streamer,
-											  bbstreamer_member *member,
-											  const char *data, int len,
-											  bbstreamer_archive_context context);
-static void bbstreamer_lz4_compressor_finalize(bbstreamer *streamer);
-static void bbstreamer_lz4_compressor_free(bbstreamer *streamer);
+static void astreamer_lz4_compressor_content(astreamer *streamer,
+											 astreamer_member *member,
+											 const char *data, int len,
+											 astreamer_archive_context context);
+static void astreamer_lz4_compressor_finalize(astreamer *streamer);
+static void astreamer_lz4_compressor_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_lz4_compressor_ops = {
-	.content = bbstreamer_lz4_compressor_content,
-	.finalize = bbstreamer_lz4_compressor_finalize,
-	.free = bbstreamer_lz4_compressor_free
+static const astreamer_ops astreamer_lz4_compressor_ops = {
+	.content = astreamer_lz4_compressor_content,
+	.finalize = astreamer_lz4_compressor_finalize,
+	.free = astreamer_lz4_compressor_free
 };
 
-static void bbstreamer_lz4_decompressor_content(bbstreamer *streamer,
-												bbstreamer_member *member,
-												const char *data, int len,
-												bbstreamer_archive_context context);
-static void bbstreamer_lz4_decompressor_finalize(bbstreamer *streamer);
-static void bbstreamer_lz4_decompressor_free(bbstreamer *streamer);
+static void astreamer_lz4_decompressor_content(astreamer *streamer,
+											   astreamer_member *member,
+											   const char *data, int len,
+											   astreamer_archive_context context);
+static void astreamer_lz4_decompressor_finalize(astreamer *streamer);
+static void astreamer_lz4_decompressor_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_lz4_decompressor_ops = {
-	.content = bbstreamer_lz4_decompressor_content,
-	.finalize = bbstreamer_lz4_decompressor_finalize,
-	.free = bbstreamer_lz4_decompressor_free
+static const astreamer_ops astreamer_lz4_decompressor_ops = {
+	.content = astreamer_lz4_decompressor_content,
+	.finalize = astreamer_lz4_decompressor_finalize,
+	.free = astreamer_lz4_decompressor_free
 };
 #endif
 
@@ -66,19 +66,19 @@ static const bbstreamer_ops bbstreamer_lz4_decompressor_ops = {
  * Create a new base backup streamer that performs lz4 compression of tar
  * blocks.
  */
-bbstreamer *
-bbstreamer_lz4_compressor_new(bbstreamer *next, pg_compress_specification *compress)
+astreamer *
+astreamer_lz4_compressor_new(astreamer *next, pg_compress_specification *compress)
 {
 #ifdef USE_LZ4
-	bbstreamer_lz4_frame *streamer;
+	astreamer_lz4_frame *streamer;
 	LZ4F_errorCode_t ctxError;
 	LZ4F_preferences_t *prefs;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_lz4_frame));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_lz4_compressor_ops;
+	streamer = palloc0(sizeof(astreamer_lz4_frame));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_lz4_compressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -113,19 +113,19 @@ bbstreamer_lz4_compressor_new(bbstreamer *next, pg_compress_specification *compr
  * of output buffer to next streamer and empty the buffer.
  */
 static void
-bbstreamer_lz4_compressor_content(bbstreamer *streamer,
-								  bbstreamer_member *member,
-								  const char *data, int len,
-								  bbstreamer_archive_context context)
+astreamer_lz4_compressor_content(astreamer *streamer,
+								 astreamer_member *member,
+								 const char *data, int len,
+								 astreamer_archive_context context)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 	uint8	   *next_in,
 			   *next_out;
 	size_t		out_bound,
 				compressed_size,
 				avail_out;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
+	mystreamer = (astreamer_lz4_frame *) streamer;
 	next_in = (uint8 *) data;
 
 	/* Write header before processing the first input chunk. */
@@ -159,10 +159,10 @@ bbstreamer_lz4_compressor_content(bbstreamer *streamer,
 	out_bound = LZ4F_compressBound(len, &mystreamer->prefs);
 	if (avail_out < out_bound)
 	{
-		bbstreamer_content(mystreamer->base.bbs_next, member,
-						   mystreamer->base.bbs_buffer.data,
-						   mystreamer->bytes_written,
-						   context);
+		astreamer_content(mystreamer->base.bbs_next, member,
+						  mystreamer->base.bbs_buffer.data,
+						  mystreamer->bytes_written,
+						  context);
 
 		/* Enlarge buffer if it falls short of out bound. */
 		if (mystreamer->base.bbs_buffer.maxlen < out_bound)
@@ -196,25 +196,25 @@ bbstreamer_lz4_compressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_lz4_compressor_finalize(bbstreamer *streamer)
+astreamer_lz4_compressor_finalize(astreamer *streamer)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 	uint8	   *next_out;
 	size_t		footer_bound,
 				compressed_size,
 				avail_out;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
+	mystreamer = (astreamer_lz4_frame *) streamer;
 
 	/* Find out the footer bound and update the output buffer. */
 	footer_bound = LZ4F_compressBound(0, &mystreamer->prefs);
 	if ((mystreamer->base.bbs_buffer.maxlen - mystreamer->bytes_written) <
 		footer_bound)
 	{
-		bbstreamer_content(mystreamer->base.bbs_next, NULL,
-						   mystreamer->base.bbs_buffer.data,
-						   mystreamer->bytes_written,
-						   BBSTREAMER_UNKNOWN);
+		astreamer_content(mystreamer->base.bbs_next, NULL,
+						  mystreamer->base.bbs_buffer.data,
+						  mystreamer->bytes_written,
+						  ASTREAMER_UNKNOWN);
 
 		/* Enlarge buffer if it falls short of footer bound. */
 		if (mystreamer->base.bbs_buffer.maxlen < footer_bound)
@@ -243,24 +243,24 @@ bbstreamer_lz4_compressor_finalize(bbstreamer *streamer)
 
 	mystreamer->bytes_written += compressed_size;
 
-	bbstreamer_content(mystreamer->base.bbs_next, NULL,
-					   mystreamer->base.bbs_buffer.data,
-					   mystreamer->bytes_written,
-					   BBSTREAMER_UNKNOWN);
+	astreamer_content(mystreamer->base.bbs_next, NULL,
+					  mystreamer->base.bbs_buffer.data,
+					  mystreamer->bytes_written,
+					  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_lz4_compressor_free(bbstreamer *streamer)
+astreamer_lz4_compressor_free(astreamer *streamer)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
-	bbstreamer_free(streamer->bbs_next);
+	mystreamer = (astreamer_lz4_frame *) streamer;
+	astreamer_free(streamer->bbs_next);
 	LZ4F_freeCompressionContext(mystreamer->cctx);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
@@ -271,18 +271,18 @@ bbstreamer_lz4_compressor_free(bbstreamer *streamer)
  * Create a new base backup streamer that performs decompression of lz4
  * compressed blocks.
  */
-bbstreamer *
-bbstreamer_lz4_decompressor_new(bbstreamer *next)
+astreamer *
+astreamer_lz4_decompressor_new(astreamer *next)
 {
 #ifdef USE_LZ4
-	bbstreamer_lz4_frame *streamer;
+	astreamer_lz4_frame *streamer;
 	LZ4F_errorCode_t ctxError;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_lz4_frame));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_lz4_decompressor_ops;
+	streamer = palloc0(sizeof(astreamer_lz4_frame));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_lz4_decompressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -307,18 +307,18 @@ bbstreamer_lz4_decompressor_new(bbstreamer *next)
  * to the next streamer.
  */
 static void
-bbstreamer_lz4_decompressor_content(bbstreamer *streamer,
-									bbstreamer_member *member,
-									const char *data, int len,
-									bbstreamer_archive_context context)
+astreamer_lz4_decompressor_content(astreamer *streamer,
+								   astreamer_member *member,
+								   const char *data, int len,
+								   astreamer_archive_context context)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 	uint8	   *next_in,
 			   *next_out;
 	size_t		avail_in,
 				avail_out;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
+	mystreamer = (astreamer_lz4_frame *) streamer;
 	next_in = (uint8 *) data;
 	next_out = (uint8 *) mystreamer->base.bbs_buffer.data;
 	avail_in = len;
@@ -366,10 +366,10 @@ bbstreamer_lz4_decompressor_content(bbstreamer *streamer,
 		 */
 		if (mystreamer->bytes_written >= mystreamer->base.bbs_buffer.maxlen)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, member,
-							   mystreamer->base.bbs_buffer.data,
-							   mystreamer->base.bbs_buffer.maxlen,
-							   context);
+			astreamer_content(mystreamer->base.bbs_next, member,
+							  mystreamer->base.bbs_buffer.data,
+							  mystreamer->base.bbs_buffer.maxlen,
+							  context);
 
 			avail_out = mystreamer->base.bbs_buffer.maxlen;
 			mystreamer->bytes_written = 0;
@@ -387,34 +387,34 @@ bbstreamer_lz4_decompressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_lz4_decompressor_finalize(bbstreamer *streamer)
+astreamer_lz4_decompressor_finalize(astreamer *streamer)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
+	mystreamer = (astreamer_lz4_frame *) streamer;
 
 	/*
 	 * End of the stream, if there is some pending data in output buffers then
 	 * we must forward it to next streamer.
 	 */
-	bbstreamer_content(mystreamer->base.bbs_next, NULL,
-					   mystreamer->base.bbs_buffer.data,
-					   mystreamer->base.bbs_buffer.maxlen,
-					   BBSTREAMER_UNKNOWN);
+	astreamer_content(mystreamer->base.bbs_next, NULL,
+					  mystreamer->base.bbs_buffer.data,
+					  mystreamer->base.bbs_buffer.maxlen,
+					  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_lz4_decompressor_free(bbstreamer *streamer)
+astreamer_lz4_decompressor_free(astreamer *streamer)
 {
-	bbstreamer_lz4_frame *mystreamer;
+	astreamer_lz4_frame *mystreamer;
 
-	mystreamer = (bbstreamer_lz4_frame *) streamer;
-	bbstreamer_free(streamer->bbs_next);
+	mystreamer = (astreamer_lz4_frame *) streamer;
+	astreamer_free(streamer->bbs_next);
 	LZ4F_freeDecompressionContext(mystreamer->dctx);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
diff --git a/src/bin/pg_basebackup/bbstreamer_tar.c b/src/bin/pg_basebackup/astreamer_tar.c
similarity index 50%
rename from src/bin/pg_basebackup/bbstreamer_tar.c
rename to src/bin/pg_basebackup/astreamer_tar.c
index 9137d17ddc1..673690cd18f 100644
--- a/src/bin/pg_basebackup/bbstreamer_tar.c
+++ b/src/bin/pg_basebackup/astreamer_tar.c
@@ -1,13 +1,13 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_tar.c
+ * astreamer_tar.c
  *
  * This module implements three types of tar processing. A tar parser
- * expects unlabelled chunks of data (e.g. BBSTREAMER_UNKNOWN) and splits
- * it into labelled chunks (any other value of bbstreamer_archive_context).
+ * expects unlabelled chunks of data (e.g. ASTREAMER_UNKNOWN) and splits
+ * it into labelled chunks (any other value of astreamer_archive_context).
  * A tar archiver does the reverse: it takes a bunch of labelled chunks
  * and produces a tarfile, optionally replacing member headers and trailers
- * so that upstream bbstreamer objects can perform surgery on the tarfile
+ * so that upstream astreamer objects can perform surgery on the tarfile
  * contents without knowing the details of the tar format. A tar terminator
  * just adds two blocks of NUL bytes to the end of the file, since older
  * server versions produce files with this terminator omitted.
@@ -15,7 +15,7 @@
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_tar.c
+ *		  src/bin/pg_basebackup/astreamer_tar.c
  *-------------------------------------------------------------------------
  */
 
@@ -23,83 +23,83 @@
 
 #include <time.h>
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/logging.h"
 #include "pgtar.h"
 
-typedef struct bbstreamer_tar_parser
+typedef struct astreamer_tar_parser
 {
-	bbstreamer	base;
-	bbstreamer_archive_context next_context;
-	bbstreamer_member member;
+	astreamer	base;
+	astreamer_archive_context next_context;
+	astreamer_member member;
 	size_t		file_bytes_sent;
 	size_t		pad_bytes_expected;
-} bbstreamer_tar_parser;
+} astreamer_tar_parser;
 
-typedef struct bbstreamer_tar_archiver
+typedef struct astreamer_tar_archiver
 {
-	bbstreamer	base;
+	astreamer	base;
 	bool		rearchive_member;
-} bbstreamer_tar_archiver;
+} astreamer_tar_archiver;
 
-static void bbstreamer_tar_parser_content(bbstreamer *streamer,
-										  bbstreamer_member *member,
-										  const char *data, int len,
-										  bbstreamer_archive_context context);
-static void bbstreamer_tar_parser_finalize(bbstreamer *streamer);
-static void bbstreamer_tar_parser_free(bbstreamer *streamer);
-static bool bbstreamer_tar_header(bbstreamer_tar_parser *mystreamer);
+static void astreamer_tar_parser_content(astreamer *streamer,
+										 astreamer_member *member,
+										 const char *data, int len,
+										 astreamer_archive_context context);
+static void astreamer_tar_parser_finalize(astreamer *streamer);
+static void astreamer_tar_parser_free(astreamer *streamer);
+static bool astreamer_tar_header(astreamer_tar_parser *mystreamer);
 
-static const bbstreamer_ops bbstreamer_tar_parser_ops = {
-	.content = bbstreamer_tar_parser_content,
-	.finalize = bbstreamer_tar_parser_finalize,
-	.free = bbstreamer_tar_parser_free
+static const astreamer_ops astreamer_tar_parser_ops = {
+	.content = astreamer_tar_parser_content,
+	.finalize = astreamer_tar_parser_finalize,
+	.free = astreamer_tar_parser_free
 };
 
-static void bbstreamer_tar_archiver_content(bbstreamer *streamer,
-											bbstreamer_member *member,
-											const char *data, int len,
-											bbstreamer_archive_context context);
-static void bbstreamer_tar_archiver_finalize(bbstreamer *streamer);
-static void bbstreamer_tar_archiver_free(bbstreamer *streamer);
+static void astreamer_tar_archiver_content(astreamer *streamer,
+										   astreamer_member *member,
+										   const char *data, int len,
+										   astreamer_archive_context context);
+static void astreamer_tar_archiver_finalize(astreamer *streamer);
+static void astreamer_tar_archiver_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_tar_archiver_ops = {
-	.content = bbstreamer_tar_archiver_content,
-	.finalize = bbstreamer_tar_archiver_finalize,
-	.free = bbstreamer_tar_archiver_free
+static const astreamer_ops astreamer_tar_archiver_ops = {
+	.content = astreamer_tar_archiver_content,
+	.finalize = astreamer_tar_archiver_finalize,
+	.free = astreamer_tar_archiver_free
 };
 
-static void bbstreamer_tar_terminator_content(bbstreamer *streamer,
-											  bbstreamer_member *member,
-											  const char *data, int len,
-											  bbstreamer_archive_context context);
-static void bbstreamer_tar_terminator_finalize(bbstreamer *streamer);
-static void bbstreamer_tar_terminator_free(bbstreamer *streamer);
+static void astreamer_tar_terminator_content(astreamer *streamer,
+											 astreamer_member *member,
+											 const char *data, int len,
+											 astreamer_archive_context context);
+static void astreamer_tar_terminator_finalize(astreamer *streamer);
+static void astreamer_tar_terminator_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_tar_terminator_ops = {
-	.content = bbstreamer_tar_terminator_content,
-	.finalize = bbstreamer_tar_terminator_finalize,
-	.free = bbstreamer_tar_terminator_free
+static const astreamer_ops astreamer_tar_terminator_ops = {
+	.content = astreamer_tar_terminator_content,
+	.finalize = astreamer_tar_terminator_finalize,
+	.free = astreamer_tar_terminator_free
 };
 
 /*
- * Create a bbstreamer that can parse a stream of content as tar data.
+ * Create a astreamer that can parse a stream of content as tar data.
  *
- * The input should be a series of BBSTREAMER_UNKNOWN chunks; the bbstreamer
+ * The input should be a series of ASTREAMER_UNKNOWN chunks; the astreamer
  * specified by 'next' will receive a series of typed chunks, as per the
- * conventions described in bbstreamer.h.
+ * conventions described in astreamer.h.
  */
-bbstreamer *
-bbstreamer_tar_parser_new(bbstreamer *next)
+astreamer *
+astreamer_tar_parser_new(astreamer *next)
 {
-	bbstreamer_tar_parser *streamer;
+	astreamer_tar_parser *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_tar_parser));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_tar_parser_ops;
+	streamer = palloc0(sizeof(astreamer_tar_parser));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_tar_parser_ops;
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
-	streamer->next_context = BBSTREAMER_MEMBER_HEADER;
+	streamer->next_context = ASTREAMER_MEMBER_HEADER;
 
 	return &streamer->base;
 }
@@ -108,29 +108,29 @@ bbstreamer_tar_parser_new(bbstreamer *next)
  * Parse unknown content as tar data.
  */
 static void
-bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
-							  const char *data, int len,
-							  bbstreamer_archive_context context)
+astreamer_tar_parser_content(astreamer *streamer, astreamer_member *member,
+							 const char *data, int len,
+							 astreamer_archive_context context)
 {
-	bbstreamer_tar_parser *mystreamer = (bbstreamer_tar_parser *) streamer;
+	astreamer_tar_parser *mystreamer = (astreamer_tar_parser *) streamer;
 	size_t		nbytes;
 
 	/* Expect unparsed input. */
 	Assert(member == NULL);
-	Assert(context == BBSTREAMER_UNKNOWN);
+	Assert(context == ASTREAMER_UNKNOWN);
 
 	while (len > 0)
 	{
 		switch (mystreamer->next_context)
 		{
-			case BBSTREAMER_MEMBER_HEADER:
+			case ASTREAMER_MEMBER_HEADER:
 
 				/*
 				 * If we're expecting an archive member header, accumulate a
 				 * full block of data before doing anything further.
 				 */
-				if (!bbstreamer_buffer_until(streamer, &data, &len,
-											 TAR_BLOCK_SIZE))
+				if (!astreamer_buffer_until(streamer, &data, &len,
+											TAR_BLOCK_SIZE))
 					return;
 
 				/*
@@ -139,32 +139,32 @@ bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
 				 * thought was the next file header is actually the start of
 				 * the archive trailer. Switch modes accordingly.
 				 */
-				if (bbstreamer_tar_header(mystreamer))
+				if (astreamer_tar_header(mystreamer))
 				{
 					if (mystreamer->member.size == 0)
 					{
 						/* No content; trailer is zero-length. */
-						bbstreamer_content(mystreamer->base.bbs_next,
-										   &mystreamer->member,
-										   NULL, 0,
-										   BBSTREAMER_MEMBER_TRAILER);
+						astreamer_content(mystreamer->base.bbs_next,
+										  &mystreamer->member,
+										  NULL, 0,
+										  ASTREAMER_MEMBER_TRAILER);
 
 						/* Expect next header. */
-						mystreamer->next_context = BBSTREAMER_MEMBER_HEADER;
+						mystreamer->next_context = ASTREAMER_MEMBER_HEADER;
 					}
 					else
 					{
 						/* Expect contents. */
-						mystreamer->next_context = BBSTREAMER_MEMBER_CONTENTS;
+						mystreamer->next_context = ASTREAMER_MEMBER_CONTENTS;
 					}
 					mystreamer->base.bbs_buffer.len = 0;
 					mystreamer->file_bytes_sent = 0;
 				}
 				else
-					mystreamer->next_context = BBSTREAMER_ARCHIVE_TRAILER;
+					mystreamer->next_context = ASTREAMER_ARCHIVE_TRAILER;
 				break;
 
-			case BBSTREAMER_MEMBER_CONTENTS:
+			case ASTREAMER_MEMBER_CONTENTS:
 
 				/*
 				 * Send as much content as we have, but not more than the
@@ -174,10 +174,10 @@ bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
 				nbytes = mystreamer->member.size - mystreamer->file_bytes_sent;
 				nbytes = Min(nbytes, len);
 				Assert(nbytes > 0);
-				bbstreamer_content(mystreamer->base.bbs_next,
-								   &mystreamer->member,
-								   data, nbytes,
-								   BBSTREAMER_MEMBER_CONTENTS);
+				astreamer_content(mystreamer->base.bbs_next,
+								  &mystreamer->member,
+								  data, nbytes,
+								  ASTREAMER_MEMBER_CONTENTS);
 				mystreamer->file_bytes_sent += nbytes;
 				data += nbytes;
 				len -= nbytes;
@@ -193,53 +193,53 @@ bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
 					if (mystreamer->pad_bytes_expected == 0)
 					{
 						/* Trailer is zero-length. */
-						bbstreamer_content(mystreamer->base.bbs_next,
-										   &mystreamer->member,
-										   NULL, 0,
-										   BBSTREAMER_MEMBER_TRAILER);
+						astreamer_content(mystreamer->base.bbs_next,
+										  &mystreamer->member,
+										  NULL, 0,
+										  ASTREAMER_MEMBER_TRAILER);
 
 						/* Expect next header. */
-						mystreamer->next_context = BBSTREAMER_MEMBER_HEADER;
+						mystreamer->next_context = ASTREAMER_MEMBER_HEADER;
 					}
 					else
 					{
 						/* Trailer is not zero-length. */
-						mystreamer->next_context = BBSTREAMER_MEMBER_TRAILER;
+						mystreamer->next_context = ASTREAMER_MEMBER_TRAILER;
 					}
 					mystreamer->base.bbs_buffer.len = 0;
 				}
 				break;
 
-			case BBSTREAMER_MEMBER_TRAILER:
+			case ASTREAMER_MEMBER_TRAILER:
 
 				/*
 				 * If we're expecting an archive member trailer, accumulate
 				 * the expected number of padding bytes before sending
 				 * anything onward.
 				 */
-				if (!bbstreamer_buffer_until(streamer, &data, &len,
-											 mystreamer->pad_bytes_expected))
+				if (!astreamer_buffer_until(streamer, &data, &len,
+											mystreamer->pad_bytes_expected))
 					return;
 
 				/* OK, now we can send it. */
-				bbstreamer_content(mystreamer->base.bbs_next,
-								   &mystreamer->member,
-								   data, mystreamer->pad_bytes_expected,
-								   BBSTREAMER_MEMBER_TRAILER);
+				astreamer_content(mystreamer->base.bbs_next,
+								  &mystreamer->member,
+								  data, mystreamer->pad_bytes_expected,
+								  ASTREAMER_MEMBER_TRAILER);
 
 				/* Expect next file header. */
-				mystreamer->next_context = BBSTREAMER_MEMBER_HEADER;
+				mystreamer->next_context = ASTREAMER_MEMBER_HEADER;
 				mystreamer->base.bbs_buffer.len = 0;
 				break;
 
-			case BBSTREAMER_ARCHIVE_TRAILER:
+			case ASTREAMER_ARCHIVE_TRAILER:
 
 				/*
 				 * We've seen an end-of-archive indicator, so anything more is
 				 * buffered and sent as part of the archive trailer. But we
 				 * don't expect more than 2 blocks.
 				 */
-				bbstreamer_buffer_bytes(streamer, &data, &len, len);
+				astreamer_buffer_bytes(streamer, &data, &len, len);
 				if (len > 2 * TAR_BLOCK_SIZE)
 					pg_fatal("tar file trailer exceeds 2 blocks");
 				return;
@@ -255,14 +255,14 @@ bbstreamer_tar_parser_content(bbstreamer *streamer, bbstreamer_member *member,
  * Parse a file header within a tar stream.
  *
  * The return value is true if we found a file header and passed it on to the
- * next bbstreamer; it is false if we have reached the archive trailer.
+ * next astreamer; it is false if we have reached the archive trailer.
  */
 static bool
-bbstreamer_tar_header(bbstreamer_tar_parser *mystreamer)
+astreamer_tar_header(astreamer_tar_parser *mystreamer)
 {
 	bool		has_nonzero_byte = false;
 	int			i;
-	bbstreamer_member *member = &mystreamer->member;
+	astreamer_member *member = &mystreamer->member;
 	char	   *buffer = mystreamer->base.bbs_buffer.data;
 
 	Assert(mystreamer->base.bbs_buffer.len == TAR_BLOCK_SIZE);
@@ -304,10 +304,10 @@ bbstreamer_tar_header(bbstreamer_tar_parser *mystreamer)
 	/* Compute number of padding bytes. */
 	mystreamer->pad_bytes_expected = tarPaddingBytesRequired(member->size);
 
-	/* Forward the entire header to the next bbstreamer. */
-	bbstreamer_content(mystreamer->base.bbs_next, member,
-					   buffer, TAR_BLOCK_SIZE,
-					   BBSTREAMER_MEMBER_HEADER);
+	/* Forward the entire header to the next astreamer. */
+	astreamer_content(mystreamer->base.bbs_next, member,
+					  buffer, TAR_BLOCK_SIZE,
+					  ASTREAMER_MEMBER_HEADER);
 
 	return true;
 }
@@ -316,50 +316,50 @@ bbstreamer_tar_header(bbstreamer_tar_parser *mystreamer)
  * End-of-stream processing for a tar parser.
  */
 static void
-bbstreamer_tar_parser_finalize(bbstreamer *streamer)
+astreamer_tar_parser_finalize(astreamer *streamer)
 {
-	bbstreamer_tar_parser *mystreamer = (bbstreamer_tar_parser *) streamer;
+	astreamer_tar_parser *mystreamer = (astreamer_tar_parser *) streamer;
 
-	if (mystreamer->next_context != BBSTREAMER_ARCHIVE_TRAILER &&
-		(mystreamer->next_context != BBSTREAMER_MEMBER_HEADER ||
+	if (mystreamer->next_context != ASTREAMER_ARCHIVE_TRAILER &&
+		(mystreamer->next_context != ASTREAMER_MEMBER_HEADER ||
 		 mystreamer->base.bbs_buffer.len > 0))
 		pg_fatal("COPY stream ended before last file was finished");
 
 	/* Send the archive trailer, even if empty. */
-	bbstreamer_content(streamer->bbs_next, NULL,
-					   streamer->bbs_buffer.data, streamer->bbs_buffer.len,
-					   BBSTREAMER_ARCHIVE_TRAILER);
+	astreamer_content(streamer->bbs_next, NULL,
+					  streamer->bbs_buffer.data, streamer->bbs_buffer.len,
+					  ASTREAMER_ARCHIVE_TRAILER);
 
 	/* Now finalize successor. */
-	bbstreamer_finalize(streamer->bbs_next);
+	astreamer_finalize(streamer->bbs_next);
 }
 
 /*
  * Free memory associated with a tar parser.
  */
 static void
-bbstreamer_tar_parser_free(bbstreamer *streamer)
+astreamer_tar_parser_free(astreamer *streamer)
 {
 	pfree(streamer->bbs_buffer.data);
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 }
 
 /*
- * Create a bbstreamer that can generate a tar archive.
+ * Create a astreamer that can generate a tar archive.
  *
  * This is intended to be usable either for generating a brand-new tar archive
  * or for modifying one on the fly. The input should be a series of typed
- * chunks (i.e. not BBSTREAMER_UNKNOWN). See also the comments for
- * bbstreamer_tar_parser_content.
+ * chunks (i.e. not ASTREAMER_UNKNOWN). See also the comments for
+ * astreamer_tar_parser_content.
  */
-bbstreamer *
-bbstreamer_tar_archiver_new(bbstreamer *next)
+astreamer *
+astreamer_tar_archiver_new(astreamer *next)
 {
-	bbstreamer_tar_archiver *streamer;
+	astreamer_tar_archiver *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer_tar_archiver));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_tar_archiver_ops;
+	streamer = palloc0(sizeof(astreamer_tar_archiver));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_tar_archiver_ops;
 	streamer->base.bbs_next = next;
 
 	return &streamer->base;
@@ -368,36 +368,36 @@ bbstreamer_tar_archiver_new(bbstreamer *next)
 /*
  * Fix up the stream of input chunks to create a valid tar file.
  *
- * If a BBSTREAMER_MEMBER_HEADER chunk is of size 0, it is replaced with a
+ * If a ASTREAMER_MEMBER_HEADER chunk is of size 0, it is replaced with a
  * newly-constructed tar header. If it is of size TAR_BLOCK_SIZE, it is
  * passed through without change. Any other size is a fatal error (and
  * indicates a bug).
  *
- * Whenever a new BBSTREAMER_MEMBER_HEADER chunk is constructed, the
- * corresponding BBSTREAMER_MEMBER_TRAILER chunk is also constructed from
+ * Whenever a new ASTREAMER_MEMBER_HEADER chunk is constructed, the
+ * corresponding ASTREAMER_MEMBER_TRAILER chunk is also constructed from
  * scratch. Specifically, we construct a block of zero bytes sufficient to
  * pad out to a block boundary, as required by the tar format. Other
- * BBSTREAMER_MEMBER_TRAILER chunks are passed through without change.
+ * ASTREAMER_MEMBER_TRAILER chunks are passed through without change.
  *
- * Any BBSTREAMER_MEMBER_CONTENTS chunks are passed through without change.
+ * Any ASTREAMER_MEMBER_CONTENTS chunks are passed through without change.
  *
- * The BBSTREAMER_ARCHIVE_TRAILER chunk is replaced with two
+ * The ASTREAMER_ARCHIVE_TRAILER chunk is replaced with two
  * blocks of zero bytes. Not all tar programs require this, but apparently
  * some do. The server does not supply this trailer. If no archive trailer is
- * present, one will be added by bbstreamer_tar_parser_finalize.
+ * present, one will be added by astreamer_tar_parser_finalize.
  */
 static void
-bbstreamer_tar_archiver_content(bbstreamer *streamer,
-								bbstreamer_member *member,
-								const char *data, int len,
-								bbstreamer_archive_context context)
+astreamer_tar_archiver_content(astreamer *streamer,
+							   astreamer_member *member,
+							   const char *data, int len,
+							   astreamer_archive_context context)
 {
-	bbstreamer_tar_archiver *mystreamer = (bbstreamer_tar_archiver *) streamer;
+	astreamer_tar_archiver *mystreamer = (astreamer_tar_archiver *) streamer;
 	char		buffer[2 * TAR_BLOCK_SIZE];
 
-	Assert(context != BBSTREAMER_UNKNOWN);
+	Assert(context != ASTREAMER_UNKNOWN);
 
-	if (context == BBSTREAMER_MEMBER_HEADER && len != TAR_BLOCK_SIZE)
+	if (context == ASTREAMER_MEMBER_HEADER && len != TAR_BLOCK_SIZE)
 	{
 		Assert(len == 0);
 
@@ -411,7 +411,7 @@ bbstreamer_tar_archiver_content(bbstreamer *streamer,
 		/* Also make a note to replace padding, in case size changed. */
 		mystreamer->rearchive_member = true;
 	}
-	else if (context == BBSTREAMER_MEMBER_TRAILER &&
+	else if (context == ASTREAMER_MEMBER_TRAILER &&
 			 mystreamer->rearchive_member)
 	{
 		int			pad_bytes = tarPaddingBytesRequired(member->size);
@@ -424,7 +424,7 @@ bbstreamer_tar_archiver_content(bbstreamer *streamer,
 		/* Don't do this again unless we replace another header. */
 		mystreamer->rearchive_member = false;
 	}
-	else if (context == BBSTREAMER_ARCHIVE_TRAILER)
+	else if (context == ASTREAMER_ARCHIVE_TRAILER)
 	{
 		/* Trailer should always be two blocks of zero bytes. */
 		memset(buffer, 0, 2 * TAR_BLOCK_SIZE);
@@ -432,40 +432,40 @@ bbstreamer_tar_archiver_content(bbstreamer *streamer,
 		len = 2 * TAR_BLOCK_SIZE;
 	}
 
-	bbstreamer_content(streamer->bbs_next, member, data, len, context);
+	astreamer_content(streamer->bbs_next, member, data, len, context);
 }
 
 /*
  * End-of-stream processing for a tar archiver.
  */
 static void
-bbstreamer_tar_archiver_finalize(bbstreamer *streamer)
+astreamer_tar_archiver_finalize(astreamer *streamer)
 {
-	bbstreamer_finalize(streamer->bbs_next);
+	astreamer_finalize(streamer->bbs_next);
 }
 
 /*
  * Free memory associated with a tar archiver.
  */
 static void
-bbstreamer_tar_archiver_free(bbstreamer *streamer)
+astreamer_tar_archiver_free(astreamer *streamer)
 {
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	pfree(streamer);
 }
 
 /*
- * Create a bbstreamer that blindly adds two blocks of NUL bytes to the
+ * Create a astreamer that blindly adds two blocks of NUL bytes to the
  * end of an incomplete tarfile that the server might send us.
  */
-bbstreamer *
-bbstreamer_tar_terminator_new(bbstreamer *next)
+astreamer *
+astreamer_tar_terminator_new(astreamer *next)
 {
-	bbstreamer *streamer;
+	astreamer  *streamer;
 
-	streamer = palloc0(sizeof(bbstreamer));
-	*((const bbstreamer_ops **) &streamer->bbs_ops) =
-		&bbstreamer_tar_terminator_ops;
+	streamer = palloc0(sizeof(astreamer));
+	*((const astreamer_ops **) &streamer->bbs_ops) =
+		&astreamer_tar_terminator_ops;
 	streamer->bbs_next = next;
 
 	return streamer;
@@ -475,17 +475,17 @@ bbstreamer_tar_terminator_new(bbstreamer *next)
  * Pass all the content through without change.
  */
 static void
-bbstreamer_tar_terminator_content(bbstreamer *streamer,
-								  bbstreamer_member *member,
-								  const char *data, int len,
-								  bbstreamer_archive_context context)
+astreamer_tar_terminator_content(astreamer *streamer,
+								 astreamer_member *member,
+								 const char *data, int len,
+								 astreamer_archive_context context)
 {
 	/* Expect unparsed input. */
 	Assert(member == NULL);
-	Assert(context == BBSTREAMER_UNKNOWN);
+	Assert(context == ASTREAMER_UNKNOWN);
 
 	/* Just forward it. */
-	bbstreamer_content(streamer->bbs_next, member, data, len, context);
+	astreamer_content(streamer->bbs_next, member, data, len, context);
 }
 
 /*
@@ -493,22 +493,22 @@ bbstreamer_tar_terminator_content(bbstreamer *streamer,
  * to supply.
  */
 static void
-bbstreamer_tar_terminator_finalize(bbstreamer *streamer)
+astreamer_tar_terminator_finalize(astreamer *streamer)
 {
 	char		buffer[2 * TAR_BLOCK_SIZE];
 
 	memset(buffer, 0, 2 * TAR_BLOCK_SIZE);
-	bbstreamer_content(streamer->bbs_next, NULL, buffer,
-					   2 * TAR_BLOCK_SIZE, BBSTREAMER_UNKNOWN);
-	bbstreamer_finalize(streamer->bbs_next);
+	astreamer_content(streamer->bbs_next, NULL, buffer,
+					  2 * TAR_BLOCK_SIZE, ASTREAMER_UNKNOWN);
+	astreamer_finalize(streamer->bbs_next);
 }
 
 /*
  * Free memory associated with a tar terminator.
  */
 static void
-bbstreamer_tar_terminator_free(bbstreamer *streamer)
+astreamer_tar_terminator_free(astreamer *streamer)
 {
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	pfree(streamer);
 }
diff --git a/src/bin/pg_basebackup/bbstreamer_zstd.c b/src/bin/pg_basebackup/astreamer_zstd.c
similarity index 64%
rename from src/bin/pg_basebackup/bbstreamer_zstd.c
rename to src/bin/pg_basebackup/astreamer_zstd.c
index 20f11d4450e..58dc679ef99 100644
--- a/src/bin/pg_basebackup/bbstreamer_zstd.c
+++ b/src/bin/pg_basebackup/astreamer_zstd.c
@@ -1,11 +1,11 @@
 /*-------------------------------------------------------------------------
  *
- * bbstreamer_zstd.c
+ * astreamer_zstd.c
  *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer_zstd.c
+ *		  src/bin/pg_basebackup/astreamer_zstd.c
  *-------------------------------------------------------------------------
  */
 
@@ -17,44 +17,44 @@
 #include <zstd.h>
 #endif
 
-#include "bbstreamer.h"
+#include "astreamer.h"
 #include "common/logging.h"
 
 #ifdef USE_ZSTD
 
-typedef struct bbstreamer_zstd_frame
+typedef struct astreamer_zstd_frame
 {
-	bbstreamer	base;
+	astreamer	base;
 
 	ZSTD_CCtx  *cctx;
 	ZSTD_DCtx  *dctx;
 	ZSTD_outBuffer zstd_outBuf;
-} bbstreamer_zstd_frame;
+} astreamer_zstd_frame;
 
-static void bbstreamer_zstd_compressor_content(bbstreamer *streamer,
-											   bbstreamer_member *member,
-											   const char *data, int len,
-											   bbstreamer_archive_context context);
-static void bbstreamer_zstd_compressor_finalize(bbstreamer *streamer);
-static void bbstreamer_zstd_compressor_free(bbstreamer *streamer);
+static void astreamer_zstd_compressor_content(astreamer *streamer,
+											  astreamer_member *member,
+											  const char *data, int len,
+											  astreamer_archive_context context);
+static void astreamer_zstd_compressor_finalize(astreamer *streamer);
+static void astreamer_zstd_compressor_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_zstd_compressor_ops = {
-	.content = bbstreamer_zstd_compressor_content,
-	.finalize = bbstreamer_zstd_compressor_finalize,
-	.free = bbstreamer_zstd_compressor_free
+static const astreamer_ops astreamer_zstd_compressor_ops = {
+	.content = astreamer_zstd_compressor_content,
+	.finalize = astreamer_zstd_compressor_finalize,
+	.free = astreamer_zstd_compressor_free
 };
 
-static void bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
-												 bbstreamer_member *member,
-												 const char *data, int len,
-												 bbstreamer_archive_context context);
-static void bbstreamer_zstd_decompressor_finalize(bbstreamer *streamer);
-static void bbstreamer_zstd_decompressor_free(bbstreamer *streamer);
+static void astreamer_zstd_decompressor_content(astreamer *streamer,
+												astreamer_member *member,
+												const char *data, int len,
+												astreamer_archive_context context);
+static void astreamer_zstd_decompressor_finalize(astreamer *streamer);
+static void astreamer_zstd_decompressor_free(astreamer *streamer);
 
-static const bbstreamer_ops bbstreamer_zstd_decompressor_ops = {
-	.content = bbstreamer_zstd_decompressor_content,
-	.finalize = bbstreamer_zstd_decompressor_finalize,
-	.free = bbstreamer_zstd_decompressor_free
+static const astreamer_ops astreamer_zstd_decompressor_ops = {
+	.content = astreamer_zstd_decompressor_content,
+	.finalize = astreamer_zstd_decompressor_finalize,
+	.free = astreamer_zstd_decompressor_free
 };
 #endif
 
@@ -62,19 +62,19 @@ static const bbstreamer_ops bbstreamer_zstd_decompressor_ops = {
  * Create a new base backup streamer that performs zstd compression of tar
  * blocks.
  */
-bbstreamer *
-bbstreamer_zstd_compressor_new(bbstreamer *next, pg_compress_specification *compress)
+astreamer *
+astreamer_zstd_compressor_new(astreamer *next, pg_compress_specification *compress)
 {
 #ifdef USE_ZSTD
-	bbstreamer_zstd_frame *streamer;
+	astreamer_zstd_frame *streamer;
 	size_t		ret;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_zstd_frame));
+	streamer = palloc0(sizeof(astreamer_zstd_frame));
 
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_zstd_compressor_ops;
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_zstd_compressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -142,12 +142,12 @@ bbstreamer_zstd_compressor_new(bbstreamer *next, pg_compress_specification *comp
  * of output buffer to next streamer and empty the buffer.
  */
 static void
-bbstreamer_zstd_compressor_content(bbstreamer *streamer,
-								   bbstreamer_member *member,
-								   const char *data, int len,
-								   bbstreamer_archive_context context)
+astreamer_zstd_compressor_content(astreamer *streamer,
+								  astreamer_member *member,
+								  const char *data, int len,
+								  astreamer_archive_context context)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 	ZSTD_inBuffer inBuf = {data, len, 0};
 
 	while (inBuf.pos < inBuf.size)
@@ -162,10 +162,10 @@ bbstreamer_zstd_compressor_content(bbstreamer *streamer,
 		if (mystreamer->zstd_outBuf.size - mystreamer->zstd_outBuf.pos <
 			max_needed)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, member,
-							   mystreamer->zstd_outBuf.dst,
-							   mystreamer->zstd_outBuf.pos,
-							   context);
+			astreamer_content(mystreamer->base.bbs_next, member,
+							  mystreamer->zstd_outBuf.dst,
+							  mystreamer->zstd_outBuf.pos,
+							  context);
 
 			/* Reset the ZSTD output buffer. */
 			mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data;
@@ -187,9 +187,9 @@ bbstreamer_zstd_compressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_zstd_compressor_finalize(bbstreamer *streamer)
+astreamer_zstd_compressor_finalize(astreamer *streamer)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 	size_t		yet_to_flush;
 
 	do
@@ -204,10 +204,10 @@ bbstreamer_zstd_compressor_finalize(bbstreamer *streamer)
 		if (mystreamer->zstd_outBuf.size - mystreamer->zstd_outBuf.pos <
 			max_needed)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, NULL,
-							   mystreamer->zstd_outBuf.dst,
-							   mystreamer->zstd_outBuf.pos,
-							   BBSTREAMER_UNKNOWN);
+			astreamer_content(mystreamer->base.bbs_next, NULL,
+							  mystreamer->zstd_outBuf.dst,
+							  mystreamer->zstd_outBuf.pos,
+							  ASTREAMER_UNKNOWN);
 
 			/* Reset the ZSTD output buffer. */
 			mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data;
@@ -227,23 +227,23 @@ bbstreamer_zstd_compressor_finalize(bbstreamer *streamer)
 
 	/* Make sure to pass any remaining bytes to the next streamer. */
 	if (mystreamer->zstd_outBuf.pos > 0)
-		bbstreamer_content(mystreamer->base.bbs_next, NULL,
-						   mystreamer->zstd_outBuf.dst,
-						   mystreamer->zstd_outBuf.pos,
-						   BBSTREAMER_UNKNOWN);
+		astreamer_content(mystreamer->base.bbs_next, NULL,
+						  mystreamer->zstd_outBuf.dst,
+						  mystreamer->zstd_outBuf.pos,
+						  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_zstd_compressor_free(bbstreamer *streamer)
+astreamer_zstd_compressor_free(astreamer *streamer)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	ZSTD_freeCCtx(mystreamer->cctx);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
@@ -254,17 +254,17 @@ bbstreamer_zstd_compressor_free(bbstreamer *streamer)
  * Create a new base backup streamer that performs decompression of zstd
  * compressed blocks.
  */
-bbstreamer *
-bbstreamer_zstd_decompressor_new(bbstreamer *next)
+astreamer *
+astreamer_zstd_decompressor_new(astreamer *next)
 {
 #ifdef USE_ZSTD
-	bbstreamer_zstd_frame *streamer;
+	astreamer_zstd_frame *streamer;
 
 	Assert(next != NULL);
 
-	streamer = palloc0(sizeof(bbstreamer_zstd_frame));
-	*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
-		&bbstreamer_zstd_decompressor_ops;
+	streamer = palloc0(sizeof(astreamer_zstd_frame));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_zstd_decompressor_ops;
 
 	streamer->base.bbs_next = next;
 	initStringInfo(&streamer->base.bbs_buffer);
@@ -293,12 +293,12 @@ bbstreamer_zstd_decompressor_new(bbstreamer *next)
  * to the next streamer.
  */
 static void
-bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
-									 bbstreamer_member *member,
-									 const char *data, int len,
-									 bbstreamer_archive_context context)
+astreamer_zstd_decompressor_content(astreamer *streamer,
+									astreamer_member *member,
+									const char *data, int len,
+									astreamer_archive_context context)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 	ZSTD_inBuffer inBuf = {data, len, 0};
 
 	while (inBuf.pos < inBuf.size)
@@ -311,10 +311,10 @@ bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
 		 */
 		if (mystreamer->zstd_outBuf.pos >= mystreamer->zstd_outBuf.size)
 		{
-			bbstreamer_content(mystreamer->base.bbs_next, member,
-							   mystreamer->zstd_outBuf.dst,
-							   mystreamer->zstd_outBuf.pos,
-							   context);
+			astreamer_content(mystreamer->base.bbs_next, member,
+							  mystreamer->zstd_outBuf.dst,
+							  mystreamer->zstd_outBuf.pos,
+							  context);
 
 			/* Reset the ZSTD output buffer. */
 			mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data;
@@ -335,32 +335,32 @@ bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
  * End-of-stream processing.
  */
 static void
-bbstreamer_zstd_decompressor_finalize(bbstreamer *streamer)
+astreamer_zstd_decompressor_finalize(astreamer *streamer)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 
 	/*
 	 * End of the stream, if there is some pending data in output buffers then
 	 * we must forward it to next streamer.
 	 */
 	if (mystreamer->zstd_outBuf.pos > 0)
-		bbstreamer_content(mystreamer->base.bbs_next, NULL,
-						   mystreamer->base.bbs_buffer.data,
-						   mystreamer->base.bbs_buffer.maxlen,
-						   BBSTREAMER_UNKNOWN);
+		astreamer_content(mystreamer->base.bbs_next, NULL,
+						  mystreamer->base.bbs_buffer.data,
+						  mystreamer->base.bbs_buffer.maxlen,
+						  ASTREAMER_UNKNOWN);
 
-	bbstreamer_finalize(mystreamer->base.bbs_next);
+	astreamer_finalize(mystreamer->base.bbs_next);
 }
 
 /*
  * Free memory.
  */
 static void
-bbstreamer_zstd_decompressor_free(bbstreamer *streamer)
+astreamer_zstd_decompressor_free(astreamer *streamer)
 {
-	bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+	astreamer_zstd_frame *mystreamer = (astreamer_zstd_frame *) streamer;
 
-	bbstreamer_free(streamer->bbs_next);
+	astreamer_free(streamer->bbs_next);
 	ZSTD_freeDCtx(mystreamer->dctx);
 	pfree(streamer->bbs_buffer.data);
 	pfree(streamer);
diff --git a/src/bin/pg_basebackup/bbstreamer.h b/src/bin/pg_basebackup/bbstreamer.h
deleted file mode 100644
index 3b820f13b51..00000000000
--- a/src/bin/pg_basebackup/bbstreamer.h
+++ /dev/null
@@ -1,226 +0,0 @@
-/*-------------------------------------------------------------------------
- *
- * bbstreamer.h
- *
- * Each tar archive returned by the server is passed to one or more
- * bbstreamer objects for further processing. The bbstreamer may do
- * something simple, like write the archive to a file, perhaps after
- * compressing it, but it can also do more complicated things, like
- * annotating the byte stream to indicate which parts of the data
- * correspond to tar headers or trailing padding, vs. which parts are
- * payload data. A subsequent bbstreamer may use this information to
- * make further decisions about how to process the data; for example,
- * it might choose to modify the archive contents.
- *
- * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
- *
- * IDENTIFICATION
- *		  src/bin/pg_basebackup/bbstreamer.h
- *-------------------------------------------------------------------------
- */
-
-#ifndef BBSTREAMER_H
-#define BBSTREAMER_H
-
-#include "common/compression.h"
-#include "lib/stringinfo.h"
-#include "pqexpbuffer.h"
-
-struct bbstreamer;
-struct bbstreamer_ops;
-typedef struct bbstreamer bbstreamer;
-typedef struct bbstreamer_ops bbstreamer_ops;
-
-/*
- * Each chunk of archive data passed to a bbstreamer is classified into one
- * of these categories. When data is first received from the remote server,
- * each chunk will be categorized as BBSTREAMER_UNKNOWN, and the chunks will
- * be of whatever size the remote server chose to send.
- *
- * If the archive is parsed (e.g. see bbstreamer_tar_parser_new()), then all
- * chunks should be labelled as one of the other types listed here. In
- * addition, there should be exactly one BBSTREAMER_MEMBER_HEADER chunk and
- * exactly one BBSTREAMER_MEMBER_TRAILER chunk per archive member, even if
- * that means a zero-length call. There can be any number of
- * BBSTREAMER_MEMBER_CONTENTS chunks in between those calls. There
- * should exactly BBSTREAMER_ARCHIVE_TRAILER chunk, and it should follow the
- * last BBSTREAMER_MEMBER_TRAILER chunk.
- *
- * In theory, we could need other classifications here, such as a way of
- * indicating an archive header, but the "tar" format doesn't need anything
- * else, so for the time being there's no point.
- */
-typedef enum
-{
-	BBSTREAMER_UNKNOWN,
-	BBSTREAMER_MEMBER_HEADER,
-	BBSTREAMER_MEMBER_CONTENTS,
-	BBSTREAMER_MEMBER_TRAILER,
-	BBSTREAMER_ARCHIVE_TRAILER,
-} bbstreamer_archive_context;
-
-/*
- * Each chunk of data that is classified as BBSTREAMER_MEMBER_HEADER,
- * BBSTREAMER_MEMBER_CONTENTS, or BBSTREAMER_MEMBER_TRAILER should also
- * pass a pointer to an instance of this struct. The details are expected
- * to be present in the archive header and used to fill the struct, after
- * which all subsequent calls for the same archive member are expected to
- * pass the same details.
- */
-typedef struct
-{
-	char		pathname[MAXPGPATH];
-	pgoff_t		size;
-	mode_t		mode;
-	uid_t		uid;
-	gid_t		gid;
-	bool		is_directory;
-	bool		is_link;
-	char		linktarget[MAXPGPATH];
-} bbstreamer_member;
-
-/*
- * Generally, each type of bbstreamer will define its own struct, but the
- * first element should be 'bbstreamer base'. A bbstreamer that does not
- * require any additional private data could use this structure directly.
- *
- * bbs_ops is a pointer to the bbstreamer_ops object which contains the
- * function pointers appropriate to this type of bbstreamer.
- *
- * bbs_next is a pointer to the successor bbstreamer, for those types of
- * bbstreamer which forward data to a successor. It need not be used and
- * should be set to NULL when not relevant.
- *
- * bbs_buffer is a buffer for accumulating data for temporary storage. Each
- * type of bbstreamer makes its own decisions about whether and how to use
- * this buffer.
- */
-struct bbstreamer
-{
-	const bbstreamer_ops *bbs_ops;
-	bbstreamer *bbs_next;
-	StringInfoData bbs_buffer;
-};
-
-/*
- * There are three callbacks for a bbstreamer. The 'content' callback is
- * called repeatedly, as described in the bbstreamer_archive_context comments.
- * Then, the 'finalize' callback is called once at the end, to give the
- * bbstreamer a chance to perform cleanup such as closing files. Finally,
- * because this code is running in a frontend environment where, as of this
- * writing, there are no memory contexts, the 'free' callback is called to
- * release memory. These callbacks should always be invoked using the static
- * inline functions defined below.
- */
-struct bbstreamer_ops
-{
-	void		(*content) (bbstreamer *streamer, bbstreamer_member *member,
-							const char *data, int len,
-							bbstreamer_archive_context context);
-	void		(*finalize) (bbstreamer *streamer);
-	void		(*free) (bbstreamer *streamer);
-};
-
-/* Send some content to a bbstreamer. */
-static inline void
-bbstreamer_content(bbstreamer *streamer, bbstreamer_member *member,
-				   const char *data, int len,
-				   bbstreamer_archive_context context)
-{
-	Assert(streamer != NULL);
-	streamer->bbs_ops->content(streamer, member, data, len, context);
-}
-
-/* Finalize a bbstreamer. */
-static inline void
-bbstreamer_finalize(bbstreamer *streamer)
-{
-	Assert(streamer != NULL);
-	streamer->bbs_ops->finalize(streamer);
-}
-
-/* Free a bbstreamer. */
-static inline void
-bbstreamer_free(bbstreamer *streamer)
-{
-	Assert(streamer != NULL);
-	streamer->bbs_ops->free(streamer);
-}
-
-/*
- * This is a convenience method for use when implementing a bbstreamer; it is
- * not for use by outside callers. It adds the amount of data specified by
- * 'nbytes' to the bbstreamer's buffer and adjusts '*len' and '*data'
- * accordingly.
- */
-static inline void
-bbstreamer_buffer_bytes(bbstreamer *streamer, const char **data, int *len,
-						int nbytes)
-{
-	Assert(nbytes <= *len);
-
-	appendBinaryStringInfo(&streamer->bbs_buffer, *data, nbytes);
-	*len -= nbytes;
-	*data += nbytes;
-}
-
-/*
- * This is a convenience method for use when implementing a bbstreamer; it is
- * not for use by outsider callers. It attempts to add enough data to the
- * bbstreamer's buffer to reach a length of target_bytes and adjusts '*len'
- * and '*data' accordingly. It returns true if the target length has been
- * reached and false otherwise.
- */
-static inline bool
-bbstreamer_buffer_until(bbstreamer *streamer, const char **data, int *len,
-						int target_bytes)
-{
-	int			buflen = streamer->bbs_buffer.len;
-
-	if (buflen >= target_bytes)
-	{
-		/* Target length already reached; nothing to do. */
-		return true;
-	}
-
-	if (buflen + *len < target_bytes)
-	{
-		/* Not enough data to reach target length; buffer all of it. */
-		bbstreamer_buffer_bytes(streamer, data, len, *len);
-		return false;
-	}
-
-	/* Buffer just enough to reach the target length. */
-	bbstreamer_buffer_bytes(streamer, data, len, target_bytes - buflen);
-	return true;
-}
-
-/*
- * Functions for creating bbstreamer objects of various types. See the header
- * comments for each of these functions for details.
- */
-extern bbstreamer *bbstreamer_plain_writer_new(char *pathname, FILE *file);
-extern bbstreamer *bbstreamer_gzip_writer_new(char *pathname, FILE *file,
-											  pg_compress_specification *compress);
-extern bbstreamer *bbstreamer_extractor_new(const char *basepath,
-											const char *(*link_map) (const char *),
-											void (*report_output_file) (const char *));
-
-extern bbstreamer *bbstreamer_gzip_decompressor_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_lz4_compressor_new(bbstreamer *next,
-												 pg_compress_specification *compress);
-extern bbstreamer *bbstreamer_lz4_decompressor_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_zstd_compressor_new(bbstreamer *next,
-												  pg_compress_specification *compress);
-extern bbstreamer *bbstreamer_zstd_decompressor_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_tar_parser_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_tar_terminator_new(bbstreamer *next);
-extern bbstreamer *bbstreamer_tar_archiver_new(bbstreamer *next);
-
-extern bbstreamer *bbstreamer_recovery_injector_new(bbstreamer *next,
-													bool is_recovery_guc_supported,
-													PQExpBuffer recoveryconfcontents);
-extern void bbstreamer_inject_file(bbstreamer *streamer, char *pathname,
-								   char *data, int len);
-
-#endif
diff --git a/src/bin/pg_basebackup/meson.build b/src/bin/pg_basebackup/meson.build
index c00acd5e118..a68dbd7837d 100644
--- a/src/bin/pg_basebackup/meson.build
+++ b/src/bin/pg_basebackup/meson.build
@@ -1,12 +1,12 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 common_sources = files(
-  'bbstreamer_file.c',
-  'bbstreamer_gzip.c',
-  'bbstreamer_inject.c',
-  'bbstreamer_lz4.c',
-  'bbstreamer_tar.c',
-  'bbstreamer_zstd.c',
+  'astreamer_file.c',
+  'astreamer_gzip.c',
+  'astreamer_inject.c',
+  'astreamer_lz4.c',
+  'astreamer_tar.c',
+  'astreamer_zstd.c',
   'receivelog.c',
   'streamutil.c',
   'walmethods.c',
diff --git a/src/bin/pg_basebackup/nls.mk b/src/bin/pg_basebackup/nls.mk
index 384dbb021e9..950b9797b1e 100644
--- a/src/bin/pg_basebackup/nls.mk
+++ b/src/bin/pg_basebackup/nls.mk
@@ -1,12 +1,12 @@
 # src/bin/pg_basebackup/nls.mk
 CATALOG_NAME     = pg_basebackup
 GETTEXT_FILES    = $(FRONTEND_COMMON_GETTEXT_FILES) \
-                   bbstreamer_file.c \
-                   bbstreamer_gzip.c \
-                   bbstreamer_inject.c \
-                   bbstreamer_lz4.c \
-                   bbstreamer_tar.c \
-                   bbstreamer_zstd.c \
+                   astreamer_file.c \
+                   astreamer_gzip.c \
+                   astreamer_inject.c \
+                   astreamer_lz4.c \
+                   astreamer_tar.c \
+                   astreamer_zstd.c \
                    pg_basebackup.c \
                    pg_createsubscriber.c \
                    pg_receivewal.c \
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 8f3dd04fd22..4179b064cbc 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -26,8 +26,8 @@
 #endif
 
 #include "access/xlog_internal.h"
+#include "astreamer.h"
 #include "backup/basebackup.h"
-#include "bbstreamer.h"
 #include "common/compression.h"
 #include "common/file_perm.h"
 #include "common/file_utils.h"
@@ -57,8 +57,8 @@ typedef struct ArchiveStreamState
 {
 	int			tablespacenum;
 	pg_compress_specification *compress;
-	bbstreamer *streamer;
-	bbstreamer *manifest_inject_streamer;
+	astreamer *streamer;
+	astreamer *manifest_inject_streamer;
 	PQExpBuffer manifest_buffer;
 	char		manifest_filename[MAXPGPATH];
 	FILE	   *manifest_file;
@@ -67,7 +67,7 @@ typedef struct ArchiveStreamState
 typedef struct WriteTarState
 {
 	int			tablespacenum;
-	bbstreamer *streamer;
+	astreamer *streamer;
 } WriteTarState;
 
 typedef struct WriteManifestState
@@ -199,8 +199,8 @@ static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *fo
 static void progress_update_filename(const char *filename);
 static void progress_report(int tablespacenum, bool force, bool finished);
 
-static bbstreamer *CreateBackupStreamer(char *archive_name, char *spclocation,
-										bbstreamer **manifest_inject_streamer_p,
+static astreamer *CreateBackupStreamer(char *archive_name, char *spclocation,
+										astreamer **manifest_inject_streamer_p,
 										bool is_recovery_guc_supported,
 										bool expect_unterminated_tarfile,
 										pg_compress_specification *compress);
@@ -1053,19 +1053,19 @@ ReceiveCopyData(PGconn *conn, WriteDataCallback callback,
  * the options selected by the user.  We may just write the results directly
  * to a file, or we might compress first, or we might extract the tar file
  * and write each member separately. This function doesn't do any of that
- * directly, but it works out what kind of bbstreamer we need to create so
+ * directly, but it works out what kind of astreamer we need to create so
  * that the right stuff happens when, down the road, we actually receive
  * the data.
  */
-static bbstreamer *
+static astreamer *
 CreateBackupStreamer(char *archive_name, char *spclocation,
-					 bbstreamer **manifest_inject_streamer_p,
+					 astreamer **manifest_inject_streamer_p,
 					 bool is_recovery_guc_supported,
 					 bool expect_unterminated_tarfile,
 					 pg_compress_specification *compress)
 {
-	bbstreamer *streamer = NULL;
-	bbstreamer *manifest_inject_streamer = NULL;
+	astreamer *streamer = NULL;
+	astreamer *manifest_inject_streamer = NULL;
 	bool		inject_manifest;
 	bool		is_tar,
 				is_tar_gz,
@@ -1160,7 +1160,7 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 			directory = psprintf("%s/%s", basedir, spclocation);
 		else
 			directory = get_tablespace_mapping(spclocation);
-		streamer = bbstreamer_extractor_new(directory,
+		streamer = astreamer_extractor_new(directory,
 											get_tablespace_mapping,
 											progress_update_filename);
 	}
@@ -1188,27 +1188,27 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 		}
 
 		if (compress->algorithm == PG_COMPRESSION_NONE)
-			streamer = bbstreamer_plain_writer_new(archive_filename,
+			streamer = astreamer_plain_writer_new(archive_filename,
 												   archive_file);
 		else if (compress->algorithm == PG_COMPRESSION_GZIP)
 		{
 			strlcat(archive_filename, ".gz", sizeof(archive_filename));
-			streamer = bbstreamer_gzip_writer_new(archive_filename,
+			streamer = astreamer_gzip_writer_new(archive_filename,
 												  archive_file, compress);
 		}
 		else if (compress->algorithm == PG_COMPRESSION_LZ4)
 		{
 			strlcat(archive_filename, ".lz4", sizeof(archive_filename));
-			streamer = bbstreamer_plain_writer_new(archive_filename,
+			streamer = astreamer_plain_writer_new(archive_filename,
 												   archive_file);
-			streamer = bbstreamer_lz4_compressor_new(streamer, compress);
+			streamer = astreamer_lz4_compressor_new(streamer, compress);
 		}
 		else if (compress->algorithm == PG_COMPRESSION_ZSTD)
 		{
 			strlcat(archive_filename, ".zst", sizeof(archive_filename));
-			streamer = bbstreamer_plain_writer_new(archive_filename,
+			streamer = astreamer_plain_writer_new(archive_filename,
 												   archive_file);
-			streamer = bbstreamer_zstd_compressor_new(streamer, compress);
+			streamer = astreamer_zstd_compressor_new(streamer, compress);
 		}
 		else
 		{
@@ -1222,7 +1222,7 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 		 * into it.
 		 */
 		if (must_parse_archive)
-			streamer = bbstreamer_tar_archiver_new(streamer);
+			streamer = astreamer_tar_archiver_new(streamer);
 		progress_update_filename(archive_filename);
 	}
 
@@ -1241,7 +1241,7 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 	if (spclocation == NULL && writerecoveryconf)
 	{
 		Assert(must_parse_archive);
-		streamer = bbstreamer_recovery_injector_new(streamer,
+		streamer = astreamer_recovery_injector_new(streamer,
 													is_recovery_guc_supported,
 													recoveryconfcontents);
 	}
@@ -1253,9 +1253,9 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 	 * we're talking to such a server we'll need to add the terminator here.
 	 */
 	if (must_parse_archive)
-		streamer = bbstreamer_tar_parser_new(streamer);
+		streamer = astreamer_tar_parser_new(streamer);
 	else if (expect_unterminated_tarfile)
-		streamer = bbstreamer_tar_terminator_new(streamer);
+		streamer = astreamer_tar_terminator_new(streamer);
 
 	/*
 	 * If the user has requested a server compressed archive along with
@@ -1264,11 +1264,11 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
 	if (format == 'p')
 	{
 		if (is_tar_gz)
-			streamer = bbstreamer_gzip_decompressor_new(streamer);
+			streamer = astreamer_gzip_decompressor_new(streamer);
 		else if (is_tar_lz4)
-			streamer = bbstreamer_lz4_decompressor_new(streamer);
+			streamer = astreamer_lz4_decompressor_new(streamer);
 		else if (is_tar_zstd)
-			streamer = bbstreamer_zstd_decompressor_new(streamer);
+			streamer = astreamer_zstd_decompressor_new(streamer);
 	}
 
 	/* Return the results. */
@@ -1307,7 +1307,7 @@ ReceiveArchiveStream(PGconn *conn, pg_compress_specification *compress)
 	if (state.manifest_inject_streamer != NULL &&
 		state.manifest_buffer != NULL)
 	{
-		bbstreamer_inject_file(state.manifest_inject_streamer,
+		astreamer_inject_file(state.manifest_inject_streamer,
 							   "backup_manifest",
 							   state.manifest_buffer->data,
 							   state.manifest_buffer->len);
@@ -1318,8 +1318,8 @@ ReceiveArchiveStream(PGconn *conn, pg_compress_specification *compress)
 	/* If there's still an archive in progress, end processing. */
 	if (state.streamer != NULL)
 	{
-		bbstreamer_finalize(state.streamer);
-		bbstreamer_free(state.streamer);
+		astreamer_finalize(state.streamer);
+		astreamer_free(state.streamer);
 		state.streamer = NULL;
 	}
 }
@@ -1383,8 +1383,8 @@ ReceiveArchiveStreamChunk(size_t r, char *copybuf, void *callback_data)
 				/* End processing of any prior archive. */
 				if (state->streamer != NULL)
 				{
-					bbstreamer_finalize(state->streamer);
-					bbstreamer_free(state->streamer);
+					astreamer_finalize(state->streamer);
+					astreamer_free(state->streamer);
 					state->streamer = NULL;
 				}
 
@@ -1437,8 +1437,8 @@ ReceiveArchiveStreamChunk(size_t r, char *copybuf, void *callback_data)
 				else if (state->streamer != NULL)
 				{
 					/* Archive data. */
-					bbstreamer_content(state->streamer, NULL, copybuf + 1,
-									   r - 1, BBSTREAMER_UNKNOWN);
+					astreamer_content(state->streamer, NULL, copybuf + 1,
+									   r - 1, ASTREAMER_UNKNOWN);
 				}
 				else
 					pg_fatal("unexpected payload data");
@@ -1600,7 +1600,7 @@ ReceiveTarFile(PGconn *conn, char *archive_name, char *spclocation,
 			   bool tablespacenum, pg_compress_specification *compress)
 {
 	WriteTarState state;
-	bbstreamer *manifest_inject_streamer;
+	astreamer *manifest_inject_streamer;
 	bool		is_recovery_guc_supported;
 	bool		expect_unterminated_tarfile;
 
@@ -1636,7 +1636,7 @@ ReceiveTarFile(PGconn *conn, char *archive_name, char *spclocation,
 			pg_fatal("out of memory");
 
 		/* Inject it into the output tarfile. */
-		bbstreamer_inject_file(manifest_inject_streamer, "backup_manifest",
+		astreamer_inject_file(manifest_inject_streamer, "backup_manifest",
 							   buf.data, buf.len);
 
 		/* Free memory. */
@@ -1644,8 +1644,8 @@ ReceiveTarFile(PGconn *conn, char *archive_name, char *spclocation,
 	}
 
 	/* Cleanup. */
-	bbstreamer_finalize(state.streamer);
-	bbstreamer_free(state.streamer);
+	astreamer_finalize(state.streamer);
+	astreamer_free(state.streamer);
 
 	progress_report(tablespacenum, true, false);
 
@@ -1663,7 +1663,7 @@ ReceiveTarCopyChunk(size_t r, char *copybuf, void *callback_data)
 {
 	WriteTarState *state = callback_data;
 
-	bbstreamer_content(state->streamer, NULL, copybuf, r, BBSTREAMER_UNKNOWN);
+	astreamer_content(state->streamer, NULL, copybuf, r, ASTREAMER_UNKNOWN);
 
 	totaldone += r;
 	progress_report(state->tablespacenum, false, false);
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 8de9978ad8d..ba9e0200b3f 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3311,19 +3311,19 @@ bbsink_shell
 bbsink_state
 bbsink_throttle
 bbsink_zstd
-bbstreamer
-bbstreamer_archive_context
-bbstreamer_extractor
-bbstreamer_gzip_decompressor
-bbstreamer_gzip_writer
-bbstreamer_lz4_frame
-bbstreamer_member
-bbstreamer_ops
-bbstreamer_plain_writer
-bbstreamer_recovery_injector
-bbstreamer_tar_archiver
-bbstreamer_tar_parser
-bbstreamer_zstd_frame
+astreamer
+astreamer_archive_context
+astreamer_extractor
+astreamer_gzip_decompressor
+astreamer_gzip_writer
+astreamer_lz4_frame
+astreamer_member
+astreamer_ops
+astreamer_plain_writer
+astreamer_recovery_injector
+astreamer_tar_archiver
+astreamer_tar_parser
+astreamer_zstd_frame
 bgworker_main_type
 bh_node_type
 binaryheap
-- 
2.18.0

From 2ae018686a1e1b9b2ada8735e4e35f214b8a75bd Mon Sep 17 00:00:00 2001
From: Robert Haas <rhaas@postgresql.org>
Date: Mon, 5 Aug 2024 12:45:58 -0400
Subject: [PATCH v6] Improve file header comments for astramer code.

Make it clear that "astreamer" stands for "archive streamer".
Generalize comments that still believe this code can only be used
by pg_basebackup. Add some comments explaining the asymmetry
between the gzip, lz4, and zstd astreamers, in the hopes of making
life easier for anyone who hacks on this code in the future.
---
 src/fe_utils/astreamer_file.c    |  4 ++++
 src/fe_utils/astreamer_gzip.c    | 15 +++++++++++++++
 src/fe_utils/astreamer_lz4.c     |  4 ++++
 src/fe_utils/astreamer_zstd.c    |  4 ++++
 src/include/fe_utils/astreamer.h | 21 +++++++++++++++------
 5 files changed, 42 insertions(+), 6 deletions(-)

diff --git a/src/fe_utils/astreamer_file.c b/src/fe_utils/astreamer_file.c
index 13d1192c6e..e75d166ebf 100644
--- a/src/fe_utils/astreamer_file.c
+++ b/src/fe_utils/astreamer_file.c
@@ -2,6 +2,10 @@
  *
  * astreamer_file.c
  *
+ * Archive streamers that write to files. astreamer_plan_writer writes
+ * the whole archive to a single file, and astreamer_extractor writes
+ * each archive member to a separate file in a given directory.
+ *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
diff --git a/src/fe_utils/astreamer_gzip.c b/src/fe_utils/astreamer_gzip.c
index dd28defac7..1c773a2384 100644
--- a/src/fe_utils/astreamer_gzip.c
+++ b/src/fe_utils/astreamer_gzip.c
@@ -2,6 +2,21 @@
  *
  * astreamer_gzip.c
  *
+ * Archive streamers that deal with data compressed using gzip.
+ * astreamer_gzip_writer applies gzip compression to the input data
+ * and writes the result to a file. astreamer_gzip_decompressor assumes
+ * that the input stream is compressed using gzip and decompresses it.
+ *
+ * Note that the code in this file is asymmetric with what we do for
+ * other compression types: for lz4 and zstd, there is a compressor and
+ * a decompressor, rather than a writer and a decompressor. The approach
+ * taken here is less flexible, because a writer can only write to a file,
+ * while a compressor can write to a subsequent astreamer which is free
+ * to do whatever it likes. The reason it's like this is because this
+ * code was adapated from old, less-modular pg_basebackup that used the
+ * same APIs that astreamer_gzip_writer uses, and it didn't seem
+ * necessary to change anything at the time.
+ *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
diff --git a/src/fe_utils/astreamer_lz4.c b/src/fe_utils/astreamer_lz4.c
index d8b2a367e4..2bf14084e7 100644
--- a/src/fe_utils/astreamer_lz4.c
+++ b/src/fe_utils/astreamer_lz4.c
@@ -2,6 +2,10 @@
  *
  * astreamer_lz4.c
  *
+ * Archive streamers that deal with data compressed using lz4.
+ * astreamer_lz4_compressor applies lz4 compression to the input stream,
+ * and astreamer_lz4_decompressor does the reverse.
+ *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
diff --git a/src/fe_utils/astreamer_zstd.c b/src/fe_utils/astreamer_zstd.c
index 45f6cb6736..4b2d42b231 100644
--- a/src/fe_utils/astreamer_zstd.c
+++ b/src/fe_utils/astreamer_zstd.c
@@ -2,6 +2,10 @@
  *
  * astreamer_zstd.c
  *
+ * Archive streamers that deal with data compressed using zstd.
+ * astreamer_zstd_compressor applies lz4 compression to the input stream,
+ * and astreamer_zstd_decompressor does the reverse.
+ *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
diff --git a/src/include/fe_utils/astreamer.h b/src/include/fe_utils/astreamer.h
index 2c014dbddb..570cfba304 100644
--- a/src/include/fe_utils/astreamer.h
+++ b/src/include/fe_utils/astreamer.h
@@ -2,9 +2,18 @@
  *
  * astreamer.h
  *
- * Each tar archive returned by the server is passed to one or more
- * astreamer objects for further processing. The astreamer may do
- * something simple, like write the archive to a file, perhaps after
+ * The "archive streamer" interface is intended to allow frontend code
+ * to stream from possibly-compressed archive files from any source and
+ * perform arbitrary actions based on the contents of those archives.
+ * Archive streamers are intended to be composable, and most tasks will
+ * require two or more archive streamers to complete. For instance,
+ * if the input is an uncompressed tar stream, a tar parser astreamer
+ * could be used to interpret it, and then an extractor astreamer could
+ * be used to write each archive member out to a file.
+ *
+ * In general, each archive streamer is relatively free to take whatever
+ * action it desires in the stream of chunks provided by the caller. It
+ * may do something simple, like write the archive to a file, perhaps after
  * compressing it, but it can also do more complicated things, like
  * annotating the byte stream to indicate which parts of the data
  * correspond to tar headers or trailing padding, vs. which parts are
@@ -33,9 +42,9 @@ typedef struct astreamer_ops astreamer_ops;
 
 /*
  * Each chunk of archive data passed to a astreamer is classified into one
- * of these categories. When data is first received from the remote server,
- * each chunk will be categorized as ASTREAMER_UNKNOWN, and the chunks will
- * be of whatever size the remote server chose to send.
+ * of these categories. When data is initially passed to an archive streamer,
+ * each chunk will be categorized as ASTREAMER_UNKNOWN, and the chunks can
+ * be of whatever size the caller finds convenient.
  *
  * If the archive is parsed (e.g. see astreamer_tar_parser_new()), then all
  * chunks should be labelled as one of the other types listed here. In
-- 
2.39.3 (Apple Git-145)

From 975b994d8ca76f3c16b7eb9119e45ec25dba410b Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 10:26:35 +0530
Subject: [PATCH v7 10/12] pg_verifybackup: Add backup format and compression
 option

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the next patch that implements tar format support.
----
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 146 +++++++++++++++++++++-
 1 file changed, 144 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 801e13886c2..f20b6e2895c 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,6 +18,7 @@
 #include <sys/stat.h>
 #include <time.h>
 
+#include "common/compression.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
@@ -56,6 +57,8 @@ static void report_manifest_error(JsonManifestParseContext *context,
 								  const char *fmt,...)
 			pg_attribute_printf(2, 3) pg_attribute_noreturn();
 
+static char find_backup_format(verifier_context *context);
+static pg_compress_algorithm find_backup_compression(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
@@ -74,6 +77,9 @@ static void usage(void);
 
 static const char *progname;
 
+static char format = '\0';		/* p(lain)/t(ar) */
+static pg_compress_algorithm compress_algorithm = PG_COMPRESSION_NONE;
+
 /*
  * Main entry point.
  */
@@ -84,11 +90,13 @@ main(int argc, char **argv)
 		{"exit-on-error", no_argument, NULL, 'e'},
 		{"ignore", required_argument, NULL, 'i'},
 		{"manifest-path", required_argument, NULL, 'm'},
+		{"format", required_argument, NULL, 'F'},
 		{"no-parse-wal", no_argument, NULL, 'n'},
 		{"progress", no_argument, NULL, 'P'},
 		{"quiet", no_argument, NULL, 'q'},
 		{"skip-checksums", no_argument, NULL, 's'},
 		{"wal-directory", required_argument, NULL, 'w'},
+		{"compress", required_argument, NULL, 'Z'},
 		{NULL, 0, NULL, 0}
 	};
 
@@ -99,6 +107,7 @@ main(int argc, char **argv)
 	bool		quiet = false;
 	char	   *wal_directory = NULL;
 	char	   *pg_waldump_path = NULL;
+	bool		tar_compression_specified = false;
 
 	pg_logging_init(argv[0]);
 	set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_verifybackup"));
@@ -141,7 +150,7 @@ main(int argc, char **argv)
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
 
-	while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1)
+	while ((c = getopt_long(argc, argv, "eF:i:m:nPqsw:Z:", long_options, NULL)) != -1)
 	{
 		switch (c)
 		{
@@ -160,6 +169,15 @@ main(int argc, char **argv)
 				manifest_path = pstrdup(optarg);
 				canonicalize_path(manifest_path);
 				break;
+			case 'F':
+				if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
+					format = 'p';
+				else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
+					format = 't';
+				else
+					pg_fatal("invalid backup format \"%s\", must be \"plain\" or \"tar\"",
+							 optarg);
+				break;
 			case 'n':
 				no_parse_wal = true;
 				break;
@@ -176,6 +194,12 @@ main(int argc, char **argv)
 				wal_directory = pstrdup(optarg);
 				canonicalize_path(wal_directory);
 				break;
+			case 'Z':
+				if (!parse_compress_algorithm(optarg, &compress_algorithm))
+					pg_fatal("unrecognized compression algorithm: \"%s\"",
+							 optarg);
+				tar_compression_specified = true;
+				break;
 			default:
 				/* getopt_long already emitted a complaint */
 				pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -207,11 +231,41 @@ main(int argc, char **argv)
 		pg_fatal("cannot specify both %s and %s",
 				 "-P/--progress", "-q/--quiet");
 
+	/* Complain if compression method specified but the format isn't tar. */
+	if (format != 't' && tar_compression_specified)
+	{
+		pg_log_error("only tar mode backups can be compressed");
+		pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+		exit(1);
+	}
+
+	/* Determine the backup format if it hasn't been specified. */
+	if (format == '\0')
+		format = find_backup_format(&context);
+
+	/*
+	 * Determine the tar backup compression method if it hasn't been
+	 * specified.
+	 */
+	if (format == 't' && !tar_compression_specified)
+		compress_algorithm = find_backup_compression(&context);
+
 	/* Unless --no-parse-wal was specified, we will need pg_waldump. */
 	if (!no_parse_wal)
 	{
 		int			ret;
 
+		/*
+		 * XXX: In the future, we should consider enhancing pg_waldump to read
+		 * WAL files from the tar archive.
+		 */
+		if (format != 'p')
+		{
+			pg_log_error("pg_waldump does not support parsing WAL files from a tar archive.");
+			pg_log_error_hint("Try \"%s --help\" to skip parse WAL files option.", progname);
+			exit(1);
+		}
+
 		pg_waldump_path = pg_malloc(MAXPGPATH);
 		ret = find_other_exec(argv[0], "pg_waldump",
 							  "pg_waldump (PostgreSQL) " PG_VERSION "\n",
@@ -279,7 +333,15 @@ main(int argc, char **argv)
 	 */
 	if (!context.skip_checksums)
 	{
-		verify_backup_checksums(&context);
+		/*
+		 * We were only checking the plain backup here. For the tar backup,
+		 * file checksums verification (if requested) will be done immediately
+		 * when the file is accessed, as we don't have random access to the
+		 * files like we do with plain backups.
+		 */
+		if (format == 'p')
+			verify_backup_checksums(&context);
+
 		progress_report(&context, true);
 	}
 
@@ -972,6 +1034,84 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
 	return false;
 }
 
+/*
+ * To detect the backup format, it checks for the PG_VERSION file in the backup
+ * directory. If found, it will be considered a plain-format backup; otherwise,
+ * it will be assumed to be a tar-format backup.
+ */
+static char
+find_backup_format(verifier_context *context)
+{
+	char		result;
+	char	   *path;
+	struct stat sb;
+
+	/* Should be here only if the backup format is unknown */
+	Assert(format == '\0');
+
+	/* Check PG_VERSION file. */
+	path = psprintf("%s/%s", context->backup_directory, "PG_VERSION");
+	result = (stat(path, &sb) == 0) ? 'p' : 't';
+	pfree(path);
+
+	return result;
+}
+
+/*
+ * To determine the compression format, we will search for the main data
+ * directory archive and its extension, which starts with base.tar, as
+ * pg_basebackup writes the main data directory to an archive file named
+ * base.tar followed by a compression type extension like .gz, .lz4, or .zst.
+ */
+static pg_compress_algorithm
+find_backup_compression(verifier_context *context)
+{
+	char	   *path;
+	struct stat sb;
+	bool		found;
+
+	/* Should be here only for tar backup */
+	Assert(format == 't');
+
+	/*
+	 * Is this a tar archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_NONE;
+
+	/*
+	 * Is this a .tar.gz archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar.gz");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_GZIP;
+
+	/*
+	 * Is this a .tar.lz4 archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar.lz4");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_LZ4;
+
+	/*
+	 * Is this a .tar.zst archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar.zst");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_ZSTD;
+
+	return PG_COMPRESSION_NONE; /* placate compiler */
+}
+
 /*
  * Print a progress report based on the variables in verifier_context.
  *
@@ -1054,11 +1194,13 @@ usage(void)
 	printf(_("  -e, --exit-on-error         exit immediately on error\n"));
 	printf(_("  -i, --ignore=RELATIVE_PATH  ignore indicated path\n"));
 	printf(_("  -m, --manifest-path=PATH    use specified path for manifest\n"));
+	printf(_("  -F, --format=p|t            backup format (plain, tar)\n"));
 	printf(_("  -n, --no-parse-wal          do not try to parse WAL files\n"));
 	printf(_("  -P, --progress              show progress information\n"));
 	printf(_("  -q, --quiet                 do not print any output, except for errors\n"));
 	printf(_("  -s, --skip-checksums        skip checksum verification\n"));
 	printf(_("  -w, --wal-directory=PATH    use specified path for WAL files\n"));
+	printf(_("  -Z, --compress=METHOD       compress method (gzip, lz4, zstd, none) \n"));
 	printf(_("  -V, --version               output version information, then exit\n"));
 	printf(_("  -?, --help                  show this help, then exit\n"));
 	printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
-- 
2.18.0

From c8ca272edb9cc433f7eb680876f5947b98fe010f Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 17:04:56 +0530
Subject: [PATCH v7 12/12] pg_verifybackup: Tests and document

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the previous patch that implements tar format support.
----
---
 doc/src/sgml/ref/pg_verifybackup.sgml         | 54 +++++++++++++-
 src/bin/pg_verifybackup/t/001_basic.pl        | 18 ++++-
 src/bin/pg_verifybackup/t/004_options.pl      |  2 +-
 src/bin/pg_verifybackup/t/005_bad_manifest.pl |  2 +-
 src/bin/pg_verifybackup/t/008_untar.pl        | 73 ++++++-------------
 src/bin/pg_verifybackup/t/010_client_untar.pl | 48 +-----------
 6 files changed, 96 insertions(+), 101 deletions(-)

diff --git a/doc/src/sgml/ref/pg_verifybackup.sgml b/doc/src/sgml/ref/pg_verifybackup.sgml
index a3f167f9f6e..c743bd89a92 100644
--- a/doc/src/sgml/ref/pg_verifybackup.sgml
+++ b/doc/src/sgml/ref/pg_verifybackup.sgml
@@ -34,8 +34,10 @@ PostgreSQL documentation
    integrity of a database cluster backup taken using
    <command>pg_basebackup</command> against a
    <literal>backup_manifest</literal> generated by the server at the time
-   of the backup.  The backup must be stored in the "plain"
-   format; a "tar" format backup can be checked after extracting it.
+   of the backup.  The backup must be stored in the "plain" or "tar"
+   format.  Verification is supported for <literal>gzip</literal>,
+   <literal>lz4</literal>, and  <literal>zstd</literal> compressed tar backup;
+   any other compressed format backups can be checked after decompressing them.
   </para>
 
   <para>
@@ -168,6 +170,42 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-F <replaceable class="parameter">format</replaceable></option></term>
+      <term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
+      <listitem>
+       <para>
+        Specifies the format of the backup. <replaceable>format</replaceable>
+        can be one of the following:
+
+        <variablelist>
+         <varlistentry>
+          <term><literal>p</literal></term>
+          <term><literal>plain</literal></term>
+          <listitem>
+           <para>
+            Backup consists of plain files with the same layout as the
+            source server's data directory and tablespaces.
+           </para>
+          </listitem>
+         </varlistentry>
+
+         <varlistentry>
+          <term><literal>t</literal></term>
+          <term><literal>tar</literal></term>
+          <listitem>
+           <para>
+            Backup consists of tar files. The main data directory's contents
+            will be written to a file named <filename>base.tar</filename>,
+            and each other tablespace will be written to a separate tar file
+            named after that tablespace's OID.
+           </para>
+           </listitem>
+         </varlistentry>
+        </variablelist></para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-n</option></term>
       <term><option>--no-parse-wal</option></term>
@@ -227,6 +265,18 @@ PostgreSQL documentation
        </para>
       </listitem>
      </varlistentry>
+
+     <varlistentry>
+      <term><option>-Z <replaceable class="parameter">method</replaceable></option></term>
+      <term><option>--compress=<replaceable class="parameter">method</replaceable></option></term>
+      <listitem>
+       <para>
+        The tar backup compression method can be <literal>gzip</literal>,
+        <literal>lz4</literal>, <literal>zstd</literal>, or
+        <literal>none</literal> if no compression.
+       </para>
+      </listitem>
+     </varlistentry>
     </variablelist>
    </para>
 
diff --git a/src/bin/pg_verifybackup/t/001_basic.pl b/src/bin/pg_verifybackup/t/001_basic.pl
index 2f3e52d296f..d47ce1f04fc 100644
--- a/src/bin/pg_verifybackup/t/001_basic.pl
+++ b/src/bin/pg_verifybackup/t/001_basic.pl
@@ -17,13 +17,25 @@ command_fails_like(
 	qr/no backup directory specified/,
 	'target directory must be specified');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir ],
 	qr/could not open file.*\/backup_manifest\"/,
 	'pg_verifybackup requires a manifest');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir, $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir, $tempdir ],
 	qr/too many command-line arguments/,
 	'multiple target directories not allowed');
+command_fails_like(
+	[ 'pg_verifybackup', '-Zgzip', $tempdir ],
+	qr/only tar mode backups can be compressed/,
+	'compression method required format option');
+command_fails_like(
+	[ 'pg_verifybackup', '-Fp', '-Zlz4', $tempdir ],
+	qr/only tar mode backups can be compressed/,
+	'compression method required tar format option');
+command_fails_like(
+	[ 'pg_verifybackup', '-Fp', '-Znon_exist', $tempdir ],
+	qr/unrecognized compression algorithm/,
+	'compression method should be valid');
 
 # create fake manifest file
 open(my $fh, '>', "$tempdir/backup_manifest") || die "open: $!";
@@ -31,7 +43,7 @@ close($fh);
 
 # but then try to use an alternate, nonexisting manifest
 command_fails_like(
-	[ 'pg_verifybackup', '-m', "$tempdir/not_the_manifest", $tempdir ],
+	[ 'pg_verifybackup', '-Fp', '-m', "$tempdir/not_the_manifest", $tempdir ],
 	qr/could not open file.*\/not_the_manifest\"/,
 	'pg_verifybackup respects -m flag');
 
diff --git a/src/bin/pg_verifybackup/t/004_options.pl b/src/bin/pg_verifybackup/t/004_options.pl
index 8ed2214408e..2f197648740 100644
--- a/src/bin/pg_verifybackup/t/004_options.pl
+++ b/src/bin/pg_verifybackup/t/004_options.pl
@@ -108,7 +108,7 @@ unlike(
 # Test valid manifest with nonexistent backup directory.
 command_fails_like(
 	[
-		'pg_verifybackup', '-m',
+		'pg_verifybackup', '-Fp', '-m',
 		"$backup_path/backup_manifest", "$backup_path/fake"
 	],
 	qr/could not open directory/,
diff --git a/src/bin/pg_verifybackup/t/005_bad_manifest.pl b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
index c4ed64b62d5..28c51b6feb0 100644
--- a/src/bin/pg_verifybackup/t/005_bad_manifest.pl
+++ b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
@@ -208,7 +208,7 @@ sub test_bad_manifest
 	print $fh $manifest_contents;
 	close($fh);
 
-	command_fails_like([ 'pg_verifybackup', $tempdir ], $regexp, $test_name);
+	command_fails_like([ 'pg_verifybackup', '-Fp', $tempdir ], $regexp, $test_name);
 	return;
 }
 
diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
index 7a09f3b75b2..9896560adc3 100644
--- a/src/bin/pg_verifybackup/t/008_untar.pl
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -16,6 +16,20 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 
+# Create a couple of directories to use as tablespaces.
+my $TS1_LOCATION = $primary->backup_dir .'/ts1';
+$TS1_LOCATION =~ s/\/\.\//\//g;    # collapse foo/./bar to foo/bar
+mkdir($TS1_LOCATION);
+
+# Create a tablespace with table in it.
+$primary->safe_psql('postgres', qq(
+		CREATE TABLESPACE regress_ts1 LOCATION '$TS1_LOCATION';
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1';
+		CREATE TABLE regress_tbl1(i int) TABLESPACE regress_ts1;
+		INSERT INTO regress_tbl1 VALUES(generate_series(1,5));));
+my $tsoid = $primary->safe_psql('postgres', qq(
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1'));
+
 my $backup_path = $primary->backup_dir . '/server-backup';
 my $extract_path = $primary->backup_dir . '/extracted-backup';
 
@@ -23,39 +37,31 @@ my @test_configuration = (
 	{
 		'compression_method' => 'none',
 		'backup_flags' => [],
-		'backup_archive' => 'base.tar',
+		'backup_archive' => ['base.tar', "$tsoid.tar"],
 		'enabled' => 1
 	},
 	{
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'server-gzip' ],
-		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.gz', "$tsoid.tar.gz" ],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'server-lz4' ],
-		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => [ '-d', '-m' ],
+		'backup_archive' => ['base.tar.lz4', "$tsoid.tar.lz4" ],
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd:level=1,long' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	});
 
@@ -86,47 +92,16 @@ for my $tc (@test_configuration)
 		my $backup_files = join(',',
 			sort grep { $_ ne '.' && $_ ne '..' } slurp_dir($backup_path));
 		my $expected_backup_files =
-		  join(',', sort ('backup_manifest', $tc->{'backup_archive'}));
+		  join(',', sort ('backup_manifest', @{ $tc->{'backup_archive'} }));
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok(['pg_verifybackup', '-n', '-e', $backup_path],
+			"verify backup, compression $method");
 
 		# Cleanup.
-		unlink($backup_path . '/backup_manifest');
-		unlink($backup_path . '/base.tar');
-		unlink($backup_path . '/' . $tc->{'backup_archive'});
+		rmtree($backup_path);
 		rmtree($extract_path);
 	}
 }
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index 8c076d46dee..6b7d7483f6e 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -29,41 +29,30 @@ my @test_configuration = (
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'client-gzip:5' ],
 		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'client-lz4:5' ],
 		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => ['-d'],
-		'output_file' => 'base.tar',
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:5' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:level=1,long' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'parallel zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:workers=3' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1"),
 		'possibly_unsupported' =>
 		  qr/could not set compression worker count to 3: Unsupported parameter/
@@ -118,40 +107,9 @@ for my $tc (@test_configuration)
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			push @decompress, $backup_path . '/' . $tc->{'output_file'}
-			  if $tc->{'output_file'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok( [ 'pg_verifybackup', '-n', '-e', $backup_path ],
+			"verify backup, compression $method");
 
 		# Cleanup.
 		rmtree($extract_path);
-- 
2.18.0

From c9415e97006e5765151d7fcd3990bcb0c4a05966 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 31 Jul 2024 16:22:07 +0530
Subject: [PATCH v7 11/12] pg_verifybackup: Read tar files and verify its
 contents

---
 src/bin/pg_verifybackup/Makefile           |   4 +-
 src/bin/pg_verifybackup/astreamer_verify.c | 367 +++++++++++++++++++++
 src/bin/pg_verifybackup/meson.build        |   6 +-
 src/bin/pg_verifybackup/pg_verifybackup.c  | 216 +++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h  |   9 +
 src/tools/pgindent/typedefs.list           |   1 +
 6 files changed, 594 insertions(+), 9 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/astreamer_verify.c

diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile
index 7c045f142e8..df7aaabd530 100644
--- a/src/bin/pg_verifybackup/Makefile
+++ b/src/bin/pg_verifybackup/Makefile
@@ -17,11 +17,13 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 # We need libpq only because fe_utils does.
+override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 
 OBJS = \
 	$(WIN32RES) \
-	pg_verifybackup.o
+	pg_verifybackup.o \
+	astreamer_verify.o
 
 all: pg_verifybackup
 
diff --git a/src/bin/pg_verifybackup/astreamer_verify.c b/src/bin/pg_verifybackup/astreamer_verify.c
new file mode 100644
index 00000000000..be40922c042
--- /dev/null
+++ b/src/bin/pg_verifybackup/astreamer_verify.c
@@ -0,0 +1,367 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer_verify.c
+ *
+ * Extend fe_utils/astreamer.h archive streaming facility to verify TAR
+ * backup.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * src/bin/pg_verifybackup/astreamer_verify.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include "common/logging.h"
+#include "fe_utils/astreamer.h"
+#include "pg_verifybackup.h"
+
+typedef struct astreamer_verify
+{
+	astreamer	base;
+	verifier_context *context;
+	char	   *archiveName;
+	Oid			tblspcOid;
+
+	/* Hold information for a member file verification */
+	manifest_file *mfile;
+	int64		receivedBytes;
+	bool		verifyChecksums;
+	bool		verifyControlData;
+	pg_checksum_context *checksum_ctx;
+} astreamer_verify;
+
+static void astreamer_verify_content(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len,
+									 astreamer_archive_context context);
+static void astreamer_verify_finalize(astreamer *streamer);
+static void astreamer_verify_free(astreamer *streamer);
+
+static void verify_member_header(astreamer *streamer, astreamer_member *member);
+static void verify_member_contents(astreamer *streamer,
+								   astreamer_member *member,
+								   const char *data, int len);
+static void verify_content_checksum(astreamer *streamer,
+									astreamer_member *member,
+									const char *buffer, int buffer_len);
+static void verify_controldata(astreamer *streamer,
+							   astreamer_member *member,
+							   const char *data, int len);
+static void reset_member_info(astreamer *streamer);
+
+static const astreamer_ops astreamer_verify_ops = {
+	.content = astreamer_verify_content,
+	.finalize = astreamer_verify_finalize,
+	.free = astreamer_verify_free
+};
+
+/*
+ * Create a astreamer that can verifies content of a TAR file.
+ */
+astreamer *
+astreamer_verify_content_new(astreamer *next, verifier_context *context,
+							 char *archive_name, Oid tblspc_oid)
+{
+	astreamer_verify *streamer;
+
+	streamer = palloc0(sizeof(astreamer_verify));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_verify_ops;
+
+	streamer->base.bbs_next = next;
+	streamer->context = context;
+	streamer->archiveName = archive_name;
+	streamer->tblspcOid = tblspc_oid;
+	initStringInfo(&streamer->base.bbs_buffer);
+
+	return &streamer->base;
+}
+
+/*
+ * It verifies each TAR member entry against the manifest data and performs
+ * checksum verification if enabled. Additionally, it validates the backup's
+ * system identifier against the backup_manifest.
+ */
+static void
+astreamer_verify_content(astreamer *streamer,
+						 astreamer_member *member, const char *data,
+						 int len, astreamer_archive_context context)
+{
+	Assert(context != ASTREAMER_UNKNOWN);
+
+	switch (context)
+	{
+		case ASTREAMER_MEMBER_HEADER:
+
+			/*
+			 * Perform the initial check and setup for the verification.
+			 */
+			verify_member_header(streamer, member);
+			break;
+
+		case ASTREAMER_MEMBER_CONTENTS:
+
+			/*
+			 * Peform the required contants verification.
+			 */
+			verify_member_contents(streamer, member, data, len);
+			break;
+
+		case ASTREAMER_MEMBER_TRAILER:
+
+			/*
+			 * Reset the temporary information stored for a verification.
+			 */
+			reset_member_info(streamer);
+			break;
+
+		case ASTREAMER_ARCHIVE_TRAILER:
+			break;
+
+		default:
+			/* Shouldn't happen. */
+			pg_fatal("unexpected state while parsing tar archive");
+	}
+}
+
+/*
+ * End-of-stream processing for a astreamer_verify stream.
+ */
+static void
+astreamer_verify_finalize(astreamer *streamer)
+{
+	Assert(streamer->bbs_next == NULL);
+}
+
+/*
+ * Free memory associated with a astreamer_verify stream.
+ */
+static void
+astreamer_verify_free(astreamer *streamer)
+{
+	pfree(streamer->bbs_buffer.data);
+	pfree(streamer);
+}
+
+/*
+ * Verify the entry if it is a file in the backup manifest. If the archive being
+ * processed is a tablespace, prepare the required file path for subsequent
+ * operations. Finally, check if it needs to perform checksum verification and
+ * control data verification during file content processing.
+ */
+static void
+verify_member_header(astreamer *streamer, astreamer_member *member)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_file *m;
+
+	/* We are only interested in files that are not in the ignore list. */
+	if (member->is_directory || member->is_link ||
+		should_ignore_relpath(mystreamer->context, member->pathname))
+		return;
+
+	/*
+	 * The backup_manifest stores a relative path to the base directory for
+	 * files belong tablespace, whereas <tablespaceoid>.tar doesn't. Prepare
+	 * the required path, otherwise, the manfiest entry verification will
+	 * fail.
+	 */
+	if (OidIsValid(mystreamer->tblspcOid))
+	{
+		char		temp[MAXPGPATH];
+
+		/* Copy original name at temporary space */
+		memcpy(temp, member->pathname, MAXPGPATH);
+
+		snprintf(member->pathname, MAXPGPATH, "%s/%d/%s",
+				 "pg_tblspc", mystreamer->tblspcOid, temp);
+	}
+
+	/* Check the manifest entry */
+	m = verify_manifest_entry(mystreamer->context, member->pathname,
+							  member->size);
+	mystreamer->mfile = (void *) m;
+
+	/*
+	 * Prepare for checksum and manifest system identifier verification.
+	 *
+	 * We could have these checks while receiving contents. However, since
+	 * contents are received in multiple iterations, this would result in
+	 * these lengthy checks being performed multiple times. Instead, having a
+	 * single flag would be more efficient.
+	 */
+	mystreamer->verifyChecksums =
+		(!mystreamer->context->skip_checksums && should_verify_checksum(m));
+	mystreamer->verifyControlData =
+		should_verify_control_data(mystreamer->context->manifest, m);
+}
+
+/*
+ * Process the member content according to the flags set by the member header
+ * processing routine for checksum and control data verification.
+ */
+static void
+verify_member_contents(astreamer *streamer, astreamer_member *member,
+					   const char *data, int len)
+
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	/* Verify the checksums */
+	if (mystreamer->verifyChecksums)
+		verify_content_checksum(streamer, member, data, len);
+
+	/* Verify pg_control information */
+	if (mystreamer->verifyControlData)
+		verify_controldata(streamer, member, data, len);
+}
+
+/*
+ * Similar to verify_file_checksum() but this function computes the checksum
+ * incrementally for the received file content. Unlike a normal backup
+ * directory, TAR format files do not allow random access, so checksum
+ * verification occurs progressively. Additionally, the function calls the
+ * routine for control data verification if the flags indicate that it is
+ * required.
+ *
+ * On the first visit, the function initializes checksum_ctx, which will be used
+ * for incremental checksum calculation. Once the complete file content is
+ * received (tracked using the receivedBytes), the routine that performs the
+ * final checksum verification is called
+ */
+static void
+verify_content_checksum(astreamer *streamer, astreamer_member *member,
+						const char *buffer, int buffer_len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	pg_checksum_context *checksum_ctx = mystreamer->checksum_ctx;
+	verifier_context *context = mystreamer->context;
+	manifest_file *m = mystreamer->mfile;
+	const char *relpath = m->pathname;
+	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
+
+	/*
+	 * Mark it false to avoid unexpected re-entrance for the same file content
+	 * (e.g. returned in error should not be revisited).
+	 */
+	Assert(mystreamer->verifyChecksums);
+	mystreamer->verifyChecksums = false;
+
+	/* Should have came for the right file */
+	Assert(strcmp(member->pathname, relpath) == 0);
+
+	/* If we were first time for this file */
+	if (!checksum_ctx)
+	{
+		checksum_ctx = pg_malloc(sizeof(pg_checksum_context));
+		mystreamer->checksum_ctx = checksum_ctx;
+
+		if (pg_checksum_init(checksum_ctx, m->checksum_type) < 0)
+		{
+			report_backup_error(context,
+								"%s: could not initialize checksum of file \"%s\"",
+								mystreamer->archiveName, relpath);
+			return;
+		}
+	}
+
+	/* Update the total count of computed checksum bytes. */
+	mystreamer->receivedBytes += buffer_len;
+
+	if (pg_checksum_update(checksum_ctx, (uint8 *) buffer, buffer_len) < 0)
+	{
+		report_backup_error(context, "could not update checksum of file \"%s\"",
+							relpath);
+		return;
+	}
+
+	/* Report progress */
+	context->done_size += buffer_len;
+	progress_report(context, false);
+
+	/* Yet to receive the full content of the file. */
+	if (mystreamer->receivedBytes < m->size)
+	{
+		mystreamer->verifyChecksums = true;
+		return;
+	}
+
+	/* Do the final computation and verification. */
+	verify_checksum(context, m, checksum_ctx, checksumbuf);
+}
+
+/*
+ * Prepare the control data from the received file contents, which are supposed
+ * to be from the pg_control file, including CRC calculation. Then, call the
+ * routines that perform the final verification of the control file information.
+ */
+static void
+verify_controldata(astreamer *streamer, astreamer_member *member,
+				   const char *data, int len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_data *manifest = mystreamer->context->manifest;
+	ControlFileData control_file;
+	pg_crc32c	crc;
+	bool		crc_ok;
+
+	/* Should be here only for control file */
+	Assert(strcmp(member->pathname, "global/pg_control") == 0);
+	Assert(manifest->version != 1);
+
+	/* Mark it as false to avoid unexpected re-entrance */
+	Assert(mystreamer->verifyControlData);
+	mystreamer->verifyControlData = false;
+
+	/* Should have whole control file data. */
+	if (!astreamer_buffer_until(streamer, &data, &len, sizeof(ControlFileData)))
+	{
+		mystreamer->verifyControlData = true;
+		return;
+	}
+
+	pg_log_debug("%s: reading \"%s\"", mystreamer->archiveName,
+				 member->pathname);
+
+	if (streamer->bbs_buffer.len != sizeof(ControlFileData))
+		report_fatal_error("%s: could not read control file: read %d of %zu",
+						   mystreamer->archiveName, streamer->bbs_buffer.len,
+						   sizeof(ControlFileData));
+
+	memcpy(&control_file, streamer->bbs_buffer.data,
+		   sizeof(ControlFileData));
+
+	/* Check the CRC. */
+	INIT_CRC32C(crc);
+	COMP_CRC32C(crc,
+				(char *) (&control_file),
+				offsetof(ControlFileData, crc));
+	FIN_CRC32C(crc);
+
+	crc_ok = EQ_CRC32C(crc, control_file.crc);
+
+	/* Do the final control data verification. */
+	verify_control_data(&control_file, member->pathname, crc_ok,
+						manifest->system_identifier);
+}
+
+/*
+ * Reset flags and free memory allocations for member file verification.
+ */
+static void
+reset_member_info(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	mystreamer->mfile = NULL;
+	mystreamer->receivedBytes = 0;
+	mystreamer->verifyChecksums = false;
+	mystreamer->verifyControlData = false;
+
+	if (mystreamer->checksum_ctx)
+		pfree(mystreamer->checksum_ctx);
+	mystreamer->checksum_ctx = NULL;
+}
diff --git a/src/bin/pg_verifybackup/meson.build b/src/bin/pg_verifybackup/meson.build
index 7c7d31a0350..1e3fcf7ee5a 100644
--- a/src/bin/pg_verifybackup/meson.build
+++ b/src/bin/pg_verifybackup/meson.build
@@ -1,7 +1,8 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 pg_verifybackup_sources = files(
-  'pg_verifybackup.c'
+  'pg_verifybackup.c',
+  'astreamer_verify.c'
 )
 
 if host_system == 'windows'
@@ -10,9 +11,10 @@ if host_system == 'windows'
     '--FILEDESC', 'pg_verifybackup - verify a backup against using a backup manifest'])
 endif
 
+pg_verifybackup_deps = [frontend_code, libpq, lz4, zlib, zstd]
 pg_verifybackup = executable('pg_verifybackup',
   pg_verifybackup_sources,
-  dependencies: [frontend_code, libpq],
+  dependencies: pg_verifybackup_deps,
   kwargs: default_bin_args,
 )
 bin_targets += pg_verifybackup
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index f20b6e2895c..ce2ba7437be 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -20,9 +20,12 @@
 
 #include "common/compression.h"
 #include "common/parse_manifest.h"
+#include "common/relpath.h"
+#include "fe_utils/astreamer.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
 #include "pg_verifybackup.h"
+#include "pgtar.h"
 #include "pgtime.h"
 
 /*
@@ -39,6 +42,11 @@
  */
 #define ESTIMATED_BYTES_PER_MANIFEST_LINE	100
 
+
+static void (*verify_backup_file_cb) (verifier_context *context,
+									  char *relpath, char *fullpath,
+									  size_t filesize);
+
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -63,6 +71,15 @@ static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
 							   char *relpath, char *fullpath);
+static void verify_plain_file_cb(verifier_context *context,
+								 char *relpath, char *fullpath,
+								 size_t filesize);
+static void verify_tar_file_cb(verifier_context *context,
+							   char *relpath, char *fullpath,
+							   size_t filesize);
+static void verify_tar_content(verifier_context *context,
+							   char *relpath, char *fullpath,
+							   astreamer *streamer);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -71,6 +88,9 @@ static void verify_file_checksum(verifier_context *context,
 static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
+static astreamer *create_archive_verifier(verifier_context *context,
+										  char *archive_name,
+										  Oid tblspc_oid);
 
 static void compute_total_size(verifier_context *context);
 static void usage(void);
@@ -146,6 +166,10 @@ main(int argc, char **argv)
 	 */
 	simple_string_list_append(&context.ignore_list, "backup_manifest");
 	simple_string_list_append(&context.ignore_list, "pg_wal");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.gz");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.lz4");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.zst");
 	simple_string_list_append(&context.ignore_list, "postgresql.auto.conf");
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
@@ -250,6 +274,15 @@ main(int argc, char **argv)
 	if (format == 't' && !tar_compression_specified)
 		compress_algorithm = find_backup_compression(&context);
 
+	/*
+	 * Setup the required callback function to verify plain or tar backup
+	 * files.
+	 */
+	if (format == 'p')
+		verify_backup_file_cb = verify_plain_file_cb;
+	else
+		verify_backup_file_cb = verify_tar_file_cb;
+
 	/* Unless --no-parse-wal was specified, we will need pg_waldump. */
 	if (!no_parse_wal)
 	{
@@ -645,7 +678,8 @@ verify_backup_directory(verifier_context *context, char *relpath,
 }
 
 /*
- * Verify one file (which might actually be a directory or a symlink).
+ * Verify one file (which might actually be a directory, a symlink or a
+ * archive).
  *
  * The arguments to this function have the same meaning as the arguments to
  * verify_backup_directory.
@@ -654,7 +688,6 @@ static void
 verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 {
 	struct stat sb;
-	manifest_file *m;
 
 	if (stat(fullpath, &sb) != 0)
 	{
@@ -687,8 +720,25 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		return;
 	}
 
+	/* Do the further verifications */
+	verify_backup_file_cb(context, relpath, fullpath, sb.st_size);
+}
+
+/*
+ * Verify one plan file or a symlink.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_backup_directory. The additional argument is the file size for
+ * verifying against manifest entry.
+ */
+static void
+verify_plain_file_cb(verifier_context *context, char *relpath,
+					 char *fullpath, size_t filesize)
+{
+	manifest_file *m;
+
 	/* Check the backup manifest entry for this file. */
-	m = verify_manifest_entry(context, relpath, sb.st_size);
+	m = verify_manifest_entry(context, relpath, filesize);
 
 	/* Validate the manifest system identifier */
 	if (should_verify_control_data(context->manifest, m))
@@ -706,6 +756,124 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 	}
 }
 
+/*
+ * Verify one tar file.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_backup_directory. The additional argument is the file size for
+ * verifying against manifest entry.
+ */
+static void
+verify_tar_file_cb(verifier_context *context, char *relpath,
+				   char *fullpath, size_t filesize)
+{
+	astreamer  *streamer;
+	Oid			tblspc_oid = InvalidOid;
+	int			file_name_len;
+	int			file_extn_len = 0;	/* placate compiler */
+	char	   *file_extn = "";
+
+	/* Should be tar backup */
+	Assert(format == 't');
+
+	/* Find the tar file extension. */
+	if (compress_algorithm == PG_COMPRESSION_NONE)
+	{
+		file_extn = ".tar";
+		file_extn_len = 4;
+
+	}
+	else if (compress_algorithm == PG_COMPRESSION_GZIP)
+	{
+		file_extn = ".tar.gz";
+		file_extn_len = 7;
+
+	}
+	else if (compress_algorithm == PG_COMPRESSION_LZ4)
+	{
+		file_extn = ".tar.lz4";
+		file_extn_len = 8;
+	}
+	else if (compress_algorithm == PG_COMPRESSION_ZSTD)
+	{
+		file_extn = ".tar.zst";
+		file_extn_len = 8;
+	}
+
+	/*
+	 * Ensure that we have the correct file type corresponding to the backup
+	 * format.
+	 */
+	file_name_len = strlen(relpath);
+	if (file_name_len < file_extn_len ||
+		strcmp(relpath + file_name_len - file_extn_len, file_extn) != 0)
+	{
+		if (compress_algorithm == PG_COMPRESSION_NONE)
+			report_backup_error(context,
+								"\"%s\" is not a valid file, expecting tar file",
+								relpath);
+		else
+			report_backup_error(context,
+								"\"%s\" is not a valid file, expecting \"%s\" compressed tar file",
+								relpath,
+								get_compress_algorithm_name(compress_algorithm));
+		return;
+	}
+
+	/*
+	 * For the tablespace, pg_basebackup writes the data out to
+	 * <tablespaceoid>.tar.  If a file matches that format, then extract the
+	 * tablespaceoid, which we need to prepare the paths of the files
+	 * belonging to that tablespace relative to the base directory.
+	 */
+	if (strspn(relpath, "0123456789") == (file_name_len - file_extn_len))
+		tblspc_oid = strtoi64(relpath, NULL, 10);
+
+	streamer = create_archive_verifier(context, relpath, tblspc_oid);
+	verify_tar_content(context, relpath, fullpath, streamer);
+
+	/* Cleanup. */
+	astreamer_finalize(streamer);
+	astreamer_free(streamer);
+}
+
+/*
+ * Reads a given tar file in predefined chunks and pass to astreamer.  Which
+ * initiates routines for decompression (if necessary) then verification
+ * of each member within the tar archive.
+ */
+static void
+verify_tar_content(verifier_context *context, char *relpath, char *fullpath,
+				   astreamer *streamer)
+{
+	int			fd;
+	int			rc;
+	char	   *buffer;
+
+	/* Open the target file. */
+	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
+	{
+		report_backup_error(context, "could not open file \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
+
+	/* Perform the reads */
+	while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
+		astreamer_content(streamer, NULL, buffer, rc, ASTREAMER_UNKNOWN);
+
+	if (rc < 0)
+		report_backup_error(context, "could not read file \"%s\": %m",
+							relpath);
+
+	/* Close the file. */
+	if (close(fd) != 0)
+		report_backup_error(context, "could not close file \"%s\": %m",
+							relpath);
+}
+
 /*
  * Verify file and its size entry in the manifest.
  */
@@ -1058,10 +1226,10 @@ find_backup_format(verifier_context *context)
 }
 
 /*
- * To determine the compression format, we will search for the main data
- * directory archive and its extension, which starts with base.tar, as
  * pg_basebackup writes the main data directory to an archive file named
- * base.tar followed by a compression type extension like .gz, .lz4, or .zst.
+ * base.tar, followed by a compression type extension such as .gz, .lz4, or
+ * .zst. To determine the compression format, we need to search for this main
+ * data directory archive file.
  */
 static pg_compress_algorithm
 find_backup_compression(verifier_context *context)
@@ -1112,6 +1280,42 @@ find_backup_compression(verifier_context *context)
 	return PG_COMPRESSION_NONE; /* placate compiler */
 }
 
+/*
+ * Identifies the necessary steps for verifying the contents of the
+ * provided tar file.
+ */
+static astreamer *
+create_archive_verifier(verifier_context *context, char *archive_name,
+						Oid tblspc_oid)
+{
+	astreamer  *streamer = NULL;
+
+	/* Should be here only for tar backup */
+	Assert(format == 't');
+
+	/*
+	 * To verify the contents of the tar file, the initial step is to parse
+	 * its content.
+	 */
+	streamer = astreamer_verify_content_new(streamer, context, archive_name,
+											tblspc_oid);
+	streamer = astreamer_tar_parser_new(streamer);
+
+	/*
+	 * If the tar file is compressed, we must perform the appropriate
+	 * decompression operation before proceeding with the verification of its
+	 * contents.
+	 */
+	if (compress_algorithm == PG_COMPRESSION_GZIP)
+		streamer = astreamer_gzip_decompressor_new(streamer);
+	else if (compress_algorithm == PG_COMPRESSION_LZ4)
+		streamer = astreamer_lz4_decompressor_new(streamer);
+	else if (compress_algorithm == PG_COMPRESSION_ZSTD)
+		streamer = astreamer_zstd_decompressor_new(streamer);
+
+	return streamer;
+}
+
 /*
  * Print a progress report based on the variables in verifier_context.
  *
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index c88f71ff14b..f0a7c8918fb 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -137,4 +137,13 @@ extern bool should_ignore_relpath(verifier_context *context,
 
 extern void progress_report(verifier_context *context, bool finished);
 
+/* Forward declarations to avoid fe_utils/astreamer.h include. */
+struct astreamer;
+typedef struct astreamer astreamer;
+
+extern astreamer *astreamer_verify_content_new(astreamer *next,
+											   verifier_context *context,
+											   char *archive_name,
+											   Oid tblspc_oid);
+
 #endif							/* PG_VERIFYBACKUP_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 547d14b3e7c..d86b28b260e 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3329,6 +3329,7 @@ astreamer_plain_writer
 astreamer_recovery_injector
 astreamer_tar_archiver
 astreamer_tar_parser
+astreamer_verify
 astreamer_zstd_frame
 bgworker_main_type
 bh_node_type
-- 
2.18.0

From 44a78699dacfa90f36a37c8868ad11a50d53cb12 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 15:47:26 +0530
Subject: [PATCH v7 08/12] Refactor: split verify_control_file.

Separated the manifest entry verification code into a new function and
introduced the should_verify_control_data() macro, similar to
should_verify_checksum().

Note that should_verify_checksum() has been slightly modified to
include a NULL check for its argument, maintaining the same code
structure as should_verify_control_data().
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 44 +++++++++++------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 18 +++++++++-
 2 files changed, 37 insertions(+), 25 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 3eddaa2468e..5f055a23a63 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,7 +18,6 @@
 #include <sys/stat.h>
 #include <time.h>
 
-#include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
@@ -61,8 +60,6 @@ static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
 							   char *relpath, char *fullpath);
-static void verify_control_file(const char *controlpath,
-								uint64 manifest_system_identifier);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -625,14 +622,20 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 	/* Check the backup manifest entry for this file. */
 	m = verify_manifest_entry(context, relpath, sb.st_size);
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0 &&
-		m != NULL && m->matched && !m->bad)
-		verify_control_file(fullpath, context->manifest->system_identifier);
+	/* Validate the manifest system identifier */
+	if (should_verify_control_data(context->manifest, m))
+	{
+		ControlFileData *control_file;
+		bool		crc_ok;
+
+		pg_log_debug("reading \"%s\"", fullpath);
+		control_file = get_controlfile_by_exact_path(fullpath, &crc_ok);
+
+		verify_control_data(control_file, fullpath, crc_ok,
+							context->manifest->system_identifier);
+		/* Release memory. */
+		pfree(control_file);
+	}
 }
 
 /*
@@ -676,18 +679,14 @@ verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
 }
 
 /*
- * Sanity check control file and validate system identifier against manifest
- * system identifier.
+ * Sanity check control file data and validate system identifier against
+ * manifest system identifier.
  */
-static void
-verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
+void
+verify_control_data(ControlFileData *control_file,
+					const char *controlpath, bool crc_ok,
+					uint64 manifest_system_identifier)
 {
-	ControlFileData *control_file;
-	bool		crc_ok;
-
-	pg_log_debug("reading \"%s\"", controlpath);
-	control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
-
 	/* Control file contents not meaningful if CRC is bad. */
 	if (!crc_ok)
 		report_fatal_error("%s: CRC is incorrect", controlpath);
@@ -703,9 +702,6 @@ verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
 						   controlpath,
 						   (unsigned long long) manifest_system_identifier,
 						   (unsigned long long) control_file->system_identifier);
-
-	/* Release memory. */
-	pfree(control_file);
 }
 
 /*
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 1bc5f7a6b4a..c88f71ff14b 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -16,6 +16,7 @@
 
 #include "common/controldata_utils.h"
 #include "common/hashfn_unstable.h"
+#include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 
@@ -44,7 +45,19 @@ typedef struct manifest_file
 } manifest_file;
 
 #define should_verify_checksum(m) \
-	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+	(((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+
+/*
+ * Validate the manifest system identifier against the control file; this
+ * feature is not available in manifest version 1.  This validation should be
+ * carried out only if the manifest entry validation is completed without any
+ * errors.
+ */
+#define should_verify_control_data(manifest, m) \
+	(((manifest)->version != 1) && \
+	 ((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (strcmp((m)->pathname, "global/pg_control") == 0))
 
 /*
  * Define a hash table which we can use to store information about the files
@@ -110,6 +123,9 @@ extern manifest_file *verify_manifest_entry(verifier_context *context,
 extern void verify_checksum(verifier_context *context, manifest_file *m,
 							pg_checksum_context *checksum_ctx,
 							uint8 *checksumbuf);
+extern void verify_control_data(ControlFileData *control_file,
+								const char *controlpath, bool crc_ok,
+								uint64 manifest_system_identifier);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From ee279a6a540a640857599951807438cda02f30d6 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Fri, 2 Aug 2024 16:37:38 +0530
Subject: [PATCH v7 09/12] Refactor: move first and last progress_report call
 to Main.

---
 src/bin/pg_verifybackup/pg_verifybackup.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 5f055a23a63..801e13886c2 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -253,7 +253,10 @@ main(int argc, char **argv)
 	 * read, which occurs only when checksum verification is enabled.
 	 */
 	if (!context.skip_checksums)
+	{
 		compute_total_size(&context);
+		progress_report(&context, false);
+	}
 
 	/*
 	 * Now scan the files in the backup directory. At this stage, we verify
@@ -275,7 +278,10 @@ main(int argc, char **argv)
 	 * told to skip it.
 	 */
 	if (!context.skip_checksums)
+	{
 		verify_backup_checksums(&context);
+		progress_report(&context, true);
+	}
 
 	/*
 	 * Try to parse the required ranges of WAL records, unless we were told
@@ -736,8 +742,6 @@ verify_backup_checksums(verifier_context *context)
 	manifest_file *m;
 	uint8	   *buffer;
 
-	progress_report(context, false);
-
 	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
 
 	manifest_files_start_iterate(manifest->files, &it);
@@ -761,8 +765,6 @@ verify_backup_checksums(verifier_context *context)
 	}
 
 	pfree(buffer);
-
-	progress_report(context, true);
 }
 
 /*
-- 
2.18.0

From 37148e750aa68092eaa3e13c0c46cb4978c3f67a Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 12:15:26 +0530
Subject: [PATCH v7 06/12] Refactor: split verify_backup_file() function.

Move the manifest entry verification code into a new function as
verify_manifest_entry(). And the total size computation code into
another new function, compute_total_size(), which is called from the
main.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 76 ++++++++++++++++++-----
 src/bin/pg_verifybackup/pg_verifybackup.h |  3 +
 2 files changed, 62 insertions(+), 17 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 4e42757c346..ab6bda8c9dc 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -72,6 +72,7 @@ static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
 
+static void compute_total_size(verifier_context *context);
 static void usage(void);
 
 static const char *progname;
@@ -250,6 +251,13 @@ main(int argc, char **argv)
 	 */
 	context.manifest = parse_manifest_file(manifest_path);
 
+	/*
+	 * For the progress report, compute the total size of the files to be
+	 * read, which occurs only when checksum verification is enabled.
+	 */
+	if (!context.skip_checksums)
+		compute_total_size(&context);
+
 	/*
 	 * Now scan the files in the backup directory. At this stage, we verify
 	 * that every file on disk is present in the manifest and that the sizes
@@ -614,6 +622,27 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		return;
 	}
 
+	/* Check the backup manifest entry for this file. */
+	m = verify_manifest_entry(context, relpath, sb.st_size);
+
+	/*
+	 * Validate the manifest system identifier, not available in manifest
+	 * version 1.
+	 */
+	if (context->manifest->version != 1 &&
+		strcmp(relpath, "global/pg_control") == 0 &&
+		m != NULL && m->matched && !m->bad)
+		verify_control_file(fullpath, context->manifest->system_identifier);
+}
+
+/*
+ * Verify file and its size entry in the manifest.
+ */
+manifest_file *
+verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
+{
+	manifest_file *m;
+
 	/* Check whether there's an entry in the manifest hash. */
 	m = manifest_files_lookup(context->manifest->files, relpath);
 	if (m == NULL)
@@ -621,40 +650,29 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		report_backup_error(context,
 							"\"%s\" is present on disk but not in the manifest",
 							relpath);
-		return;
+		return NULL;
 	}
 
 	/* Flag this entry as having been encountered in the filesystem. */
 	m->matched = true;
 
 	/* Check that the size matches. */
-	if (m->size != sb.st_size)
+	if (m->size != filesize)
 	{
 		report_backup_error(context,
 							"\"%s\" has size %lld on disk but size %zu in the manifest",
-							relpath, (long long int) sb.st_size, m->size);
+							relpath, (long long int) filesize, m->size);
 		m->bad = true;
 	}
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0)
-		verify_control_file(fullpath, context->manifest->system_identifier);
-
-	/* Update statistics for progress report, if necessary */
-	if (context->show_progress && !context->skip_checksums &&
-		should_verify_checksum(m))
-		context->total_size += m->size;
-
 	/*
 	 * We don't verify checksums at this stage. We first finish verifying that
 	 * we have the expected set of files with the expected sizes, and only
 	 * afterwards verify the checksums. That's because computing checksums may
 	 * take a while, and we'd like to report more obvious problems quickly.
 	 */
+
+	return m;
 }
 
 /*
@@ -817,7 +835,7 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 
 	/*
 	 * Double-check that we read the expected number of bytes from the file.
-	 * Normally, a file size mismatch would be caught in verify_backup_file
+	 * Normally, a file size mismatch would be caught in verify_manifest_entry
 	 * and this check would never be reached, but this provides additional
 	 * safety and clarity in the event of concurrent modifications or
 	 * filesystem misbehavior.
@@ -988,6 +1006,30 @@ progress_report(verifier_context *context, bool finished)
 	fputc((!finished && isatty(fileno(stderr))) ? '\r' : '\n', stderr);
 }
 
+/*
+ * Compute the total size of backup files for progress reporting.
+ */
+static void
+compute_total_size(verifier_context *context)
+{
+	manifest_data *manifest = context->manifest;
+	manifest_files_iterator it;
+	manifest_file *m;
+	uint64		total_size = 0;
+
+	if (!context->show_progress)
+		return;
+
+	manifest_files_start_iterate(manifest->files, &it);
+	while ((m = manifest_files_iterate(manifest->files, &it)) != NULL)
+	{
+		if (!should_ignore_relpath(context, m->pathname))
+			total_size += m->size;
+	}
+
+	context->total_size = total_size;
+}
+
 /*
  * Print out usage information and exit.
  */
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 90900048547..98c75916255 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -105,6 +105,9 @@ typedef struct verifier_context
 	uint64		done_size;
 } verifier_context;
 
+extern manifest_file *verify_manifest_entry(verifier_context *context,
+											char *relpath, int64 filesize);
+
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
 			pg_attribute_printf(2, 3);
-- 
2.18.0

From 6ec5031183fe3c42f0dbc69ba5374ea302116ef7 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 16:45:55 +0530
Subject: [PATCH v7 07/12] Refactor: split verify_file_checksum() function.

Move the core functionality of verify_file_checksum to a new function
to reuse it instead of duplicating the code.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 18 ++++++++++++++++--
 src/bin/pg_verifybackup/pg_verifybackup.h |  3 +++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index ab6bda8c9dc..3eddaa2468e 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -782,7 +782,6 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	int			rc;
 	size_t		bytes_read = 0;
 	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
-	int			checksumlen;
 
 	/* Open the target file. */
 	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
@@ -848,8 +847,23 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 		return;
 	}
 
+	/* Do the final computation and verification. */
+	verify_checksum(context, m, &checksum_ctx, checksumbuf);
+}
+
+/*
+ * A helper function to finalize checksum computation and verify it against the
+ * backup manifest information.
+ */
+void
+verify_checksum(verifier_context *context, manifest_file *m,
+				pg_checksum_context *checksum_ctx, uint8 *checksumbuf)
+{
+	int			checksumlen;
+	const char *relpath = m->pathname;
+
 	/* Get the final checksum. */
-	checksumlen = pg_checksum_final(&checksum_ctx, checksumbuf);
+	checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
 	if (checksumlen < 0)
 	{
 		report_backup_error(context,
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 98c75916255..1bc5f7a6b4a 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -107,6 +107,9 @@ typedef struct verifier_context
 
 extern manifest_file *verify_manifest_entry(verifier_context *context,
 											char *relpath, int64 filesize);
+extern void verify_checksum(verifier_context *context, manifest_file *m,
+							pg_checksum_context *checksum_ctx,
+							uint8 *checksumbuf);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From bb127a0c8f25d15ee9c10d111be7611ce58dcb39 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 12:10:34 +0530
Subject: [PATCH v7 05/12] Refactor: move some part of pg_verifybackup.c to
 pg_verifybackup.h

---
 src/bin/pg_verifybackup/pg_verifybackup.c | 102 +------------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 118 ++++++++++++++++++++++
 2 files changed, 123 insertions(+), 97 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/pg_verifybackup.h

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 71585ffc50e..4e42757c346 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,12 +18,11 @@
 #include <sys/stat.h>
 #include <time.h>
 
-#include "common/controldata_utils.h"
-#include "common/hashfn_unstable.h"
 #include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
+#include "pg_verifybackup.h"
 #include "pgtime.h"
 
 /*
@@ -40,89 +39,6 @@
  */
 #define ESTIMATED_BYTES_PER_MANIFEST_LINE	100
 
-/*
- * How many bytes should we try to read from a file at once?
- */
-#define READ_CHUNK_SIZE				(128 * 1024)
-
-/*
- * Each file described by the manifest file is parsed to produce an object
- * like this.
- */
-typedef struct manifest_file
-{
-	uint32		status;			/* hash status */
-	const char *pathname;
-	size_t		size;
-	pg_checksum_type checksum_type;
-	int			checksum_length;
-	uint8	   *checksum_payload;
-	bool		matched;
-	bool		bad;
-} manifest_file;
-
-#define should_verify_checksum(m) \
-	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
-
-/*
- * Define a hash table which we can use to store information about the files
- * mentioned in the backup manifest.
- */
-#define SH_PREFIX		manifest_files
-#define SH_ELEMENT_TYPE	manifest_file
-#define SH_KEY_TYPE		const char *
-#define	SH_KEY			pathname
-#define SH_HASH_KEY(tb, key)	hash_string(key)
-#define SH_EQUAL(tb, a, b)		(strcmp(a, b) == 0)
-#define	SH_SCOPE		static inline
-#define SH_RAW_ALLOCATOR	pg_malloc0
-#define SH_DECLARE
-#define SH_DEFINE
-#include "lib/simplehash.h"
-
-/*
- * Each WAL range described by the manifest file is parsed to produce an
- * object like this.
- */
-typedef struct manifest_wal_range
-{
-	TimeLineID	tli;
-	XLogRecPtr	start_lsn;
-	XLogRecPtr	end_lsn;
-	struct manifest_wal_range *next;
-	struct manifest_wal_range *prev;
-} manifest_wal_range;
-
-/*
- * All the data parsed from a backup_manifest file.
- */
-typedef struct manifest_data
-{
-	int			version;
-	uint64		system_identifier;
-	manifest_files_hash *files;
-	manifest_wal_range *first_wal_range;
-	manifest_wal_range *last_wal_range;
-} manifest_data;
-
-/*
- * All of the context information we need while checking a backup manifest.
- */
-typedef struct verifier_context
-{
-	manifest_data *manifest;
-	char	   *backup_directory;
-	SimpleStringList ignore_list;
-	bool		skip_checksums;
-	bool		exit_on_error;
-	bool		saw_any_error;
-
-	/* Progress indicators */
-	bool		show_progress;
-	uint64		total_size;
-	uint64		done_size;
-} verifier_context;
-
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -156,14 +72,6 @@ static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
 
-static void report_backup_error(verifier_context *context,
-								const char *pg_restrict fmt,...)
-			pg_attribute_printf(2, 3);
-static void report_fatal_error(const char *pg_restrict fmt,...)
-			pg_attribute_printf(1, 2) pg_attribute_noreturn();
-static bool should_ignore_relpath(verifier_context *context, const char *relpath);
-
-static void progress_report(verifier_context *context, bool finished);
 static void usage(void);
 
 static const char *progname;
@@ -978,7 +886,7 @@ parse_required_wal(verifier_context *context, char *pg_waldump_path,
  * Update the context to indicate that we saw an error, and exit if the
  * context says we should.
  */
-static void
+void
 report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
 {
 	va_list		ap;
@@ -995,7 +903,7 @@ report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
 /*
  * Report a fatal error and exit
  */
-static void
+void
 report_fatal_error(const char *pg_restrict fmt,...)
 {
 	va_list		ap;
@@ -1014,7 +922,7 @@ report_fatal_error(const char *pg_restrict fmt,...)
  * Note that by "prefix" we mean a parent directory; for this purpose,
  * "aa/bb" is not a prefix of "aa/bbb", but it is a prefix of "aa/bb/cc".
  */
-static bool
+bool
 should_ignore_relpath(verifier_context *context, const char *relpath)
 {
 	SimpleStringListCell *cell;
@@ -1043,7 +951,7 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
  * If finished is set to true, this is the last progress report. The cursor
  * is moved to the next line.
  */
-static void
+void
 progress_report(verifier_context *context, bool finished)
 {
 	static pg_time_t last_progress_report = 0;
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
new file mode 100644
index 00000000000..90900048547
--- /dev/null
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -0,0 +1,118 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_verifybackup.h
+ *	  Verify a backup against a backup manifest.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_verifybackup/pg_verifybackup.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef PG_VERIFYBACKUP_H
+#define PG_VERIFYBACKUP_H
+
+#include "common/controldata_utils.h"
+#include "common/hashfn_unstable.h"
+#include "common/parse_manifest.h"
+#include "fe_utils/simple_list.h"
+
+extern bool show_progress;
+extern bool skip_checksums;
+
+/*
+ * How many bytes should we try to read from a file at once?
+ */
+#define READ_CHUNK_SIZE				(128 * 1024)
+
+/*
+ * Each file described by the manifest file is parsed to produce an object
+ * like this.
+ */
+typedef struct manifest_file
+{
+	uint32		status;			/* hash status */
+	const char *pathname;
+	size_t		size;
+	pg_checksum_type checksum_type;
+	int			checksum_length;
+	uint8	   *checksum_payload;
+	bool		matched;
+	bool		bad;
+} manifest_file;
+
+#define should_verify_checksum(m) \
+	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+
+/*
+ * Define a hash table which we can use to store information about the files
+ * mentioned in the backup manifest.
+ */
+#define SH_PREFIX		manifest_files
+#define SH_ELEMENT_TYPE	manifest_file
+#define SH_KEY_TYPE		const char *
+#define	SH_KEY			pathname
+#define SH_HASH_KEY(tb, key)	hash_string(key)
+#define SH_EQUAL(tb, a, b)		(strcmp(a, b) == 0)
+#define	SH_SCOPE		static inline
+#define SH_RAW_ALLOCATOR	pg_malloc0
+#define SH_DECLARE
+#define SH_DEFINE
+#include "lib/simplehash.h"
+
+/*
+ * Each WAL range described by the manifest file is parsed to produce an
+ * object like this.
+ */
+typedef struct manifest_wal_range
+{
+	TimeLineID	tli;
+	XLogRecPtr	start_lsn;
+	XLogRecPtr	end_lsn;
+	struct manifest_wal_range *next;
+	struct manifest_wal_range *prev;
+} manifest_wal_range;
+
+/*
+ * All the data parsed from a backup_manifest file.
+ */
+typedef struct manifest_data
+{
+	int			version;
+	uint64		system_identifier;
+	manifest_files_hash *files;
+	manifest_wal_range *first_wal_range;
+	manifest_wal_range *last_wal_range;
+} manifest_data;
+
+/*
+ * All of the context information we need while checking a backup manifest.
+ */
+typedef struct verifier_context
+{
+	manifest_data *manifest;
+	char	   *backup_directory;
+	SimpleStringList ignore_list;
+	bool		skip_checksums;
+	bool		exit_on_error;
+	bool		saw_any_error;
+
+	/* Progress indicators */
+	bool		show_progress;
+	uint64		total_size;
+	uint64		done_size;
+} verifier_context;
+
+extern void report_backup_error(verifier_context *context,
+								const char *pg_restrict fmt,...)
+			pg_attribute_printf(2, 3);
+extern void report_fatal_error(const char *pg_restrict fmt,...)
+			pg_attribute_printf(1, 2) pg_attribute_noreturn();
+extern bool should_ignore_relpath(verifier_context *context,
+								  const char *relpath);
+
+extern void progress_report(verifier_context *context, bool finished);
+
+#endif							/* PG_VERIFYBACKUP_H */
-- 
2.18.0

From 0ef14ab4be362f6ab48c6ebd501d3036ba4d21d9 Mon Sep 17 00:00:00 2001
From: Robert Haas <rhaas@postgresql.org>
Date: Tue, 6 Aug 2024 10:23:45 +0530
Subject: [PATCH v7] Improve file header comments for astramer code.

Make it clear that "astreamer" stands for "archive streamer".
Generalize comments that still believe this code can only be used
by pg_basebackup. Add some comments explaining the asymmetry
between the gzip, lz4, and zstd astreamers, in the hopes of making
life easier for anyone who hacks on this code in the future.
---
 src/fe_utils/astreamer_file.c    |  4 ++++
 src/fe_utils/astreamer_gzip.c    | 15 +++++++++++++++
 src/fe_utils/astreamer_lz4.c     |  4 ++++
 src/fe_utils/astreamer_zstd.c    |  4 ++++
 src/include/fe_utils/astreamer.h | 21 +++++++++++++++------
 5 files changed, 42 insertions(+), 6 deletions(-)

diff --git a/src/fe_utils/astreamer_file.c b/src/fe_utils/astreamer_file.c
index 13d1192c6e6..c9a030853bc 100644
--- a/src/fe_utils/astreamer_file.c
+++ b/src/fe_utils/astreamer_file.c
@@ -2,6 +2,10 @@
  *
  * astreamer_file.c
  *
+ * Archive streamers that write to files. astreamer_plain_writer writes
+ * the whole archive to a single file, and astreamer_extractor writes
+ * each archive member to a separate file in a given directory.
+ *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
diff --git a/src/fe_utils/astreamer_gzip.c b/src/fe_utils/astreamer_gzip.c
index dd28defac7b..1c773a23848 100644
--- a/src/fe_utils/astreamer_gzip.c
+++ b/src/fe_utils/astreamer_gzip.c
@@ -2,6 +2,21 @@
  *
  * astreamer_gzip.c
  *
+ * Archive streamers that deal with data compressed using gzip.
+ * astreamer_gzip_writer applies gzip compression to the input data
+ * and writes the result to a file. astreamer_gzip_decompressor assumes
+ * that the input stream is compressed using gzip and decompresses it.
+ *
+ * Note that the code in this file is asymmetric with what we do for
+ * other compression types: for lz4 and zstd, there is a compressor and
+ * a decompressor, rather than a writer and a decompressor. The approach
+ * taken here is less flexible, because a writer can only write to a file,
+ * while a compressor can write to a subsequent astreamer which is free
+ * to do whatever it likes. The reason it's like this is because this
+ * code was adapated from old, less-modular pg_basebackup that used the
+ * same APIs that astreamer_gzip_writer uses, and it didn't seem
+ * necessary to change anything at the time.
+ *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
diff --git a/src/fe_utils/astreamer_lz4.c b/src/fe_utils/astreamer_lz4.c
index d8b2a367e47..2bf14084e7f 100644
--- a/src/fe_utils/astreamer_lz4.c
+++ b/src/fe_utils/astreamer_lz4.c
@@ -2,6 +2,10 @@
  *
  * astreamer_lz4.c
  *
+ * Archive streamers that deal with data compressed using lz4.
+ * astreamer_lz4_compressor applies lz4 compression to the input stream,
+ * and astreamer_lz4_decompressor does the reverse.
+ *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
diff --git a/src/fe_utils/astreamer_zstd.c b/src/fe_utils/astreamer_zstd.c
index 45f6cb67363..4b2d42b2311 100644
--- a/src/fe_utils/astreamer_zstd.c
+++ b/src/fe_utils/astreamer_zstd.c
@@ -2,6 +2,10 @@
  *
  * astreamer_zstd.c
  *
+ * Archive streamers that deal with data compressed using zstd.
+ * astreamer_zstd_compressor applies lz4 compression to the input stream,
+ * and astreamer_zstd_decompressor does the reverse.
+ *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
diff --git a/src/include/fe_utils/astreamer.h b/src/include/fe_utils/astreamer.h
index 2c014dbddbe..570cfba3040 100644
--- a/src/include/fe_utils/astreamer.h
+++ b/src/include/fe_utils/astreamer.h
@@ -2,9 +2,18 @@
  *
  * astreamer.h
  *
- * Each tar archive returned by the server is passed to one or more
- * astreamer objects for further processing. The astreamer may do
- * something simple, like write the archive to a file, perhaps after
+ * The "archive streamer" interface is intended to allow frontend code
+ * to stream from possibly-compressed archive files from any source and
+ * perform arbitrary actions based on the contents of those archives.
+ * Archive streamers are intended to be composable, and most tasks will
+ * require two or more archive streamers to complete. For instance,
+ * if the input is an uncompressed tar stream, a tar parser astreamer
+ * could be used to interpret it, and then an extractor astreamer could
+ * be used to write each archive member out to a file.
+ *
+ * In general, each archive streamer is relatively free to take whatever
+ * action it desires in the stream of chunks provided by the caller. It
+ * may do something simple, like write the archive to a file, perhaps after
  * compressing it, but it can also do more complicated things, like
  * annotating the byte stream to indicate which parts of the data
  * correspond to tar headers or trailing padding, vs. which parts are
@@ -33,9 +42,9 @@ typedef struct astreamer_ops astreamer_ops;
 
 /*
  * Each chunk of archive data passed to a astreamer is classified into one
- * of these categories. When data is first received from the remote server,
- * each chunk will be categorized as ASTREAMER_UNKNOWN, and the chunks will
- * be of whatever size the remote server chose to send.
+ * of these categories. When data is initially passed to an archive streamer,
+ * each chunk will be categorized as ASTREAMER_UNKNOWN, and the chunks can
+ * be of whatever size the caller finds convenient.
  *
  * If the archive is parsed (e.g. see astreamer_tar_parser_new()), then all
  * chunks should be labelled as one of the other types listed here. In
-- 
2.18.0

From 43fb489aecb872cc6f9a59ebbdca9c5a1110ea72 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 31 Jul 2024 11:43:52 +0530
Subject: [PATCH v7 04/12] Refactor: move few global variable to
 verifier_context struct

Global variables are:
	1. show_progress
	2. skip_checksums
	3. total_size
	4. done_size
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 50 +++++++++++------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index d77e70fbe38..71585ffc50e 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -113,8 +113,14 @@ typedef struct verifier_context
 	manifest_data *manifest;
 	char	   *backup_directory;
 	SimpleStringList ignore_list;
+	bool		skip_checksums;
 	bool		exit_on_error;
 	bool		saw_any_error;
+
+	/* Progress indicators */
+	bool		show_progress;
+	uint64		total_size;
+	uint64		done_size;
 } verifier_context;
 
 static manifest_data *parse_manifest_file(char *manifest_path);
@@ -157,19 +163,11 @@ static void report_fatal_error(const char *pg_restrict fmt,...)
 			pg_attribute_printf(1, 2) pg_attribute_noreturn();
 static bool should_ignore_relpath(verifier_context *context, const char *relpath);
 
-static void progress_report(bool finished);
+static void progress_report(verifier_context *context, bool finished);
 static void usage(void);
 
 static const char *progname;
 
-/* options */
-static bool show_progress = false;
-static bool skip_checksums = false;
-
-/* Progress indicators */
-static uint64 total_size = 0;
-static uint64 done_size = 0;
-
 /*
  * Main entry point.
  */
@@ -260,13 +258,13 @@ main(int argc, char **argv)
 				no_parse_wal = true;
 				break;
 			case 'P':
-				show_progress = true;
+				context.show_progress = true;
 				break;
 			case 'q':
 				quiet = true;
 				break;
 			case 's':
-				skip_checksums = true;
+				context.skip_checksums = true;
 				break;
 			case 'w':
 				wal_directory = pstrdup(optarg);
@@ -299,7 +297,7 @@ main(int argc, char **argv)
 	}
 
 	/* Complain if the specified arguments conflict */
-	if (show_progress && quiet)
+	if (context.show_progress && quiet)
 		pg_fatal("cannot specify both %s and %s",
 				 "-P/--progress", "-q/--quiet");
 
@@ -363,7 +361,7 @@ main(int argc, char **argv)
 	 * Now do the expensive work of verifying file checksums, unless we were
 	 * told to skip it.
 	 */
-	if (!skip_checksums)
+	if (!context.skip_checksums)
 		verify_backup_checksums(&context);
 
 	/*
@@ -739,8 +737,9 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		verify_control_file(fullpath, context->manifest->system_identifier);
 
 	/* Update statistics for progress report, if necessary */
-	if (show_progress && !skip_checksums && should_verify_checksum(m))
-		total_size += m->size;
+	if (context->show_progress && !context->skip_checksums &&
+		should_verify_checksum(m))
+		context->total_size += m->size;
 
 	/*
 	 * We don't verify checksums at this stage. We first finish verifying that
@@ -815,7 +814,7 @@ verify_backup_checksums(verifier_context *context)
 	manifest_file *m;
 	uint8	   *buffer;
 
-	progress_report(false);
+	progress_report(context, false);
 
 	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
 
@@ -841,7 +840,7 @@ verify_backup_checksums(verifier_context *context)
 
 	pfree(buffer);
 
-	progress_report(true);
+	progress_report(context, true);
 }
 
 /*
@@ -889,8 +888,8 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 		}
 
 		/* Report progress */
-		done_size += rc;
-		progress_report(false);
+		context->done_size += rc;
+		progress_report(context, false);
 	}
 	if (rc < 0)
 		report_backup_error(context, "could not read file \"%s\": %m",
@@ -1036,7 +1035,7 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
 }
 
 /*
- * Print a progress report based on the global variables.
+ * Print a progress report based on the variables in verifier_context.
  *
  * Progress report is written at maximum once per second, unless the finished
  * parameter is set to true.
@@ -1045,7 +1044,7 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
  * is moved to the next line.
  */
 static void
-progress_report(bool finished)
+progress_report(verifier_context *context, bool finished)
 {
 	static pg_time_t last_progress_report = 0;
 	pg_time_t	now;
@@ -1053,7 +1052,7 @@ progress_report(bool finished)
 	char		totalsize_str[32];
 	char		donesize_str[32];
 
-	if (!show_progress)
+	if (!context->show_progress)
 		return;
 
 	now = time(NULL);
@@ -1061,12 +1060,13 @@ progress_report(bool finished)
 		return;					/* Max once per second */
 
 	last_progress_report = now;
-	percent_size = total_size ? (int) ((done_size * 100 / total_size)) : 0;
+	percent_size = context->total_size ?
+		(int) ((context->done_size * 100 / context->total_size)) : 0;
 
 	snprintf(totalsize_str, sizeof(totalsize_str), UINT64_FORMAT,
-			 total_size / 1024);
+			 context->total_size / 1024);
 	snprintf(donesize_str, sizeof(donesize_str), UINT64_FORMAT,
-			 done_size / 1024);
+			 context->done_size / 1024);
 
 	fprintf(stderr,
 			_("%*s/%s kB (%d%%) verified"),
-- 
2.18.0

From 6b6192100ff02e04e50d9400a0d2b14216f6615e Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 10:26:35 +0530
Subject: [PATCH v8 10/12] pg_verifybackup: Add backup format and compression
 option

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the next patch that implements tar format support.
----
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 77 ++++++++++++++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h |  1 +
 2 files changed, 76 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index cb4094c8138..8b92b26f4d7 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,6 +18,7 @@
 #include <sys/stat.h>
 #include <time.h>
 
+#include "common/compression.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
@@ -56,6 +57,7 @@ static void report_manifest_error(JsonManifestParseContext *context,
 								  const char *fmt,...)
 			pg_attribute_printf(2, 3) pg_attribute_noreturn();
 
+static char find_backup_format(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
@@ -84,6 +86,7 @@ main(int argc, char **argv)
 		{"exit-on-error", no_argument, NULL, 'e'},
 		{"ignore", required_argument, NULL, 'i'},
 		{"manifest-path", required_argument, NULL, 'm'},
+		{"format", required_argument, NULL, 'F'},
 		{"no-parse-wal", no_argument, NULL, 'n'},
 		{"progress", no_argument, NULL, 'P'},
 		{"quiet", no_argument, NULL, 'q'},
@@ -105,6 +108,7 @@ main(int argc, char **argv)
 	progname = get_progname(argv[0]);
 
 	memset(&context, 0, sizeof(context));
+	context.format = '\0';
 
 	if (argc > 1)
 	{
@@ -141,7 +145,7 @@ main(int argc, char **argv)
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
 
-	while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1)
+	while ((c = getopt_long(argc, argv, "eF:i:m:nPqsw:", long_options, NULL)) != -1)
 	{
 		switch (c)
 		{
@@ -160,6 +164,15 @@ main(int argc, char **argv)
 				manifest_path = pstrdup(optarg);
 				canonicalize_path(manifest_path);
 				break;
+			case 'F':
+				if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
+					context.format = 'p';
+				else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
+					context.format = 't';
+				else
+					pg_fatal("invalid backup format \"%s\", must be \"plain\" or \"tar\"",
+							 optarg);
+				break;
 			case 'n':
 				no_parse_wal = true;
 				break;
@@ -207,11 +220,26 @@ main(int argc, char **argv)
 		pg_fatal("cannot specify both %s and %s",
 				 "-P/--progress", "-q/--quiet");
 
+	/* Determine the backup format if it hasn't been specified. */
+	if (context.format == '\0')
+		context.format = find_backup_format(&context);
+
 	/* Unless --no-parse-wal was specified, we will need pg_waldump. */
 	if (!no_parse_wal)
 	{
 		int			ret;
 
+		/*
+		 * XXX: In the future, we should consider enhancing pg_waldump to read
+		 * WAL files from the tar archive.
+		 */
+		if (context.format != 'p')
+		{
+			pg_log_error("pg_waldump does not support parsing WAL files from a tar archive.");
+			pg_log_error_hint("Try \"%s --help\" to skip parse WAL files option.", progname);
+			exit(1);
+		}
+
 		pg_waldump_path = pg_malloc(MAXPGPATH);
 		ret = find_other_exec(argv[0], "pg_waldump",
 							  "pg_waldump (PostgreSQL) " PG_VERSION "\n",
@@ -279,7 +307,15 @@ main(int argc, char **argv)
 	 */
 	if (!context.skip_checksums)
 	{
-		verify_backup_checksums(&context);
+		/*
+		 * We were only checking the plain backup here. For the tar backup,
+		 * file checksums verification (if requested) will be done immediately
+		 * when the file is accessed, as we don't have random access to the
+		 * files like we do with plain backups.
+		 */
+		if (context.format == 'p')
+			verify_backup_checksums(&context);
+
 		progress_report(&context, true);
 	}
 
@@ -972,6 +1008,42 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
 	return false;
 }
 
+/*
+ * To detect the backup format, it checks for the PG_VERSION file in the backup
+ * directory. If found, it will be considered a plain-format backup; otherwise,
+ * it will be assumed to be a tar-format backup.
+ */
+static char
+find_backup_format(verifier_context *context)
+{
+	char		result;
+	char	   *path;
+	struct stat sb;
+
+	/* Should be here only if the backup format is unknown */
+	Assert(context->format == '\0');
+
+	/* Check PG_VERSION file. */
+	path = psprintf("%s/%s", context->backup_directory, "PG_VERSION");
+	if (stat(path, &sb) == 0)
+		result = 'p';
+	else
+	{
+		if (errno != ENOENT)
+		{
+			pg_log_error("cannot determine backup format");
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+
+		/* Otherwise, it is assumed to be a TAR backup. */
+		result = 't';
+	}
+	pfree(path);
+
+	return result;
+}
+
 /*
  * Print a progress report based on the variables in verifier_context.
  *
@@ -1060,6 +1132,7 @@ usage(void)
 	printf(_("  -e, --exit-on-error         exit immediately on error\n"));
 	printf(_("  -i, --ignore=RELATIVE_PATH  ignore indicated path\n"));
 	printf(_("  -m, --manifest-path=PATH    use specified path for manifest\n"));
+	printf(_("  -F, --format=p|t            backup format (plain, tar)\n"));
 	printf(_("  -n, --no-parse-wal          do not try to parse WAL files\n"));
 	printf(_("  -P, --progress              show progress information\n"));
 	printf(_("  -q, --quiet                 do not print any output, except for errors\n"));
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 8a4046b0e33..fcef972d9ad 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -107,6 +107,7 @@ typedef struct verifier_context
 	manifest_data *manifest;
 	char	   *backup_directory;
 	SimpleStringList ignore_list;
+	char		format;			/* backup format:  p(lain)/t(ar) */
 	bool		skip_checksums;
 	bool		exit_on_error;
 	bool		saw_any_error;
-- 
2.18.0

From 83357e1fcccb433a5e4c343232f66431998e9e59 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Fri, 2 Aug 2024 16:37:38 +0530
Subject: [PATCH v8 09/12] Refactor: move first and last progress_report call
 to Main.

The progress_report() is currently called at the start and end of
verify_backup_checksums(), which is used only for plain backups. Since
we also need to report progress for TAR backups, the progress_report()
has been moved from verify_backup_checksums() to a common
location.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index d518f995298..cb4094c8138 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -253,7 +253,10 @@ main(int argc, char **argv)
 	 * read, which occurs only when checksum verification is enabled.
 	 */
 	if (!context.skip_checksums)
+	{
 		compute_total_size(&context);
+		progress_report(&context, false);
+	}
 
 	/*
 	 * Now scan the files in the backup directory. At this stage, we verify
@@ -275,7 +278,10 @@ main(int argc, char **argv)
 	 * told to skip it.
 	 */
 	if (!context.skip_checksums)
+	{
 		verify_backup_checksums(&context);
+		progress_report(&context, true);
+	}
 
 	/*
 	 * Try to parse the required ranges of WAL records, unless we were told
@@ -736,8 +742,6 @@ verify_backup_checksums(verifier_context *context)
 	manifest_file *m;
 	uint8	   *buffer;
 
-	progress_report(context, false);
-
 	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
 
 	manifest_files_start_iterate(manifest->files, &it);
@@ -761,8 +765,6 @@ verify_backup_checksums(verifier_context *context)
 	}
 
 	pfree(buffer);
-
-	progress_report(context, true);
 }
 
 /*
-- 
2.18.0

From a8018ab64411eca3663761ca35913f30176b9102 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 15:47:26 +0530
Subject: [PATCH v8 08/12] Refactor: split verify_control_file.

Separated the manifest entry verification code into a new function and
introduced the should_verify_control_data() macro, similar to
should_verify_checksum().

Like verify_file_checksum(), verify_control_file() is too design to
accept the pg_control file patch which will be opened and respective
information will be verified. But, in case of tar backup we would be
having pg_control file containt instead, that needs to be verified in
the same way.  For that reason the code that doing the verification is
separated into separate function to so that can be reused for the tar
backup verification as well.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 44 +++++++++++------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 15 ++++++++
 2 files changed, 35 insertions(+), 24 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 44b2cd49e0c..d518f995298 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,7 +18,6 @@
 #include <sys/stat.h>
 #include <time.h>
 
-#include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
@@ -61,8 +60,6 @@ static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
 							   char *relpath, char *fullpath);
-static void verify_control_file(const char *controlpath,
-								uint64 manifest_system_identifier);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -625,14 +622,20 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 	/* Check the backup manifest entry for this file. */
 	m = verify_manifest_entry(context, relpath, sb.st_size);
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0 &&
-		m != NULL && m->matched && !m->bad)
-		verify_control_file(fullpath, context->manifest->system_identifier);
+	/* Validate the manifest system identifier */
+	if (should_verify_control_data(context->manifest, m))
+	{
+		ControlFileData *control_file;
+		bool		crc_ok;
+
+		pg_log_debug("reading \"%s\"", fullpath);
+		control_file = get_controlfile_by_exact_path(fullpath, &crc_ok);
+
+		verify_control_data(control_file, fullpath, crc_ok,
+							context->manifest->system_identifier);
+		/* Release memory. */
+		pfree(control_file);
+	}
 }
 
 /*
@@ -676,18 +679,14 @@ verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
 }
 
 /*
- * Sanity check control file and validate system identifier against manifest
- * system identifier.
+ * Sanity check control file data and validate system identifier against
+ * manifest system identifier.
  */
-static void
-verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
+void
+verify_control_data(ControlFileData *control_file,
+					const char *controlpath, bool crc_ok,
+					uint64 manifest_system_identifier)
 {
-	ControlFileData *control_file;
-	bool		crc_ok;
-
-	pg_log_debug("reading \"%s\"", controlpath);
-	control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
-
 	/* Control file contents not meaningful if CRC is bad. */
 	if (!crc_ok)
 		report_fatal_error("%s: CRC is incorrect", controlpath);
@@ -703,9 +702,6 @@ verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
 						   controlpath,
 						   (unsigned long long) manifest_system_identifier,
 						   (unsigned long long) control_file->system_identifier);
-
-	/* Release memory. */
-	pfree(control_file);
 }
 
 /*
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 1bc5f7a6b4a..8a4046b0e33 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -16,6 +16,7 @@
 
 #include "common/controldata_utils.h"
 #include "common/hashfn_unstable.h"
+#include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 
@@ -46,6 +47,17 @@ typedef struct manifest_file
 #define should_verify_checksum(m) \
 	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
 
+/*
+ * Validate the manifest system identifier against the control file; this
+ * feature is not available in manifest version 1.  This validation should be
+ * carried out only if the manifest entry validation is completed without any
+ * errors.
+ */
+#define should_verify_control_data(manifest, m) \
+	(((manifest)->version != 1) && \
+	 ((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (strcmp((m)->pathname, "global/pg_control") == 0))
+
 /*
  * Define a hash table which we can use to store information about the files
  * mentioned in the backup manifest.
@@ -110,6 +122,9 @@ extern manifest_file *verify_manifest_entry(verifier_context *context,
 extern void verify_checksum(verifier_context *context, manifest_file *m,
 							pg_checksum_context *checksum_ctx,
 							uint8 *checksumbuf);
+extern void verify_control_data(ControlFileData *control_file,
+								const char *controlpath, bool crc_ok,
+								uint64 manifest_system_identifier);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From 4108699d558f2d935aff10944361b4dc93b5fcc3 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 31 Jul 2024 16:22:07 +0530
Subject: [PATCH v8 11/12] pg_verifybackup: Read tar files and verify its
 contents

This patch implements TAR format backup verification.

For reading and decompression, fe_utils/astreamer.h is used. For
verification, a new archive streamer has been added in
astreamer_verify.c to handle TAR member files and their contents; see
astreamer_verify_content() for details. The stack of astreamers will
be set up for each TAR file in verify_tar_file, depending on its
compression type which will be auto-deteced.

When information about a TAR member file (i.e., ASTREAMER_MEMBER_HEADER)
is received, we first verify its entry against the backup manifest. We
then decide if further checks are needed, such as checksum
verification and control data verification (if it is a pg_control
file), once the member file contents are received. Although this
decision could be made when the contents are received, it is more
efficient to make it earlier since the member file contents are
received in multiple iterations. In short, we process
ASTREAMER_MEMBER_CONTENTS multiple times but only once for other
ASTREAMER_MEMBER_* cases. We maintain this information in the
astreamer_verify structure for each member file, which is reset when
the file ends.
---
 src/bin/pg_verifybackup/Makefile           |   4 +-
 src/bin/pg_verifybackup/astreamer_verify.c | 377 +++++++++++++++++++++
 src/bin/pg_verifybackup/meson.build        |   6 +-
 src/bin/pg_verifybackup/pg_verifybackup.c  | 208 +++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h  |  12 +-
 src/tools/pgindent/typedefs.list           |   1 +
 6 files changed, 601 insertions(+), 7 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/astreamer_verify.c

diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile
index 7c045f142e8..df7aaabd530 100644
--- a/src/bin/pg_verifybackup/Makefile
+++ b/src/bin/pg_verifybackup/Makefile
@@ -17,11 +17,13 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 # We need libpq only because fe_utils does.
+override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 
 OBJS = \
 	$(WIN32RES) \
-	pg_verifybackup.o
+	pg_verifybackup.o \
+	astreamer_verify.o
 
 all: pg_verifybackup
 
diff --git a/src/bin/pg_verifybackup/astreamer_verify.c b/src/bin/pg_verifybackup/astreamer_verify.c
new file mode 100644
index 00000000000..91d324fddce
--- /dev/null
+++ b/src/bin/pg_verifybackup/astreamer_verify.c
@@ -0,0 +1,377 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer_verify.c
+ *
+ * Extend fe_utils/astreamer.h archive streaming facility to verify TAR
+ * backup.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * src/bin/pg_verifybackup/astreamer_verify.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include "common/logging.h"
+#include "fe_utils/astreamer.h"
+#include "pg_verifybackup.h"
+
+typedef struct astreamer_verify
+{
+	astreamer	base;
+	verifier_context *context;
+	char	   *archive_name;
+	Oid			tblspc_oid;
+	pg_checksum_context *checksum_ctx;
+
+	/* Hold information for a member file verification */
+	manifest_file *mfile;
+	int64		received_bytes;
+	bool		verify_checksum;
+	bool		verify_control_data;
+} astreamer_verify;
+
+static void astreamer_verify_content(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len,
+									 astreamer_archive_context context);
+static void astreamer_verify_finalize(astreamer *streamer);
+static void astreamer_verify_free(astreamer *streamer);
+
+static void verify_member_header(astreamer *streamer, astreamer_member *member);
+static void verify_member_contents(astreamer *streamer,
+								   astreamer_member *member,
+								   const char *data, int len);
+static void verify_content_checksum(astreamer *streamer,
+									astreamer_member *member,
+									const char *buffer, int buffer_len);
+static void verify_controldata(astreamer *streamer,
+							   astreamer_member *member,
+							   const char *data, int len);
+static void reset_member_info(astreamer *streamer);
+
+static const astreamer_ops astreamer_verify_ops = {
+	.content = astreamer_verify_content,
+	.finalize = astreamer_verify_finalize,
+	.free = astreamer_verify_free
+};
+
+/*
+ * Create a astreamer that can verifies content of a TAR file.
+ */
+astreamer *
+astreamer_verify_content_new(astreamer *next, verifier_context *context,
+							 char *archive_name, Oid tblspc_oid)
+{
+	astreamer_verify *streamer;
+
+	streamer = palloc0(sizeof(astreamer_verify));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_verify_ops;
+
+	streamer->base.bbs_next = next;
+	streamer->context = context;
+	streamer->archive_name = archive_name;
+	streamer->tblspc_oid = tblspc_oid;
+	initStringInfo(&streamer->base.bbs_buffer);
+
+	if (!context->skip_checksums)
+		streamer->checksum_ctx = pg_malloc(sizeof(pg_checksum_context));
+
+	return &streamer->base;
+}
+
+/*
+ * It verifies each TAR member entry against the manifest data and performs
+ * checksum verification if enabled. Additionally, it validates the backup's
+ * system identifier against the backup_manifest.
+ */
+static void
+astreamer_verify_content(astreamer *streamer, astreamer_member *member,
+						 const char *data, int len,
+						 astreamer_archive_context context)
+{
+	Assert(context != ASTREAMER_UNKNOWN);
+
+	switch (context)
+	{
+		case ASTREAMER_MEMBER_HEADER:
+
+			/*
+			 * Perform the initial check and setup for the verification.
+			 */
+			verify_member_header(streamer, member);
+			break;
+
+		case ASTREAMER_MEMBER_CONTENTS:
+
+			/*
+			 * Peform the required contants verification.
+			 */
+			verify_member_contents(streamer, member, data, len);
+			break;
+
+		case ASTREAMER_MEMBER_TRAILER:
+
+			/*
+			 * Reset the temporary information stored for a verification.
+			 */
+			reset_member_info(streamer);
+			break;
+
+		case ASTREAMER_ARCHIVE_TRAILER:
+			break;
+
+		default:
+			/* Shouldn't happen. */
+			pg_fatal("unexpected state while parsing tar archive");
+	}
+}
+
+/*
+ * End-of-stream processing for a astreamer_verify stream.
+ */
+static void
+astreamer_verify_finalize(astreamer *streamer)
+{
+	Assert(streamer->bbs_next == NULL);
+}
+
+/*
+ * Free memory associated with a astreamer_verify stream.
+ */
+static void
+astreamer_verify_free(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	if (mystreamer->checksum_ctx)
+		pfree(mystreamer->checksum_ctx);
+
+	pfree(streamer->bbs_buffer.data);
+	pfree(streamer);
+}
+
+/*
+ * Verify the entry if it is a file in the backup manifest. If the archive being
+ * processed is a tablespace, prepare the required file path for subsequent
+ * operations. Finally, check if it needs to perform checksum verification and
+ * control data verification during file content processing.
+ */
+static void
+verify_member_header(astreamer *streamer, astreamer_member *member)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_file *m;
+
+	/* We are only interested in files that are not in the ignore list. */
+	if (member->is_directory || member->is_link ||
+		should_ignore_relpath(mystreamer->context, member->pathname))
+		return;
+
+	/*
+	 * The backup_manifest stores a relative path to the base directory for
+	 * files belong tablespace, whereas <tablespaceoid>.tar doesn't. Prepare
+	 * the required path, otherwise, the manfiest entry verification will
+	 * fail.
+	 */
+	if (OidIsValid(mystreamer->tblspc_oid))
+	{
+		char		temp[MAXPGPATH];
+
+		/* Copy original name at temporary space */
+		memcpy(temp, member->pathname, MAXPGPATH);
+
+		snprintf(member->pathname, MAXPGPATH, "%s/%d/%s",
+				 "pg_tblspc", mystreamer->tblspc_oid, temp);
+	}
+
+	/* Check the manifest entry */
+	m = verify_manifest_entry(mystreamer->context, member->pathname,
+							  member->size);
+	mystreamer->mfile = (void *) m;
+
+	/*
+	 * Prepare for checksum and control data verification.
+	 *
+	 * We could have these checks while receiving contents. However, since
+	 * contents are received in multiple iterations, this would result in
+	 * these lengthy checks being performed multiple times. Instead, having a
+	 * single flag would be more efficient.
+	 */
+	mystreamer->verify_checksum =
+		(!mystreamer->context->skip_checksums && should_verify_checksum(m));
+	mystreamer->verify_control_data =
+		should_verify_control_data(mystreamer->context->manifest, m);
+
+	/* Initialize the context required for checksum verification. */
+	if (mystreamer->verify_checksum &&
+		pg_checksum_init(mystreamer->checksum_ctx, m->checksum_type) < 0)
+	{
+		report_backup_error(mystreamer->context,
+							"%s: could not initialize checksum of file \"%s\"",
+							mystreamer->archive_name, m->pathname);
+
+		/*
+		 * Checksum verification cannot be performed without proper context
+		 * initialization.
+		 */
+		mystreamer->verify_checksum = false;
+	}
+}
+
+/*
+ * Process the member content according to the flags set by the member header
+ * processing routine for checksum and control data verification.
+ */
+static void
+verify_member_contents(astreamer *streamer, astreamer_member *member,
+					   const char *data, int len)
+
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	/* Verify the checksums */
+	if (mystreamer->verify_checksum)
+		verify_content_checksum(streamer, member, data, len);
+
+	/* Verify pg_control information */
+	if (mystreamer->verify_control_data)
+		verify_controldata(streamer, member, data, len);
+}
+
+/*
+ * Similar to verify_file_checksum() but this function computes the checksum
+ * incrementally for the received file content. Unlike a normal backup
+ * directory, TAR format files do not allow random access, so checksum
+ * verification occurs progressively. Additionally, the function calls the
+ * routine for control data verification if the flags indicate that it is
+ * required.
+ *
+ * Caller should pass correctly initialised checksum_ctx, which will be used
+ * for incremental checksum calculation. Once the complete file content is
+ * received (tracked using the received_bytes), the routine that performs the
+ * final checksum verification is called
+ */
+static void
+verify_content_checksum(astreamer *streamer, astreamer_member *member,
+						const char *buffer, int buffer_len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	pg_checksum_context *checksum_ctx = mystreamer->checksum_ctx;
+	verifier_context *context = mystreamer->context;
+	manifest_file *m = mystreamer->mfile;
+	const char *relpath = m->pathname;
+	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
+
+	/*
+	 * Mark it false to avoid unexpected re-entrance for the same file content
+	 * (e.g. returned in error should not be revisited).
+	 */
+	Assert(mystreamer->verify_checksum);
+	mystreamer->verify_checksum = false;
+
+	/* Should have came for the right file */
+	Assert(strcmp(member->pathname, relpath) == 0);
+
+	/*
+	 * The checksum context should match the type noted in the backup
+	 * manifest.
+	 */
+	Assert(checksum_ctx->type == m->checksum_type);
+
+	/* Update the total count of computed checksum bytes. */
+	mystreamer->received_bytes += buffer_len;
+
+	if (pg_checksum_update(checksum_ctx, (uint8 *) buffer, buffer_len) < 0)
+	{
+		report_backup_error(context, "could not update checksum of file \"%s\"",
+							relpath);
+		return;
+	}
+
+	/* Report progress */
+	context->done_size += buffer_len;
+	progress_report(context, false);
+
+	/* Yet to receive the full content of the file. */
+	if (mystreamer->received_bytes < m->size)
+	{
+		mystreamer->verify_checksum = true;
+		return;
+	}
+
+	/* Do the final computation and verification. */
+	verify_checksum(context, m, checksum_ctx, checksumbuf);
+}
+
+/*
+ * Prepare the control data from the received file contents, which are supposed
+ * to be from the pg_control file, including CRC calculation. Then, call the
+ * routines that perform the final verification of the control file information.
+ */
+static void
+verify_controldata(astreamer *streamer, astreamer_member *member,
+				   const char *data, int len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_data *manifest = mystreamer->context->manifest;
+	ControlFileData control_file;
+	pg_crc32c	crc;
+	bool		crc_ok;
+
+	/* Should be here only for control file */
+	Assert(strcmp(member->pathname, "global/pg_control") == 0);
+	Assert(manifest->version != 1);
+
+	/* Mark it as false to avoid unexpected re-entrance */
+	Assert(mystreamer->verify_control_data);
+	mystreamer->verify_control_data = false;
+
+	/* Should have whole control file data. */
+	if (!astreamer_buffer_until(streamer, &data, &len, sizeof(ControlFileData)))
+	{
+		mystreamer->verify_control_data = true;
+		return;
+	}
+
+	pg_log_debug("%s: reading \"%s\"", mystreamer->archive_name,
+				 member->pathname);
+
+	if (streamer->bbs_buffer.len != sizeof(ControlFileData))
+		report_fatal_error("%s: could not read control file: read %d of %zu",
+						   mystreamer->archive_name, streamer->bbs_buffer.len,
+						   sizeof(ControlFileData));
+
+	memcpy(&control_file, streamer->bbs_buffer.data,
+		   sizeof(ControlFileData));
+
+	/* Check the CRC. */
+	INIT_CRC32C(crc);
+	COMP_CRC32C(crc,
+				(char *) (&control_file),
+				offsetof(ControlFileData, crc));
+	FIN_CRC32C(crc);
+
+	crc_ok = EQ_CRC32C(crc, control_file.crc);
+
+	/* Do the final control data verification. */
+	verify_control_data(&control_file, member->pathname, crc_ok,
+						manifest->system_identifier);
+}
+
+/*
+ * Reset flags and free memory allocations for member file verification.
+ */
+static void
+reset_member_info(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	mystreamer->mfile = NULL;
+	mystreamer->received_bytes = 0;
+	mystreamer->verify_checksum = false;
+	mystreamer->verify_control_data = false;
+}
diff --git a/src/bin/pg_verifybackup/meson.build b/src/bin/pg_verifybackup/meson.build
index 7c7d31a0350..1e3fcf7ee5a 100644
--- a/src/bin/pg_verifybackup/meson.build
+++ b/src/bin/pg_verifybackup/meson.build
@@ -1,7 +1,8 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 pg_verifybackup_sources = files(
-  'pg_verifybackup.c'
+  'pg_verifybackup.c',
+  'astreamer_verify.c'
 )
 
 if host_system == 'windows'
@@ -10,9 +11,10 @@ if host_system == 'windows'
     '--FILEDESC', 'pg_verifybackup - verify a backup against using a backup manifest'])
 endif
 
+pg_verifybackup_deps = [frontend_code, libpq, lz4, zlib, zstd]
 pg_verifybackup = executable('pg_verifybackup',
   pg_verifybackup_sources,
-  dependencies: [frontend_code, libpq],
+  dependencies: pg_verifybackup_deps,
   kwargs: default_bin_args,
 )
 bin_targets += pg_verifybackup
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 8b92b26f4d7..2a058938986 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -20,9 +20,12 @@
 
 #include "common/compression.h"
 #include "common/parse_manifest.h"
+#include "common/relpath.h"
+#include "fe_utils/astreamer.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
 #include "pg_verifybackup.h"
+#include "pgtar.h"
 #include "pgtime.h"
 
 /*
@@ -62,6 +65,15 @@ static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
 							   char *relpath, char *fullpath);
+static void verify_plain_file(verifier_context *context,
+							  char *relpath, char *fullpath,
+							  size_t filesize);
+static void verify_tar_file(verifier_context *context,
+							char *relpath, char *fullpath,
+							size_t filesize);
+static void verify_tar_content(verifier_context *context,
+							   char *relpath, char *fullpath,
+							   astreamer *streamer);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -70,6 +82,10 @@ static void verify_file_checksum(verifier_context *context,
 static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
+static astreamer *create_archive_verifier(verifier_context *context,
+										  char *archive_name,
+										  Oid tblspc_oid,
+										  pg_compress_algorithm compress_algo);
 
 static void compute_total_size(verifier_context *context);
 static void usage(void);
@@ -141,6 +157,10 @@ main(int argc, char **argv)
 	 */
 	simple_string_list_append(&context.ignore_list, "backup_manifest");
 	simple_string_list_append(&context.ignore_list, "pg_wal");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.gz");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.lz4");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.zst");
 	simple_string_list_append(&context.ignore_list, "postgresql.auto.conf");
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
@@ -619,7 +639,8 @@ verify_backup_directory(verifier_context *context, char *relpath,
 }
 
 /*
- * Verify one file (which might actually be a directory or a symlink).
+ * Verify one file (which might actually be a directory, a symlink or a
+ * archive).
  *
  * The arguments to this function have the same meaning as the arguments to
  * verify_backup_directory.
@@ -628,7 +649,6 @@ static void
 verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 {
 	struct stat sb;
-	manifest_file *m;
 
 	if (stat(fullpath, &sb) != 0)
 	{
@@ -661,8 +681,28 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		return;
 	}
 
+	/* Do the further verifications */
+	if (context->format == 'p')
+		verify_plain_file(context, relpath, fullpath, sb.st_size);
+	else
+		verify_tar_file(context, relpath, fullpath, sb.st_size);
+}
+
+/*
+ * Verify one plan file or a symlink.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_backup_directory. The additional argument is the file size for
+ * verifying against manifest entry.
+ */
+static void
+verify_plain_file(verifier_context *context, char *relpath, char *fullpath,
+				  size_t filesize)
+{
+	manifest_file *m;
+
 	/* Check the backup manifest entry for this file. */
-	m = verify_manifest_entry(context, relpath, sb.st_size);
+	m = verify_manifest_entry(context, relpath, filesize);
 
 	/* Validate the manifest system identifier */
 	if (should_verify_control_data(context->manifest, m))
@@ -680,6 +720,132 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 	}
 }
 
+/*
+ * Verify one tar file.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_backup_directory. The additional argument is the file size for
+ * verifying against manifest entry.
+ */
+static void
+verify_tar_file(verifier_context *context, char *relpath, char *fullpath,
+				size_t filesize)
+{
+	astreamer  *streamer;
+	Oid			tblspc_oid = InvalidOid;
+	int			file_name_len;
+	int			file_extn_len;
+	pg_compress_algorithm compress_algorithm;
+
+	/* Should be tar backup */
+	Assert(context->format == 't');
+
+	/* Find the compression type of the tar file */
+	if (strstr(relpath, ".tgz") != NULL)
+	{
+		compress_algorithm = PG_COMPRESSION_GZIP;
+		file_extn_len = 4;		/* length of ".tgz" */
+	}
+	else if (strstr(relpath, ".tar.gz") != NULL)
+	{
+		compress_algorithm = PG_COMPRESSION_GZIP;
+		file_extn_len = 7;		/* length of ".tar.gz" */
+	}
+	else if (strstr(relpath, ".tar.lz4") != NULL)
+	{
+		compress_algorithm = PG_COMPRESSION_LZ4;
+		file_extn_len = 8;		/* length of ".tar.lz4" */
+	}
+	else if (strstr(relpath, ".tar.zst") != NULL)
+	{
+		compress_algorithm = PG_COMPRESSION_ZSTD;
+		file_extn_len = 8;		/* length of ".tar.zst" */
+	}
+	else if (strstr(relpath, ".tar") != NULL)
+	{
+		compress_algorithm = PG_COMPRESSION_NONE;
+		file_extn_len = 4;		/* length of ".tar" */
+	}
+	else
+	{
+		report_backup_error(context,
+							"\"%s\" unexpected file in the tar format backup",
+							relpath);
+		return;
+	}
+
+	/*
+	 * We expect TAR files to back up the main directory and tablespace.
+	 *
+	 * pg_basebackup writes the main data directory to an archive file named
+	 * base.tar and the tablespace directory to <tablespaceoid>.tar, followed
+	 * by a compression type extension such as .gz, .lz4, or .zst.
+	 */
+	file_name_len = strlen(relpath);
+	if (strspn(relpath, "0123456789") == (file_name_len - file_extn_len))
+	{
+		/*
+		 * Since the file matches the <tablespaceoid>.tar format, extract the
+		 * tablespaceoid, which is needed to prepare the paths of the files
+		 * belonging to that tablespace relative to the base directory.
+		 */
+		tblspc_oid = strtoi64(relpath, NULL, 10);
+	}
+	/* Otherwise, it should be a base.tar file; if not, raise an error. */
+	else if (strncmp("base", relpath, file_name_len - file_extn_len) != 0)
+	{
+		report_backup_error(context,
+							"\"%s\" unexpected file in the tar format backup",
+							relpath);
+		return;
+	}
+
+	streamer = create_archive_verifier(context, relpath, tblspc_oid,
+									   compress_algorithm);
+	verify_tar_content(context, relpath, fullpath, streamer);
+
+	/* Cleanup. */
+	astreamer_finalize(streamer);
+	astreamer_free(streamer);
+}
+
+/*
+ * Reads a given tar file in predefined chunks and pass to astreamer.  Which
+ * initiates routines for decompression (if necessary) then verification
+ * of each member within the tar archive.
+ */
+static void
+verify_tar_content(verifier_context *context, char *relpath, char *fullpath,
+				   astreamer *streamer)
+{
+	int			fd;
+	int			rc;
+	char	   *buffer;
+
+	/* Open the target file. */
+	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
+	{
+		report_backup_error(context, "could not open file \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
+
+	/* Perform the reads */
+	while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
+		astreamer_content(streamer, NULL, buffer, rc, ASTREAMER_UNKNOWN);
+
+	if (rc < 0)
+		report_backup_error(context, "could not read file \"%s\": %m",
+							relpath);
+
+	/* Close the file. */
+	if (close(fd) != 0)
+		report_backup_error(context, "could not close file \"%s\": %m",
+							relpath);
+}
+
 /*
  * Verify file and its size entry in the manifest.
  */
@@ -1044,6 +1210,42 @@ find_backup_format(verifier_context *context)
 	return result;
 }
 
+/*
+ * Identifies the necessary steps for verifying the contents of the
+ * provided tar file.
+ */
+static astreamer *
+create_archive_verifier(verifier_context *context, char *archive_name,
+						Oid tblspc_oid, pg_compress_algorithm compress_algo)
+{
+	astreamer  *streamer = NULL;
+
+	/* Should be here only for tar backup */
+	Assert(context->format == 't');
+
+	/*
+	 * To verify the contents of the tar file, the initial step is to parse
+	 * its content.
+	 */
+	streamer = astreamer_verify_content_new(streamer, context, archive_name,
+											tblspc_oid);
+	streamer = astreamer_tar_parser_new(streamer);
+
+	/*
+	 * If the tar file is compressed, we must perform the appropriate
+	 * decompression operation before proceeding with the verification of its
+	 * contents.
+	 */
+	if (compress_algo == PG_COMPRESSION_GZIP)
+		streamer = astreamer_gzip_decompressor_new(streamer);
+	else if (compress_algo == PG_COMPRESSION_LZ4)
+		streamer = astreamer_lz4_decompressor_new(streamer);
+	else if (compress_algo == PG_COMPRESSION_ZSTD)
+		streamer = astreamer_zstd_decompressor_new(streamer);
+
+	return streamer;
+}
+
 /*
  * Print a progress report based on the variables in verifier_context.
  *
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index fcef972d9ad..82f845b449f 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -45,7 +45,8 @@ typedef struct manifest_file
 } manifest_file;
 
 #define should_verify_checksum(m) \
-	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+	(((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
 
 /*
  * Validate the manifest system identifier against the control file; this
@@ -137,4 +138,13 @@ extern bool should_ignore_relpath(verifier_context *context,
 
 extern void progress_report(verifier_context *context, bool finished);
 
+/* Forward declarations to avoid fe_utils/astreamer.h include. */
+struct astreamer;
+typedef struct astreamer astreamer;
+
+extern astreamer *astreamer_verify_content_new(astreamer *next,
+											   verifier_context *context,
+											   char *archive_name,
+											   Oid tblspc_oid);
+
 #endif							/* PG_VERIFYBACKUP_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 547d14b3e7c..d86b28b260e 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3329,6 +3329,7 @@ astreamer_plain_writer
 astreamer_recovery_injector
 astreamer_tar_archiver
 astreamer_tar_parser
+astreamer_verify
 astreamer_zstd_frame
 bgworker_main_type
 bh_node_type
-- 
2.18.0

From 583c90336870a528d4df5ed83296ff2c995ae385 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 7 Aug 2024 18:15:29 +0530
Subject: [PATCH v8 12/12] pg_verifybackup: Tests and document

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the previous patch that implements tar format support.
----
---
 doc/src/sgml/ref/pg_verifybackup.sgml         | 42 ++++++++++-
 src/bin/pg_verifybackup/t/001_basic.pl        |  6 +-
 src/bin/pg_verifybackup/t/004_options.pl      |  2 +-
 src/bin/pg_verifybackup/t/005_bad_manifest.pl |  2 +-
 src/bin/pg_verifybackup/t/008_untar.pl        | 73 ++++++-------------
 src/bin/pg_verifybackup/t/010_client_untar.pl | 48 +-----------
 6 files changed, 72 insertions(+), 101 deletions(-)

diff --git a/doc/src/sgml/ref/pg_verifybackup.sgml b/doc/src/sgml/ref/pg_verifybackup.sgml
index a3f167f9f6e..60f771c7663 100644
--- a/doc/src/sgml/ref/pg_verifybackup.sgml
+++ b/doc/src/sgml/ref/pg_verifybackup.sgml
@@ -34,8 +34,10 @@ PostgreSQL documentation
    integrity of a database cluster backup taken using
    <command>pg_basebackup</command> against a
    <literal>backup_manifest</literal> generated by the server at the time
-   of the backup.  The backup must be stored in the "plain"
-   format; a "tar" format backup can be checked after extracting it.
+   of the backup.  The backup must be stored in the "plain" or "tar"
+   format.  Verification is supported for <literal>gzip</literal>,
+   <literal>lz4</literal>, and  <literal>zstd</literal> compressed tar backup;
+   any other compressed format backups can be checked after decompressing them.
   </para>
 
   <para>
@@ -168,6 +170,42 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-F <replaceable class="parameter">format</replaceable></option></term>
+      <term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
+      <listitem>
+       <para>
+        Specifies the format of the backup. <replaceable>format</replaceable>
+        can be one of the following:
+
+        <variablelist>
+         <varlistentry>
+          <term><literal>p</literal></term>
+          <term><literal>plain</literal></term>
+          <listitem>
+           <para>
+            Backup consists of plain files with the same layout as the
+            source server's data directory and tablespaces.
+           </para>
+          </listitem>
+         </varlistentry>
+
+         <varlistentry>
+          <term><literal>t</literal></term>
+          <term><literal>tar</literal></term>
+          <listitem>
+           <para>
+            Backup consists of tar files. The main data directory's contents
+            will be written to a file named <filename>base.tar</filename>,
+            and each other tablespace will be written to a separate tar file
+            named after that tablespace's OID.
+           </para>
+           </listitem>
+         </varlistentry>
+        </variablelist></para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-n</option></term>
       <term><option>--no-parse-wal</option></term>
diff --git a/src/bin/pg_verifybackup/t/001_basic.pl b/src/bin/pg_verifybackup/t/001_basic.pl
index 2f3e52d296f..ca5b0402b7d 100644
--- a/src/bin/pg_verifybackup/t/001_basic.pl
+++ b/src/bin/pg_verifybackup/t/001_basic.pl
@@ -17,11 +17,11 @@ command_fails_like(
 	qr/no backup directory specified/,
 	'target directory must be specified');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir ],
 	qr/could not open file.*\/backup_manifest\"/,
 	'pg_verifybackup requires a manifest');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir, $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir, $tempdir ],
 	qr/too many command-line arguments/,
 	'multiple target directories not allowed');
 
@@ -31,7 +31,7 @@ close($fh);
 
 # but then try to use an alternate, nonexisting manifest
 command_fails_like(
-	[ 'pg_verifybackup', '-m', "$tempdir/not_the_manifest", $tempdir ],
+	[ 'pg_verifybackup', '-Fp', '-m', "$tempdir/not_the_manifest", $tempdir ],
 	qr/could not open file.*\/not_the_manifest\"/,
 	'pg_verifybackup respects -m flag');
 
diff --git a/src/bin/pg_verifybackup/t/004_options.pl b/src/bin/pg_verifybackup/t/004_options.pl
index 8ed2214408e..2f197648740 100644
--- a/src/bin/pg_verifybackup/t/004_options.pl
+++ b/src/bin/pg_verifybackup/t/004_options.pl
@@ -108,7 +108,7 @@ unlike(
 # Test valid manifest with nonexistent backup directory.
 command_fails_like(
 	[
-		'pg_verifybackup', '-m',
+		'pg_verifybackup', '-Fp', '-m',
 		"$backup_path/backup_manifest", "$backup_path/fake"
 	],
 	qr/could not open directory/,
diff --git a/src/bin/pg_verifybackup/t/005_bad_manifest.pl b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
index c4ed64b62d5..28c51b6feb0 100644
--- a/src/bin/pg_verifybackup/t/005_bad_manifest.pl
+++ b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
@@ -208,7 +208,7 @@ sub test_bad_manifest
 	print $fh $manifest_contents;
 	close($fh);
 
-	command_fails_like([ 'pg_verifybackup', $tempdir ], $regexp, $test_name);
+	command_fails_like([ 'pg_verifybackup', '-Fp', $tempdir ], $regexp, $test_name);
 	return;
 }
 
diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
index 7a09f3b75b2..9896560adc3 100644
--- a/src/bin/pg_verifybackup/t/008_untar.pl
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -16,6 +16,20 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 
+# Create a couple of directories to use as tablespaces.
+my $TS1_LOCATION = $primary->backup_dir .'/ts1';
+$TS1_LOCATION =~ s/\/\.\//\//g;    # collapse foo/./bar to foo/bar
+mkdir($TS1_LOCATION);
+
+# Create a tablespace with table in it.
+$primary->safe_psql('postgres', qq(
+		CREATE TABLESPACE regress_ts1 LOCATION '$TS1_LOCATION';
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1';
+		CREATE TABLE regress_tbl1(i int) TABLESPACE regress_ts1;
+		INSERT INTO regress_tbl1 VALUES(generate_series(1,5));));
+my $tsoid = $primary->safe_psql('postgres', qq(
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1'));
+
 my $backup_path = $primary->backup_dir . '/server-backup';
 my $extract_path = $primary->backup_dir . '/extracted-backup';
 
@@ -23,39 +37,31 @@ my @test_configuration = (
 	{
 		'compression_method' => 'none',
 		'backup_flags' => [],
-		'backup_archive' => 'base.tar',
+		'backup_archive' => ['base.tar', "$tsoid.tar"],
 		'enabled' => 1
 	},
 	{
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'server-gzip' ],
-		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.gz', "$tsoid.tar.gz" ],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'server-lz4' ],
-		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => [ '-d', '-m' ],
+		'backup_archive' => ['base.tar.lz4', "$tsoid.tar.lz4" ],
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd:level=1,long' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	});
 
@@ -86,47 +92,16 @@ for my $tc (@test_configuration)
 		my $backup_files = join(',',
 			sort grep { $_ ne '.' && $_ ne '..' } slurp_dir($backup_path));
 		my $expected_backup_files =
-		  join(',', sort ('backup_manifest', $tc->{'backup_archive'}));
+		  join(',', sort ('backup_manifest', @{ $tc->{'backup_archive'} }));
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok(['pg_verifybackup', '-n', '-e', $backup_path],
+			"verify backup, compression $method");
 
 		# Cleanup.
-		unlink($backup_path . '/backup_manifest');
-		unlink($backup_path . '/base.tar');
-		unlink($backup_path . '/' . $tc->{'backup_archive'});
+		rmtree($backup_path);
 		rmtree($extract_path);
 	}
 }
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index 8c076d46dee..6b7d7483f6e 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -29,41 +29,30 @@ my @test_configuration = (
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'client-gzip:5' ],
 		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'client-lz4:5' ],
 		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => ['-d'],
-		'output_file' => 'base.tar',
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:5' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:level=1,long' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'parallel zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:workers=3' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1"),
 		'possibly_unsupported' =>
 		  qr/could not set compression worker count to 3: Unsupported parameter/
@@ -118,40 +107,9 @@ for my $tc (@test_configuration)
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			push @decompress, $backup_path . '/' . $tc->{'output_file'}
-			  if $tc->{'output_file'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok( [ 'pg_verifybackup', '-n', '-e', $backup_path ],
+			"verify backup, compression $method");
 
 		# Cleanup.
 		rmtree($extract_path);
-- 
2.18.0

From 4be84391a1e05b70ca6fcb51e050a1a08de2b94d Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 12:15:26 +0530
Subject: [PATCH v8 06/12] Refactor: split verify_backup_file() function.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Move the manifest entry verification code into a new function called
verify_manifest_entry(). Also, move the total size computation code
into another new function called compute_total_size(), which will be
called from the main function.

The current computation is designed for plain backups, which operate
in two rounds. In the first round, only the files are checked against
the manifest backup, and their sizes are added to the total size for
progress reporting. In the second round, the actual files are read for
the checksums verification, and the read size progress is noted and
reported.

However, for tar backups, we do not operate in two rounds because TAR
format files do not allow random access like plain backups. Therefore,
we verify the file entries against the backup manifest and right
after, perform the checksum verification in a single pass. That’s why
we need to compute the total size before starting the TAR backup
verification pass.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 82 ++++++++++++++++++-----
 src/bin/pg_verifybackup/pg_verifybackup.h |  3 +
 2 files changed, 68 insertions(+), 17 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 4e42757c346..8eadaac72e3 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -72,6 +72,7 @@ static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
 
+static void compute_total_size(verifier_context *context);
 static void usage(void);
 
 static const char *progname;
@@ -250,6 +251,13 @@ main(int argc, char **argv)
 	 */
 	context.manifest = parse_manifest_file(manifest_path);
 
+	/*
+	 * For the progress report, compute the total size of the files to be
+	 * read, which occurs only when checksum verification is enabled.
+	 */
+	if (!context.skip_checksums)
+		compute_total_size(&context);
+
 	/*
 	 * Now scan the files in the backup directory. At this stage, we verify
 	 * that every file on disk is present in the manifest and that the sizes
@@ -614,6 +622,27 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		return;
 	}
 
+	/* Check the backup manifest entry for this file. */
+	m = verify_manifest_entry(context, relpath, sb.st_size);
+
+	/*
+	 * Validate the manifest system identifier, not available in manifest
+	 * version 1.
+	 */
+	if (context->manifest->version != 1 &&
+		strcmp(relpath, "global/pg_control") == 0 &&
+		m != NULL && m->matched && !m->bad)
+		verify_control_file(fullpath, context->manifest->system_identifier);
+}
+
+/*
+ * Verify file and its size entry in the manifest.
+ */
+manifest_file *
+verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
+{
+	manifest_file *m;
+
 	/* Check whether there's an entry in the manifest hash. */
 	m = manifest_files_lookup(context->manifest->files, relpath);
 	if (m == NULL)
@@ -621,40 +650,29 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		report_backup_error(context,
 							"\"%s\" is present on disk but not in the manifest",
 							relpath);
-		return;
+		return NULL;
 	}
 
 	/* Flag this entry as having been encountered in the filesystem. */
 	m->matched = true;
 
 	/* Check that the size matches. */
-	if (m->size != sb.st_size)
+	if (m->size != filesize)
 	{
 		report_backup_error(context,
 							"\"%s\" has size %lld on disk but size %zu in the manifest",
-							relpath, (long long int) sb.st_size, m->size);
+							relpath, (long long int) filesize, m->size);
 		m->bad = true;
 	}
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0)
-		verify_control_file(fullpath, context->manifest->system_identifier);
-
-	/* Update statistics for progress report, if necessary */
-	if (context->show_progress && !context->skip_checksums &&
-		should_verify_checksum(m))
-		context->total_size += m->size;
-
 	/*
 	 * We don't verify checksums at this stage. We first finish verifying that
 	 * we have the expected set of files with the expected sizes, and only
 	 * afterwards verify the checksums. That's because computing checksums may
 	 * take a while, and we'd like to report more obvious problems quickly.
 	 */
+
+	return m;
 }
 
 /*
@@ -817,7 +835,7 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 
 	/*
 	 * Double-check that we read the expected number of bytes from the file.
-	 * Normally, a file size mismatch would be caught in verify_backup_file
+	 * Normally, a file size mismatch would be caught in verify_manifest_entry
 	 * and this check would never be reached, but this provides additional
 	 * safety and clarity in the event of concurrent modifications or
 	 * filesystem misbehavior.
@@ -988,6 +1006,36 @@ progress_report(verifier_context *context, bool finished)
 	fputc((!finished && isatty(fileno(stderr))) ? '\r' : '\n', stderr);
 }
 
+/*
+ * Compute the total size of backup files for progress reporting.
+ */
+static void
+compute_total_size(verifier_context *context)
+{
+	manifest_data *manifest = context->manifest;
+	manifest_files_iterator it;
+	manifest_file *m;
+	uint64		total_size = 0;
+
+	if (!context->show_progress)
+		return;
+
+	manifest_files_start_iterate(manifest->files, &it);
+	while ((m = manifest_files_iterate(manifest->files, &it)) != NULL)
+	{
+		/*
+		 * We are not going to read files that are ignored or whose checksums
+		 * are not calculated, so their sizes should be excluded from the
+		 * total.
+		 */
+		if (!should_ignore_relpath(context, m->pathname) &&
+			m->checksum_type != CHECKSUM_TYPE_NONE)
+			total_size += m->size;
+	}
+
+	context->total_size = total_size;
+}
+
 /*
  * Print out usage information and exit.
  */
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 90900048547..98c75916255 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -105,6 +105,9 @@ typedef struct verifier_context
 	uint64		done_size;
 } verifier_context;
 
+extern manifest_file *verify_manifest_entry(verifier_context *context,
+											char *relpath, int64 filesize);
+
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
 			pg_attribute_printf(2, 3);
-- 
2.18.0

From 62ecfedd2666b23d171539fed738281702f824fd Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 16:45:55 +0530
Subject: [PATCH v8 07/12] Refactor: split verify_file_checksum() function.

Move the core functionality of verify_file_checksum to a new function
to reuse it instead of duplicating the code.

The verify_file_checksum() function is designed to take a file path,
open and read the file contents, and then calculate the checksum.
However, for TAR backups, instead of a file path, we receive the file
content in chunks, and the checksum needs to be calculated
incrementally. While the initial operations for plain and TAR backup
checksum calculations differ, the final checks and error handling are
the same. By moving the shared logic to a separate function, we can
reuse the code for both types of backups.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 18 ++++++++++++++++--
 src/bin/pg_verifybackup/pg_verifybackup.h |  3 +++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 8eadaac72e3..44b2cd49e0c 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -782,7 +782,6 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	int			rc;
 	size_t		bytes_read = 0;
 	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
-	int			checksumlen;
 
 	/* Open the target file. */
 	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
@@ -848,8 +847,23 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 		return;
 	}
 
+	/* Do the final computation and verification. */
+	verify_checksum(context, m, &checksum_ctx, checksumbuf);
+}
+
+/*
+ * A helper function to finalize checksum computation and verify it against the
+ * backup manifest information.
+ */
+void
+verify_checksum(verifier_context *context, manifest_file *m,
+				pg_checksum_context *checksum_ctx, uint8 *checksumbuf)
+{
+	int			checksumlen;
+	const char *relpath = m->pathname;
+
 	/* Get the final checksum. */
-	checksumlen = pg_checksum_final(&checksum_ctx, checksumbuf);
+	checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
 	if (checksumlen < 0)
 	{
 		report_backup_error(context,
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 98c75916255..1bc5f7a6b4a 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -107,6 +107,9 @@ typedef struct verifier_context
 
 extern manifest_file *verify_manifest_entry(verifier_context *context,
 											char *relpath, int64 filesize);
+extern void verify_checksum(verifier_context *context, manifest_file *m,
+							pg_checksum_context *checksum_ctx,
+							uint8 *checksumbuf);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From c630430b0ec31674091be6952811934ebc4cff3d Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 31 Jul 2024 11:43:52 +0530
Subject: [PATCH v8 04/12] Refactor: move few global variable to
 verifier_context struct

Global variables are:
	1. show_progress
	2. skip_checksums
	3. total_size
	4. done_size
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 50 +++++++++++------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index d77e70fbe38..71585ffc50e 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -113,8 +113,14 @@ typedef struct verifier_context
 	manifest_data *manifest;
 	char	   *backup_directory;
 	SimpleStringList ignore_list;
+	bool		skip_checksums;
 	bool		exit_on_error;
 	bool		saw_any_error;
+
+	/* Progress indicators */
+	bool		show_progress;
+	uint64		total_size;
+	uint64		done_size;
 } verifier_context;
 
 static manifest_data *parse_manifest_file(char *manifest_path);
@@ -157,19 +163,11 @@ static void report_fatal_error(const char *pg_restrict fmt,...)
 			pg_attribute_printf(1, 2) pg_attribute_noreturn();
 static bool should_ignore_relpath(verifier_context *context, const char *relpath);
 
-static void progress_report(bool finished);
+static void progress_report(verifier_context *context, bool finished);
 static void usage(void);
 
 static const char *progname;
 
-/* options */
-static bool show_progress = false;
-static bool skip_checksums = false;
-
-/* Progress indicators */
-static uint64 total_size = 0;
-static uint64 done_size = 0;
-
 /*
  * Main entry point.
  */
@@ -260,13 +258,13 @@ main(int argc, char **argv)
 				no_parse_wal = true;
 				break;
 			case 'P':
-				show_progress = true;
+				context.show_progress = true;
 				break;
 			case 'q':
 				quiet = true;
 				break;
 			case 's':
-				skip_checksums = true;
+				context.skip_checksums = true;
 				break;
 			case 'w':
 				wal_directory = pstrdup(optarg);
@@ -299,7 +297,7 @@ main(int argc, char **argv)
 	}
 
 	/* Complain if the specified arguments conflict */
-	if (show_progress && quiet)
+	if (context.show_progress && quiet)
 		pg_fatal("cannot specify both %s and %s",
 				 "-P/--progress", "-q/--quiet");
 
@@ -363,7 +361,7 @@ main(int argc, char **argv)
 	 * Now do the expensive work of verifying file checksums, unless we were
 	 * told to skip it.
 	 */
-	if (!skip_checksums)
+	if (!context.skip_checksums)
 		verify_backup_checksums(&context);
 
 	/*
@@ -739,8 +737,9 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		verify_control_file(fullpath, context->manifest->system_identifier);
 
 	/* Update statistics for progress report, if necessary */
-	if (show_progress && !skip_checksums && should_verify_checksum(m))
-		total_size += m->size;
+	if (context->show_progress && !context->skip_checksums &&
+		should_verify_checksum(m))
+		context->total_size += m->size;
 
 	/*
 	 * We don't verify checksums at this stage. We first finish verifying that
@@ -815,7 +814,7 @@ verify_backup_checksums(verifier_context *context)
 	manifest_file *m;
 	uint8	   *buffer;
 
-	progress_report(false);
+	progress_report(context, false);
 
 	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
 
@@ -841,7 +840,7 @@ verify_backup_checksums(verifier_context *context)
 
 	pfree(buffer);
 
-	progress_report(true);
+	progress_report(context, true);
 }
 
 /*
@@ -889,8 +888,8 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 		}
 
 		/* Report progress */
-		done_size += rc;
-		progress_report(false);
+		context->done_size += rc;
+		progress_report(context, false);
 	}
 	if (rc < 0)
 		report_backup_error(context, "could not read file \"%s\": %m",
@@ -1036,7 +1035,7 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
 }
 
 /*
- * Print a progress report based on the global variables.
+ * Print a progress report based on the variables in verifier_context.
  *
  * Progress report is written at maximum once per second, unless the finished
  * parameter is set to true.
@@ -1045,7 +1044,7 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
  * is moved to the next line.
  */
 static void
-progress_report(bool finished)
+progress_report(verifier_context *context, bool finished)
 {
 	static pg_time_t last_progress_report = 0;
 	pg_time_t	now;
@@ -1053,7 +1052,7 @@ progress_report(bool finished)
 	char		totalsize_str[32];
 	char		donesize_str[32];
 
-	if (!show_progress)
+	if (!context->show_progress)
 		return;
 
 	now = time(NULL);
@@ -1061,12 +1060,13 @@ progress_report(bool finished)
 		return;					/* Max once per second */
 
 	last_progress_report = now;
-	percent_size = total_size ? (int) ((done_size * 100 / total_size)) : 0;
+	percent_size = context->total_size ?
+		(int) ((context->done_size * 100 / context->total_size)) : 0;
 
 	snprintf(totalsize_str, sizeof(totalsize_str), UINT64_FORMAT,
-			 total_size / 1024);
+			 context->total_size / 1024);
 	snprintf(donesize_str, sizeof(donesize_str), UINT64_FORMAT,
-			 done_size / 1024);
+			 context->done_size / 1024);
 
 	fprintf(stderr,
 			_("%*s/%s kB (%d%%) verified"),
-- 
2.18.0

From aadd222cb50197b79e152daaa6f728ced368ae38 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 12:10:34 +0530
Subject: [PATCH v8 05/12] Refactor: move some part of pg_verifybackup.c to
 pg_verifybackup.h

---
 src/bin/pg_verifybackup/pg_verifybackup.c | 102 +------------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 118 ++++++++++++++++++++++
 2 files changed, 123 insertions(+), 97 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/pg_verifybackup.h

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 71585ffc50e..4e42757c346 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,12 +18,11 @@
 #include <sys/stat.h>
 #include <time.h>
 
-#include "common/controldata_utils.h"
-#include "common/hashfn_unstable.h"
 #include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
+#include "pg_verifybackup.h"
 #include "pgtime.h"
 
 /*
@@ -40,89 +39,6 @@
  */
 #define ESTIMATED_BYTES_PER_MANIFEST_LINE	100
 
-/*
- * How many bytes should we try to read from a file at once?
- */
-#define READ_CHUNK_SIZE				(128 * 1024)
-
-/*
- * Each file described by the manifest file is parsed to produce an object
- * like this.
- */
-typedef struct manifest_file
-{
-	uint32		status;			/* hash status */
-	const char *pathname;
-	size_t		size;
-	pg_checksum_type checksum_type;
-	int			checksum_length;
-	uint8	   *checksum_payload;
-	bool		matched;
-	bool		bad;
-} manifest_file;
-
-#define should_verify_checksum(m) \
-	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
-
-/*
- * Define a hash table which we can use to store information about the files
- * mentioned in the backup manifest.
- */
-#define SH_PREFIX		manifest_files
-#define SH_ELEMENT_TYPE	manifest_file
-#define SH_KEY_TYPE		const char *
-#define	SH_KEY			pathname
-#define SH_HASH_KEY(tb, key)	hash_string(key)
-#define SH_EQUAL(tb, a, b)		(strcmp(a, b) == 0)
-#define	SH_SCOPE		static inline
-#define SH_RAW_ALLOCATOR	pg_malloc0
-#define SH_DECLARE
-#define SH_DEFINE
-#include "lib/simplehash.h"
-
-/*
- * Each WAL range described by the manifest file is parsed to produce an
- * object like this.
- */
-typedef struct manifest_wal_range
-{
-	TimeLineID	tli;
-	XLogRecPtr	start_lsn;
-	XLogRecPtr	end_lsn;
-	struct manifest_wal_range *next;
-	struct manifest_wal_range *prev;
-} manifest_wal_range;
-
-/*
- * All the data parsed from a backup_manifest file.
- */
-typedef struct manifest_data
-{
-	int			version;
-	uint64		system_identifier;
-	manifest_files_hash *files;
-	manifest_wal_range *first_wal_range;
-	manifest_wal_range *last_wal_range;
-} manifest_data;
-
-/*
- * All of the context information we need while checking a backup manifest.
- */
-typedef struct verifier_context
-{
-	manifest_data *manifest;
-	char	   *backup_directory;
-	SimpleStringList ignore_list;
-	bool		skip_checksums;
-	bool		exit_on_error;
-	bool		saw_any_error;
-
-	/* Progress indicators */
-	bool		show_progress;
-	uint64		total_size;
-	uint64		done_size;
-} verifier_context;
-
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -156,14 +72,6 @@ static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
 
-static void report_backup_error(verifier_context *context,
-								const char *pg_restrict fmt,...)
-			pg_attribute_printf(2, 3);
-static void report_fatal_error(const char *pg_restrict fmt,...)
-			pg_attribute_printf(1, 2) pg_attribute_noreturn();
-static bool should_ignore_relpath(verifier_context *context, const char *relpath);
-
-static void progress_report(verifier_context *context, bool finished);
 static void usage(void);
 
 static const char *progname;
@@ -978,7 +886,7 @@ parse_required_wal(verifier_context *context, char *pg_waldump_path,
  * Update the context to indicate that we saw an error, and exit if the
  * context says we should.
  */
-static void
+void
 report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
 {
 	va_list		ap;
@@ -995,7 +903,7 @@ report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
 /*
  * Report a fatal error and exit
  */
-static void
+void
 report_fatal_error(const char *pg_restrict fmt,...)
 {
 	va_list		ap;
@@ -1014,7 +922,7 @@ report_fatal_error(const char *pg_restrict fmt,...)
  * Note that by "prefix" we mean a parent directory; for this purpose,
  * "aa/bb" is not a prefix of "aa/bbb", but it is a prefix of "aa/bb/cc".
  */
-static bool
+bool
 should_ignore_relpath(verifier_context *context, const char *relpath)
 {
 	SimpleStringListCell *cell;
@@ -1043,7 +951,7 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
  * If finished is set to true, this is the last progress report. The cursor
  * is moved to the next line.
  */
-static void
+void
 progress_report(verifier_context *context, bool finished)
 {
 	static pg_time_t last_progress_report = 0;
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
new file mode 100644
index 00000000000..90900048547
--- /dev/null
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -0,0 +1,118 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_verifybackup.h
+ *	  Verify a backup against a backup manifest.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_verifybackup/pg_verifybackup.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef PG_VERIFYBACKUP_H
+#define PG_VERIFYBACKUP_H
+
+#include "common/controldata_utils.h"
+#include "common/hashfn_unstable.h"
+#include "common/parse_manifest.h"
+#include "fe_utils/simple_list.h"
+
+extern bool show_progress;
+extern bool skip_checksums;
+
+/*
+ * How many bytes should we try to read from a file at once?
+ */
+#define READ_CHUNK_SIZE				(128 * 1024)
+
+/*
+ * Each file described by the manifest file is parsed to produce an object
+ * like this.
+ */
+typedef struct manifest_file
+{
+	uint32		status;			/* hash status */
+	const char *pathname;
+	size_t		size;
+	pg_checksum_type checksum_type;
+	int			checksum_length;
+	uint8	   *checksum_payload;
+	bool		matched;
+	bool		bad;
+} manifest_file;
+
+#define should_verify_checksum(m) \
+	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+
+/*
+ * Define a hash table which we can use to store information about the files
+ * mentioned in the backup manifest.
+ */
+#define SH_PREFIX		manifest_files
+#define SH_ELEMENT_TYPE	manifest_file
+#define SH_KEY_TYPE		const char *
+#define	SH_KEY			pathname
+#define SH_HASH_KEY(tb, key)	hash_string(key)
+#define SH_EQUAL(tb, a, b)		(strcmp(a, b) == 0)
+#define	SH_SCOPE		static inline
+#define SH_RAW_ALLOCATOR	pg_malloc0
+#define SH_DECLARE
+#define SH_DEFINE
+#include "lib/simplehash.h"
+
+/*
+ * Each WAL range described by the manifest file is parsed to produce an
+ * object like this.
+ */
+typedef struct manifest_wal_range
+{
+	TimeLineID	tli;
+	XLogRecPtr	start_lsn;
+	XLogRecPtr	end_lsn;
+	struct manifest_wal_range *next;
+	struct manifest_wal_range *prev;
+} manifest_wal_range;
+
+/*
+ * All the data parsed from a backup_manifest file.
+ */
+typedef struct manifest_data
+{
+	int			version;
+	uint64		system_identifier;
+	manifest_files_hash *files;
+	manifest_wal_range *first_wal_range;
+	manifest_wal_range *last_wal_range;
+} manifest_data;
+
+/*
+ * All of the context information we need while checking a backup manifest.
+ */
+typedef struct verifier_context
+{
+	manifest_data *manifest;
+	char	   *backup_directory;
+	SimpleStringList ignore_list;
+	bool		skip_checksums;
+	bool		exit_on_error;
+	bool		saw_any_error;
+
+	/* Progress indicators */
+	bool		show_progress;
+	uint64		total_size;
+	uint64		done_size;
+} verifier_context;
+
+extern void report_backup_error(verifier_context *context,
+								const char *pg_restrict fmt,...)
+			pg_attribute_printf(2, 3);
+extern void report_fatal_error(const char *pg_restrict fmt,...)
+			pg_attribute_printf(1, 2) pg_attribute_noreturn();
+extern bool should_ignore_relpath(verifier_context *context,
+								  const char *relpath);
+
+extern void progress_report(verifier_context *context, bool finished);
+
+#endif							/* PG_VERIFYBACKUP_H */
-- 
2.18.0

From 87de8fb8600c7e373a44e0b3dddd25044c524352 Mon Sep 17 00:00:00 2001
From: Robert Haas <rhaas@postgresql.org>
Date: Tue, 6 Aug 2024 10:23:45 +0530
Subject: [PATCH v8] Improve file header comments for astramer code.

Make it clear that "astreamer" stands for "archive streamer".
Generalize comments that still believe this code can only be used
by pg_basebackup. Add some comments explaining the asymmetry
between the gzip, lz4, and zstd astreamers, in the hopes of making
life easier for anyone who hacks on this code in the future.
---
 src/fe_utils/astreamer_file.c    |  4 ++++
 src/fe_utils/astreamer_gzip.c    | 15 +++++++++++++++
 src/fe_utils/astreamer_lz4.c     |  4 ++++
 src/fe_utils/astreamer_zstd.c    |  4 ++++
 src/include/fe_utils/astreamer.h | 21 +++++++++++++++------
 5 files changed, 42 insertions(+), 6 deletions(-)

diff --git a/src/fe_utils/astreamer_file.c b/src/fe_utils/astreamer_file.c
index 13d1192c6e6..c9a030853bc 100644
--- a/src/fe_utils/astreamer_file.c
+++ b/src/fe_utils/astreamer_file.c
@@ -2,6 +2,10 @@
  *
  * astreamer_file.c
  *
+ * Archive streamers that write to files. astreamer_plain_writer writes
+ * the whole archive to a single file, and astreamer_extractor writes
+ * each archive member to a separate file in a given directory.
+ *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
diff --git a/src/fe_utils/astreamer_gzip.c b/src/fe_utils/astreamer_gzip.c
index dd28defac7b..1c773a23848 100644
--- a/src/fe_utils/astreamer_gzip.c
+++ b/src/fe_utils/astreamer_gzip.c
@@ -2,6 +2,21 @@
  *
  * astreamer_gzip.c
  *
+ * Archive streamers that deal with data compressed using gzip.
+ * astreamer_gzip_writer applies gzip compression to the input data
+ * and writes the result to a file. astreamer_gzip_decompressor assumes
+ * that the input stream is compressed using gzip and decompresses it.
+ *
+ * Note that the code in this file is asymmetric with what we do for
+ * other compression types: for lz4 and zstd, there is a compressor and
+ * a decompressor, rather than a writer and a decompressor. The approach
+ * taken here is less flexible, because a writer can only write to a file,
+ * while a compressor can write to a subsequent astreamer which is free
+ * to do whatever it likes. The reason it's like this is because this
+ * code was adapated from old, less-modular pg_basebackup that used the
+ * same APIs that astreamer_gzip_writer uses, and it didn't seem
+ * necessary to change anything at the time.
+ *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
diff --git a/src/fe_utils/astreamer_lz4.c b/src/fe_utils/astreamer_lz4.c
index d8b2a367e47..2bf14084e7f 100644
--- a/src/fe_utils/astreamer_lz4.c
+++ b/src/fe_utils/astreamer_lz4.c
@@ -2,6 +2,10 @@
  *
  * astreamer_lz4.c
  *
+ * Archive streamers that deal with data compressed using lz4.
+ * astreamer_lz4_compressor applies lz4 compression to the input stream,
+ * and astreamer_lz4_decompressor does the reverse.
+ *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
diff --git a/src/fe_utils/astreamer_zstd.c b/src/fe_utils/astreamer_zstd.c
index 45f6cb67363..4b2d42b2311 100644
--- a/src/fe_utils/astreamer_zstd.c
+++ b/src/fe_utils/astreamer_zstd.c
@@ -2,6 +2,10 @@
  *
  * astreamer_zstd.c
  *
+ * Archive streamers that deal with data compressed using zstd.
+ * astreamer_zstd_compressor applies lz4 compression to the input stream,
+ * and astreamer_zstd_decompressor does the reverse.
+ *
  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
diff --git a/src/include/fe_utils/astreamer.h b/src/include/fe_utils/astreamer.h
index 2c014dbddbe..570cfba3040 100644
--- a/src/include/fe_utils/astreamer.h
+++ b/src/include/fe_utils/astreamer.h
@@ -2,9 +2,18 @@
  *
  * astreamer.h
  *
- * Each tar archive returned by the server is passed to one or more
- * astreamer objects for further processing. The astreamer may do
- * something simple, like write the archive to a file, perhaps after
+ * The "archive streamer" interface is intended to allow frontend code
+ * to stream from possibly-compressed archive files from any source and
+ * perform arbitrary actions based on the contents of those archives.
+ * Archive streamers are intended to be composable, and most tasks will
+ * require two or more archive streamers to complete. For instance,
+ * if the input is an uncompressed tar stream, a tar parser astreamer
+ * could be used to interpret it, and then an extractor astreamer could
+ * be used to write each archive member out to a file.
+ *
+ * In general, each archive streamer is relatively free to take whatever
+ * action it desires in the stream of chunks provided by the caller. It
+ * may do something simple, like write the archive to a file, perhaps after
  * compressing it, but it can also do more complicated things, like
  * annotating the byte stream to indicate which parts of the data
  * correspond to tar headers or trailing padding, vs. which parts are
@@ -33,9 +42,9 @@ typedef struct astreamer_ops astreamer_ops;
 
 /*
  * Each chunk of archive data passed to a astreamer is classified into one
- * of these categories. When data is first received from the remote server,
- * each chunk will be categorized as ASTREAMER_UNKNOWN, and the chunks will
- * be of whatever size the remote server chose to send.
+ * of these categories. When data is initially passed to an archive streamer,
+ * each chunk will be categorized as ASTREAMER_UNKNOWN, and the chunks can
+ * be of whatever size the caller finds convenient.
  *
  * If the archive is parsed (e.g. see astreamer_tar_parser_new()), then all
  * chunks should be labelled as one of the other types listed here. In
-- 
2.18.0

From 98ecaf7d965d44e4c5d1e558b1406230da31a79c Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 31 Jul 2024 16:22:07 +0530
Subject: [PATCH v9 11/12] pg_verifybackup: Read tar files and verify its
 contents

This patch implements TAR format backup verification.

For progress reporting support, we perform this verification in two
passes: the first pass calculates total_size, and the second pass
updates done_size as verification progresses.

For the verification, in the first pass, we call verify_tar_file(),
which performs basic verification by expecting only base.tar or
<tablespaceoid>.tar files and raises an error for any other files.  It
also determines the compression type of the archive file. All this
information is stored in a newly added tarFile struct, which is
appended to a list that will be used in the second pass (by
verify_tar_content()) for the final verification. In the second pass,
the tar archives are read, decompressed, and the required verification
is carried out.

For reading and decompression, fe_utils/astreamer.h is used. For
verification, a new archive streamer has been added in
astreamer_verify.c to handle TAR member files and their contents; see
astreamer_verify_content() for details. The stack of astreamers will
be set up for each TAR file in verify_tar_content(), depending on its
compression type which is detected in the first pass.

When information about a TAR member file (i.e., ASTREAMER_MEMBER_HEADER)
is received, we first verify its entry against the backup manifest. We
then decide if further checks are needed, such as checksum
verification and control data verification (if it is a pg_control
file), once the member file contents are received. Although this
decision could be made when the contents are received, it is more
efficient to make it earlier since the member file contents are
received in multiple iterations. In short, we process
ASTREAMER_MEMBER_CONTENTS multiple times but only once for other
ASTREAMER_MEMBER_* cases. We maintain this information in the
astreamer_verify structure for each member file, which is reset when
the file ends.
---
 src/bin/pg_verifybackup/Makefile           |   4 +-
 src/bin/pg_verifybackup/astreamer_verify.c | 354 +++++++++++++++++++++
 src/bin/pg_verifybackup/meson.build        |   6 +-
 src/bin/pg_verifybackup/pg_verifybackup.c  | 313 +++++++++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h  |   6 +
 src/tools/pgindent/typedefs.list           |   2 +
 6 files changed, 675 insertions(+), 10 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/astreamer_verify.c

diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile
index 7c045f142e8..df7aaabd530 100644
--- a/src/bin/pg_verifybackup/Makefile
+++ b/src/bin/pg_verifybackup/Makefile
@@ -17,11 +17,13 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 # We need libpq only because fe_utils does.
+override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 
 OBJS = \
 	$(WIN32RES) \
-	pg_verifybackup.o
+	pg_verifybackup.o \
+	astreamer_verify.o
 
 all: pg_verifybackup
 
diff --git a/src/bin/pg_verifybackup/astreamer_verify.c b/src/bin/pg_verifybackup/astreamer_verify.c
new file mode 100644
index 00000000000..0983dffde8e
--- /dev/null
+++ b/src/bin/pg_verifybackup/astreamer_verify.c
@@ -0,0 +1,354 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer_verify.c
+ *
+ * Extend fe_utils/astreamer.h archive streaming facility to verify TAR
+ * backup.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * src/bin/pg_verifybackup/astreamer_verify.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include "common/logging.h"
+#include "pg_verifybackup.h"
+
+typedef struct astreamer_verify
+{
+	astreamer	base;
+	verifier_context *context;
+	char	   *archive_name;
+	Oid			tblspc_oid;
+	pg_checksum_context *checksum_ctx;
+
+	/* Hold information for a member file verification */
+	manifest_file *mfile;
+	int64		received_bytes;
+	bool		verify_checksum;
+	bool		verify_control_data;
+} astreamer_verify;
+
+static void astreamer_verify_content(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len,
+									 astreamer_archive_context context);
+static void astreamer_verify_finalize(astreamer *streamer);
+static void astreamer_verify_free(astreamer *streamer);
+
+static void verify_member_header(astreamer *streamer, astreamer_member *member);
+static void verify_member_checksum(astreamer *streamer,
+								   astreamer_member *member,
+								   const char *buffer, int buffer_len);
+static void verify_member_control_data(astreamer *streamer,
+									   astreamer_member *member,
+									   const char *data, int len);
+static void reset_member_info(astreamer *streamer);
+
+static const astreamer_ops astreamer_verify_ops = {
+	.content = astreamer_verify_content,
+	.finalize = astreamer_verify_finalize,
+	.free = astreamer_verify_free
+};
+
+/*
+ * Create a astreamer that can verifies content of a TAR file.
+ */
+astreamer *
+astreamer_verify_content_new(astreamer *next, verifier_context *context,
+							 char *archive_name, Oid tblspc_oid)
+{
+	astreamer_verify *streamer;
+
+	streamer = palloc0(sizeof(astreamer_verify));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_verify_ops;
+
+	streamer->base.bbs_next = next;
+	streamer->context = context;
+	streamer->archive_name = archive_name;
+	streamer->tblspc_oid = tblspc_oid;
+	initStringInfo(&streamer->base.bbs_buffer);
+
+	if (!context->skip_checksums)
+		streamer->checksum_ctx = pg_malloc(sizeof(pg_checksum_context));
+
+	return &streamer->base;
+}
+
+/*
+ * The main entry point of the archive streamer for verifying tar members.
+ */
+static void
+astreamer_verify_content(astreamer *streamer, astreamer_member *member,
+						 const char *data, int len,
+						 astreamer_archive_context context)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	Assert(context != ASTREAMER_UNKNOWN);
+
+	switch (context)
+	{
+		case ASTREAMER_MEMBER_HEADER:
+
+			/*
+			 * Perform the initial check and setup verification steps.
+			 */
+			verify_member_header(streamer, member);
+			break;
+
+		case ASTREAMER_MEMBER_CONTENTS:
+
+			/*
+			 * Process the member content according to the flags set by the
+			 * member header processing routine for checksum and control data
+			 * verification.
+			 */
+			if (mystreamer->verify_checksum)
+				verify_member_checksum(streamer, member, data, len);
+
+			if (mystreamer->verify_control_data)
+				verify_member_control_data(streamer, member, data, len);
+			break;
+
+		case ASTREAMER_MEMBER_TRAILER:
+
+			/*
+			 * Reset the temporary information stored for the verification.
+			 */
+			reset_member_info(streamer);
+			break;
+
+		case ASTREAMER_ARCHIVE_TRAILER:
+			break;
+
+		default:
+			/* Shouldn't happen. */
+			pg_fatal("unexpected state while parsing tar archive");
+	}
+}
+
+/*
+ * End-of-stream processing for a astreamer_verify stream.
+ */
+static void
+astreamer_verify_finalize(astreamer *streamer)
+{
+	Assert(streamer->bbs_next == NULL);
+}
+
+/*
+ * Free memory associated with a astreamer_verify stream.
+ */
+static void
+astreamer_verify_free(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	if (mystreamer->checksum_ctx)
+		pfree(mystreamer->checksum_ctx);
+
+	pfree(streamer->bbs_buffer.data);
+	pfree(streamer);
+}
+
+/*
+ * Verifies the tar member entry if it corresponds to a file in the backup
+ * manifest. If the archive being processed is a tablespace, prepares the
+ * required file path for subsequent operations. Finally, determines if
+ * checksum verification and control data verification need to be performed
+ * during file content processing
+ */
+static void
+verify_member_header(astreamer *streamer, astreamer_member *member)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_file *m;
+
+	/* We are only interested in files that are not in the ignore list. */
+	if (member->is_directory || member->is_link ||
+		should_ignore_relpath(mystreamer->context, member->pathname))
+		return;
+
+	/*
+	 * The backup_manifest stores a relative path to the base directory for
+	 * files belong tablespace, whereas <tablespaceoid>.tar doesn't. Prepare
+	 * the required path, otherwise, the manfiest entry verification will
+	 * fail.
+	 */
+	if (OidIsValid(mystreamer->tblspc_oid))
+	{
+		char		temp[MAXPGPATH];
+
+		/* Copy original name at temporary space */
+		memcpy(temp, member->pathname, MAXPGPATH);
+
+		snprintf(member->pathname, MAXPGPATH, "%s/%d/%s",
+				 "pg_tblspc", mystreamer->tblspc_oid, temp);
+	}
+
+	/* Check the manifest entry */
+	m = verify_manifest_entry(mystreamer->context, member->pathname,
+							  member->size);
+	mystreamer->mfile = (void *) m;
+
+	/*
+	 * Prepare for checksum and control data verification.
+	 *
+	 * We could have these checks while receiving contents. However, since
+	 * contents are received in multiple iterations, this would result in
+	 * these lengthy checks being performed multiple times. Instead, having a
+	 * single flag would be more efficient.
+	 */
+	mystreamer->verify_checksum =
+		(!mystreamer->context->skip_checksums && should_verify_checksum(m));
+	mystreamer->verify_control_data =
+		should_verify_control_data(mystreamer->context->manifest, m);
+
+	/* Initialize the context required for checksum verification. */
+	if (mystreamer->verify_checksum &&
+		pg_checksum_init(mystreamer->checksum_ctx, m->checksum_type) < 0)
+	{
+		report_backup_error(mystreamer->context,
+							"%s: could not initialize checksum of file \"%s\"",
+							mystreamer->archive_name, m->pathname);
+
+		/*
+		 * Checksum verification cannot be performed without proper context
+		 * initialization.
+		 */
+		mystreamer->verify_checksum = false;
+	}
+}
+
+/*
+ * Computes the checksum incrementally for the received file content, and
+ * finally calls the routine for checksum verification, similar to
+ * verify_file_checksum().
+ *
+ * The caller should pass a correctly initialized checksum_ctx, which will be
+ * used for incremental checksum computation. Once the complete file content is
+ * received (tracked using received_bytes), the final checksum verification
+ * happens.
+ */
+static void
+verify_member_checksum(astreamer *streamer, astreamer_member *member,
+					   const char *buffer, int buffer_len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	pg_checksum_context *checksum_ctx = mystreamer->checksum_ctx;
+	verifier_context *context = mystreamer->context;
+	manifest_file *m = mystreamer->mfile;
+	const char *relpath = m->pathname;
+	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
+
+	/*
+	 * Mark it false to avoid unexpected re-entrance for the same file content
+	 * (e.g. returned in error should not be revisited).
+	 */
+	Assert(mystreamer->verify_checksum);
+	mystreamer->verify_checksum = false;
+
+	/* Should have came for the right file */
+	Assert(strcmp(member->pathname, relpath) == 0);
+
+	/*
+	 * The checksum context should match the type noted in the backup
+	 * manifest.
+	 */
+	Assert(checksum_ctx->type == m->checksum_type);
+
+	/* Update the total count of computed checksum bytes. */
+	mystreamer->received_bytes += buffer_len;
+
+	if (pg_checksum_update(checksum_ctx, (uint8 *) buffer, buffer_len) < 0)
+	{
+		report_backup_error(context, "could not update checksum of file \"%s\"",
+							relpath);
+		return;
+	}
+
+	/* Yet to receive the full content of the file. */
+	if (mystreamer->received_bytes < m->size)
+	{
+		mystreamer->verify_checksum = true;
+		return;
+	}
+
+	/* Do the final computation and verification. */
+	verify_checksum(context, m, checksum_ctx, checksumbuf);
+}
+
+/*
+ * Prepare the control data from the received tar member contents, which are
+ * supposed to be from the pg_control file, including CRC calculation. Then,
+ * call the routines that perform the final verification of the control file
+ * information.
+ */
+static void
+verify_member_control_data(astreamer *streamer, astreamer_member *member,
+						   const char *data, int len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_data *manifest = mystreamer->context->manifest;
+	ControlFileData control_file;
+	pg_crc32c	crc;
+	bool		crc_ok;
+
+	/* Should be here only for control file */
+	Assert(strcmp(member->pathname, "global/pg_control") == 0);
+	Assert(manifest->version != 1);
+
+	/* Mark it as false to avoid unexpected re-entrance */
+	Assert(mystreamer->verify_control_data);
+	mystreamer->verify_control_data = false;
+
+	/* Should have whole control file data. */
+	if (!astreamer_buffer_until(streamer, &data, &len, sizeof(ControlFileData)))
+	{
+		mystreamer->verify_control_data = true;
+		return;
+	}
+
+	pg_log_debug("%s: reading \"%s\"", mystreamer->archive_name,
+				 member->pathname);
+
+	if (streamer->bbs_buffer.len != sizeof(ControlFileData))
+		report_fatal_error("%s: could not read control file: read %d of %zu",
+						   mystreamer->archive_name, streamer->bbs_buffer.len,
+						   sizeof(ControlFileData));
+
+	memcpy(&control_file, streamer->bbs_buffer.data,
+		   sizeof(ControlFileData));
+
+	/* Check the CRC. */
+	INIT_CRC32C(crc);
+	COMP_CRC32C(crc,
+				(char *) (&control_file),
+				offsetof(ControlFileData, crc));
+	FIN_CRC32C(crc);
+
+	crc_ok = EQ_CRC32C(crc, control_file.crc);
+
+	/* Do the final control data verification. */
+	verify_control_data(&control_file, member->pathname, crc_ok,
+						manifest->system_identifier);
+}
+
+/*
+ * Reset flags and free memory allocations for member file verification.
+ */
+static void
+reset_member_info(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	mystreamer->mfile = NULL;
+	mystreamer->received_bytes = 0;
+	mystreamer->verify_checksum = false;
+	mystreamer->verify_control_data = false;
+}
diff --git a/src/bin/pg_verifybackup/meson.build b/src/bin/pg_verifybackup/meson.build
index 7c7d31a0350..1e3fcf7ee5a 100644
--- a/src/bin/pg_verifybackup/meson.build
+++ b/src/bin/pg_verifybackup/meson.build
@@ -1,7 +1,8 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 pg_verifybackup_sources = files(
-  'pg_verifybackup.c'
+  'pg_verifybackup.c',
+  'astreamer_verify.c'
 )
 
 if host_system == 'windows'
@@ -10,9 +11,10 @@ if host_system == 'windows'
     '--FILEDESC', 'pg_verifybackup - verify a backup against using a backup manifest'])
 endif
 
+pg_verifybackup_deps = [frontend_code, libpq, lz4, zlib, zstd]
 pg_verifybackup = executable('pg_verifybackup',
   pg_verifybackup_sources,
-  dependencies: [frontend_code, libpq],
+  dependencies: pg_verifybackup_deps,
   kwargs: default_bin_args,
 )
 bin_targets += pg_verifybackup
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 88196cca4e0..42b776eda18 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -20,9 +20,11 @@
 
 #include "common/compression.h"
 #include "common/parse_manifest.h"
+#include "common/relpath.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
 #include "pg_verifybackup.h"
+#include "pgtar.h"
 #include "pgtime.h"
 
 /*
@@ -39,6 +41,16 @@
  */
 #define ESTIMATED_BYTES_PER_MANIFEST_LINE	100
 
+/*
+ * Tar archive information needed for content verification.
+ */
+typedef struct tarFile
+{
+	char	   *relpath;
+	Oid			tblspc_oid;
+	pg_compress_algorithm compress_algorithm;
+} tarFile;
+
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -61,7 +73,18 @@ static char find_backup_format(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
-							   char *relpath, char *fullpath);
+							   char *relpath, char *fullpath,
+							   SimplePtrList *tarFiles);
+static void verify_plain_file(verifier_context *context,
+							  char *relpath, char *fullpath,
+							  size_t filesize);
+static void verify_tar_file_name(verifier_context *context, char *relpath,
+								 char *fullpath, int64 filesize,
+								 SimplePtrList *tarFiles);
+static void verify_tar_file_contents(verifier_context *context,
+									 SimplePtrList *tarFiles);
+static void verify_tar_contents(verifier_context *context, char *relpath,
+								char *fullpath, astreamer *streamer);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -70,6 +93,10 @@ static void verify_file_checksum(verifier_context *context,
 static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
+static astreamer *create_archive_verifier(verifier_context *context,
+										  char *archive_name,
+										  Oid tblspc_oid,
+										  pg_compress_algorithm compress_algo);
 
 static void progress_report(bool finished);
 static void usage(void);
@@ -148,6 +175,10 @@ main(int argc, char **argv)
 	 */
 	simple_string_list_append(&context.ignore_list, "backup_manifest");
 	simple_string_list_append(&context.ignore_list, "pg_wal");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.gz");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.lz4");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.zst");
 	simple_string_list_append(&context.ignore_list, "postgresql.auto.conf");
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
@@ -556,6 +587,7 @@ verify_backup_directory(verifier_context *context, char *relpath,
 {
 	DIR		   *dir;
 	struct dirent *dirent;
+	SimplePtrList tarFiles = {NULL, NULL};
 
 	dir = opendir(fullpath);
 	if (dir == NULL)
@@ -595,12 +627,17 @@ verify_backup_directory(verifier_context *context, char *relpath,
 			newrelpath = psprintf("%s/%s", relpath, filename);
 
 		if (!should_ignore_relpath(context, newrelpath))
-			verify_backup_file(context, newrelpath, newfullpath);
+			verify_backup_file(context, newrelpath, newfullpath, &tarFiles);
 
 		pfree(newfullpath);
 		pfree(newrelpath);
 	}
 
+	/* Perform the final verification of the tar contents, if any. */
+	Assert(tarFiles.head == NULL || context->format == 't');
+	if (tarFiles.head != NULL)
+		verify_tar_file_contents(context, &tarFiles);
+
 	if (closedir(dir))
 	{
 		report_backup_error(context,
@@ -610,16 +647,18 @@ verify_backup_directory(verifier_context *context, char *relpath,
 }
 
 /*
- * Verify one file (which might actually be a directory or a symlink).
+ * Verify one file (which might actually be a directory, a symlink or a
+ * archive).
  *
  * The arguments to this function have the same meaning as the arguments to
- * verify_backup_directory.
+ * verify_backup_directory. The additional argument outputs the list of tar
+ * archive information if any.
  */
 static void
-verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
+verify_backup_file(verifier_context *context, char *relpath, char *fullpath,
+				   SimplePtrList *tarFiles)
 {
 	struct stat sb;
-	manifest_file *m;
 
 	if (stat(fullpath, &sb) != 0)
 	{
@@ -652,8 +691,36 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		return;
 	}
 
+	/* Do the further verifications */
+	if (context->format == 'p')
+		verify_plain_file(context, relpath, fullpath, sb.st_size);
+	else
+	{
+		/*
+		 * This is preparatory work for the tar format backup verification,
+		 * where we verify only the archive file name and its compression
+		 * type. The final verification will be carried out after listing all
+		 * the archives from the backup directory.
+		 */
+		verify_tar_file_name(context, relpath, fullpath, sb.st_size, tarFiles);
+	}
+}
+
+/*
+ * Verify one plain file or a symlink.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_backup_directory. The additional argument is the file size for
+ * verifying against manifest entry.
+ */
+static void
+verify_plain_file(verifier_context *context, char *relpath, char *fullpath,
+				  size_t filesize)
+{
+	manifest_file *m;
+
 	/* Check the backup manifest entry for this file. */
-	m = verify_manifest_entry(context, relpath, sb.st_size);
+	m = verify_manifest_entry(context, relpath, filesize);
 
 	/* Validate the manifest system identifier */
 	if (should_verify_control_data(context->manifest, m))
@@ -676,6 +743,202 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		total_size += m->size;
 }
 
+/*
+ * Verify one tar archive file.
+ *
+ * This does not perform a complete verification; it only performs basic
+ * validation of the tar format backup file, detects the compression type, and
+ * appends that information to the tarFiles list. An error will be reported if
+ * the tar archive name or compression type is not as expected.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_backup_file. The additional argument is the file size for progress
+ * report.
+ */
+static void
+verify_tar_file_name(verifier_context *context, char *relpath, char *fullpath,
+					 int64 filesize, SimplePtrList *tarFiles)
+{
+	Oid			tblspc_oid = InvalidOid;
+	int			file_name_len;
+	int			file_extn_len;
+	pg_compress_algorithm compress_algorithm;
+	tarFile    *tar_file;
+
+	/* Should be tar format backup */
+	Assert(context->format == 't');
+
+	/* Find the compression type of the tar file */
+	if (strstr(relpath, ".tgz") != NULL)
+	{
+		compress_algorithm = PG_COMPRESSION_GZIP;
+		file_extn_len = 4;		/* length of ".tgz" */
+	}
+	else if (strstr(relpath, ".tar.gz") != NULL)
+	{
+		compress_algorithm = PG_COMPRESSION_GZIP;
+		file_extn_len = 7;		/* length of ".tar.gz" */
+	}
+	else if (strstr(relpath, ".tar.lz4") != NULL)
+	{
+		compress_algorithm = PG_COMPRESSION_LZ4;
+		file_extn_len = 8;		/* length of ".tar.lz4" */
+	}
+	else if (strstr(relpath, ".tar.zst") != NULL)
+	{
+		compress_algorithm = PG_COMPRESSION_ZSTD;
+		file_extn_len = 8;		/* length of ".tar.zst" */
+	}
+	else if (strstr(relpath, ".tar") != NULL)
+	{
+		compress_algorithm = PG_COMPRESSION_NONE;
+		file_extn_len = 4;		/* length of ".tar" */
+	}
+	else
+	{
+		report_backup_error(context,
+							"\"%s\" unexpected file in the tar format backup",
+							relpath);
+		return;
+	}
+
+	/*
+	 * We expect tar archive files of backing up the main directory and
+	 * tablespace.
+	 *
+	 * pg_basebackup writes the main data directory to an archive file named
+	 * base.tar and the tablespace directory to <tablespaceoid>.tar, followed
+	 * by a compression type extension such as .gz, .lz4, or .zst.
+	 */
+	file_name_len = strlen(relpath);
+	if (strspn(relpath, "0123456789") == (file_name_len - file_extn_len))
+	{
+		/*
+		 * Since the file matches the <tablespaceoid>.tar format, extract the
+		 * tablespaceoid, which is needed to prepare the paths of the files
+		 * belonging to that tablespace relative to the base directory.
+		 */
+		tblspc_oid = strtoi64(relpath, NULL, 10);
+	}
+	/* Otherwise, it should be a base.tar file; if not, raise an error. */
+	else if (strncmp("base", relpath, file_name_len - file_extn_len) != 0)
+	{
+		report_backup_error(context,
+							"\"%s\" unexpected file in the tar format backup",
+							relpath);
+		return;
+	}
+
+	/*
+	 * Append the information to the list for complete verification at a later
+	 * stage.
+	 */
+	tar_file = pg_malloc(sizeof(tarFile));
+	tar_file->relpath = pstrdup(relpath);
+	tar_file->tblspc_oid = tblspc_oid;
+	tar_file->compress_algorithm = compress_algorithm;
+
+	simple_ptr_list_append(tarFiles, tar_file);
+
+	/* Update statistics for progress report, if necessary */
+	if (show_progress)
+		total_size += filesize;
+}
+
+/*
+ * This is the final part of tar file verification, which prepares the archive
+ * streamer stack according to the tar file compression format for each tar
+ * archive and invokes them for reading, decompressing, and ultimately
+ * verifying the contents.
+ *
+ * The arguments to this function should be a list of valid tar archives to
+ * verify, and the allocation will be freed once the verification is complete.
+ */
+static void
+verify_tar_file_contents(verifier_context *context, SimplePtrList *tarFiles)
+{
+	SimplePtrListCell *cell;
+
+	progress_report(false);
+
+	for (cell = tarFiles->head; cell != NULL; cell = cell->next)
+	{
+		tarFile    *tar_file = (tarFile *) cell->ptr;
+		astreamer  *streamer;
+		char	   *fullpath;
+
+		/* Prepare archive streamer stack */
+		streamer = create_archive_verifier(context,
+										   tar_file->relpath,
+										   tar_file->tblspc_oid,
+										   tar_file->compress_algorithm);
+
+		/* Compute the full pathname to the target file. */
+		fullpath = psprintf("%s/%s", context->backup_directory,
+							tar_file->relpath);
+
+		/* Invoke the streamer for reading, decompressing, and verifying. */
+		verify_tar_contents(context, tar_file->relpath, fullpath, streamer);
+
+		/* Cleanup. */
+		pfree(tar_file->relpath);
+		pfree(tar_file);
+		pfree(fullpath);
+
+		astreamer_finalize(streamer);
+		astreamer_free(streamer);
+	}
+	simple_ptr_list_destroy(tarFiles);
+
+	progress_report(true);
+}
+
+/*
+ * Performs the actual work for tar content verification. It reads a given tar
+ * file in predefined chunks and passes it to the streamer, which initiates
+ * routines for decompression (if necessary) and then verifies each member
+ * within the tar archive.
+ */
+static void
+verify_tar_contents(verifier_context *context, char *relpath, char *fullpath,
+					astreamer *streamer)
+{
+	int			fd;
+	int			rc;
+	char	   *buffer;
+
+	pg_log_debug("reading \"%s\"", fullpath);
+
+	/* Open the target file. */
+	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
+	{
+		report_backup_error(context, "could not open file \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
+
+	/* Perform the reads */
+	while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
+	{
+		astreamer_content(streamer, NULL, buffer, rc, ASTREAMER_UNKNOWN);
+
+		/* Report progress */
+		done_size += rc;
+		progress_report(false);
+	}
+
+	if (rc < 0)
+		report_backup_error(context, "could not read file \"%s\": %m",
+							relpath);
+
+	/* Close the file. */
+	if (close(fd) != 0)
+		report_backup_error(context, "could not close file \"%s\": %m",
+							relpath);
+}
+
 /*
  * Verify file and its size entry in the manifest.
  */
@@ -1044,6 +1307,42 @@ find_backup_format(verifier_context *context)
 	return result;
 }
 
+/*
+ * Identifies the necessary steps for verifying the contents of the
+ * provided tar file.
+ */
+static astreamer *
+create_archive_verifier(verifier_context *context, char *archive_name,
+						Oid tblspc_oid, pg_compress_algorithm compress_algo)
+{
+	astreamer  *streamer = NULL;
+
+	/* Should be here only for tar backup */
+	Assert(context->format == 't');
+
+	/*
+	 * To verify the contents of the tar file, the initial step is to parse
+	 * its content.
+	 */
+	streamer = astreamer_verify_content_new(streamer, context, archive_name,
+											tblspc_oid);
+	streamer = astreamer_tar_parser_new(streamer);
+
+	/*
+	 * If the tar file is compressed, we must perform the appropriate
+	 * decompression operation before proceeding with the verification of its
+	 * contents.
+	 */
+	if (compress_algo == PG_COMPRESSION_GZIP)
+		streamer = astreamer_gzip_decompressor_new(streamer);
+	else if (compress_algo == PG_COMPRESSION_LZ4)
+		streamer = astreamer_lz4_decompressor_new(streamer);
+	else if (compress_algo == PG_COMPRESSION_ZSTD)
+		streamer = astreamer_zstd_decompressor_new(streamer);
+
+	return streamer;
+}
+
 /*
  * Print a progress report based on the global variables.
  *
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 856e8947c1d..db847a59657 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -18,6 +18,7 @@
 #include "common/hashfn_unstable.h"
 #include "common/logging.h"
 #include "common/parse_manifest.h"
+#include "fe_utils/astreamer.h"
 #include "fe_utils/simple_list.h"
 
 /*
@@ -128,4 +129,9 @@ extern void report_fatal_error(const char *pg_restrict fmt,...)
 extern bool should_ignore_relpath(verifier_context *context,
 								  const char *relpath);
 
+extern astreamer *astreamer_verify_content_new(astreamer *next,
+											   verifier_context *context,
+											   char *archive_name,
+											   Oid tblspc_oid);
+
 #endif							/* PG_VERIFYBACKUP_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 547d14b3e7c..47b5f0edcc7 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3329,6 +3329,7 @@ astreamer_plain_writer
 astreamer_recovery_injector
 astreamer_tar_archiver
 astreamer_tar_parser
+astreamer_verify
 astreamer_zstd_frame
 bgworker_main_type
 bh_node_type
@@ -3950,6 +3951,7 @@ substitute_phv_relids_context
 subxids_array_status
 symbol
 tablespaceinfo
+tarFile
 td_entry
 teSection
 temp_tablespaces_extra
-- 
2.18.0

From 3ba9b317cc313be4c3927b59d1d967458f5bcb66 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 15:47:26 +0530
Subject: [PATCH v9 08/12] Refactor: split verify_control_file.

Separated the manifest entry verification code into a new function and
introduced the should_verify_control_data() macro, similar to
should_verify_checksum().

Like verify_file_checksum(), verify_control_file() is too design to
accept the pg_control file patch which will be opened and respective
information will be verified. But, in case of tar backup we would be
having pg_control file containt instead, that needs to be verified in
the same way.  For that reason the code that doing the verification is
separated into separate function to so that can be reused for the tar
backup verification as well.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 44 +++++++++++------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 15 ++++++++
 2 files changed, 35 insertions(+), 24 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index e4f499fcd37..5adf24e8f90 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,7 +18,6 @@
 #include <sys/stat.h>
 #include <time.h>
 
-#include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
@@ -61,8 +60,6 @@ static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
 							   char *relpath, char *fullpath);
-static void verify_control_file(const char *controlpath,
-								uint64 manifest_system_identifier);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -625,14 +622,20 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 	/* Check the backup manifest entry for this file. */
 	m = verify_manifest_entry(context, relpath, sb.st_size);
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0 &&
-		m != NULL && m->matched && !m->bad)
-		verify_control_file(fullpath, context->manifest->system_identifier);
+	/* Validate the manifest system identifier */
+	if (should_verify_control_data(context->manifest, m))
+	{
+		ControlFileData *control_file;
+		bool		crc_ok;
+
+		pg_log_debug("reading \"%s\"", fullpath);
+		control_file = get_controlfile_by_exact_path(fullpath, &crc_ok);
+
+		verify_control_data(control_file, fullpath, crc_ok,
+							context->manifest->system_identifier);
+		/* Release memory. */
+		pfree(control_file);
+	}
 
 	/* Update statistics for progress report, if necessary */
 	if (show_progress && !context->skip_checksums &&
@@ -681,18 +684,14 @@ verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
 }
 
 /*
- * Sanity check control file and validate system identifier against manifest
- * system identifier.
+ * Sanity check control file data and validate system identifier against
+ * manifest system identifier.
  */
-static void
-verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
+void
+verify_control_data(ControlFileData *control_file,
+					const char *controlpath, bool crc_ok,
+					uint64 manifest_system_identifier)
 {
-	ControlFileData *control_file;
-	bool		crc_ok;
-
-	pg_log_debug("reading \"%s\"", controlpath);
-	control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
-
 	/* Control file contents not meaningful if CRC is bad. */
 	if (!crc_ok)
 		report_fatal_error("%s: CRC is incorrect", controlpath);
@@ -708,9 +707,6 @@ verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
 						   controlpath,
 						   (unsigned long long) manifest_system_identifier,
 						   (unsigned long long) control_file->system_identifier);
-
-	/* Release memory. */
-	pfree(control_file);
 }
 
 /*
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 12812cf5584..42d01c26466 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -16,6 +16,7 @@
 
 #include "common/controldata_utils.h"
 #include "common/hashfn_unstable.h"
+#include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 
@@ -44,6 +45,17 @@ typedef struct manifest_file
 	(((m) != NULL) && ((m)->matched) && !((m)->bad) && \
 	 (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
 
+/*
+ * Validate the manifest system identifier against the control file; this
+ * feature is not available in manifest version 1.  This validation should be
+ * carried out only if the manifest entry validation is completed without any
+ * errors.
+ */
+#define should_verify_control_data(manifest, m) \
+	(((manifest)->version != 1) && \
+	 ((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (strcmp((m)->pathname, "global/pg_control") == 0))
+
 /*
  * Define a hash table which we can use to store information about the files
  * mentioned in the backup manifest.
@@ -103,6 +115,9 @@ extern manifest_file *verify_manifest_entry(verifier_context *context,
 extern void verify_checksum(verifier_context *context, manifest_file *m,
 							pg_checksum_context *checksum_ctx,
 							uint8 *checksumbuf);
+extern void verify_control_data(ControlFileData *control_file,
+								const char *controlpath, bool crc_ok,
+								uint64 manifest_system_identifier);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From 6004858acf1740710f529d526ef70c96dee0bf9a Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 7 Aug 2024 18:15:29 +0530
Subject: [PATCH v9 12/12] pg_verifybackup: Tests and document

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the previous patch that implements tar format support.
----
---
 doc/src/sgml/ref/pg_verifybackup.sgml         | 42 ++++++++++-
 src/bin/pg_verifybackup/t/001_basic.pl        |  6 +-
 src/bin/pg_verifybackup/t/004_options.pl      |  2 +-
 src/bin/pg_verifybackup/t/005_bad_manifest.pl |  2 +-
 src/bin/pg_verifybackup/t/008_untar.pl        | 73 ++++++-------------
 src/bin/pg_verifybackup/t/010_client_untar.pl | 48 +-----------
 6 files changed, 72 insertions(+), 101 deletions(-)

diff --git a/doc/src/sgml/ref/pg_verifybackup.sgml b/doc/src/sgml/ref/pg_verifybackup.sgml
index a3f167f9f6e..60f771c7663 100644
--- a/doc/src/sgml/ref/pg_verifybackup.sgml
+++ b/doc/src/sgml/ref/pg_verifybackup.sgml
@@ -34,8 +34,10 @@ PostgreSQL documentation
    integrity of a database cluster backup taken using
    <command>pg_basebackup</command> against a
    <literal>backup_manifest</literal> generated by the server at the time
-   of the backup.  The backup must be stored in the "plain"
-   format; a "tar" format backup can be checked after extracting it.
+   of the backup.  The backup must be stored in the "plain" or "tar"
+   format.  Verification is supported for <literal>gzip</literal>,
+   <literal>lz4</literal>, and  <literal>zstd</literal> compressed tar backup;
+   any other compressed format backups can be checked after decompressing them.
   </para>
 
   <para>
@@ -168,6 +170,42 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-F <replaceable class="parameter">format</replaceable></option></term>
+      <term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
+      <listitem>
+       <para>
+        Specifies the format of the backup. <replaceable>format</replaceable>
+        can be one of the following:
+
+        <variablelist>
+         <varlistentry>
+          <term><literal>p</literal></term>
+          <term><literal>plain</literal></term>
+          <listitem>
+           <para>
+            Backup consists of plain files with the same layout as the
+            source server's data directory and tablespaces.
+           </para>
+          </listitem>
+         </varlistentry>
+
+         <varlistentry>
+          <term><literal>t</literal></term>
+          <term><literal>tar</literal></term>
+          <listitem>
+           <para>
+            Backup consists of tar files. The main data directory's contents
+            will be written to a file named <filename>base.tar</filename>,
+            and each other tablespace will be written to a separate tar file
+            named after that tablespace's OID.
+           </para>
+           </listitem>
+         </varlistentry>
+        </variablelist></para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-n</option></term>
       <term><option>--no-parse-wal</option></term>
diff --git a/src/bin/pg_verifybackup/t/001_basic.pl b/src/bin/pg_verifybackup/t/001_basic.pl
index 2f3e52d296f..ca5b0402b7d 100644
--- a/src/bin/pg_verifybackup/t/001_basic.pl
+++ b/src/bin/pg_verifybackup/t/001_basic.pl
@@ -17,11 +17,11 @@ command_fails_like(
 	qr/no backup directory specified/,
 	'target directory must be specified');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir ],
 	qr/could not open file.*\/backup_manifest\"/,
 	'pg_verifybackup requires a manifest');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir, $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir, $tempdir ],
 	qr/too many command-line arguments/,
 	'multiple target directories not allowed');
 
@@ -31,7 +31,7 @@ close($fh);
 
 # but then try to use an alternate, nonexisting manifest
 command_fails_like(
-	[ 'pg_verifybackup', '-m', "$tempdir/not_the_manifest", $tempdir ],
+	[ 'pg_verifybackup', '-Fp', '-m', "$tempdir/not_the_manifest", $tempdir ],
 	qr/could not open file.*\/not_the_manifest\"/,
 	'pg_verifybackup respects -m flag');
 
diff --git a/src/bin/pg_verifybackup/t/004_options.pl b/src/bin/pg_verifybackup/t/004_options.pl
index 8ed2214408e..2f197648740 100644
--- a/src/bin/pg_verifybackup/t/004_options.pl
+++ b/src/bin/pg_verifybackup/t/004_options.pl
@@ -108,7 +108,7 @@ unlike(
 # Test valid manifest with nonexistent backup directory.
 command_fails_like(
 	[
-		'pg_verifybackup', '-m',
+		'pg_verifybackup', '-Fp', '-m',
 		"$backup_path/backup_manifest", "$backup_path/fake"
 	],
 	qr/could not open directory/,
diff --git a/src/bin/pg_verifybackup/t/005_bad_manifest.pl b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
index c4ed64b62d5..28c51b6feb0 100644
--- a/src/bin/pg_verifybackup/t/005_bad_manifest.pl
+++ b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
@@ -208,7 +208,7 @@ sub test_bad_manifest
 	print $fh $manifest_contents;
 	close($fh);
 
-	command_fails_like([ 'pg_verifybackup', $tempdir ], $regexp, $test_name);
+	command_fails_like([ 'pg_verifybackup', '-Fp', $tempdir ], $regexp, $test_name);
 	return;
 }
 
diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
index 7a09f3b75b2..9896560adc3 100644
--- a/src/bin/pg_verifybackup/t/008_untar.pl
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -16,6 +16,20 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 
+# Create a couple of directories to use as tablespaces.
+my $TS1_LOCATION = $primary->backup_dir .'/ts1';
+$TS1_LOCATION =~ s/\/\.\//\//g;    # collapse foo/./bar to foo/bar
+mkdir($TS1_LOCATION);
+
+# Create a tablespace with table in it.
+$primary->safe_psql('postgres', qq(
+		CREATE TABLESPACE regress_ts1 LOCATION '$TS1_LOCATION';
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1';
+		CREATE TABLE regress_tbl1(i int) TABLESPACE regress_ts1;
+		INSERT INTO regress_tbl1 VALUES(generate_series(1,5));));
+my $tsoid = $primary->safe_psql('postgres', qq(
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1'));
+
 my $backup_path = $primary->backup_dir . '/server-backup';
 my $extract_path = $primary->backup_dir . '/extracted-backup';
 
@@ -23,39 +37,31 @@ my @test_configuration = (
 	{
 		'compression_method' => 'none',
 		'backup_flags' => [],
-		'backup_archive' => 'base.tar',
+		'backup_archive' => ['base.tar', "$tsoid.tar"],
 		'enabled' => 1
 	},
 	{
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'server-gzip' ],
-		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.gz', "$tsoid.tar.gz" ],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'server-lz4' ],
-		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => [ '-d', '-m' ],
+		'backup_archive' => ['base.tar.lz4', "$tsoid.tar.lz4" ],
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd:level=1,long' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	});
 
@@ -86,47 +92,16 @@ for my $tc (@test_configuration)
 		my $backup_files = join(',',
 			sort grep { $_ ne '.' && $_ ne '..' } slurp_dir($backup_path));
 		my $expected_backup_files =
-		  join(',', sort ('backup_manifest', $tc->{'backup_archive'}));
+		  join(',', sort ('backup_manifest', @{ $tc->{'backup_archive'} }));
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok(['pg_verifybackup', '-n', '-e', $backup_path],
+			"verify backup, compression $method");
 
 		# Cleanup.
-		unlink($backup_path . '/backup_manifest');
-		unlink($backup_path . '/base.tar');
-		unlink($backup_path . '/' . $tc->{'backup_archive'});
+		rmtree($backup_path);
 		rmtree($extract_path);
 	}
 }
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index 8c076d46dee..6b7d7483f6e 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -29,41 +29,30 @@ my @test_configuration = (
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'client-gzip:5' ],
 		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'client-lz4:5' ],
 		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => ['-d'],
-		'output_file' => 'base.tar',
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:5' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:level=1,long' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'parallel zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:workers=3' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1"),
 		'possibly_unsupported' =>
 		  qr/could not set compression worker count to 3: Unsupported parameter/
@@ -118,40 +107,9 @@ for my $tc (@test_configuration)
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			push @decompress, $backup_path . '/' . $tc->{'output_file'}
-			  if $tc->{'output_file'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok( [ 'pg_verifybackup', '-n', '-e', $backup_path ],
+			"verify backup, compression $method");
 
 		# Cleanup.
 		rmtree($extract_path);
-- 
2.18.0

From 0420df794f89344b4b96df0a767e1ea186bcacce Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 10:26:35 +0530
Subject: [PATCH v9 10/12] pg_verifybackup: Add backup format and compression
 option

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the next patch that implements tar format support.
----
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 74 ++++++++++++++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h |  1 +
 2 files changed, 73 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 5adf24e8f90..88196cca4e0 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,6 +18,7 @@
 #include <sys/stat.h>
 #include <time.h>
 
+#include "common/compression.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
@@ -56,6 +57,7 @@ static void report_manifest_error(JsonManifestParseContext *context,
 								  const char *fmt,...)
 			pg_attribute_printf(2, 3) pg_attribute_noreturn();
 
+static char find_backup_format(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
@@ -91,6 +93,7 @@ main(int argc, char **argv)
 		{"exit-on-error", no_argument, NULL, 'e'},
 		{"ignore", required_argument, NULL, 'i'},
 		{"manifest-path", required_argument, NULL, 'm'},
+		{"format", required_argument, NULL, 'F'},
 		{"no-parse-wal", no_argument, NULL, 'n'},
 		{"progress", no_argument, NULL, 'P'},
 		{"quiet", no_argument, NULL, 'q'},
@@ -112,6 +115,7 @@ main(int argc, char **argv)
 	progname = get_progname(argv[0]);
 
 	memset(&context, 0, sizeof(context));
+	context.format = '\0';
 
 	if (argc > 1)
 	{
@@ -148,7 +152,7 @@ main(int argc, char **argv)
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
 
-	while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1)
+	while ((c = getopt_long(argc, argv, "eF:i:m:nPqsw:", long_options, NULL)) != -1)
 	{
 		switch (c)
 		{
@@ -167,6 +171,15 @@ main(int argc, char **argv)
 				manifest_path = pstrdup(optarg);
 				canonicalize_path(manifest_path);
 				break;
+			case 'F':
+				if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
+					context.format = 'p';
+				else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
+					context.format = 't';
+				else
+					pg_fatal("invalid backup format \"%s\", must be \"plain\" or \"tar\"",
+							 optarg);
+				break;
 			case 'n':
 				no_parse_wal = true;
 				break;
@@ -214,11 +227,26 @@ main(int argc, char **argv)
 		pg_fatal("cannot specify both %s and %s",
 				 "-P/--progress", "-q/--quiet");
 
+	/* Determine the backup format if it hasn't been specified. */
+	if (context.format == '\0')
+		context.format = find_backup_format(&context);
+
 	/* Unless --no-parse-wal was specified, we will need pg_waldump. */
 	if (!no_parse_wal)
 	{
 		int			ret;
 
+		/*
+		 * XXX: In the future, we should consider enhancing pg_waldump to read
+		 * WAL files from the tar archive.
+		 */
+		if (context.format != 'p')
+		{
+			pg_log_error("pg_waldump does not support parsing WAL files from a tar archive.");
+			pg_log_error_hint("Try \"%s --help\" to skip parse WAL files option.", progname);
+			exit(1);
+		}
+
 		pg_waldump_path = pg_malloc(MAXPGPATH);
 		ret = find_other_exec(argv[0], "pg_waldump",
 							  "pg_waldump (PostgreSQL) " PG_VERSION "\n",
@@ -273,8 +301,13 @@ main(int argc, char **argv)
 	/*
 	 * Now do the expensive work of verifying file checksums, unless we were
 	 * told to skip it.
+	 *
+	 * We were only checking the plain backup here. For the tar backup, file
+	 * checksums verification (if requested) will be done immediately when the
+	 * file is accessed, as we don't have random access to the files like we
+	 * do with plain backups.
 	 */
-	if (!context.skip_checksums)
+	if (!context.skip_checksums && context.format == 'p')
 		verify_backup_checksums(&context);
 
 	/*
@@ -975,6 +1008,42 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
 	return false;
 }
 
+/*
+ * To detect the backup format, it checks for the PG_VERSION file in the backup
+ * directory. If found, it will be considered a plain-format backup; otherwise,
+ * it will be assumed to be a tar-format backup.
+ */
+static char
+find_backup_format(verifier_context *context)
+{
+	char		result;
+	char	   *path;
+	struct stat sb;
+
+	/* Should be here only if the backup format is unknown */
+	Assert(context->format == '\0');
+
+	/* Check PG_VERSION file. */
+	path = psprintf("%s/%s", context->backup_directory, "PG_VERSION");
+	if (stat(path, &sb) == 0)
+		result = 'p';
+	else
+	{
+		if (errno != ENOENT)
+		{
+			pg_log_error("cannot determine backup format");
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+
+		/* Otherwise, it is assumed to be a TAR backup. */
+		result = 't';
+	}
+	pfree(path);
+
+	return result;
+}
+
 /*
  * Print a progress report based on the global variables.
  *
@@ -1032,6 +1101,7 @@ usage(void)
 	printf(_("  -e, --exit-on-error         exit immediately on error\n"));
 	printf(_("  -i, --ignore=RELATIVE_PATH  ignore indicated path\n"));
 	printf(_("  -m, --manifest-path=PATH    use specified path for manifest\n"));
+	printf(_("  -F, --format=p|t            backup format (plain, tar)\n"));
 	printf(_("  -n, --no-parse-wal          do not try to parse WAL files\n"));
 	printf(_("  -P, --progress              show progress information\n"));
 	printf(_("  -q, --quiet                 do not print any output, except for errors\n"));
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 42d01c26466..856e8947c1d 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -105,6 +105,7 @@ typedef struct verifier_context
 	manifest_data *manifest;
 	char	   *backup_directory;
 	SimpleStringList ignore_list;
+	char		format;			/* backup format:  p(lain)/t(ar) */
 	bool		skip_checksums;
 	bool		exit_on_error;
 	bool		saw_any_error;
-- 
2.18.0

From 18887cb590f950cdbc7f4e5bc11812f4e85ccfdd Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 8 Aug 2024 16:01:33 +0530
Subject: [PATCH v9 09/12] Add simple_ptr_list_destroy() and
 simple_ptr_list_destroy_deep() API.

We didn't have any helper function to destroy SimplePtrList, likely
because it wasn't needed before, but it's required in a later patch in
this set.  I've added two functions for this purpose, inspired by
list_free() and list_free_deep().
---
 src/fe_utils/simple_list.c         | 39 ++++++++++++++++++++++++++++++
 src/include/fe_utils/simple_list.h |  2 ++
 2 files changed, 41 insertions(+)

diff --git a/src/fe_utils/simple_list.c b/src/fe_utils/simple_list.c
index 2d88eb54067..9d218911c31 100644
--- a/src/fe_utils/simple_list.c
+++ b/src/fe_utils/simple_list.c
@@ -173,3 +173,42 @@ simple_ptr_list_append(SimplePtrList *list, void *ptr)
 		list->head = cell;
 	list->tail = cell;
 }
+
+/*
+ * Destroy a pointer list and optionally the pointed-to element
+ */
+static void
+simple_ptr_list_destroy_private(SimplePtrList *list, bool deep)
+{
+	SimplePtrListCell *cell;
+
+	cell = list->head;
+	while (cell != NULL)
+	{
+		SimplePtrListCell *next;
+
+		next = cell->next;
+		if (deep)
+			pg_free(cell->ptr);
+		pg_free(cell);
+		cell = next;
+	}
+}
+
+/*
+ * Destroy a pointer list and the pointed-to element
+ */
+void
+simple_ptr_list_destroy_deep(SimplePtrList *list)
+{
+	simple_ptr_list_destroy_private(list, true);
+}
+
+/*
+ * Destroy only pointer list and not the pointed-to element
+ */
+void
+simple_ptr_list_destroy(SimplePtrList *list)
+{
+	simple_ptr_list_destroy_private(list, false);
+}
diff --git a/src/include/fe_utils/simple_list.h b/src/include/fe_utils/simple_list.h
index d42ecded8ed..5b7cbec8a62 100644
--- a/src/include/fe_utils/simple_list.h
+++ b/src/include/fe_utils/simple_list.h
@@ -66,5 +66,7 @@ extern void simple_string_list_destroy(SimpleStringList *list);
 extern const char *simple_string_list_not_touched(SimpleStringList *list);
 
 extern void simple_ptr_list_append(SimplePtrList *list, void *ptr);
+extern void simple_ptr_list_destroy_deep(SimplePtrList *list);
+extern void simple_ptr_list_destroy(SimplePtrList *list);
 
 #endif							/* SIMPLE_LIST_H */
-- 
2.18.0

From 6ef19e787db0174040cd0d020085d2a78c2c78f7 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 12:10:34 +0530
Subject: [PATCH v9 05/12] Refactor: move some part of pg_verifybackup.c to
 pg_verifybackup.h

---
 src/bin/pg_verifybackup/pg_verifybackup.c |  94 +------------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 108 ++++++++++++++++++++++
 2 files changed, 112 insertions(+), 90 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/pg_verifybackup.h

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index c6d01d52335..384a4dd3500 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,12 +18,11 @@
 #include <sys/stat.h>
 #include <time.h>
 
-#include "common/controldata_utils.h"
-#include "common/hashfn_unstable.h"
 #include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
+#include "pg_verifybackup.h"
 #include "pgtime.h"
 
 /*
@@ -40,84 +39,6 @@
  */
 #define ESTIMATED_BYTES_PER_MANIFEST_LINE	100
 
-/*
- * How many bytes should we try to read from a file at once?
- */
-#define READ_CHUNK_SIZE				(128 * 1024)
-
-/*
- * Each file described by the manifest file is parsed to produce an object
- * like this.
- */
-typedef struct manifest_file
-{
-	uint32		status;			/* hash status */
-	const char *pathname;
-	size_t		size;
-	pg_checksum_type checksum_type;
-	int			checksum_length;
-	uint8	   *checksum_payload;
-	bool		matched;
-	bool		bad;
-} manifest_file;
-
-#define should_verify_checksum(m) \
-	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
-
-/*
- * Define a hash table which we can use to store information about the files
- * mentioned in the backup manifest.
- */
-#define SH_PREFIX		manifest_files
-#define SH_ELEMENT_TYPE	manifest_file
-#define SH_KEY_TYPE		const char *
-#define	SH_KEY			pathname
-#define SH_HASH_KEY(tb, key)	hash_string(key)
-#define SH_EQUAL(tb, a, b)		(strcmp(a, b) == 0)
-#define	SH_SCOPE		static inline
-#define SH_RAW_ALLOCATOR	pg_malloc0
-#define SH_DECLARE
-#define SH_DEFINE
-#include "lib/simplehash.h"
-
-/*
- * Each WAL range described by the manifest file is parsed to produce an
- * object like this.
- */
-typedef struct manifest_wal_range
-{
-	TimeLineID	tli;
-	XLogRecPtr	start_lsn;
-	XLogRecPtr	end_lsn;
-	struct manifest_wal_range *next;
-	struct manifest_wal_range *prev;
-} manifest_wal_range;
-
-/*
- * All the data parsed from a backup_manifest file.
- */
-typedef struct manifest_data
-{
-	int			version;
-	uint64		system_identifier;
-	manifest_files_hash *files;
-	manifest_wal_range *first_wal_range;
-	manifest_wal_range *last_wal_range;
-} manifest_data;
-
-/*
- * All of the context information we need while checking a backup manifest.
- */
-typedef struct verifier_context
-{
-	manifest_data *manifest;
-	char	   *backup_directory;
-	SimpleStringList ignore_list;
-	bool		skip_checksums;
-	bool		exit_on_error;
-	bool		saw_any_error;
-} verifier_context;
-
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -151,13 +72,6 @@ static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
 
-static void report_backup_error(verifier_context *context,
-								const char *pg_restrict fmt,...)
-			pg_attribute_printf(2, 3);
-static void report_fatal_error(const char *pg_restrict fmt,...)
-			pg_attribute_printf(1, 2) pg_attribute_noreturn();
-static bool should_ignore_relpath(verifier_context *context, const char *relpath);
-
 static void progress_report(bool finished);
 static void usage(void);
 
@@ -980,7 +894,7 @@ parse_required_wal(verifier_context *context, char *pg_waldump_path,
  * Update the context to indicate that we saw an error, and exit if the
  * context says we should.
  */
-static void
+void
 report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
 {
 	va_list		ap;
@@ -997,7 +911,7 @@ report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
 /*
  * Report a fatal error and exit
  */
-static void
+void
 report_fatal_error(const char *pg_restrict fmt,...)
 {
 	va_list		ap;
@@ -1016,7 +930,7 @@ report_fatal_error(const char *pg_restrict fmt,...)
  * Note that by "prefix" we mean a parent directory; for this purpose,
  * "aa/bb" is not a prefix of "aa/bbb", but it is a prefix of "aa/bb/cc".
  */
-static bool
+bool
 should_ignore_relpath(verifier_context *context, const char *relpath)
 {
 	SimpleStringListCell *cell;
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
new file mode 100644
index 00000000000..bd9c95c477a
--- /dev/null
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -0,0 +1,108 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_verifybackup.h
+ *	  Verify a backup against a backup manifest.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_verifybackup/pg_verifybackup.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef PG_VERIFYBACKUP_H
+#define PG_VERIFYBACKUP_H
+
+#include "common/controldata_utils.h"
+#include "common/hashfn_unstable.h"
+#include "common/parse_manifest.h"
+#include "fe_utils/simple_list.h"
+
+/*
+ * How many bytes should we try to read from a file at once?
+ */
+#define READ_CHUNK_SIZE				(128 * 1024)
+
+/*
+ * Each file described by the manifest file is parsed to produce an object
+ * like this.
+ */
+typedef struct manifest_file
+{
+	uint32		status;			/* hash status */
+	const char *pathname;
+	size_t		size;
+	pg_checksum_type checksum_type;
+	int			checksum_length;
+	uint8	   *checksum_payload;
+	bool		matched;
+	bool		bad;
+} manifest_file;
+
+#define should_verify_checksum(m) \
+	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+
+/*
+ * Define a hash table which we can use to store information about the files
+ * mentioned in the backup manifest.
+ */
+#define SH_PREFIX		manifest_files
+#define SH_ELEMENT_TYPE	manifest_file
+#define SH_KEY_TYPE		const char *
+#define	SH_KEY			pathname
+#define SH_HASH_KEY(tb, key)	hash_string(key)
+#define SH_EQUAL(tb, a, b)		(strcmp(a, b) == 0)
+#define	SH_SCOPE		static inline
+#define SH_RAW_ALLOCATOR	pg_malloc0
+#define SH_DECLARE
+#define SH_DEFINE
+#include "lib/simplehash.h"
+
+/*
+ * Each WAL range described by the manifest file is parsed to produce an
+ * object like this.
+ */
+typedef struct manifest_wal_range
+{
+	TimeLineID	tli;
+	XLogRecPtr	start_lsn;
+	XLogRecPtr	end_lsn;
+	struct manifest_wal_range *next;
+	struct manifest_wal_range *prev;
+} manifest_wal_range;
+
+/*
+ * All the data parsed from a backup_manifest file.
+ */
+typedef struct manifest_data
+{
+	int			version;
+	uint64		system_identifier;
+	manifest_files_hash *files;
+	manifest_wal_range *first_wal_range;
+	manifest_wal_range *last_wal_range;
+} manifest_data;
+
+/*
+ * All of the context information we need while checking a backup manifest.
+ */
+typedef struct verifier_context
+{
+	manifest_data *manifest;
+	char	   *backup_directory;
+	SimpleStringList ignore_list;
+	bool		skip_checksums;
+	bool		exit_on_error;
+	bool		saw_any_error;
+} verifier_context;
+
+extern void report_backup_error(verifier_context *context,
+								const char *pg_restrict fmt,...)
+			pg_attribute_printf(2, 3);
+extern void report_fatal_error(const char *pg_restrict fmt,...)
+			pg_attribute_printf(1, 2) pg_attribute_noreturn();
+extern bool should_ignore_relpath(verifier_context *context,
+								  const char *relpath);
+
+#endif							/* PG_VERIFYBACKUP_H */
-- 
2.18.0

From 424537524164197abacfb9ce81001ee5da04d9c5 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 16:45:55 +0530
Subject: [PATCH v9 07/12] Refactor: split verify_file_checksum() function.

Move the core functionality of verify_file_checksum to a new function
to reuse it instead of duplicating the code.

The verify_file_checksum() function is designed to take a file path,
open and read the file contents, and then calculate the checksum.
However, for TAR backups, instead of a file path, we receive the file
content in chunks, and the checksum needs to be calculated
incrementally. While the initial operations for plain and TAR backup
checksum calculations differ, the final checks and error handling are
the same. By moving the shared logic to a separate function, we can
reuse the code for both types of backups.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 18 ++++++++++++++++--
 src/bin/pg_verifybackup/pg_verifybackup.h |  3 +++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index e4288f453ef..e4f499fcd37 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -787,7 +787,6 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	int			rc;
 	size_t		bytes_read = 0;
 	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
-	int			checksumlen;
 
 	/* Open the target file. */
 	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
@@ -853,8 +852,23 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 		return;
 	}
 
+	/* Do the final computation and verification. */
+	verify_checksum(context, m, &checksum_ctx, checksumbuf);
+}
+
+/*
+ * A helper function to finalize checksum computation and verify it against the
+ * backup manifest information.
+ */
+void
+verify_checksum(verifier_context *context, manifest_file *m,
+				pg_checksum_context *checksum_ctx, uint8 *checksumbuf)
+{
+	int			checksumlen;
+	const char *relpath = m->pathname;
+
 	/* Get the final checksum. */
-	checksumlen = pg_checksum_final(&checksum_ctx, checksumbuf);
+	checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
 	if (checksumlen < 0)
 	{
 		report_backup_error(context,
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 2e71f14669b..12812cf5584 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -100,6 +100,9 @@ typedef struct verifier_context
 
 extern manifest_file *verify_manifest_entry(verifier_context *context,
 											char *relpath, int64 filesize);
+extern void verify_checksum(verifier_context *context, manifest_file *m,
+							pg_checksum_context *checksum_ctx,
+							uint8 *checksumbuf);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From 47c999edada3ce31009351ed2393b04d7ea9f67e Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 8 Aug 2024 15:10:43 +0530
Subject: [PATCH v9 04/12] Refactor: move skip_checksums global variable to
 verifier_context struct

To enable access to this flag in another file.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index d77e70fbe38..c6d01d52335 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -113,6 +113,7 @@ typedef struct verifier_context
 	manifest_data *manifest;
 	char	   *backup_directory;
 	SimpleStringList ignore_list;
+	bool		skip_checksums;
 	bool		exit_on_error;
 	bool		saw_any_error;
 } verifier_context;
@@ -164,7 +165,6 @@ static const char *progname;
 
 /* options */
 static bool show_progress = false;
-static bool skip_checksums = false;
 
 /* Progress indicators */
 static uint64 total_size = 0;
@@ -266,7 +266,7 @@ main(int argc, char **argv)
 				quiet = true;
 				break;
 			case 's':
-				skip_checksums = true;
+				context.skip_checksums = true;
 				break;
 			case 'w':
 				wal_directory = pstrdup(optarg);
@@ -363,7 +363,7 @@ main(int argc, char **argv)
 	 * Now do the expensive work of verifying file checksums, unless we were
 	 * told to skip it.
 	 */
-	if (!skip_checksums)
+	if (!context.skip_checksums)
 		verify_backup_checksums(&context);
 
 	/*
@@ -739,7 +739,8 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		verify_control_file(fullpath, context->manifest->system_identifier);
 
 	/* Update statistics for progress report, if necessary */
-	if (show_progress && !skip_checksums && should_verify_checksum(m))
+	if (show_progress && !context->skip_checksums &&
+		should_verify_checksum(m))
 		total_size += m->size;
 
 	/*
-- 
2.18.0

From 5a323df24f2d0b76fb06ac58c1c70de46056e4ee Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 8 Aug 2024 14:45:04 +0530
Subject: [PATCH v9 06/12] Refactor: split verify_backup_file() function.

Move the manifest entry verification code into a new function called
verify_manifest_entry() so that it can be reused for tar backup
verification. If verify_manifest_entry() doesn't find an entry, it
reports an error as before and returns NULL to the caller. This is why
a NULL check is added to should_verify_checksum().
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 49 +++++++++++++++--------
 src/bin/pg_verifybackup/pg_verifybackup.h |  6 ++-
 2 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 384a4dd3500..e4288f453ef 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -622,6 +622,32 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		return;
 	}
 
+	/* Check the backup manifest entry for this file. */
+	m = verify_manifest_entry(context, relpath, sb.st_size);
+
+	/*
+	 * Validate the manifest system identifier, not available in manifest
+	 * version 1.
+	 */
+	if (context->manifest->version != 1 &&
+		strcmp(relpath, "global/pg_control") == 0 &&
+		m != NULL && m->matched && !m->bad)
+		verify_control_file(fullpath, context->manifest->system_identifier);
+
+	/* Update statistics for progress report, if necessary */
+	if (show_progress && !context->skip_checksums &&
+		should_verify_checksum(m))
+		total_size += m->size;
+}
+
+/*
+ * Verify file and its size entry in the manifest.
+ */
+manifest_file *
+verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
+{
+	manifest_file *m;
+
 	/* Check whether there's an entry in the manifest hash. */
 	m = manifest_files_lookup(context->manifest->files, relpath);
 	if (m == NULL)
@@ -629,40 +655,29 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		report_backup_error(context,
 							"\"%s\" is present on disk but not in the manifest",
 							relpath);
-		return;
+		return NULL;
 	}
 
 	/* Flag this entry as having been encountered in the filesystem. */
 	m->matched = true;
 
 	/* Check that the size matches. */
-	if (m->size != sb.st_size)
+	if (m->size != filesize)
 	{
 		report_backup_error(context,
 							"\"%s\" has size %lld on disk but size %zu in the manifest",
-							relpath, (long long int) sb.st_size, m->size);
+							relpath, (long long int) filesize, m->size);
 		m->bad = true;
 	}
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0)
-		verify_control_file(fullpath, context->manifest->system_identifier);
-
-	/* Update statistics for progress report, if necessary */
-	if (show_progress && !context->skip_checksums &&
-		should_verify_checksum(m))
-		total_size += m->size;
-
 	/*
 	 * We don't verify checksums at this stage. We first finish verifying that
 	 * we have the expected set of files with the expected sizes, and only
 	 * afterwards verify the checksums. That's because computing checksums may
 	 * take a while, and we'd like to report more obvious problems quickly.
 	 */
+
+	return m;
 }
 
 /*
@@ -825,7 +840,7 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 
 	/*
 	 * Double-check that we read the expected number of bytes from the file.
-	 * Normally, a file size mismatch would be caught in verify_backup_file
+	 * Normally, a file size mismatch would be caught in verify_manifest_entry
 	 * and this check would never be reached, but this provides additional
 	 * safety and clarity in the event of concurrent modifications or
 	 * filesystem misbehavior.
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index bd9c95c477a..2e71f14669b 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -41,7 +41,8 @@ typedef struct manifest_file
 } manifest_file;
 
 #define should_verify_checksum(m) \
-	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+	(((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
 
 /*
  * Define a hash table which we can use to store information about the files
@@ -97,6 +98,9 @@ typedef struct verifier_context
 	bool		saw_any_error;
 } verifier_context;
 
+extern manifest_file *verify_manifest_entry(verifier_context *context,
+											char *relpath, int64 filesize);
+
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
 			pg_attribute_printf(2, 3);
-- 
2.18.0

From acabc34488287ab3346551db23b4c99fac0850a9 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 7 Aug 2024 18:15:29 +0530
Subject: [PATCH v10 12/12] pg_verifybackup: Tests and document

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the previous patch that implements tar format support.
----
---
 doc/src/sgml/ref/pg_verifybackup.sgml         | 42 ++++++++++-
 src/bin/pg_verifybackup/t/001_basic.pl        |  6 +-
 src/bin/pg_verifybackup/t/004_options.pl      |  2 +-
 src/bin/pg_verifybackup/t/005_bad_manifest.pl |  2 +-
 src/bin/pg_verifybackup/t/008_untar.pl        | 73 ++++++-------------
 src/bin/pg_verifybackup/t/010_client_untar.pl | 48 +-----------
 6 files changed, 72 insertions(+), 101 deletions(-)

diff --git a/doc/src/sgml/ref/pg_verifybackup.sgml b/doc/src/sgml/ref/pg_verifybackup.sgml
index a3f167f9f6e..60f771c7663 100644
--- a/doc/src/sgml/ref/pg_verifybackup.sgml
+++ b/doc/src/sgml/ref/pg_verifybackup.sgml
@@ -34,8 +34,10 @@ PostgreSQL documentation
    integrity of a database cluster backup taken using
    <command>pg_basebackup</command> against a
    <literal>backup_manifest</literal> generated by the server at the time
-   of the backup.  The backup must be stored in the "plain"
-   format; a "tar" format backup can be checked after extracting it.
+   of the backup.  The backup must be stored in the "plain" or "tar"
+   format.  Verification is supported for <literal>gzip</literal>,
+   <literal>lz4</literal>, and  <literal>zstd</literal> compressed tar backup;
+   any other compressed format backups can be checked after decompressing them.
   </para>
 
   <para>
@@ -168,6 +170,42 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-F <replaceable class="parameter">format</replaceable></option></term>
+      <term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
+      <listitem>
+       <para>
+        Specifies the format of the backup. <replaceable>format</replaceable>
+        can be one of the following:
+
+        <variablelist>
+         <varlistentry>
+          <term><literal>p</literal></term>
+          <term><literal>plain</literal></term>
+          <listitem>
+           <para>
+            Backup consists of plain files with the same layout as the
+            source server's data directory and tablespaces.
+           </para>
+          </listitem>
+         </varlistentry>
+
+         <varlistentry>
+          <term><literal>t</literal></term>
+          <term><literal>tar</literal></term>
+          <listitem>
+           <para>
+            Backup consists of tar files. The main data directory's contents
+            will be written to a file named <filename>base.tar</filename>,
+            and each other tablespace will be written to a separate tar file
+            named after that tablespace's OID.
+           </para>
+           </listitem>
+         </varlistentry>
+        </variablelist></para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-n</option></term>
       <term><option>--no-parse-wal</option></term>
diff --git a/src/bin/pg_verifybackup/t/001_basic.pl b/src/bin/pg_verifybackup/t/001_basic.pl
index 2f3e52d296f..ca5b0402b7d 100644
--- a/src/bin/pg_verifybackup/t/001_basic.pl
+++ b/src/bin/pg_verifybackup/t/001_basic.pl
@@ -17,11 +17,11 @@ command_fails_like(
 	qr/no backup directory specified/,
 	'target directory must be specified');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir ],
 	qr/could not open file.*\/backup_manifest\"/,
 	'pg_verifybackup requires a manifest');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir, $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir, $tempdir ],
 	qr/too many command-line arguments/,
 	'multiple target directories not allowed');
 
@@ -31,7 +31,7 @@ close($fh);
 
 # but then try to use an alternate, nonexisting manifest
 command_fails_like(
-	[ 'pg_verifybackup', '-m', "$tempdir/not_the_manifest", $tempdir ],
+	[ 'pg_verifybackup', '-Fp', '-m', "$tempdir/not_the_manifest", $tempdir ],
 	qr/could not open file.*\/not_the_manifest\"/,
 	'pg_verifybackup respects -m flag');
 
diff --git a/src/bin/pg_verifybackup/t/004_options.pl b/src/bin/pg_verifybackup/t/004_options.pl
index 8ed2214408e..2f197648740 100644
--- a/src/bin/pg_verifybackup/t/004_options.pl
+++ b/src/bin/pg_verifybackup/t/004_options.pl
@@ -108,7 +108,7 @@ unlike(
 # Test valid manifest with nonexistent backup directory.
 command_fails_like(
 	[
-		'pg_verifybackup', '-m',
+		'pg_verifybackup', '-Fp', '-m',
 		"$backup_path/backup_manifest", "$backup_path/fake"
 	],
 	qr/could not open directory/,
diff --git a/src/bin/pg_verifybackup/t/005_bad_manifest.pl b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
index c4ed64b62d5..28c51b6feb0 100644
--- a/src/bin/pg_verifybackup/t/005_bad_manifest.pl
+++ b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
@@ -208,7 +208,7 @@ sub test_bad_manifest
 	print $fh $manifest_contents;
 	close($fh);
 
-	command_fails_like([ 'pg_verifybackup', $tempdir ], $regexp, $test_name);
+	command_fails_like([ 'pg_verifybackup', '-Fp', $tempdir ], $regexp, $test_name);
 	return;
 }
 
diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
index 7a09f3b75b2..9896560adc3 100644
--- a/src/bin/pg_verifybackup/t/008_untar.pl
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -16,6 +16,20 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 
+# Create a couple of directories to use as tablespaces.
+my $TS1_LOCATION = $primary->backup_dir .'/ts1';
+$TS1_LOCATION =~ s/\/\.\//\//g;    # collapse foo/./bar to foo/bar
+mkdir($TS1_LOCATION);
+
+# Create a tablespace with table in it.
+$primary->safe_psql('postgres', qq(
+		CREATE TABLESPACE regress_ts1 LOCATION '$TS1_LOCATION';
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1';
+		CREATE TABLE regress_tbl1(i int) TABLESPACE regress_ts1;
+		INSERT INTO regress_tbl1 VALUES(generate_series(1,5));));
+my $tsoid = $primary->safe_psql('postgres', qq(
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1'));
+
 my $backup_path = $primary->backup_dir . '/server-backup';
 my $extract_path = $primary->backup_dir . '/extracted-backup';
 
@@ -23,39 +37,31 @@ my @test_configuration = (
 	{
 		'compression_method' => 'none',
 		'backup_flags' => [],
-		'backup_archive' => 'base.tar',
+		'backup_archive' => ['base.tar', "$tsoid.tar"],
 		'enabled' => 1
 	},
 	{
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'server-gzip' ],
-		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.gz', "$tsoid.tar.gz" ],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'server-lz4' ],
-		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => [ '-d', '-m' ],
+		'backup_archive' => ['base.tar.lz4', "$tsoid.tar.lz4" ],
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd:level=1,long' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	});
 
@@ -86,47 +92,16 @@ for my $tc (@test_configuration)
 		my $backup_files = join(',',
 			sort grep { $_ ne '.' && $_ ne '..' } slurp_dir($backup_path));
 		my $expected_backup_files =
-		  join(',', sort ('backup_manifest', $tc->{'backup_archive'}));
+		  join(',', sort ('backup_manifest', @{ $tc->{'backup_archive'} }));
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok(['pg_verifybackup', '-n', '-e', $backup_path],
+			"verify backup, compression $method");
 
 		# Cleanup.
-		unlink($backup_path . '/backup_manifest');
-		unlink($backup_path . '/base.tar');
-		unlink($backup_path . '/' . $tc->{'backup_archive'});
+		rmtree($backup_path);
 		rmtree($extract_path);
 	}
 }
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index 8c076d46dee..6b7d7483f6e 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -29,41 +29,30 @@ my @test_configuration = (
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'client-gzip:5' ],
 		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'client-lz4:5' ],
 		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => ['-d'],
-		'output_file' => 'base.tar',
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:5' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:level=1,long' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'parallel zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:workers=3' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1"),
 		'possibly_unsupported' =>
 		  qr/could not set compression worker count to 3: Unsupported parameter/
@@ -118,40 +107,9 @@ for my $tc (@test_configuration)
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			push @decompress, $backup_path . '/' . $tc->{'output_file'}
-			  if $tc->{'output_file'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok( [ 'pg_verifybackup', '-n', '-e', $backup_path ],
+			"verify backup, compression $method");
 
 		# Cleanup.
 		rmtree($extract_path);
-- 
2.18.0

From 52e11acc54815e9e5db92171b3fba9522f6da683 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 31 Jul 2024 16:22:07 +0530
Subject: [PATCH v10 11/12] pg_verifybackup: Read tar files and verify its
 contents

This patch implements TAR format backup verification.

For progress reporting support, we perform this verification in two
passes: the first pass calculates total_size, and the second pass
updates done_size as verification progresses.

For the verification, in the first pass, we call verify_tar_backup_file(),
which performs basic verification by expecting only base.tar or
<tablespaceoid>.tar files and raises an error for any other files.  It
also determines the compression type of the archive file. All this
information is stored in a newly added tarFile struct, which is
appended to a list that will be used in the second pass (by
verify_tar_content()) for the final verification. In the second pass,
the tar archives are read, decompressed, and the required verification
is carried out.

For reading and decompression, fe_utils/astreamer.h is used. For
verification, a new archive streamer has been added in
astreamer_verify.c to handle TAR member files and their contents; see
astreamer_verify_content() for details. The stack of astreamers will
be set up for each TAR file in verify_tar_content(), depending on its
compression type which is detected in the first pass.

When information about a TAR member file (i.e., ASTREAMER_MEMBER_HEADER)
is received, we first verify its entry against the backup manifest. We
then decide if further checks are needed, such as checksum
verification and control data verification (if it is a pg_control
file), once the member file contents are received. Although this
decision could be made when the contents are received, it is more
efficient to make it earlier since the member file contents are
received in multiple iterations. In short, we process
ASTREAMER_MEMBER_CONTENTS multiple times but only once for other
ASTREAMER_MEMBER_* cases. We maintain this information in the
astreamer_verify structure for each member file, which is reset when
the file ends.

Unlike in a plain backup, checksum verification here occurs in two
steps. First, as the contents are received, the checksum is computed
incrementally (see member_compute_checksum). Then, at the end of
processing the member file, the final verification is performed (see
member_verify_checksum).

Similarly, during the content receiving stage, if the file is
pg_control, the data will be copied into a local buffer (see
member_copy_control_data).  The verification will then be carried out
at the end of the member file processing (see member_verify_control_data)
---
 src/bin/pg_verifybackup/Makefile           |   4 +-
 src/bin/pg_verifybackup/astreamer_verify.c | 355 +++++++++++++++++++++
 src/bin/pg_verifybackup/meson.build        |   6 +-
 src/bin/pg_verifybackup/pg_verifybackup.c  | 283 +++++++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h  |   6 +
 src/tools/pgindent/typedefs.list           |   2 +
 6 files changed, 649 insertions(+), 7 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/astreamer_verify.c

diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile
index 7c045f142e8..df7aaabd530 100644
--- a/src/bin/pg_verifybackup/Makefile
+++ b/src/bin/pg_verifybackup/Makefile
@@ -17,11 +17,13 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 # We need libpq only because fe_utils does.
+override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 
 OBJS = \
 	$(WIN32RES) \
-	pg_verifybackup.o
+	pg_verifybackup.o \
+	astreamer_verify.o
 
 all: pg_verifybackup
 
diff --git a/src/bin/pg_verifybackup/astreamer_verify.c b/src/bin/pg_verifybackup/astreamer_verify.c
new file mode 100644
index 00000000000..28a3f976877
--- /dev/null
+++ b/src/bin/pg_verifybackup/astreamer_verify.c
@@ -0,0 +1,355 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer_verify.c
+ *
+ * Extend fe_utils/astreamer.h archive streaming facility to verify TAR
+ * backup.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * src/bin/pg_verifybackup/astreamer_verify.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include "common/logging.h"
+#include "pg_verifybackup.h"
+
+typedef struct astreamer_verify
+{
+	astreamer	base;
+	verifier_context *context;
+	char	   *archive_name;
+	Oid			tblspc_oid;
+	pg_checksum_context *checksum_ctx;
+
+	/* Hold information for a member file verification */
+	manifest_file *mfile;
+	int64		received_bytes;
+	bool		verify_checksum;
+	bool		verify_control_data;
+} astreamer_verify;
+
+static void astreamer_verify_content(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len,
+									 astreamer_archive_context context);
+static void astreamer_verify_finalize(astreamer *streamer);
+static void astreamer_verify_free(astreamer *streamer);
+
+static void member_verify_header(astreamer *streamer, astreamer_member *member);
+static void member_compute_checksum(astreamer *streamer,
+									astreamer_member *member,
+									const char *data, int len);
+static void member_verify_checksum(astreamer *streamer);
+static void member_copy_control_data(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len);
+static void member_verify_control_data(astreamer *streamer);
+static void member_reset_info(astreamer *streamer);
+
+static const astreamer_ops astreamer_verify_ops = {
+	.content = astreamer_verify_content,
+	.finalize = astreamer_verify_finalize,
+	.free = astreamer_verify_free
+};
+
+/*
+ * Create a astreamer that can verifies content of a TAR file.
+ */
+astreamer *
+astreamer_verify_content_new(astreamer *next, verifier_context *context,
+							 char *archive_name, Oid tblspc_oid)
+{
+	astreamer_verify *streamer;
+
+	streamer = palloc0(sizeof(astreamer_verify));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_verify_ops;
+
+	streamer->base.bbs_next = next;
+	streamer->context = context;
+	streamer->archive_name = archive_name;
+	streamer->tblspc_oid = tblspc_oid;
+	initStringInfo(&streamer->base.bbs_buffer);
+
+	if (!context->skip_checksums)
+		streamer->checksum_ctx = pg_malloc(sizeof(pg_checksum_context));
+
+	return &streamer->base;
+}
+
+/*
+ * The main entry point of the archive streamer for verifying tar members.
+ */
+static void
+astreamer_verify_content(astreamer *streamer, astreamer_member *member,
+						 const char *data, int len,
+						 astreamer_archive_context context)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	Assert(context != ASTREAMER_UNKNOWN);
+
+	switch (context)
+	{
+		case ASTREAMER_MEMBER_HEADER:
+
+			/*
+			 * Perform the initial check and setup verification steps.
+			 */
+			member_verify_header(streamer, member);
+			break;
+
+		case ASTREAMER_MEMBER_CONTENTS:
+
+			/*
+			 * Process the member content according to the flags set by the
+			 * member header processing routine for checksum computation and
+			 * copy control data to local buffer.
+			 */
+			if (mystreamer->verify_checksum)
+				member_compute_checksum(streamer, member, data, len);
+
+			if (mystreamer->verify_control_data)
+				member_copy_control_data(streamer, member, data, len);
+			break;
+
+		case ASTREAMER_MEMBER_TRAILER:
+
+			/* Do the final checksum verification. */
+			if (mystreamer->verify_checksum)
+				member_verify_checksum(streamer);
+
+			/* Do the control data verification */
+			if (mystreamer->verify_control_data)
+				member_verify_control_data(streamer);
+
+			/*
+			 * Reset the temporary information stored for the verification.
+			 */
+			member_reset_info(streamer);
+			break;
+
+		case ASTREAMER_ARCHIVE_TRAILER:
+			break;
+
+		default:
+			/* Shouldn't happen. */
+			pg_fatal("unexpected state while parsing tar archive");
+	}
+}
+
+/*
+ * End-of-stream processing for a astreamer_verify stream.
+ */
+static void
+astreamer_verify_finalize(astreamer *streamer)
+{
+	Assert(streamer->bbs_next == NULL);
+}
+
+/*
+ * Free memory associated with a astreamer_verify stream.
+ */
+static void
+astreamer_verify_free(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	if (mystreamer->checksum_ctx)
+		pfree(mystreamer->checksum_ctx);
+
+	pfree(streamer->bbs_buffer.data);
+	pfree(streamer);
+}
+
+/*
+ * Verifies the tar member entry if it corresponds to a file in the backup
+ * manifest. If the archive being processed is a tablespace, prepares the
+ * required file path for subsequent operations. Finally, determines if
+ * checksum verification and control data verification need to be performed
+ * during file content processing
+ */
+static void
+member_verify_header(astreamer *streamer, astreamer_member *member)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_file *m;
+
+	/* We are only interested in files that are not in the ignore list. */
+	if (member->is_directory || member->is_link ||
+		should_ignore_relpath(mystreamer->context, member->pathname))
+		return;
+
+	/*
+	 * The backup_manifest stores a relative path to the base directory for
+	 * files belong tablespace, whereas <tablespaceoid>.tar doesn't. Prepare
+	 * the required path, otherwise, the manfiest entry verification will
+	 * fail.
+	 */
+	if (OidIsValid(mystreamer->tblspc_oid))
+	{
+		char		temp[MAXPGPATH];
+
+		/* Copy original name at temporary space */
+		memcpy(temp, member->pathname, MAXPGPATH);
+
+		snprintf(member->pathname, MAXPGPATH, "%s/%d/%s",
+				 "pg_tblspc", mystreamer->tblspc_oid, temp);
+	}
+
+	/* Check the manifest entry */
+	m = verify_manifest_entry(mystreamer->context, member->pathname,
+							  member->size);
+	mystreamer->mfile = (void *) m;
+
+	/*
+	 * Prepare for checksum and control data verification.
+	 *
+	 * We could have these checks while receiving contents. However, since
+	 * contents are received in multiple iterations, this would result in
+	 * these lengthy checks being performed multiple times. Instead, having a
+	 * single flag would be more efficient.
+	 */
+	mystreamer->verify_checksum =
+		(!mystreamer->context->skip_checksums && should_verify_checksum(m));
+	mystreamer->verify_control_data =
+		should_verify_control_data(mystreamer->context->manifest, m);
+
+	/* Initialize the context required for checksum verification. */
+	if (mystreamer->verify_checksum &&
+		pg_checksum_init(mystreamer->checksum_ctx, m->checksum_type) < 0)
+	{
+		report_backup_error(mystreamer->context,
+							"%s: could not initialize checksum of file \"%s\"",
+							mystreamer->archive_name, m->pathname);
+
+		/*
+		 * Checksum verification cannot be performed without proper context
+		 * initialization.
+		 */
+		mystreamer->verify_checksum = false;
+	}
+}
+
+/*
+ * Computes the checksum incrementally for the received file content.
+ *
+ * The caller should pass a correctly initialized checksum_ctx, which will be
+ * used for incremental checksum computation.
+ */
+static void
+member_compute_checksum(astreamer *streamer, astreamer_member *member,
+						const char *data, int len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	pg_checksum_context *checksum_ctx = mystreamer->checksum_ctx;
+	manifest_file *m = mystreamer->mfile;
+
+	Assert(mystreamer->verify_checksum);
+
+	/* Should have came for the right file */
+	Assert(strcmp(member->pathname, m->pathname) == 0);
+
+	/*
+	 * The checksum context should match the type noted in the backup
+	 * manifest.
+	 */
+	Assert(checksum_ctx->type == m->checksum_type);
+
+	/* Update the total count of computed checksum bytes. */
+	mystreamer->received_bytes += len;
+
+	if (pg_checksum_update(checksum_ctx, (uint8 *) data, len) < 0)
+	{
+		report_backup_error(mystreamer->context,
+							"could not update checksum of file \"%s\"",
+							m->pathname);
+		mystreamer->verify_checksum = false;
+	}
+}
+
+/*
+ * Perform the final computation and checksum verification after the entire
+ * file content has been processed.
+ */
+static void
+member_verify_checksum(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	Assert(mystreamer->verify_checksum);
+
+	verify_checksum(mystreamer->context, mystreamer->mfile,
+					mystreamer->checksum_ctx, mystreamer->received_bytes);
+}
+
+/*
+ * Stores the pg_control file contents into a local buffer; we need the entire
+ * control file data for verification.
+ */
+static void
+member_copy_control_data(astreamer *streamer, astreamer_member *member,
+						 const char *data, int len)
+{
+	/* Should be here only for control file */
+	Assert(strcmp(member->pathname, "global/pg_control") == 0);
+	Assert(((astreamer_verify *) streamer)->verify_control_data);
+
+	/* Copy enough control file data needed for verification. */
+	astreamer_buffer_until(streamer, &data, &len, sizeof(ControlFileData));
+}
+
+/*
+ * Performs the CRC calculation of pg_control data and then calls the routines
+ * that execute the final verification of the control file information.
+ */
+static void
+member_verify_control_data(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_data *manifest = mystreamer->context->manifest;
+	ControlFileData control_file;
+	pg_crc32c	crc;
+	bool		crc_ok;
+
+	/* Should be here only for control file */
+	Assert(strcmp(mystreamer->mfile->pathname, "global/pg_control") == 0);
+	Assert(mystreamer->verify_control_data);
+
+	/* Should have enough control file data needed for verification. */
+	if (streamer->bbs_buffer.len != sizeof(ControlFileData))
+		report_fatal_error("%s: unexpected control file size: %d, should be %zu",
+						   mystreamer->archive_name, streamer->bbs_buffer.len,
+						   sizeof(ControlFileData));
+
+	memcpy(&control_file, streamer->bbs_buffer.data, sizeof(ControlFileData));
+
+	/* Check the CRC. */
+	INIT_CRC32C(crc);
+	COMP_CRC32C(crc, (char *) (&control_file), offsetof(ControlFileData, crc));
+	FIN_CRC32C(crc);
+
+	crc_ok = EQ_CRC32C(crc, control_file.crc);
+
+	/* Do the final control data verification. */
+	verify_control_data(&control_file, mystreamer->mfile->pathname, crc_ok,
+						manifest->system_identifier);
+}
+
+/*
+ * Reset flags and free memory allocations for member file verification.
+ */
+static void
+member_reset_info(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	mystreamer->mfile = NULL;
+	mystreamer->received_bytes = 0;
+	mystreamer->verify_checksum = false;
+	mystreamer->verify_control_data = false;
+}
diff --git a/src/bin/pg_verifybackup/meson.build b/src/bin/pg_verifybackup/meson.build
index 7c7d31a0350..1e3fcf7ee5a 100644
--- a/src/bin/pg_verifybackup/meson.build
+++ b/src/bin/pg_verifybackup/meson.build
@@ -1,7 +1,8 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 pg_verifybackup_sources = files(
-  'pg_verifybackup.c'
+  'pg_verifybackup.c',
+  'astreamer_verify.c'
 )
 
 if host_system == 'windows'
@@ -10,9 +11,10 @@ if host_system == 'windows'
     '--FILEDESC', 'pg_verifybackup - verify a backup against using a backup manifest'])
 endif
 
+pg_verifybackup_deps = [frontend_code, libpq, lz4, zlib, zstd]
 pg_verifybackup = executable('pg_verifybackup',
   pg_verifybackup_sources,
-  dependencies: [frontend_code, libpq],
+  dependencies: pg_verifybackup_deps,
   kwargs: default_bin_args,
 )
 bin_targets += pg_verifybackup
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 3dcb174e0a9..aed1adef4e7 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -20,9 +20,12 @@
 
 #include "common/compression.h"
 #include "common/parse_manifest.h"
+#include "common/relpath.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
+#include "limits.h"
 #include "pg_verifybackup.h"
+#include "pgtar.h"
 #include "pgtime.h"
 
 /*
@@ -39,6 +42,16 @@
  */
 #define ESTIMATED_BYTES_PER_MANIFEST_LINE	100
 
+/*
+ * Tar archive information needed for content verification.
+ */
+typedef struct tarFile
+{
+	char	   *relpath;
+	Oid			tblspc_oid;
+	pg_compress_algorithm compress_algorithm;
+} tarFile;
+
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -60,8 +73,14 @@ static void report_manifest_error(JsonManifestParseContext *context,
 static char find_backup_format(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
-static void verify_plain_backup_file(verifier_context *context,
-									 char *relpath, char *fullpath);
+static void verify_plain_backup_file(verifier_context *context, char *relpath,
+									 char *fullpath);
+static void verify_tar_backup_file(verifier_context *context, char *relpath,
+								   char *fullpath, SimplePtrList *tarFiles);
+static void verify_tar_file_contents(verifier_context *context,
+									 SimplePtrList *tarFiles);
+static void verify_tar_contents(verifier_context *context, char *relpath,
+								char *fullpath, astreamer *streamer);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -70,6 +89,10 @@ static void verify_file_checksum(verifier_context *context,
 static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
+static astreamer *create_archive_verifier(verifier_context *context,
+										  char *archive_name,
+										  Oid tblspc_oid,
+										  pg_compress_algorithm compress_algo);
 
 static void progress_report(bool finished);
 static void usage(void);
@@ -148,6 +171,10 @@ main(int argc, char **argv)
 	 */
 	simple_string_list_append(&context.ignore_list, "backup_manifest");
 	simple_string_list_append(&context.ignore_list, "pg_wal");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.gz");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.lz4");
+	simple_string_list_append(&context.ignore_list, "pg_wal.tar.zst");
 	simple_string_list_append(&context.ignore_list, "postgresql.auto.conf");
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
@@ -556,6 +583,7 @@ verify_backup_directory(verifier_context *context, char *relpath,
 {
 	DIR		   *dir;
 	struct dirent *dirent;
+	SimplePtrList tarFiles = {NULL, NULL};
 
 	dir = opendir(fullpath);
 	if (dir == NULL)
@@ -595,12 +623,23 @@ verify_backup_directory(verifier_context *context, char *relpath,
 			newrelpath = psprintf("%s/%s", relpath, filename);
 
 		if (!should_ignore_relpath(context, newrelpath))
-			verify_plain_backup_file(context, newrelpath, newfullpath);
+		{
+			if (context->format == 'p')
+				verify_plain_backup_file(context, newrelpath, newfullpath);
+			else
+				verify_tar_backup_file(context, newrelpath, newfullpath,
+									   &tarFiles);
+		}
 
 		pfree(newfullpath);
 		pfree(newrelpath);
 	}
 
+	/* Perform the final verification of the tar contents, if any. */
+	Assert(tarFiles.head == NULL || context->format == 't');
+	if (tarFiles.head != NULL)
+		verify_tar_file_contents(context, &tarFiles);
+
 	if (closedir(dir))
 	{
 		report_backup_error(context,
@@ -610,7 +649,8 @@ verify_backup_directory(verifier_context *context, char *relpath,
 }
 
 /*
- * Verify one file (which might actually be a directory or a symlink).
+ * Verify one file (which might actually be a directory, a symlink or a
+ * archive).
  *
  * The arguments to this function have the same meaning as the arguments to
  * verify_backup_directory.
@@ -677,6 +717,205 @@ verify_plain_backup_file(verifier_context *context, char *relpath,
 		total_size += m->size;
 }
 
+/*
+ * Verify one tar archive file.
+ *
+ * This function does not perform a complete verification; it only carries out
+ * basic validation of the tar format backup file, detects the compression
+ * type, and appends that information to the tarFiles list. An error will be
+ * reported if the tar archive is inaccessible, or if the file type, name, or
+ * compression type is not as expected.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_plain_backup_file. The additional argument is the file size for
+ * progress report.
+ */
+static void
+verify_tar_backup_file(verifier_context *context, char *relpath,
+					   char *fullpath, SimplePtrList *tarFiles)
+{
+	struct stat sb;
+	Oid			tblspc_oid = InvalidOid;
+	pg_compress_algorithm compress_algorithm;
+	tarFile    *tar_file;
+	char	   *suffix = NULL;
+
+	/* Should be tar format backup */
+	Assert(context->format == 't');
+
+	/* Get file information */
+	if (stat(fullpath, &sb) != 0)
+	{
+		report_backup_error(context,
+							"could not stat file or directory \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	/* In a tar format backup, we expect only plain files. */
+	if (!S_ISREG(sb.st_mode))
+	{
+		report_backup_error(context,
+							"\"%s\" is not a plain file",
+							relpath);
+		return;
+	}
+
+	/*
+	 * We expect tar archive files of backing up the main directory and
+	 * tablespace.
+	 *
+	 * pg_basebackup writes the main data directory to an archive file named
+	 * base.tar and the tablespace directory to <tablespaceoid>.tar, followed
+	 * by a compression type extension such as .gz, .lz4, or .zst.
+	 */
+	if (strncmp("base", relpath, 4) == 0)
+		suffix = relpath + 4;
+	else
+	{
+		/* Expected a <tablespaceoid>.tar file here. */
+		int64 num = strtoi64(relpath, &suffix, 10);
+
+		/*
+		 * Report an error if we didn't consume at least one character, if the
+		 * result is 0, or if the value is too large to be a valid OID.
+		 */
+		if (suffix == NULL || (num <= 0) || (num > OID_MAX))
+			report_backup_error(context,
+								"\"%s\" unexpected file in the tar format backup",
+								relpath);
+		tblspc_oid = (Oid) num;
+	}
+
+	/* Now, check the compression type of the tar file */
+	if (strcmp(suffix, ".tar") == 0)
+		compress_algorithm = PG_COMPRESSION_NONE;
+	else if (strcmp(suffix, ".tgz") == 0)
+		compress_algorithm = PG_COMPRESSION_GZIP;
+	else if (strcmp(suffix, ".tar.gz") == 0)
+		compress_algorithm = PG_COMPRESSION_GZIP;
+	else if (strcmp(suffix, ".tar.lz4") == 0)
+		compress_algorithm = PG_COMPRESSION_LZ4;
+	else if (strcmp(suffix, ".tar.zst") == 0)
+		compress_algorithm = PG_COMPRESSION_ZSTD;
+	else
+	{
+		report_backup_error(context,
+							"\"%s\" unexpected file in the tar format backup",
+							relpath);
+		return;
+	}
+
+	/*
+	 * Append the information to the list for complete verification at a later
+	 * stage.
+	 */
+	tar_file = pg_malloc(sizeof(tarFile));
+	tar_file->relpath = pstrdup(relpath);
+	tar_file->tblspc_oid = tblspc_oid;
+	tar_file->compress_algorithm = compress_algorithm;
+
+	simple_ptr_list_append(tarFiles, tar_file);
+
+	/* Update statistics for progress report, if necessary */
+	if (show_progress)
+		total_size += sb.st_size;
+}
+
+/*
+ * This is the final part of tar file verification, which prepares the archive
+ * streamer stack according to the tar file compression format for each tar
+ * archive and invokes them for reading, decompressing, and ultimately
+ * verifying the contents.
+ *
+ * The arguments to this function should be a list of valid tar archives to
+ * verify, and the allocation will be freed once the verification is complete.
+ */
+static void
+verify_tar_file_contents(verifier_context *context, SimplePtrList *tarFiles)
+{
+	SimplePtrListCell *cell;
+
+	progress_report(false);
+
+	for (cell = tarFiles->head; cell != NULL; cell = cell->next)
+	{
+		tarFile    *tar_file = (tarFile *) cell->ptr;
+		astreamer  *streamer;
+		char	   *fullpath;
+
+		/* Prepare archive streamer stack */
+		streamer = create_archive_verifier(context,
+										   tar_file->relpath,
+										   tar_file->tblspc_oid,
+										   tar_file->compress_algorithm);
+
+		/* Compute the full pathname to the target file. */
+		fullpath = psprintf("%s/%s", context->backup_directory,
+							tar_file->relpath);
+
+		/* Invoke the streamer for reading, decompressing, and verifying. */
+		verify_tar_contents(context, tar_file->relpath, fullpath, streamer);
+
+		/* Cleanup. */
+		pfree(tar_file->relpath);
+		pfree(tar_file);
+		pfree(fullpath);
+
+		astreamer_finalize(streamer);
+		astreamer_free(streamer);
+	}
+	simple_ptr_list_destroy(tarFiles);
+
+	progress_report(true);
+}
+
+/*
+ * Performs the actual work for tar content verification. It reads a given tar
+ * file in predefined chunks and passes it to the streamer, which initiates
+ * routines for decompression (if necessary) and then verifies each member
+ * within the tar archive.
+ */
+static void
+verify_tar_contents(verifier_context *context, char *relpath, char *fullpath,
+					astreamer *streamer)
+{
+	int			fd;
+	int			rc;
+	char	   *buffer;
+
+	pg_log_debug("reading \"%s\"", fullpath);
+
+	/* Open the target file. */
+	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
+	{
+		report_backup_error(context, "could not open file \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
+
+	/* Perform the reads */
+	while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
+	{
+		astreamer_content(streamer, NULL, buffer, rc, ASTREAMER_UNKNOWN);
+
+		/* Report progress */
+		done_size += rc;
+		progress_report(false);
+	}
+
+	if (rc < 0)
+		report_backup_error(context, "could not read file \"%s\": %m",
+							relpath);
+
+	/* Close the file. */
+	if (close(fd) != 0)
+		report_backup_error(context, "could not close file \"%s\": %m",
+							relpath);
+}
+
 /*
  * Verify file and its size entry in the manifest.
  */
@@ -1045,6 +1284,42 @@ find_backup_format(verifier_context *context)
 	return result;
 }
 
+/*
+ * Identifies the necessary steps for verifying the contents of the
+ * provided tar file.
+ */
+static astreamer *
+create_archive_verifier(verifier_context *context, char *archive_name,
+						Oid tblspc_oid, pg_compress_algorithm compress_algo)
+{
+	astreamer  *streamer = NULL;
+
+	/* Should be here only for tar backup */
+	Assert(context->format == 't');
+
+	/*
+	 * To verify the contents of the tar file, the initial step is to parse
+	 * its content.
+	 */
+	streamer = astreamer_verify_content_new(streamer, context, archive_name,
+											tblspc_oid);
+	streamer = astreamer_tar_parser_new(streamer);
+
+	/*
+	 * If the tar file is compressed, we must perform the appropriate
+	 * decompression operation before proceeding with the verification of its
+	 * contents.
+	 */
+	if (compress_algo == PG_COMPRESSION_GZIP)
+		streamer = astreamer_gzip_decompressor_new(streamer);
+	else if (compress_algo == PG_COMPRESSION_LZ4)
+		streamer = astreamer_lz4_decompressor_new(streamer);
+	else if (compress_algo == PG_COMPRESSION_ZSTD)
+		streamer = astreamer_zstd_decompressor_new(streamer);
+
+	return streamer;
+}
+
 /*
  * Print a progress report based on the global variables.
  *
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 1bba4e7ea92..963ec71e270 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -18,6 +18,7 @@
 #include "common/hashfn_unstable.h"
 #include "common/logging.h"
 #include "common/parse_manifest.h"
+#include "fe_utils/astreamer.h"
 #include "fe_utils/simple_list.h"
 
 /*
@@ -128,4 +129,9 @@ extern void report_fatal_error(const char *pg_restrict fmt,...)
 extern bool should_ignore_relpath(verifier_context *context,
 								  const char *relpath);
 
+extern astreamer *astreamer_verify_content_new(astreamer *next,
+											   verifier_context *context,
+											   char *archive_name,
+											   Oid tblspc_oid);
+
 #endif							/* PG_VERIFYBACKUP_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 547d14b3e7c..47b5f0edcc7 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3329,6 +3329,7 @@ astreamer_plain_writer
 astreamer_recovery_injector
 astreamer_tar_archiver
 astreamer_tar_parser
+astreamer_verify
 astreamer_zstd_frame
 bgworker_main_type
 bh_node_type
@@ -3950,6 +3951,7 @@ substitute_phv_relids_context
 subxids_array_status
 symbol
 tablespaceinfo
+tarFile
 td_entry
 teSection
 temp_tablespaces_extra
-- 
2.18.0

From 1b07c42e3ea2b4a1b3b463611d93f8fa94426260 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 10:26:35 +0530
Subject: [PATCH v10 10/12] pg_verifybackup: Add backup format and compression
 option

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the next patch that implements tar format support.
----
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 74 ++++++++++++++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h |  1 +
 2 files changed, 73 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index ddc6ed7471b..3dcb174e0a9 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,6 +18,7 @@
 #include <sys/stat.h>
 #include <time.h>
 
+#include "common/compression.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
@@ -56,6 +57,7 @@ static void report_manifest_error(JsonManifestParseContext *context,
 								  const char *fmt,...)
 			pg_attribute_printf(2, 3) pg_attribute_noreturn();
 
+static char find_backup_format(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_plain_backup_file(verifier_context *context,
@@ -91,6 +93,7 @@ main(int argc, char **argv)
 		{"exit-on-error", no_argument, NULL, 'e'},
 		{"ignore", required_argument, NULL, 'i'},
 		{"manifest-path", required_argument, NULL, 'm'},
+		{"format", required_argument, NULL, 'F'},
 		{"no-parse-wal", no_argument, NULL, 'n'},
 		{"progress", no_argument, NULL, 'P'},
 		{"quiet", no_argument, NULL, 'q'},
@@ -112,6 +115,7 @@ main(int argc, char **argv)
 	progname = get_progname(argv[0]);
 
 	memset(&context, 0, sizeof(context));
+	context.format = '\0';
 
 	if (argc > 1)
 	{
@@ -148,7 +152,7 @@ main(int argc, char **argv)
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
 
-	while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1)
+	while ((c = getopt_long(argc, argv, "eF:i:m:nPqsw:", long_options, NULL)) != -1)
 	{
 		switch (c)
 		{
@@ -167,6 +171,15 @@ main(int argc, char **argv)
 				manifest_path = pstrdup(optarg);
 				canonicalize_path(manifest_path);
 				break;
+			case 'F':
+				if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
+					context.format = 'p';
+				else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
+					context.format = 't';
+				else
+					pg_fatal("invalid backup format \"%s\", must be \"plain\" or \"tar\"",
+							 optarg);
+				break;
 			case 'n':
 				no_parse_wal = true;
 				break;
@@ -214,11 +227,26 @@ main(int argc, char **argv)
 		pg_fatal("cannot specify both %s and %s",
 				 "-P/--progress", "-q/--quiet");
 
+	/* Determine the backup format if it hasn't been specified. */
+	if (context.format == '\0')
+		context.format = find_backup_format(&context);
+
 	/* Unless --no-parse-wal was specified, we will need pg_waldump. */
 	if (!no_parse_wal)
 	{
 		int			ret;
 
+		/*
+		 * XXX: In the future, we should consider enhancing pg_waldump to read
+		 * WAL files from the tar archive.
+		 */
+		if (context.format != 'p')
+		{
+			pg_log_error("pg_waldump does not support parsing WAL files from a tar archive.");
+			pg_log_error_hint("Try \"%s --help\" to skip parse WAL files option.", progname);
+			exit(1);
+		}
+
 		pg_waldump_path = pg_malloc(MAXPGPATH);
 		ret = find_other_exec(argv[0], "pg_waldump",
 							  "pg_waldump (PostgreSQL) " PG_VERSION "\n",
@@ -273,8 +301,13 @@ main(int argc, char **argv)
 	/*
 	 * Now do the expensive work of verifying file checksums, unless we were
 	 * told to skip it.
+	 *
+	 * We were only checking the plain backup here. For the tar backup, file
+	 * checksums verification (if requested) will be done immediately when the
+	 * file is accessed, as we don't have random access to the files like we
+	 * do with plain backups.
 	 */
-	if (!context.skip_checksums)
+	if (!context.skip_checksums && context.format == 'p')
 		verify_backup_checksums(&context);
 
 	/*
@@ -976,6 +1009,42 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
 	return false;
 }
 
+/*
+ * To detect the backup format, it checks for the PG_VERSION file in the backup
+ * directory. If found, it will be considered a plain-format backup; otherwise,
+ * it will be assumed to be a tar-format backup.
+ */
+static char
+find_backup_format(verifier_context *context)
+{
+	char		result;
+	char	   *path;
+	struct stat sb;
+
+	/* Should be here only if the backup format is unknown */
+	Assert(context->format == '\0');
+
+	/* Check PG_VERSION file. */
+	path = psprintf("%s/%s", context->backup_directory, "PG_VERSION");
+	if (stat(path, &sb) == 0)
+		result = 'p';
+	else
+	{
+		if (errno != ENOENT)
+		{
+			pg_log_error("cannot determine backup format");
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+
+		/* Otherwise, it is assumed to be a TAR backup. */
+		result = 't';
+	}
+	pfree(path);
+
+	return result;
+}
+
 /*
  * Print a progress report based on the global variables.
  *
@@ -1033,6 +1102,7 @@ usage(void)
 	printf(_("  -e, --exit-on-error         exit immediately on error\n"));
 	printf(_("  -i, --ignore=RELATIVE_PATH  ignore indicated path\n"));
 	printf(_("  -m, --manifest-path=PATH    use specified path for manifest\n"));
+	printf(_("  -F, --format=p|t            backup format (plain, tar)\n"));
 	printf(_("  -n, --no-parse-wal          do not try to parse WAL files\n"));
 	printf(_("  -P, --progress              show progress information\n"));
 	printf(_("  -q, --quiet                 do not print any output, except for errors\n"));
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 56fbb731337..1bba4e7ea92 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -105,6 +105,7 @@ typedef struct verifier_context
 	manifest_data *manifest;
 	char	   *backup_directory;
 	SimpleStringList ignore_list;
+	char		format;			/* backup format:  p(lain)/t(ar) */
 	bool		skip_checksums;
 	bool		exit_on_error;
 	bool		saw_any_error;
-- 
2.18.0

From 6a5086e67647bbc90e71e469d5195b76b6ba9f4b Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 8 Aug 2024 16:01:33 +0530
Subject: [PATCH v10 09/12] Add simple_ptr_list_destroy() and
 simple_ptr_list_destroy_deep() API.

We didn't have any helper function to destroy SimplePtrList, likely
because it wasn't needed before, but it's required in a later patch in
this set.  I've added two functions for this purpose, inspired by
list_free() and list_free_deep().
---
 src/fe_utils/simple_list.c         | 39 ++++++++++++++++++++++++++++++
 src/include/fe_utils/simple_list.h |  2 ++
 2 files changed, 41 insertions(+)

diff --git a/src/fe_utils/simple_list.c b/src/fe_utils/simple_list.c
index 2d88eb54067..9d218911c31 100644
--- a/src/fe_utils/simple_list.c
+++ b/src/fe_utils/simple_list.c
@@ -173,3 +173,42 @@ simple_ptr_list_append(SimplePtrList *list, void *ptr)
 		list->head = cell;
 	list->tail = cell;
 }
+
+/*
+ * Destroy a pointer list and optionally the pointed-to element
+ */
+static void
+simple_ptr_list_destroy_private(SimplePtrList *list, bool deep)
+{
+	SimplePtrListCell *cell;
+
+	cell = list->head;
+	while (cell != NULL)
+	{
+		SimplePtrListCell *next;
+
+		next = cell->next;
+		if (deep)
+			pg_free(cell->ptr);
+		pg_free(cell);
+		cell = next;
+	}
+}
+
+/*
+ * Destroy a pointer list and the pointed-to element
+ */
+void
+simple_ptr_list_destroy_deep(SimplePtrList *list)
+{
+	simple_ptr_list_destroy_private(list, true);
+}
+
+/*
+ * Destroy only pointer list and not the pointed-to element
+ */
+void
+simple_ptr_list_destroy(SimplePtrList *list)
+{
+	simple_ptr_list_destroy_private(list, false);
+}
diff --git a/src/include/fe_utils/simple_list.h b/src/include/fe_utils/simple_list.h
index d42ecded8ed..5b7cbec8a62 100644
--- a/src/include/fe_utils/simple_list.h
+++ b/src/include/fe_utils/simple_list.h
@@ -66,5 +66,7 @@ extern void simple_string_list_destroy(SimpleStringList *list);
 extern const char *simple_string_list_not_touched(SimpleStringList *list);
 
 extern void simple_ptr_list_append(SimplePtrList *list, void *ptr);
+extern void simple_ptr_list_destroy_deep(SimplePtrList *list);
+extern void simple_ptr_list_destroy(SimplePtrList *list);
 
 #endif							/* SIMPLE_LIST_H */
-- 
2.18.0

From 0542b8d4d620a181f06a16339202928b7ec3ec8c Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 15:47:26 +0530
Subject: [PATCH v10 08/12] Refactor: split verify_control_file.

Separated the manifest entry verification code into a new function and
introduced the should_verify_control_data() macro, similar to
should_verify_checksum().

Like verify_file_checksum(), verify_control_file() is too design to
accept the pg_control file patch which will be opened and respective
information will be verified. But, in case of tar backup we would be
having pg_control file contents instead, that needs to be verified in
the same way.  For that reason the code that doing the verification is
separated into separate function to so that can be reused for the tar
backup verification as well.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 44 +++++++++++------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 15 ++++++++
 2 files changed, 35 insertions(+), 24 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index c8543cb4f7f..ddc6ed7471b 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,7 +18,6 @@
 #include <sys/stat.h>
 #include <time.h>
 
-#include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
@@ -61,8 +60,6 @@ static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_plain_backup_file(verifier_context *context,
 									 char *relpath, char *fullpath);
-static void verify_control_file(const char *controlpath,
-								uint64 manifest_system_identifier);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -626,14 +623,20 @@ verify_plain_backup_file(verifier_context *context, char *relpath,
 	/* Check the backup manifest entry for this file. */
 	m = verify_manifest_entry(context, relpath, sb.st_size);
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0 &&
-		m != NULL && m->matched && !m->bad)
-		verify_control_file(fullpath, context->manifest->system_identifier);
+	/* Validate the manifest system identifier */
+	if (should_verify_control_data(context->manifest, m))
+	{
+		ControlFileData *control_file;
+		bool		crc_ok;
+
+		pg_log_debug("reading \"%s\"", fullpath);
+		control_file = get_controlfile_by_exact_path(fullpath, &crc_ok);
+
+		verify_control_data(control_file, fullpath, crc_ok,
+							context->manifest->system_identifier);
+		/* Release memory. */
+		pfree(control_file);
+	}
 
 	/* Update statistics for progress report, if necessary */
 	if (show_progress && !context->skip_checksums &&
@@ -682,18 +685,14 @@ verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
 }
 
 /*
- * Sanity check control file and validate system identifier against manifest
- * system identifier.
+ * Sanity check control file data and validate system identifier against
+ * manifest system identifier.
  */
-static void
-verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
+void
+verify_control_data(ControlFileData *control_file,
+					const char *controlpath, bool crc_ok,
+					uint64 manifest_system_identifier)
 {
-	ControlFileData *control_file;
-	bool		crc_ok;
-
-	pg_log_debug("reading \"%s\"", controlpath);
-	control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
-
 	/* Control file contents not meaningful if CRC is bad. */
 	if (!crc_ok)
 		report_fatal_error("%s: CRC is incorrect", controlpath);
@@ -709,9 +708,6 @@ verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
 						   controlpath,
 						   (unsigned long long) manifest_system_identifier,
 						   (unsigned long long) control_file->system_identifier);
-
-	/* Release memory. */
-	pfree(control_file);
 }
 
 /*
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 93859d9d541..56fbb731337 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -16,6 +16,7 @@
 
 #include "common/controldata_utils.h"
 #include "common/hashfn_unstable.h"
+#include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 
@@ -44,6 +45,17 @@ typedef struct manifest_file
 	(((m) != NULL) && ((m)->matched) && !((m)->bad) && \
 	 (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
 
+/*
+ * Validate the manifest system identifier against the control file; this
+ * feature is not available in manifest version 1.  This validation should be
+ * carried out only if the manifest entry validation is completed without any
+ * errors.
+ */
+#define should_verify_control_data(manifest, m) \
+	(((manifest)->version != 1) && \
+	 ((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (strcmp((m)->pathname, "global/pg_control") == 0))
+
 /*
  * Define a hash table which we can use to store information about the files
  * mentioned in the backup manifest.
@@ -103,6 +115,9 @@ extern manifest_file *verify_manifest_entry(verifier_context *context,
 extern void verify_checksum(verifier_context *context, manifest_file *m,
 							pg_checksum_context *checksum_ctx,
 							int64 bytes_read);
+extern void verify_control_data(ControlFileData *control_file,
+								const char *controlpath, bool crc_ok,
+								uint64 manifest_system_identifier);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From 6c9cb1c2725b9a60c5e549808555b15b06ea44e9 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 14 Aug 2024 15:14:15 +0530
Subject: [PATCH v10 07/12] Refactor: split verify_file_checksum() function.

Move the core functionality of verify_file_checksum to a new function
to reuse it instead of duplicating the code.

The verify_file_checksum() function is designed to take a file path,
open and read the file contents, and then calculate the checksum.
However, for TAR backups, instead of a file path, we receive the file
content in chunks, and the checksum needs to be calculated
incrementally. While the initial operations for plain and TAR backup
checksum calculations differ, the final checks and error handling are
the same. By moving the shared logic to a separate function, we can
reuse the code for both types of backups.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 20 +++++++++++++++++---
 src/bin/pg_verifybackup/pg_verifybackup.h |  3 +++
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index f85dfefab81..c8543cb4f7f 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -787,8 +787,6 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	int			fd;
 	int			rc;
 	size_t		bytes_read = 0;
-	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
-	int			checksumlen;
 
 	/* Open the target file. */
 	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
@@ -839,6 +837,22 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	if (rc < 0)
 		return;
 
+	/* Do the final computation and verification. */
+	verify_checksum(context, m, &checksum_ctx, bytes_read);
+}
+
+/*
+ * A helper function to finalize checksum computation and verify it against the
+ * backup manifest information.
+ */
+void
+verify_checksum(verifier_context *context, manifest_file *m,
+				pg_checksum_context *checksum_ctx, int64 bytes_read)
+{
+	const char *relpath = m->pathname;
+	int			checksumlen;
+	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
+
 	/*
 	 * Double-check that we read the expected number of bytes from the file.
 	 * Normally, a file size mismatch would be caught in verify_manifest_entry
@@ -855,7 +869,7 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	}
 
 	/* Get the final checksum. */
-	checksumlen = pg_checksum_final(&checksum_ctx, checksumbuf);
+	checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
 	if (checksumlen < 0)
 	{
 		report_backup_error(context,
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 2e71f14669b..93859d9d541 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -100,6 +100,9 @@ typedef struct verifier_context
 
 extern manifest_file *verify_manifest_entry(verifier_context *context,
 											char *relpath, int64 filesize);
+extern void verify_checksum(verifier_context *context, manifest_file *m,
+							pg_checksum_context *checksum_ctx,
+							int64 bytes_read);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From deba61f678fb652056d51889103e547d016dc6da Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 14 Aug 2024 10:42:37 +0530
Subject: [PATCH v10 06/12] Refactor: split verify_backup_file() function and
 rename it.

The function verify_backup_file() has now been renamed to
verify_plain_backup_file() to make it clearer that it is specifically
used for verifying files in a plain backup. Similarly, in a future
patch, we would have a verify_tar_backup_file() function for
verifying TAR backup files.

In addition to that, moved the manifest entry verification code into a
new function called verify_manifest_entry() so that it can be reused
for tar backup verification. If verify_manifest_entry() doesn't find
an entry, it reports an error as before and returns NULL to the
caller. This is why a NULL check is added to should_verify_checksum().
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 58 +++++++++++++++--------
 src/bin/pg_verifybackup/pg_verifybackup.h |  6 ++-
 2 files changed, 42 insertions(+), 22 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 384a4dd3500..f85dfefab81 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -59,8 +59,8 @@ static void report_manifest_error(JsonManifestParseContext *context,
 
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
-static void verify_backup_file(verifier_context *context,
-							   char *relpath, char *fullpath);
+static void verify_plain_backup_file(verifier_context *context,
+									 char *relpath, char *fullpath);
 static void verify_control_file(const char *controlpath,
 								uint64 manifest_system_identifier);
 static void report_extra_backup_files(verifier_context *context);
@@ -565,7 +565,7 @@ verify_backup_directory(verifier_context *context, char *relpath,
 			newrelpath = psprintf("%s/%s", relpath, filename);
 
 		if (!should_ignore_relpath(context, newrelpath))
-			verify_backup_file(context, newrelpath, newfullpath);
+			verify_plain_backup_file(context, newrelpath, newfullpath);
 
 		pfree(newfullpath);
 		pfree(newrelpath);
@@ -586,7 +586,8 @@ verify_backup_directory(verifier_context *context, char *relpath,
  * verify_backup_directory.
  */
 static void
-verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
+verify_plain_backup_file(verifier_context *context, char *relpath,
+						 char *fullpath)
 {
 	struct stat sb;
 	manifest_file *m;
@@ -622,6 +623,32 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		return;
 	}
 
+	/* Check the backup manifest entry for this file. */
+	m = verify_manifest_entry(context, relpath, sb.st_size);
+
+	/*
+	 * Validate the manifest system identifier, not available in manifest
+	 * version 1.
+	 */
+	if (context->manifest->version != 1 &&
+		strcmp(relpath, "global/pg_control") == 0 &&
+		m != NULL && m->matched && !m->bad)
+		verify_control_file(fullpath, context->manifest->system_identifier);
+
+	/* Update statistics for progress report, if necessary */
+	if (show_progress && !context->skip_checksums &&
+		should_verify_checksum(m))
+		total_size += m->size;
+}
+
+/*
+ * Verify file and its size entry in the manifest.
+ */
+manifest_file *
+verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
+{
+	manifest_file *m;
+
 	/* Check whether there's an entry in the manifest hash. */
 	m = manifest_files_lookup(context->manifest->files, relpath);
 	if (m == NULL)
@@ -629,40 +656,29 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		report_backup_error(context,
 							"\"%s\" is present on disk but not in the manifest",
 							relpath);
-		return;
+		return NULL;
 	}
 
 	/* Flag this entry as having been encountered in the filesystem. */
 	m->matched = true;
 
 	/* Check that the size matches. */
-	if (m->size != sb.st_size)
+	if (m->size != filesize)
 	{
 		report_backup_error(context,
 							"\"%s\" has size %lld on disk but size %zu in the manifest",
-							relpath, (long long int) sb.st_size, m->size);
+							relpath, (long long int) filesize, m->size);
 		m->bad = true;
 	}
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0)
-		verify_control_file(fullpath, context->manifest->system_identifier);
-
-	/* Update statistics for progress report, if necessary */
-	if (show_progress && !context->skip_checksums &&
-		should_verify_checksum(m))
-		total_size += m->size;
-
 	/*
 	 * We don't verify checksums at this stage. We first finish verifying that
 	 * we have the expected set of files with the expected sizes, and only
 	 * afterwards verify the checksums. That's because computing checksums may
 	 * take a while, and we'd like to report more obvious problems quickly.
 	 */
+
+	return m;
 }
 
 /*
@@ -825,7 +841,7 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 
 	/*
 	 * Double-check that we read the expected number of bytes from the file.
-	 * Normally, a file size mismatch would be caught in verify_backup_file
+	 * Normally, a file size mismatch would be caught in verify_manifest_entry
 	 * and this check would never be reached, but this provides additional
 	 * safety and clarity in the event of concurrent modifications or
 	 * filesystem misbehavior.
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index bd9c95c477a..2e71f14669b 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -41,7 +41,8 @@ typedef struct manifest_file
 } manifest_file;
 
 #define should_verify_checksum(m) \
-	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+	(((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
 
 /*
  * Define a hash table which we can use to store information about the files
@@ -97,6 +98,9 @@ typedef struct verifier_context
 	bool		saw_any_error;
 } verifier_context;
 
+extern manifest_file *verify_manifest_entry(verifier_context *context,
+											char *relpath, int64 filesize);
+
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
 			pg_attribute_printf(2, 3);
-- 
2.18.0

From dea8e0cd58a41c0912a5dff3ed32ada80ce0034b Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 12:10:34 +0530
Subject: [PATCH v10 05/12] Refactor: move some part of pg_verifybackup.c to
 pg_verifybackup.h

---
 src/bin/pg_verifybackup/pg_verifybackup.c |  94 +------------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 108 ++++++++++++++++++++++
 2 files changed, 112 insertions(+), 90 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/pg_verifybackup.h

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index c6d01d52335..384a4dd3500 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,12 +18,11 @@
 #include <sys/stat.h>
 #include <time.h>
 
-#include "common/controldata_utils.h"
-#include "common/hashfn_unstable.h"
 #include "common/logging.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
+#include "pg_verifybackup.h"
 #include "pgtime.h"
 
 /*
@@ -40,84 +39,6 @@
  */
 #define ESTIMATED_BYTES_PER_MANIFEST_LINE	100
 
-/*
- * How many bytes should we try to read from a file at once?
- */
-#define READ_CHUNK_SIZE				(128 * 1024)
-
-/*
- * Each file described by the manifest file is parsed to produce an object
- * like this.
- */
-typedef struct manifest_file
-{
-	uint32		status;			/* hash status */
-	const char *pathname;
-	size_t		size;
-	pg_checksum_type checksum_type;
-	int			checksum_length;
-	uint8	   *checksum_payload;
-	bool		matched;
-	bool		bad;
-} manifest_file;
-
-#define should_verify_checksum(m) \
-	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
-
-/*
- * Define a hash table which we can use to store information about the files
- * mentioned in the backup manifest.
- */
-#define SH_PREFIX		manifest_files
-#define SH_ELEMENT_TYPE	manifest_file
-#define SH_KEY_TYPE		const char *
-#define	SH_KEY			pathname
-#define SH_HASH_KEY(tb, key)	hash_string(key)
-#define SH_EQUAL(tb, a, b)		(strcmp(a, b) == 0)
-#define	SH_SCOPE		static inline
-#define SH_RAW_ALLOCATOR	pg_malloc0
-#define SH_DECLARE
-#define SH_DEFINE
-#include "lib/simplehash.h"
-
-/*
- * Each WAL range described by the manifest file is parsed to produce an
- * object like this.
- */
-typedef struct manifest_wal_range
-{
-	TimeLineID	tli;
-	XLogRecPtr	start_lsn;
-	XLogRecPtr	end_lsn;
-	struct manifest_wal_range *next;
-	struct manifest_wal_range *prev;
-} manifest_wal_range;
-
-/*
- * All the data parsed from a backup_manifest file.
- */
-typedef struct manifest_data
-{
-	int			version;
-	uint64		system_identifier;
-	manifest_files_hash *files;
-	manifest_wal_range *first_wal_range;
-	manifest_wal_range *last_wal_range;
-} manifest_data;
-
-/*
- * All of the context information we need while checking a backup manifest.
- */
-typedef struct verifier_context
-{
-	manifest_data *manifest;
-	char	   *backup_directory;
-	SimpleStringList ignore_list;
-	bool		skip_checksums;
-	bool		exit_on_error;
-	bool		saw_any_error;
-} verifier_context;
-
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -151,13 +72,6 @@ static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
 
-static void report_backup_error(verifier_context *context,
-								const char *pg_restrict fmt,...)
-			pg_attribute_printf(2, 3);
-static void report_fatal_error(const char *pg_restrict fmt,...)
-			pg_attribute_printf(1, 2) pg_attribute_noreturn();
-static bool should_ignore_relpath(verifier_context *context, const char *relpath);
-
 static void progress_report(bool finished);
 static void usage(void);
 
@@ -980,7 +894,7 @@ parse_required_wal(verifier_context *context, char *pg_waldump_path,
  * Update the context to indicate that we saw an error, and exit if the
  * context says we should.
  */
-static void
+void
 report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
 {
 	va_list		ap;
@@ -997,7 +911,7 @@ report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
 /*
  * Report a fatal error and exit
  */
-static void
+void
 report_fatal_error(const char *pg_restrict fmt,...)
 {
 	va_list		ap;
@@ -1016,7 +930,7 @@ report_fatal_error(const char *pg_restrict fmt,...)
  * Note that by "prefix" we mean a parent directory; for this purpose,
  * "aa/bb" is not a prefix of "aa/bbb", but it is a prefix of "aa/bb/cc".
  */
-static bool
+bool
 should_ignore_relpath(verifier_context *context, const char *relpath)
 {
 	SimpleStringListCell *cell;
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
new file mode 100644
index 00000000000..bd9c95c477a
--- /dev/null
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -0,0 +1,108 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_verifybackup.h
+ *	  Verify a backup against a backup manifest.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_verifybackup/pg_verifybackup.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef PG_VERIFYBACKUP_H
+#define PG_VERIFYBACKUP_H
+
+#include "common/controldata_utils.h"
+#include "common/hashfn_unstable.h"
+#include "common/parse_manifest.h"
+#include "fe_utils/simple_list.h"
+
+/*
+ * How many bytes should we try to read from a file at once?
+ */
+#define READ_CHUNK_SIZE				(128 * 1024)
+
+/*
+ * Each file described by the manifest file is parsed to produce an object
+ * like this.
+ */
+typedef struct manifest_file
+{
+	uint32		status;			/* hash status */
+	const char *pathname;
+	size_t		size;
+	pg_checksum_type checksum_type;
+	int			checksum_length;
+	uint8	   *checksum_payload;
+	bool		matched;
+	bool		bad;
+} manifest_file;
+
+#define should_verify_checksum(m) \
+	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+
+/*
+ * Define a hash table which we can use to store information about the files
+ * mentioned in the backup manifest.
+ */
+#define SH_PREFIX		manifest_files
+#define SH_ELEMENT_TYPE	manifest_file
+#define SH_KEY_TYPE		const char *
+#define	SH_KEY			pathname
+#define SH_HASH_KEY(tb, key)	hash_string(key)
+#define SH_EQUAL(tb, a, b)		(strcmp(a, b) == 0)
+#define	SH_SCOPE		static inline
+#define SH_RAW_ALLOCATOR	pg_malloc0
+#define SH_DECLARE
+#define SH_DEFINE
+#include "lib/simplehash.h"
+
+/*
+ * Each WAL range described by the manifest file is parsed to produce an
+ * object like this.
+ */
+typedef struct manifest_wal_range
+{
+	TimeLineID	tli;
+	XLogRecPtr	start_lsn;
+	XLogRecPtr	end_lsn;
+	struct manifest_wal_range *next;
+	struct manifest_wal_range *prev;
+} manifest_wal_range;
+
+/*
+ * All the data parsed from a backup_manifest file.
+ */
+typedef struct manifest_data
+{
+	int			version;
+	uint64		system_identifier;
+	manifest_files_hash *files;
+	manifest_wal_range *first_wal_range;
+	manifest_wal_range *last_wal_range;
+} manifest_data;
+
+/*
+ * All of the context information we need while checking a backup manifest.
+ */
+typedef struct verifier_context
+{
+	manifest_data *manifest;
+	char	   *backup_directory;
+	SimpleStringList ignore_list;
+	bool		skip_checksums;
+	bool		exit_on_error;
+	bool		saw_any_error;
+} verifier_context;
+
+extern void report_backup_error(verifier_context *context,
+								const char *pg_restrict fmt,...)
+			pg_attribute_printf(2, 3);
+extern void report_fatal_error(const char *pg_restrict fmt,...)
+			pg_attribute_printf(1, 2) pg_attribute_noreturn();
+extern bool should_ignore_relpath(verifier_context *context,
+								  const char *relpath);
+
+#endif							/* PG_VERIFYBACKUP_H */
-- 
2.18.0

From bcd2f84eb984badfdaa28ad84806c8e401259f9c Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 8 Aug 2024 15:10:43 +0530
Subject: [PATCH v10 04/12] Refactor: move skip_checksums global variable to
 verifier_context struct

To enable access to this flag in another file.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index d77e70fbe38..c6d01d52335 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -113,6 +113,7 @@ typedef struct verifier_context
 	manifest_data *manifest;
 	char	   *backup_directory;
 	SimpleStringList ignore_list;
+	bool		skip_checksums;
 	bool		exit_on_error;
 	bool		saw_any_error;
 } verifier_context;
@@ -164,7 +165,6 @@ static const char *progname;
 
 /* options */
 static bool show_progress = false;
-static bool skip_checksums = false;
 
 /* Progress indicators */
 static uint64 total_size = 0;
@@ -266,7 +266,7 @@ main(int argc, char **argv)
 				quiet = true;
 				break;
 			case 's':
-				skip_checksums = true;
+				context.skip_checksums = true;
 				break;
 			case 'w':
 				wal_directory = pstrdup(optarg);
@@ -363,7 +363,7 @@ main(int argc, char **argv)
 	 * Now do the expensive work of verifying file checksums, unless we were
 	 * told to skip it.
 	 */
-	if (!skip_checksums)
+	if (!context.skip_checksums)
 		verify_backup_checksums(&context);
 
 	/*
@@ -739,7 +739,8 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		verify_control_file(fullpath, context->manifest->system_identifier);
 
 	/* Update statistics for progress report, if necessary */
-	if (show_progress && !skip_checksums && should_verify_checksum(m))
+	if (show_progress && !context->skip_checksums &&
+		should_verify_checksum(m))
 		total_size += m->size;
 
 	/*
-- 
2.18.0

From dfaeebdc09fd689b7e45a705e32111cb226a0657 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 20 Aug 2024 13:14:22 +0530
Subject: [PATCH v11 12/12] pg_verifybackup: Tests and document

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the previous patch that implements tar format support.
----
---
 doc/src/sgml/ref/pg_verifybackup.sgml         | 43 ++++++++++-
 src/bin/pg_verifybackup/t/001_basic.pl        |  6 +-
 src/bin/pg_verifybackup/t/004_options.pl      |  2 +-
 src/bin/pg_verifybackup/t/005_bad_manifest.pl |  2 +-
 src/bin/pg_verifybackup/t/008_untar.pl        | 73 ++++++-------------
 src/bin/pg_verifybackup/t/010_client_untar.pl | 48 +-----------
 6 files changed, 73 insertions(+), 101 deletions(-)

diff --git a/doc/src/sgml/ref/pg_verifybackup.sgml b/doc/src/sgml/ref/pg_verifybackup.sgml
index a3f167f9f6e..ea6bc3ccb12 100644
--- a/doc/src/sgml/ref/pg_verifybackup.sgml
+++ b/doc/src/sgml/ref/pg_verifybackup.sgml
@@ -34,8 +34,10 @@ PostgreSQL documentation
    integrity of a database cluster backup taken using
    <command>pg_basebackup</command> against a
    <literal>backup_manifest</literal> generated by the server at the time
-   of the backup.  The backup must be stored in the "plain"
-   format; a "tar" format backup can be checked after extracting it.
+   of the backup.  The backup must be stored in the "plain" or "tar"
+   format.  Verification is supported for <literal>gzip</literal>,
+   <literal>lz4</literal>, and  <literal>zstd</literal> compressed tar backup;
+   any other compressed format backups can be checked after decompressing them.
   </para>
 
   <para>
@@ -168,6 +170,43 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-F <replaceable class="parameter">format</replaceable></option></term>
+      <term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
+      <listitem>
+       <para>
+        Specifies the format of the backup. <replaceable>format</replaceable>
+        can be one of the following:
+
+        <variablelist>
+         <varlistentry>
+          <term><literal>p</literal></term>
+          <term><literal>plain</literal></term>
+          <listitem>
+           <para>
+            Backup consists of plain files with the same layout as the
+            source server's data directory and tablespaces.
+           </para>
+          </listitem>
+         </varlistentry>
+
+         <varlistentry>
+          <term><literal>t</literal></term>
+          <term><literal>tar</literal></term>
+          <listitem>
+           <para>
+            Backup consists of tar files.  A valid backup includes the main data
+            directory in a file named <filename>base.tar</filename>, the WAL
+            files in <filename>pg_wal.tar</filename>, and separate tar files for
+            each tablespace, named after the tablespace's OID, followed by the
+            compression extension.
+           </para>
+           </listitem>
+         </varlistentry>
+        </variablelist></para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-n</option></term>
       <term><option>--no-parse-wal</option></term>
diff --git a/src/bin/pg_verifybackup/t/001_basic.pl b/src/bin/pg_verifybackup/t/001_basic.pl
index 2f3e52d296f..ca5b0402b7d 100644
--- a/src/bin/pg_verifybackup/t/001_basic.pl
+++ b/src/bin/pg_verifybackup/t/001_basic.pl
@@ -17,11 +17,11 @@ command_fails_like(
 	qr/no backup directory specified/,
 	'target directory must be specified');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir ],
 	qr/could not open file.*\/backup_manifest\"/,
 	'pg_verifybackup requires a manifest');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir, $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir, $tempdir ],
 	qr/too many command-line arguments/,
 	'multiple target directories not allowed');
 
@@ -31,7 +31,7 @@ close($fh);
 
 # but then try to use an alternate, nonexisting manifest
 command_fails_like(
-	[ 'pg_verifybackup', '-m', "$tempdir/not_the_manifest", $tempdir ],
+	[ 'pg_verifybackup', '-Fp', '-m', "$tempdir/not_the_manifest", $tempdir ],
 	qr/could not open file.*\/not_the_manifest\"/,
 	'pg_verifybackup respects -m flag');
 
diff --git a/src/bin/pg_verifybackup/t/004_options.pl b/src/bin/pg_verifybackup/t/004_options.pl
index 8ed2214408e..2f197648740 100644
--- a/src/bin/pg_verifybackup/t/004_options.pl
+++ b/src/bin/pg_verifybackup/t/004_options.pl
@@ -108,7 +108,7 @@ unlike(
 # Test valid manifest with nonexistent backup directory.
 command_fails_like(
 	[
-		'pg_verifybackup', '-m',
+		'pg_verifybackup', '-Fp', '-m',
 		"$backup_path/backup_manifest", "$backup_path/fake"
 	],
 	qr/could not open directory/,
diff --git a/src/bin/pg_verifybackup/t/005_bad_manifest.pl b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
index c4ed64b62d5..28c51b6feb0 100644
--- a/src/bin/pg_verifybackup/t/005_bad_manifest.pl
+++ b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
@@ -208,7 +208,7 @@ sub test_bad_manifest
 	print $fh $manifest_contents;
 	close($fh);
 
-	command_fails_like([ 'pg_verifybackup', $tempdir ], $regexp, $test_name);
+	command_fails_like([ 'pg_verifybackup', '-Fp', $tempdir ], $regexp, $test_name);
 	return;
 }
 
diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
index 7a09f3b75b2..9896560adc3 100644
--- a/src/bin/pg_verifybackup/t/008_untar.pl
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -16,6 +16,20 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 
+# Create a couple of directories to use as tablespaces.
+my $TS1_LOCATION = $primary->backup_dir .'/ts1';
+$TS1_LOCATION =~ s/\/\.\//\//g;    # collapse foo/./bar to foo/bar
+mkdir($TS1_LOCATION);
+
+# Create a tablespace with table in it.
+$primary->safe_psql('postgres', qq(
+		CREATE TABLESPACE regress_ts1 LOCATION '$TS1_LOCATION';
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1';
+		CREATE TABLE regress_tbl1(i int) TABLESPACE regress_ts1;
+		INSERT INTO regress_tbl1 VALUES(generate_series(1,5));));
+my $tsoid = $primary->safe_psql('postgres', qq(
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1'));
+
 my $backup_path = $primary->backup_dir . '/server-backup';
 my $extract_path = $primary->backup_dir . '/extracted-backup';
 
@@ -23,39 +37,31 @@ my @test_configuration = (
 	{
 		'compression_method' => 'none',
 		'backup_flags' => [],
-		'backup_archive' => 'base.tar',
+		'backup_archive' => ['base.tar', "$tsoid.tar"],
 		'enabled' => 1
 	},
 	{
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'server-gzip' ],
-		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.gz', "$tsoid.tar.gz" ],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'server-lz4' ],
-		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => [ '-d', '-m' ],
+		'backup_archive' => ['base.tar.lz4', "$tsoid.tar.lz4" ],
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd:level=1,long' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	});
 
@@ -86,47 +92,16 @@ for my $tc (@test_configuration)
 		my $backup_files = join(',',
 			sort grep { $_ ne '.' && $_ ne '..' } slurp_dir($backup_path));
 		my $expected_backup_files =
-		  join(',', sort ('backup_manifest', $tc->{'backup_archive'}));
+		  join(',', sort ('backup_manifest', @{ $tc->{'backup_archive'} }));
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok(['pg_verifybackup', '-n', '-e', $backup_path],
+			"verify backup, compression $method");
 
 		# Cleanup.
-		unlink($backup_path . '/backup_manifest');
-		unlink($backup_path . '/base.tar');
-		unlink($backup_path . '/' . $tc->{'backup_archive'});
+		rmtree($backup_path);
 		rmtree($extract_path);
 	}
 }
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index 8c076d46dee..6b7d7483f6e 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -29,41 +29,30 @@ my @test_configuration = (
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'client-gzip:5' ],
 		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'client-lz4:5' ],
 		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => ['-d'],
-		'output_file' => 'base.tar',
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:5' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:level=1,long' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'parallel zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:workers=3' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1"),
 		'possibly_unsupported' =>
 		  qr/could not set compression worker count to 3: Unsupported parameter/
@@ -118,40 +107,9 @@ for my $tc (@test_configuration)
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			push @decompress, $backup_path . '/' . $tc->{'output_file'}
-			  if $tc->{'output_file'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok( [ 'pg_verifybackup', '-n', '-e', $backup_path ],
+			"verify backup, compression $method");
 
 		# Cleanup.
 		rmtree($extract_path);
-- 
2.18.0

From afc610cc11c3ec4afdbc354a0ea08340b473b254 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 31 Jul 2024 16:22:07 +0530
Subject: [PATCH v11 11/12] pg_verifybackup: Read tar files and verify its
 contents

This patch implements TAR format backup verification.

For progress reporting support, we perform this verification in two
passes: the first pass calculates total_size, and the second pass
updates done_size as verification progresses.

For the verification, in the first pass, we call verify_tar_backup_file(),
which performs basic verification by expecting only base.tar, pg_wal.tar, or
<tablespaceoid>.tar files and raises an error for any other files.  It
also determines the compression type of the archive file. All this
information is stored in a newly added tarFile struct, which is
appended to a list that will be used in the second pass (by
verify_tar_content()) for the final verification. In the second pass,
the tar archives are read, decompressed, and the required verification
is carried out.

For reading and decompression, fe_utils/astreamer.h is used. For
verification, a new archive streamer has been added in
astreamer_verify.c to handle TAR member files and their contents; see
astreamer_verify_content() for details. The stack of astreamers will
be set up for each TAR file in verify_tar_content(), depending on its
compression type which is detected in the first pass.

When information about a TAR member file (i.e., ASTREAMER_MEMBER_HEADER)
is received, we first verify its entry against the backup manifest. We
then decide if further checks are needed, such as checksum
verification and control data verification (if it is a pg_control
file), once the member file contents are received. Although this
decision could be made when the contents are received, it is more
efficient to make it earlier since the member file contents are
received in multiple iterations. In short, we process
ASTREAMER_MEMBER_CONTENTS multiple times but only once for other
ASTREAMER_MEMBER_* cases. We maintain this information in the
astreamer_verify structure for each member file, which is reset when
the file ends.

Unlike in a plain backup, checksum verification here occurs in two
steps. First, as the contents are received, the checksum is computed
incrementally (see member_compute_checksum). Then, at the end of
processing the member file, the final verification is performed (see
member_verify_checksum).

Similarly, during the content receiving stage, if the file is
pg_control, the data will be copied into a local buffer (see
member_copy_control_data).  The verification will then be carried out
at the end of the member file processing (see member_verify_control_data)
---
 src/bin/pg_verifybackup/Makefile           |   4 +-
 src/bin/pg_verifybackup/astreamer_verify.c | 363 +++++++++++++++++++++
 src/bin/pg_verifybackup/meson.build        |   6 +-
 src/bin/pg_verifybackup/pg_verifybackup.c  | 284 +++++++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h  |   6 +
 src/tools/pgindent/typedefs.list           |   2 +
 6 files changed, 659 insertions(+), 6 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/astreamer_verify.c

diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile
index 7c045f142e8..df7aaabd530 100644
--- a/src/bin/pg_verifybackup/Makefile
+++ b/src/bin/pg_verifybackup/Makefile
@@ -17,11 +17,13 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 # We need libpq only because fe_utils does.
+override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 
 OBJS = \
 	$(WIN32RES) \
-	pg_verifybackup.o
+	pg_verifybackup.o \
+	astreamer_verify.o
 
 all: pg_verifybackup
 
diff --git a/src/bin/pg_verifybackup/astreamer_verify.c b/src/bin/pg_verifybackup/astreamer_verify.c
new file mode 100644
index 00000000000..3d3659195a9
--- /dev/null
+++ b/src/bin/pg_verifybackup/astreamer_verify.c
@@ -0,0 +1,363 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer_verify.c
+ *
+ * Extend fe_utils/astreamer.h archive streaming facility to verify TAR
+ * backup.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * src/bin/pg_verifybackup/astreamer_verify.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include "common/logging.h"
+#include "pg_verifybackup.h"
+
+typedef struct astreamer_verify
+{
+	astreamer	base;
+	verifier_context *context;
+	char	   *archive_name;
+	Oid			tblspc_oid;
+	pg_checksum_context *checksum_ctx;
+
+	/* Hold information for a member file verification */
+	manifest_file *mfile;
+	int64		received_bytes;
+	bool		verify_checksum;
+	bool		verify_control_data;
+} astreamer_verify;
+
+static void astreamer_verify_content(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len,
+									 astreamer_archive_context context);
+static void astreamer_verify_finalize(astreamer *streamer);
+static void astreamer_verify_free(astreamer *streamer);
+
+static void member_verify_header(astreamer *streamer, astreamer_member *member);
+static void member_compute_checksum(astreamer *streamer,
+									astreamer_member *member,
+									const char *data, int len);
+static void member_verify_checksum(astreamer *streamer);
+static void member_copy_control_data(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len);
+static void member_verify_control_data(astreamer *streamer);
+static void member_reset_info(astreamer *streamer);
+
+static const astreamer_ops astreamer_verify_ops = {
+	.content = astreamer_verify_content,
+	.finalize = astreamer_verify_finalize,
+	.free = astreamer_verify_free
+};
+
+/*
+ * Create a astreamer that can verifies content of a TAR file.
+ */
+astreamer *
+astreamer_verify_content_new(astreamer *next, verifier_context *context,
+							 char *archive_name, Oid tblspc_oid)
+{
+	astreamer_verify *streamer;
+
+	streamer = palloc0(sizeof(astreamer_verify));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_verify_ops;
+
+	streamer->base.bbs_next = next;
+	streamer->context = context;
+	streamer->archive_name = archive_name;
+	streamer->tblspc_oid = tblspc_oid;
+	initStringInfo(&streamer->base.bbs_buffer);
+
+	if (!context->skip_checksums)
+		streamer->checksum_ctx = pg_malloc(sizeof(pg_checksum_context));
+
+	return &streamer->base;
+}
+
+/*
+ * The main entry point of the archive streamer for verifying tar members.
+ */
+static void
+astreamer_verify_content(astreamer *streamer, astreamer_member *member,
+						 const char *data, int len,
+						 astreamer_archive_context context)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	Assert(context != ASTREAMER_UNKNOWN);
+
+	switch (context)
+	{
+		case ASTREAMER_MEMBER_HEADER:
+
+			/*
+			 * Perform the initial check and setup verification steps.
+			 */
+			member_verify_header(streamer, member);
+			break;
+
+		case ASTREAMER_MEMBER_CONTENTS:
+
+			/*
+			 * Since we are receiving the file in chunks, we need to process
+			 * the member content according to the flags set by the member
+			 * header processing routine, which includes checksum computation
+			 * and copying control data to the local buffer.
+			 */
+			if (mystreamer->verify_checksum)
+				member_compute_checksum(streamer, member, data, len);
+
+			if (mystreamer->verify_control_data)
+				member_copy_control_data(streamer, member, data, len);
+			break;
+
+		case ASTREAMER_MEMBER_TRAILER:
+
+			/*
+			 * We have reached the end of the member file. By this point, we
+			 * should have successfully computed the checksum of the received
+			 * content and copied the entire pg_control file data into our
+			 * local buffer. We can now proceed with the final verification.
+			 */
+			if (mystreamer->verify_checksum)
+				member_verify_checksum(streamer);
+
+			if (mystreamer->verify_control_data)
+				member_verify_control_data(streamer);
+
+			/*
+			 * Reset the temporary information stored for the verification.
+			 */
+			member_reset_info(streamer);
+			break;
+
+		case ASTREAMER_ARCHIVE_TRAILER:
+			break;
+
+		default:
+			/* Shouldn't happen. */
+			pg_fatal("unexpected state while parsing tar archive");
+	}
+}
+
+/*
+ * End-of-stream processing for a astreamer_verify stream.
+ */
+static void
+astreamer_verify_finalize(astreamer *streamer)
+{
+	Assert(streamer->bbs_next == NULL);
+}
+
+/*
+ * Free memory associated with a astreamer_verify stream.
+ */
+static void
+astreamer_verify_free(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	if (mystreamer->checksum_ctx)
+		pfree(mystreamer->checksum_ctx);
+
+	pfree(streamer->bbs_buffer.data);
+	pfree(streamer);
+}
+
+/*
+ * Verifies the tar member entry if it corresponds to a file in the backup
+ * manifest. If the archive being processed is a tablespace, prepares the
+ * required file path for subsequent operations. Finally, determines if
+ * checksum verification and control data verification need to be performed
+ * during file content processing
+ */
+static void
+member_verify_header(astreamer *streamer, astreamer_member *member)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_file *m;
+
+	/* We are only interested in files that are not in the ignore list. */
+	if (member->is_directory || member->is_link ||
+		should_ignore_relpath(mystreamer->context, member->pathname))
+		return;
+
+	/*
+	 * The backup_manifest stores a relative path to the base directory for
+	 * files belong tablespace, whereas <tablespaceoid>.tar doesn't. Prepare
+	 * the required path, otherwise, the manfiest entry verification will
+	 * fail.
+	 */
+	if (OidIsValid(mystreamer->tblspc_oid))
+	{
+		char		temp[MAXPGPATH];
+
+		/* Copy original name at temporary space */
+		memcpy(temp, member->pathname, MAXPGPATH);
+
+		snprintf(member->pathname, MAXPGPATH, "%s/%d/%s",
+				 "pg_tblspc", mystreamer->tblspc_oid, temp);
+	}
+
+	/* Check the manifest entry */
+	m = verify_manifest_entry(mystreamer->context, member->pathname,
+							  member->size);
+	mystreamer->mfile = (void *) m;
+
+	/*
+	 * Prepare for checksum and control data verification.
+	 *
+	 * We could have these checks while receiving contents. However, since
+	 * contents are received in multiple iterations, this would result in
+	 * these lengthy checks being performed multiple times. Instead, having a
+	 * single flag would be more efficient.
+	 */
+	mystreamer->verify_checksum =
+		(!mystreamer->context->skip_checksums && should_verify_checksum(m));
+	mystreamer->verify_control_data =
+		should_verify_control_data(mystreamer->context->manifest, m);
+
+	/* Initialize the context required for checksum verification. */
+	if (mystreamer->verify_checksum &&
+		pg_checksum_init(mystreamer->checksum_ctx, m->checksum_type) < 0)
+	{
+		report_backup_error(mystreamer->context,
+							"%s: could not initialize checksum of file \"%s\"",
+							mystreamer->archive_name, m->pathname);
+
+		/*
+		 * Checksum verification cannot be performed without proper context
+		 * initialization.
+		 */
+		mystreamer->verify_checksum = false;
+	}
+}
+
+/*
+ * Computes the checksum incrementally for the received file content.
+ *
+ * The caller should pass a correctly initialized checksum_ctx, which will be
+ * used for incremental checksum computation.
+ */
+static void
+member_compute_checksum(astreamer *streamer, astreamer_member *member,
+						const char *data, int len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	pg_checksum_context *checksum_ctx = mystreamer->checksum_ctx;
+	manifest_file *m = mystreamer->mfile;
+
+	Assert(mystreamer->verify_checksum);
+
+	/* Should have came for the right file */
+	Assert(strcmp(member->pathname, m->pathname) == 0);
+
+	/*
+	 * The checksum context should match the type noted in the backup
+	 * manifest.
+	 */
+	Assert(checksum_ctx->type == m->checksum_type);
+
+	/*
+	 * Update the total count of computed checksum bytes for cross-checking
+	 * with the file size in the final verification stage.
+	 */
+	mystreamer->received_bytes += len;
+
+	if (pg_checksum_update(checksum_ctx, (uint8 *) data, len) < 0)
+	{
+		report_backup_error(mystreamer->context,
+							"could not update checksum of file \"%s\"",
+							m->pathname);
+		mystreamer->verify_checksum = false;
+	}
+}
+
+/*
+ * Perform the final computation and checksum verification after the entire
+ * file content has been processed.
+ */
+static void
+member_verify_checksum(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	Assert(mystreamer->verify_checksum);
+
+	verify_checksum(mystreamer->context, mystreamer->mfile,
+					mystreamer->checksum_ctx, mystreamer->received_bytes);
+}
+
+/*
+ * Stores the pg_control file contents into a local buffer; we need the entire
+ * control file data for verification.
+ */
+static void
+member_copy_control_data(astreamer *streamer, astreamer_member *member,
+						 const char *data, int len)
+{
+	/* Should be here only for control file */
+	Assert(strcmp(member->pathname, "global/pg_control") == 0);
+	Assert(((astreamer_verify *) streamer)->verify_control_data);
+
+	/* Copy enough control file data needed for verification. */
+	astreamer_buffer_until(streamer, &data, &len, sizeof(ControlFileData));
+}
+
+/*
+ * Performs the CRC calculation of pg_control data and then calls the routines
+ * that execute the final verification of the control file information.
+ */
+static void
+member_verify_control_data(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_data *manifest = mystreamer->context->manifest;
+	ControlFileData control_file;
+	pg_crc32c	crc;
+	bool		crc_ok;
+
+	/* Should be here only for control file */
+	Assert(strcmp(mystreamer->mfile->pathname, "global/pg_control") == 0);
+	Assert(mystreamer->verify_control_data);
+
+	/* Should have enough control file data needed for verification. */
+	if (streamer->bbs_buffer.len != sizeof(ControlFileData))
+		report_fatal_error("%s: unexpected control file size: %d, should be %zu",
+						   mystreamer->archive_name, streamer->bbs_buffer.len,
+						   sizeof(ControlFileData));
+
+	memcpy(&control_file, streamer->bbs_buffer.data, sizeof(ControlFileData));
+
+	/* Check the CRC. */
+	INIT_CRC32C(crc);
+	COMP_CRC32C(crc, (char *) (&control_file), offsetof(ControlFileData, crc));
+	FIN_CRC32C(crc);
+
+	crc_ok = EQ_CRC32C(crc, control_file.crc);
+
+	/* Do the final control data verification. */
+	verify_control_data(&control_file, mystreamer->mfile->pathname, crc_ok,
+						manifest->system_identifier);
+}
+
+/*
+ * Reset flags and free memory allocations for member file verification.
+ */
+static void
+member_reset_info(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	mystreamer->mfile = NULL;
+	mystreamer->received_bytes = 0;
+	mystreamer->verify_checksum = false;
+	mystreamer->verify_control_data = false;
+}
diff --git a/src/bin/pg_verifybackup/meson.build b/src/bin/pg_verifybackup/meson.build
index 7c7d31a0350..1e3fcf7ee5a 100644
--- a/src/bin/pg_verifybackup/meson.build
+++ b/src/bin/pg_verifybackup/meson.build
@@ -1,7 +1,8 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 pg_verifybackup_sources = files(
-  'pg_verifybackup.c'
+  'pg_verifybackup.c',
+  'astreamer_verify.c'
 )
 
 if host_system == 'windows'
@@ -10,9 +11,10 @@ if host_system == 'windows'
     '--FILEDESC', 'pg_verifybackup - verify a backup against using a backup manifest'])
 endif
 
+pg_verifybackup_deps = [frontend_code, libpq, lz4, zlib, zstd]
 pg_verifybackup = executable('pg_verifybackup',
   pg_verifybackup_sources,
-  dependencies: [frontend_code, libpq],
+  dependencies: pg_verifybackup_deps,
   kwargs: default_bin_args,
 )
 bin_targets += pg_verifybackup
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 4c0367038b4..09840a2cef4 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -22,6 +22,7 @@
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
+#include "limits.h"
 #include "pg_verifybackup.h"
 #include "pgtime.h"
 
@@ -44,6 +45,16 @@
  */
 #define READ_CHUNK_SIZE				(128 * 1024)
 
+/*
+ * Tar archive information needed for content verification.
+ */
+typedef struct tar_archive
+{
+	char	   *relpath;
+	Oid			tblspc_oid;
+	pg_compress_algorithm compress_algorithm;
+} tar_archive;
+
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -65,8 +76,14 @@ static void report_manifest_error(JsonManifestParseContext *context,
 static char find_backup_format(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
-static void verify_plain_backup_file(verifier_context *context,
-									 char *relpath, char *fullpath);
+static void verify_plain_backup_file(verifier_context *context, char *relpath,
+									 char *fullpath);
+static void verify_tar_backup_file(verifier_context *context, char *relpath,
+								   char *fullpath, SimplePtrList *tarfiles);
+static void verify_tar_file_contents(verifier_context *context,
+									 SimplePtrList *tarfiles);
+static void verify_tar_contents(verifier_context *context, char *relpath,
+								char *fullpath, astreamer *streamer);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -75,6 +92,10 @@ static void verify_file_checksum(verifier_context *context,
 static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
+static astreamer *create_archive_verifier(verifier_context *context,
+										  char *archive_name,
+										  Oid tblspc_oid,
+										  pg_compress_algorithm compress_algo);
 
 static void progress_report(bool finished);
 static void usage(void);
@@ -561,6 +582,7 @@ verify_backup_directory(verifier_context *context, char *relpath,
 {
 	DIR		   *dir;
 	struct dirent *dirent;
+	SimplePtrList tarfiles = {NULL, NULL};
 
 	dir = opendir(fullpath);
 	if (dir == NULL)
@@ -600,12 +622,23 @@ verify_backup_directory(verifier_context *context, char *relpath,
 			newrelpath = psprintf("%s/%s", relpath, filename);
 
 		if (!should_ignore_relpath(context, newrelpath))
-			verify_plain_backup_file(context, newrelpath, newfullpath);
+		{
+			if (context->format == 'p')
+				verify_plain_backup_file(context, newrelpath, newfullpath);
+			else
+				verify_tar_backup_file(context, newrelpath, newfullpath,
+									   &tarfiles);
+		}
 
 		pfree(newfullpath);
 		pfree(newrelpath);
 	}
 
+	/* Perform the final verification of the tar contents, if any. */
+	Assert(tarfiles.head == NULL || context->format == 't');
+	if (tarfiles.head != NULL)
+		verify_tar_file_contents(context, &tarfiles);
+
 	if (closedir(dir))
 	{
 		report_backup_error(context,
@@ -682,6 +715,215 @@ verify_plain_backup_file(verifier_context *context, char *relpath,
 		total_size += m->size;
 }
 
+/*
+ * Verify one tar archive file.
+ *
+ * This function does not perform a complete verification; it only carries out
+ * basic validation of the tar format backup file, detects the compression
+ * type, and appends that information to the tarfiles list. An error will be
+ * reported if the tar archive is inaccessible, or if the file type, name, or
+ * compression type is not as expected.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_plain_backup_file. The additional argument is the file size for
+ * progress report.
+ */
+static void
+verify_tar_backup_file(verifier_context *context, char *relpath,
+					   char *fullpath, SimplePtrList *tarfiles)
+{
+	struct stat sb;
+	Oid			tblspc_oid = InvalidOid;
+	pg_compress_algorithm compress_algorithm;
+	tar_archive *tar_file;
+	char	   *suffix = NULL;
+
+	/* Should be tar format backup */
+	Assert(context->format == 't');
+
+	/* Get file information */
+	if (stat(fullpath, &sb) != 0)
+	{
+		report_backup_error(context,
+							"could not stat file or directory \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	/* In a tar format backup, we expect only plain files. */
+	if (!S_ISREG(sb.st_mode))
+	{
+		report_backup_error(context,
+							"\"%s\" is not a plain file",
+							relpath);
+		return;
+	}
+
+	/*
+	 * We expect tar archive files for backing up the main directory,
+	 * tablespace, and pg_wal directory.
+	 *
+	 * pg_basebackup writes the main data directory to an archive file named
+	 * base.tar, the pg_wal directory to pg_wal.tar, and the tablespace
+	 * directory to <tablespaceoid>.tar, each followed by a compression type
+	 * extension such as .gz, .lz4, or .zst.
+	 */
+	if (strncmp("base", relpath, 4) == 0)
+		suffix = relpath + 4;
+	else if (strncmp("pg_wal", relpath, 6) == 0)
+		suffix = relpath + 6;
+	else
+	{
+		/* Expected a <tablespaceoid>.tar file here. */
+		uint64		num = strtoul(relpath, &suffix, 10);
+
+		/*
+		 * Report an error if we didn't consume at least one character, if the
+		 * result is 0, or if the value is too large to be a valid OID.
+		 */
+		if (suffix == NULL || num <= 0 || num > OID_MAX)
+			report_backup_error(context,
+								"\"%s\" unexpected file in the tar format backup",
+								relpath);
+		tblspc_oid = (Oid) num;
+	}
+
+	/* Now, check the compression type of the tar file */
+	if (strcmp(suffix, ".tar") == 0)
+		compress_algorithm = PG_COMPRESSION_NONE;
+	else if (strcmp(suffix, ".tgz") == 0)
+		compress_algorithm = PG_COMPRESSION_GZIP;
+	else if (strcmp(suffix, ".tar.gz") == 0)
+		compress_algorithm = PG_COMPRESSION_GZIP;
+	else if (strcmp(suffix, ".tar.lz4") == 0)
+		compress_algorithm = PG_COMPRESSION_LZ4;
+	else if (strcmp(suffix, ".tar.zst") == 0)
+		compress_algorithm = PG_COMPRESSION_ZSTD;
+	else
+	{
+		report_backup_error(context,
+							"\"%s\" unexpected file in the tar format backup",
+							relpath);
+		return;
+	}
+
+	/*
+	 * Ignore WALs, as reading and verification will be handled through
+	 * pg_waldump.
+	 */
+	if (strncmp("pg_wal", relpath, 6) == 0)
+		return;
+
+	/*
+	 * Append the information to the list for complete verification at a later
+	 * stage.
+	 */
+	tar_file = pg_malloc(sizeof(tar_archive));
+	tar_file->relpath = pstrdup(relpath);
+	tar_file->tblspc_oid = tblspc_oid;
+	tar_file->compress_algorithm = compress_algorithm;
+
+	simple_ptr_list_append(tarfiles, tar_file);
+
+	/* Update statistics for progress report, if necessary */
+	if (show_progress)
+		total_size += sb.st_size;
+}
+
+/*
+ * This is the final part of tar file verification, which prepares the archive
+ * streamer stack according to the tar file compression format for each tar
+ * archive and invokes them for reading, decompressing, and ultimately
+ * verifying the contents.
+ *
+ * The arguments to this function should be a list of valid tar archives to
+ * verify, and the allocation will be freed once the verification is complete.
+ */
+static void
+verify_tar_file_contents(verifier_context *context, SimplePtrList *tarfiles)
+{
+	SimplePtrListCell *cell;
+
+	progress_report(false);
+
+	for (cell = tarfiles->head; cell != NULL; cell = cell->next)
+	{
+		tar_archive *tar_file = (tar_archive *) cell->ptr;
+		astreamer  *streamer;
+		char	   *fullpath;
+
+		/* Prepare archive streamer stack */
+		streamer = create_archive_verifier(context,
+										   tar_file->relpath,
+										   tar_file->tblspc_oid,
+										   tar_file->compress_algorithm);
+
+		/* Compute the full pathname to the target file. */
+		fullpath = psprintf("%s/%s", context->backup_directory,
+							tar_file->relpath);
+
+		/* Invoke the streamer for reading, decompressing, and verifying. */
+		verify_tar_contents(context, tar_file->relpath, fullpath, streamer);
+
+		/* Cleanup. */
+		pfree(tar_file->relpath);
+		pfree(tar_file);
+		pfree(fullpath);
+
+		astreamer_finalize(streamer);
+		astreamer_free(streamer);
+	}
+	simple_ptr_list_destroy(tarfiles);
+
+	progress_report(true);
+}
+
+/*
+ * Performs the actual work for tar content verification. It reads a given tar
+ * file in predefined chunks and passes it to the streamer, which initiates
+ * routines for decompression (if necessary) and then verifies each member
+ * within the tar archive.
+ */
+static void
+verify_tar_contents(verifier_context *context, char *relpath, char *fullpath,
+					astreamer *streamer)
+{
+	int			fd;
+	int			rc;
+	char	   *buffer;
+
+	pg_log_debug("reading \"%s\"", fullpath);
+
+	/* Open the target file. */
+	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
+	{
+		report_backup_error(context, "could not open file \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
+
+	/* Perform the reads */
+	while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
+	{
+		astreamer_content(streamer, NULL, buffer, rc, ASTREAMER_UNKNOWN);
+
+		/* Report progress */
+		done_size += rc;
+		progress_report(false);
+	}
+
+	if (rc < 0)
+		report_backup_error(context, "could not read file \"%s\": %m",
+							relpath);
+
+	/* Close the file. */
+	if (close(fd) != 0)
+		report_backup_error(context, "could not close file \"%s\": %m",
+							relpath);
+}
+
 /*
  * Verify file and its size entry in the manifest.
  */
@@ -1050,6 +1292,42 @@ find_backup_format(verifier_context *context)
 	return result;
 }
 
+/*
+ * Identifies the necessary steps for verifying the contents of the
+ * provided tar file.
+ */
+static astreamer *
+create_archive_verifier(verifier_context *context, char *archive_name,
+						Oid tblspc_oid, pg_compress_algorithm compress_algo)
+{
+	astreamer  *streamer = NULL;
+
+	/* Should be here only for tar backup */
+	Assert(context->format == 't');
+
+	/*
+	 * To verify the contents of the tar file, the initial step is to parse
+	 * its content.
+	 */
+	streamer = astreamer_verify_content_new(streamer, context, archive_name,
+											tblspc_oid);
+	streamer = astreamer_tar_parser_new(streamer);
+
+	/*
+	 * If the tar file is compressed, we must perform the appropriate
+	 * decompression operation before proceeding with the verification of its
+	 * contents.
+	 */
+	if (compress_algo == PG_COMPRESSION_GZIP)
+		streamer = astreamer_gzip_decompressor_new(streamer);
+	else if (compress_algo == PG_COMPRESSION_LZ4)
+		streamer = astreamer_lz4_decompressor_new(streamer);
+	else if (compress_algo == PG_COMPRESSION_ZSTD)
+		streamer = astreamer_zstd_decompressor_new(streamer);
+
+	return streamer;
+}
+
 /*
  * Print a progress report based on the global variables.
  *
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index d3b9f733087..006d457511d 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -18,6 +18,7 @@
 #include "common/hashfn_unstable.h"
 #include "common/logging.h"
 #include "common/parse_manifest.h"
+#include "fe_utils/astreamer.h"
 #include "fe_utils/simple_list.h"
 
 /*
@@ -123,4 +124,9 @@ extern void report_fatal_error(const char *pg_restrict fmt,...)
 extern bool should_ignore_relpath(verifier_context *context,
 								  const char *relpath);
 
+extern astreamer *astreamer_verify_content_new(astreamer *next,
+											   verifier_context *context,
+											   char *archive_name,
+											   Oid tblspc_oid);
+
 #endif							/* PG_VERIFYBACKUP_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 6d424c89186..143dea2feaf 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3330,6 +3330,7 @@ astreamer_plain_writer
 astreamer_recovery_injector
 astreamer_tar_archiver
 astreamer_tar_parser
+astreamer_verify
 astreamer_zstd_frame
 bgworker_main_type
 bh_node_type
@@ -3951,6 +3952,7 @@ substitute_phv_relids_context
 subxids_array_status
 symbol
 tablespaceinfo
+tar_archive
 td_entry
 teSection
 temp_tablespaces_extra
-- 
2.18.0

From 34d3478011bb94672378b3860724a9a89e2023c0 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 10:26:35 +0530
Subject: [PATCH v11 10/12] pg_verifybackup: Add backup format and compression
 option

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the next patch that implements tar format support.
----
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 73 ++++++++++++++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h |  1 +
 2 files changed, 72 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index a19f344ea67..4c0367038b4 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -62,6 +62,7 @@ static void report_manifest_error(JsonManifestParseContext *context,
 								  const char *fmt,...)
 			pg_attribute_printf(2, 3) pg_attribute_noreturn();
 
+static char find_backup_format(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_plain_backup_file(verifier_context *context,
@@ -97,6 +98,7 @@ main(int argc, char **argv)
 		{"exit-on-error", no_argument, NULL, 'e'},
 		{"ignore", required_argument, NULL, 'i'},
 		{"manifest-path", required_argument, NULL, 'm'},
+		{"format", required_argument, NULL, 'F'},
 		{"no-parse-wal", no_argument, NULL, 'n'},
 		{"progress", no_argument, NULL, 'P'},
 		{"quiet", no_argument, NULL, 'q'},
@@ -118,6 +120,7 @@ main(int argc, char **argv)
 	progname = get_progname(argv[0]);
 
 	memset(&context, 0, sizeof(context));
+	context.format = '\0';
 
 	if (argc > 1)
 	{
@@ -154,7 +157,7 @@ main(int argc, char **argv)
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
 
-	while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1)
+	while ((c = getopt_long(argc, argv, "eF:i:m:nPqsw:", long_options, NULL)) != -1)
 	{
 		switch (c)
 		{
@@ -173,6 +176,15 @@ main(int argc, char **argv)
 				manifest_path = pstrdup(optarg);
 				canonicalize_path(manifest_path);
 				break;
+			case 'F':
+				if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
+					context.format = 'p';
+				else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
+					context.format = 't';
+				else
+					pg_fatal("invalid backup format \"%s\", must be \"plain\" or \"tar\"",
+							 optarg);
+				break;
 			case 'n':
 				no_parse_wal = true;
 				break;
@@ -220,11 +232,26 @@ main(int argc, char **argv)
 		pg_fatal("cannot specify both %s and %s",
 				 "-P/--progress", "-q/--quiet");
 
+	/* Determine the backup format if it hasn't been specified. */
+	if (context.format == '\0')
+		context.format = find_backup_format(&context);
+
 	/* Unless --no-parse-wal was specified, we will need pg_waldump. */
 	if (!no_parse_wal)
 	{
 		int			ret;
 
+		/*
+		 * XXX: In the future, we should consider enhancing pg_waldump to read
+		 * WAL files from the tar archive.
+		 */
+		if (context.format != 'p')
+		{
+			pg_log_error("pg_waldump cannot read from a tar archive");
+			pg_log_error_hint("You must use -n or --no-parse-wal when verifying a tar-format backup.");
+			exit(1);
+		}
+
 		pg_waldump_path = pg_malloc(MAXPGPATH);
 		ret = find_other_exec(argv[0], "pg_waldump",
 							  "pg_waldump (PostgreSQL) " PG_VERSION "\n",
@@ -279,8 +306,13 @@ main(int argc, char **argv)
 	/*
 	 * Now do the expensive work of verifying file checksums, unless we were
 	 * told to skip it.
+	 *
+	 * We were only checking the plain backup here. For the tar backup, file
+	 * checksums verification (if requested) will be done immediately when the
+	 * file is accessed, as we don't have random access to the files like we
+	 * do with plain backups.
 	 */
-	if (!context.skip_checksums)
+	if (!context.skip_checksums && context.format == 'p')
 		verify_backup_checksums(&context);
 
 	/*
@@ -982,6 +1014,42 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
 	return false;
 }
 
+/*
+ * To detect the backup format, it checks for the PG_VERSION file in the backup
+ * directory. If found, it will be considered a plain-format backup; otherwise,
+ * it will be assumed to be a tar-format backup.
+ */
+static char
+find_backup_format(verifier_context *context)
+{
+	char		result;
+	char	   *path;
+	struct stat sb;
+
+	/* Should be here only if the backup format is unknown */
+	Assert(context->format == '\0');
+
+	/* Check PG_VERSION file. */
+	path = psprintf("%s/%s", context->backup_directory, "PG_VERSION");
+	if (stat(path, &sb) == 0)
+		result = 'p';
+	else
+	{
+		if (errno != ENOENT)
+		{
+			pg_log_error("cannot determine backup format");
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+
+		/* Otherwise, it is assumed to be a tar format backup. */
+		result = 't';
+	}
+	pfree(path);
+
+	return result;
+}
+
 /*
  * Print a progress report based on the global variables.
  *
@@ -1039,6 +1107,7 @@ usage(void)
 	printf(_("  -e, --exit-on-error         exit immediately on error\n"));
 	printf(_("  -i, --ignore=RELATIVE_PATH  ignore indicated path\n"));
 	printf(_("  -m, --manifest-path=PATH    use specified path for manifest\n"));
+	printf(_("  -F, --format=p|t            backup format (plain, tar)\n"));
 	printf(_("  -n, --no-parse-wal          do not try to parse WAL files\n"));
 	printf(_("  -P, --progress              show progress information\n"));
 	printf(_("  -q, --quiet                 do not print any output, except for errors\n"));
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 2901b00870a..d3b9f733087 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -100,6 +100,7 @@ typedef struct verifier_context
 	manifest_data *manifest;
 	char	   *backup_directory;
 	SimpleStringList ignore_list;
+	char		format;			/* backup format:  p(lain)/t(ar) */
 	bool		skip_checksums;
 	bool		exit_on_error;
 	bool		saw_any_error;
-- 
2.18.0

From 7322f5ee8d461eb0c3d54086946850adee14a001 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 8 Aug 2024 16:01:33 +0530
Subject: [PATCH v11 09/12] Add simple_ptr_list_destroy() and
 simple_ptr_list_destroy_deep() API.

We didn't have any helper function to destroy SimplePtrList, likely
because it wasn't needed before, but it's required in a later patch in
this set.  I've added two functions for this purpose, inspired by
list_free() and list_free_deep().
---
 src/fe_utils/simple_list.c         | 39 ++++++++++++++++++++++++++++++
 src/include/fe_utils/simple_list.h |  2 ++
 2 files changed, 41 insertions(+)

diff --git a/src/fe_utils/simple_list.c b/src/fe_utils/simple_list.c
index 2d88eb54067..9d218911c31 100644
--- a/src/fe_utils/simple_list.c
+++ b/src/fe_utils/simple_list.c
@@ -173,3 +173,42 @@ simple_ptr_list_append(SimplePtrList *list, void *ptr)
 		list->head = cell;
 	list->tail = cell;
 }
+
+/*
+ * Destroy a pointer list and optionally the pointed-to element
+ */
+static void
+simple_ptr_list_destroy_private(SimplePtrList *list, bool deep)
+{
+	SimplePtrListCell *cell;
+
+	cell = list->head;
+	while (cell != NULL)
+	{
+		SimplePtrListCell *next;
+
+		next = cell->next;
+		if (deep)
+			pg_free(cell->ptr);
+		pg_free(cell);
+		cell = next;
+	}
+}
+
+/*
+ * Destroy a pointer list and the pointed-to element
+ */
+void
+simple_ptr_list_destroy_deep(SimplePtrList *list)
+{
+	simple_ptr_list_destroy_private(list, true);
+}
+
+/*
+ * Destroy only pointer list and not the pointed-to element
+ */
+void
+simple_ptr_list_destroy(SimplePtrList *list)
+{
+	simple_ptr_list_destroy_private(list, false);
+}
diff --git a/src/include/fe_utils/simple_list.h b/src/include/fe_utils/simple_list.h
index d42ecded8ed..5b7cbec8a62 100644
--- a/src/include/fe_utils/simple_list.h
+++ b/src/include/fe_utils/simple_list.h
@@ -66,5 +66,7 @@ extern void simple_string_list_destroy(SimpleStringList *list);
 extern const char *simple_string_list_not_touched(SimpleStringList *list);
 
 extern void simple_ptr_list_append(SimplePtrList *list, void *ptr);
+extern void simple_ptr_list_destroy_deep(SimplePtrList *list);
+extern void simple_ptr_list_destroy(SimplePtrList *list);
 
 #endif							/* SIMPLE_LIST_H */
-- 
2.18.0

From 792b4f4a19e458de284e062adc16caa0528539bd Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 1 Aug 2024 15:47:26 +0530
Subject: [PATCH v11 08/12] Refactor: split verify_control_file.

Separated the manifest entry verification code into a new function and
introduced the should_verify_control_data() macro, similar to
should_verify_checksum().

Like verify_file_checksum(), verify_control_file() is too design to
accept the pg_control file patch which will be opened and respective
information will be verified. But, in case of tar backup we would be
having pg_control file contents instead, that needs to be verified in
the same way.  For that reason the code that doing the verification is
separated into separate function to so that can be reused for the tar
backup verification as well.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 43 +++++++++++------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 14 ++++++++
 2 files changed, 34 insertions(+), 23 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index e44d0377cd5..a19f344ea67 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -66,8 +66,6 @@ static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_plain_backup_file(verifier_context *context,
 									 char *relpath, char *fullpath);
-static void verify_control_file(const char *controlpath,
-								uint64 manifest_system_identifier);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -631,14 +629,20 @@ verify_plain_backup_file(verifier_context *context, char *relpath,
 	/* Check the backup manifest entry for this file. */
 	m = verify_manifest_entry(context, relpath, sb.st_size);
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0 &&
-		m != NULL && m->matched && !m->bad)
-		verify_control_file(fullpath, context->manifest->system_identifier);
+	/* Validate the pg_control file information */
+	if (should_verify_control_data(context->manifest, m))
+	{
+		ControlFileData *control_file;
+		bool		crc_ok;
+
+		pg_log_debug("reading \"%s\"", fullpath);
+		control_file = get_controlfile_by_exact_path(fullpath, &crc_ok);
+
+		verify_control_data(control_file, fullpath, crc_ok,
+							context->manifest->system_identifier);
+		/* Release memory. */
+		pfree(control_file);
+	}
 
 	/* Update statistics for progress report, if necessary */
 	if (show_progress && !context->skip_checksums &&
@@ -687,18 +691,14 @@ verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
 }
 
 /*
- * Sanity check control file and validate system identifier against manifest
- * system identifier.
+ * Sanity check control file data and validate system identifier against
+ * manifest system identifier.
  */
-static void
-verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
+void
+verify_control_data(ControlFileData *control_file,
+					const char *controlpath, bool crc_ok,
+					uint64 manifest_system_identifier)
 {
-	ControlFileData *control_file;
-	bool		crc_ok;
-
-	pg_log_debug("reading \"%s\"", controlpath);
-	control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
-
 	/* Control file contents not meaningful if CRC is bad. */
 	if (!crc_ok)
 		report_fatal_error("%s: CRC is incorrect", controlpath);
@@ -714,9 +714,6 @@ verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
 						   controlpath,
 						   (unsigned long long) manifest_system_identifier,
 						   (unsigned long long) control_file->system_identifier);
-
-	/* Release memory. */
-	pfree(control_file);
 }
 
 /*
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index fe0ce8a89aa..2901b00870a 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -40,6 +40,17 @@ typedef struct manifest_file
 	(((m) != NULL) && ((m)->matched) && !((m)->bad) && \
 	 (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
 
+/*
+ * Validate the control file and its system identifier against the manifest
+ * system identifier. Note that this feature is not available in manifest
+ * version 1. This validation should only be performed if the manifest entry
+ * validation has been completed without errors.
+ */
+#define should_verify_control_data(manifest, m) \
+	(((manifest)->version != 1) && \
+	 ((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (strcmp((m)->pathname, "global/pg_control") == 0))
+
 /*
  * Define a hash table which we can use to store information about the files
  * mentioned in the backup manifest.
@@ -99,6 +110,9 @@ extern manifest_file *verify_manifest_entry(verifier_context *context,
 extern void verify_checksum(verifier_context *context, manifest_file *m,
 							pg_checksum_context *checksum_ctx,
 							int64 bytes_read);
+extern void verify_control_data(ControlFileData *control_file,
+								const char *controlpath, bool crc_ok,
+								uint64 manifest_system_identifier);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From a297985e7b7815f7bda545a50b05b19877b01cea Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 14 Aug 2024 15:14:15 +0530
Subject: [PATCH v11 07/12] Refactor: split verify_file_checksum() function.

Move the core functionality of verify_file_checksum to a new function
to reuse it instead of duplicating the code.

The verify_file_checksum() function is designed to take a file path,
open and read the file contents, and then calculate the checksum.
However, for TAR backups, instead of a file path, we receive the file
content in chunks, and the checksum needs to be calculated
incrementally. While the initial operations for plain and TAR backup
checksum calculations differ, the final checks and error handling are
the same. By moving the shared logic to a separate function, we can
reuse the code for both types of backups.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 20 +++++++++++++++++---
 src/bin/pg_verifybackup/pg_verifybackup.h |  3 +++
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 5bfc98e7874..e44d0377cd5 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -792,8 +792,6 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	int			fd;
 	int			rc;
 	size_t		bytes_read = 0;
-	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
-	int			checksumlen;
 
 	/* Open the target file. */
 	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
@@ -844,6 +842,22 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	if (rc < 0)
 		return;
 
+	/* Do the final computation and verification. */
+	verify_checksum(context, m, &checksum_ctx, bytes_read);
+}
+
+/*
+ * A helper function to finalize checksum computation and verify it against the
+ * backup manifest information.
+ */
+void
+verify_checksum(verifier_context *context, manifest_file *m,
+				pg_checksum_context *checksum_ctx, int64 bytes_read)
+{
+	const char *relpath = m->pathname;
+	int			checksumlen;
+	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
+
 	/*
 	 * Double-check that we read the expected number of bytes from the file.
 	 * Normally, a file size mismatch would be caught in verify_manifest_entry
@@ -860,7 +874,7 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	}
 
 	/* Get the final checksum. */
-	checksumlen = pg_checksum_final(&checksum_ctx, checksumbuf);
+	checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
 	if (checksumlen < 0)
 	{
 		report_backup_error(context,
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index ff9476e356e..fe0ce8a89aa 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -96,6 +96,9 @@ typedef struct verifier_context
 
 extern manifest_file *verify_manifest_entry(verifier_context *context,
 											char *relpath, int64 filesize);
+extern void verify_checksum(verifier_context *context, manifest_file *m,
+							pg_checksum_context *checksum_ctx,
+							int64 bytes_read);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From 7173d77e6da29ddc35bf8d87e20db9b16f06b26c Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 14 Aug 2024 10:42:37 +0530
Subject: [PATCH v11 06/12] Refactor: split verify_backup_file() function and
 rename it.

The function verify_backup_file() has now been renamed to
verify_plain_backup_file() to make it clearer that it is specifically
used for verifying files in a plain backup. Similarly, in a future
patch, we would have a verify_tar_backup_file() function for
verifying TAR backup files.

In addition to that, moved the manifest entry verification code into a
new function called verify_manifest_entry() so that it can be reused
for tar backup verification. If verify_manifest_entry() doesn't find
an entry, it reports an error as before and returns NULL to the
caller. This is why a NULL check is added to should_verify_checksum().
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 58 +++++++++++++++--------
 src/bin/pg_verifybackup/pg_verifybackup.h |  6 ++-
 2 files changed, 42 insertions(+), 22 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 3fcfb167217..5bfc98e7874 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -64,8 +64,8 @@ static void report_manifest_error(JsonManifestParseContext *context,
 
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
-static void verify_backup_file(verifier_context *context,
-							   char *relpath, char *fullpath);
+static void verify_plain_backup_file(verifier_context *context,
+									 char *relpath, char *fullpath);
 static void verify_control_file(const char *controlpath,
 								uint64 manifest_system_identifier);
 static void report_extra_backup_files(verifier_context *context);
@@ -570,7 +570,7 @@ verify_backup_directory(verifier_context *context, char *relpath,
 			newrelpath = psprintf("%s/%s", relpath, filename);
 
 		if (!should_ignore_relpath(context, newrelpath))
-			verify_backup_file(context, newrelpath, newfullpath);
+			verify_plain_backup_file(context, newrelpath, newfullpath);
 
 		pfree(newfullpath);
 		pfree(newrelpath);
@@ -591,7 +591,8 @@ verify_backup_directory(verifier_context *context, char *relpath,
  * verify_backup_directory.
  */
 static void
-verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
+verify_plain_backup_file(verifier_context *context, char *relpath,
+						 char *fullpath)
 {
 	struct stat sb;
 	manifest_file *m;
@@ -627,6 +628,32 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		return;
 	}
 
+	/* Check the backup manifest entry for this file. */
+	m = verify_manifest_entry(context, relpath, sb.st_size);
+
+	/*
+	 * Validate the manifest system identifier, not available in manifest
+	 * version 1.
+	 */
+	if (context->manifest->version != 1 &&
+		strcmp(relpath, "global/pg_control") == 0 &&
+		m != NULL && m->matched && !m->bad)
+		verify_control_file(fullpath, context->manifest->system_identifier);
+
+	/* Update statistics for progress report, if necessary */
+	if (show_progress && !context->skip_checksums &&
+		should_verify_checksum(m))
+		total_size += m->size;
+}
+
+/*
+ * Verify file and its size entry in the manifest.
+ */
+manifest_file *
+verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
+{
+	manifest_file *m;
+
 	/* Check whether there's an entry in the manifest hash. */
 	m = manifest_files_lookup(context->manifest->files, relpath);
 	if (m == NULL)
@@ -634,40 +661,29 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		report_backup_error(context,
 							"\"%s\" is present on disk but not in the manifest",
 							relpath);
-		return;
+		return NULL;
 	}
 
 	/* Flag this entry as having been encountered in the filesystem. */
 	m->matched = true;
 
 	/* Check that the size matches. */
-	if (m->size != sb.st_size)
+	if (m->size != filesize)
 	{
 		report_backup_error(context,
 							"\"%s\" has size %lld on disk but size %zu in the manifest",
-							relpath, (long long int) sb.st_size, m->size);
+							relpath, (long long int) filesize, m->size);
 		m->bad = true;
 	}
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0)
-		verify_control_file(fullpath, context->manifest->system_identifier);
-
-	/* Update statistics for progress report, if necessary */
-	if (show_progress && !context->skip_checksums &&
-		should_verify_checksum(m))
-		total_size += m->size;
-
 	/*
 	 * We don't verify checksums at this stage. We first finish verifying that
 	 * we have the expected set of files with the expected sizes, and only
 	 * afterwards verify the checksums. That's because computing checksums may
 	 * take a while, and we'd like to report more obvious problems quickly.
 	 */
+
+	return m;
 }
 
 /*
@@ -830,7 +846,7 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 
 	/*
 	 * Double-check that we read the expected number of bytes from the file.
-	 * Normally, a file size mismatch would be caught in verify_backup_file
+	 * Normally, a file size mismatch would be caught in verify_manifest_entry
 	 * and this check would never be reached, but this provides additional
 	 * safety and clarity in the event of concurrent modifications or
 	 * filesystem misbehavior.
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index d8c566ed587..ff9476e356e 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -37,7 +37,8 @@ typedef struct manifest_file
 } manifest_file;
 
 #define should_verify_checksum(m) \
-	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+	(((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
 
 /*
  * Define a hash table which we can use to store information about the files
@@ -93,6 +94,9 @@ typedef struct verifier_context
 	bool		saw_any_error;
 } verifier_context;
 
+extern manifest_file *verify_manifest_entry(verifier_context *context,
+											char *relpath, int64 filesize);
+
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
 			pg_attribute_printf(2, 3);
-- 
2.18.0

From 8c5c624fa89eaa42e8812dad840842b30ec35ec7 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 14 Aug 2024 10:42:37 +0530
Subject: [PATCH v12 06/12] Refactor: split verify_backup_file() function and
 rename it.

The function verify_backup_file() has now been renamed to
verify_plain_backup_file() to make it clearer that it is specifically
used for verifying files in a plain backup. Similarly, in a future
patch, we would have a verify_tar_backup_file() function for
verifying TAR backup files.

In addition to that, moved the manifest entry verification code into a
new function called verify_manifest_entry() so that it can be reused
for tar backup verification. If verify_manifest_entry() doesn't find
an entry, it reports an error as before and returns NULL to the
caller. This is why a NULL check is added to should_verify_checksum().
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 58 +++++++++++++++--------
 src/bin/pg_verifybackup/pg_verifybackup.h |  6 ++-
 2 files changed, 42 insertions(+), 22 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 3fcfb167217..5bfc98e7874 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -64,8 +64,8 @@ static void report_manifest_error(JsonManifestParseContext *context,
 
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
-static void verify_backup_file(verifier_context *context,
-							   char *relpath, char *fullpath);
+static void verify_plain_backup_file(verifier_context *context,
+									 char *relpath, char *fullpath);
 static void verify_control_file(const char *controlpath,
 								uint64 manifest_system_identifier);
 static void report_extra_backup_files(verifier_context *context);
@@ -570,7 +570,7 @@ verify_backup_directory(verifier_context *context, char *relpath,
 			newrelpath = psprintf("%s/%s", relpath, filename);
 
 		if (!should_ignore_relpath(context, newrelpath))
-			verify_backup_file(context, newrelpath, newfullpath);
+			verify_plain_backup_file(context, newrelpath, newfullpath);
 
 		pfree(newfullpath);
 		pfree(newrelpath);
@@ -591,7 +591,8 @@ verify_backup_directory(verifier_context *context, char *relpath,
  * verify_backup_directory.
  */
 static void
-verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
+verify_plain_backup_file(verifier_context *context, char *relpath,
+						 char *fullpath)
 {
 	struct stat sb;
 	manifest_file *m;
@@ -627,6 +628,32 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		return;
 	}
 
+	/* Check the backup manifest entry for this file. */
+	m = verify_manifest_entry(context, relpath, sb.st_size);
+
+	/*
+	 * Validate the manifest system identifier, not available in manifest
+	 * version 1.
+	 */
+	if (context->manifest->version != 1 &&
+		strcmp(relpath, "global/pg_control") == 0 &&
+		m != NULL && m->matched && !m->bad)
+		verify_control_file(fullpath, context->manifest->system_identifier);
+
+	/* Update statistics for progress report, if necessary */
+	if (show_progress && !context->skip_checksums &&
+		should_verify_checksum(m))
+		total_size += m->size;
+}
+
+/*
+ * Verify file and its size entry in the manifest.
+ */
+manifest_file *
+verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
+{
+	manifest_file *m;
+
 	/* Check whether there's an entry in the manifest hash. */
 	m = manifest_files_lookup(context->manifest->files, relpath);
 	if (m == NULL)
@@ -634,40 +661,29 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		report_backup_error(context,
 							"\"%s\" is present on disk but not in the manifest",
 							relpath);
-		return;
+		return NULL;
 	}
 
 	/* Flag this entry as having been encountered in the filesystem. */
 	m->matched = true;
 
 	/* Check that the size matches. */
-	if (m->size != sb.st_size)
+	if (m->size != filesize)
 	{
 		report_backup_error(context,
 							"\"%s\" has size %lld on disk but size %zu in the manifest",
-							relpath, (long long int) sb.st_size, m->size);
+							relpath, (long long int) filesize, m->size);
 		m->bad = true;
 	}
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0)
-		verify_control_file(fullpath, context->manifest->system_identifier);
-
-	/* Update statistics for progress report, if necessary */
-	if (show_progress && !context->skip_checksums &&
-		should_verify_checksum(m))
-		total_size += m->size;
-
 	/*
 	 * We don't verify checksums at this stage. We first finish verifying that
 	 * we have the expected set of files with the expected sizes, and only
 	 * afterwards verify the checksums. That's because computing checksums may
 	 * take a while, and we'd like to report more obvious problems quickly.
 	 */
+
+	return m;
 }
 
 /*
@@ -830,7 +846,7 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 
 	/*
 	 * Double-check that we read the expected number of bytes from the file.
-	 * Normally, a file size mismatch would be caught in verify_backup_file
+	 * Normally, a file size mismatch would be caught in verify_manifest_entry
 	 * and this check would never be reached, but this provides additional
 	 * safety and clarity in the event of concurrent modifications or
 	 * filesystem misbehavior.
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index d8c566ed587..ff9476e356e 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -37,7 +37,8 @@ typedef struct manifest_file
 } manifest_file;
 
 #define should_verify_checksum(m) \
-	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+	(((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
 
 /*
  * Define a hash table which we can use to store information about the files
@@ -93,6 +94,9 @@ typedef struct verifier_context
 	bool		saw_any_error;
 } verifier_context;
 
+extern manifest_file *verify_manifest_entry(verifier_context *context,
+											char *relpath, int64 filesize);
+
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
 			pg_attribute_printf(2, 3);
-- 
2.18.0

From 0d01f43ee33c20cbd1341a3454e312d85612104e Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 14 Aug 2024 15:14:15 +0530
Subject: [PATCH v12 07/12] Refactor: split verify_file_checksum() function.

Move the core functionality of verify_file_checksum to a new function
to reuse it instead of duplicating the code.

The verify_file_checksum() function is designed to take a file path,
open and read the file contents, and then calculate the checksum.
However, for TAR backups, instead of a file path, we receive the file
content in chunks, and the checksum needs to be calculated
incrementally. While the initial operations for plain and TAR backup
checksum calculations differ, the final checks and error handling are
the same. By moving the shared logic to a separate function, we can
reuse the code for both types of backups.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 20 +++++++++++++++++---
 src/bin/pg_verifybackup/pg_verifybackup.h |  3 +++
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 5bfc98e7874..e44d0377cd5 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -792,8 +792,6 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	int			fd;
 	int			rc;
 	size_t		bytes_read = 0;
-	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
-	int			checksumlen;
 
 	/* Open the target file. */
 	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
@@ -844,6 +842,22 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	if (rc < 0)
 		return;
 
+	/* Do the final computation and verification. */
+	verify_checksum(context, m, &checksum_ctx, bytes_read);
+}
+
+/*
+ * A helper function to finalize checksum computation and verify it against the
+ * backup manifest information.
+ */
+void
+verify_checksum(verifier_context *context, manifest_file *m,
+				pg_checksum_context *checksum_ctx, int64 bytes_read)
+{
+	const char *relpath = m->pathname;
+	int			checksumlen;
+	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
+
 	/*
 	 * Double-check that we read the expected number of bytes from the file.
 	 * Normally, a file size mismatch would be caught in verify_manifest_entry
@@ -860,7 +874,7 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	}
 
 	/* Get the final checksum. */
-	checksumlen = pg_checksum_final(&checksum_ctx, checksumbuf);
+	checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
 	if (checksumlen < 0)
 	{
 		report_backup_error(context,
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index ff9476e356e..fe0ce8a89aa 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -96,6 +96,9 @@ typedef struct verifier_context
 
 extern manifest_file *verify_manifest_entry(verifier_context *context,
 											char *relpath, int64 filesize);
+extern void verify_checksum(verifier_context *context, manifest_file *m,
+							pg_checksum_context *checksum_ctx,
+							int64 bytes_read);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From b877ed19cdb43bdce5aabb715b4dd6a4a3642d1a Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 21 Aug 2024 10:51:23 +0530
Subject: [PATCH v12 08/12] Refactor: split verify_control_file.

Separated the manifest entry verification code into a new function and
introduced the should_verify_control_data() macro, similar to
should_verify_checksum().

Like verify_file_checksum(), verify_control_file() is too design to
accept the pg_control file patch which will be opened and respective
information will be verified. But, in case of tar backup we would be
having pg_control file contents instead, that needs to be verified in
the same way.  For that reason the code that doing the verification is
separated into separate function to so that can be reused for the tar
backup verification as well.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 42 ++++++++++-------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 14 ++++++++
 2 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index e44d0377cd5..d04e1d8c8ac 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -66,8 +66,6 @@ static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_plain_backup_file(verifier_context *context,
 									 char *relpath, char *fullpath);
-static void verify_control_file(const char *controlpath,
-								uint64 manifest_system_identifier);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -631,14 +629,20 @@ verify_plain_backup_file(verifier_context *context, char *relpath,
 	/* Check the backup manifest entry for this file. */
 	m = verify_manifest_entry(context, relpath, sb.st_size);
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0 &&
-		m != NULL && m->matched && !m->bad)
-		verify_control_file(fullpath, context->manifest->system_identifier);
+	/* Validate the pg_control information */
+	if (should_verify_control_data(context->manifest, m))
+	{
+		ControlFileData *control_file;
+		bool		crc_ok;
+
+		pg_log_debug("reading \"%s\"", fullpath);
+		control_file = get_controlfile_by_exact_path(fullpath, &crc_ok);
+
+		verify_control_data(control_file, fullpath, crc_ok,
+							context->manifest->system_identifier);
+		/* Release memory. */
+		pfree(control_file);
+	}
 
 	/* Update statistics for progress report, if necessary */
 	if (show_progress && !context->skip_checksums &&
@@ -687,18 +691,13 @@ verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
 }
 
 /*
- * Sanity check control file and validate system identifier against manifest
- * system identifier.
+ * Sanity check control file data and validate system identifier against
+ * manifest system identifier.
  */
-static void
-verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
+void
+verify_control_data(ControlFileData *control_file, const char *controlpath,
+					bool crc_ok, uint64 manifest_system_identifier)
 {
-	ControlFileData *control_file;
-	bool		crc_ok;
-
-	pg_log_debug("reading \"%s\"", controlpath);
-	control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
-
 	/* Control file contents not meaningful if CRC is bad. */
 	if (!crc_ok)
 		report_fatal_error("%s: CRC is incorrect", controlpath);
@@ -714,9 +713,6 @@ verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
 						   controlpath,
 						   (unsigned long long) manifest_system_identifier,
 						   (unsigned long long) control_file->system_identifier);
-
-	/* Release memory. */
-	pfree(control_file);
 }
 
 /*
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index fe0ce8a89aa..818064c6eed 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -40,6 +40,17 @@ typedef struct manifest_file
 	(((m) != NULL) && ((m)->matched) && !((m)->bad) && \
 	 (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
 
+/*
+ * Validate the control file and its system identifier against the manifest
+ * system identifier. Note that this feature is not available in manifest
+ * version 1. This validation should only be performed after the manifest entry
+ * validation for the pg_control file has been completed without errors.
+ */
+#define should_verify_control_data(manifest, m) \
+	(((manifest)->version != 1) && \
+	 ((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (strcmp((m)->pathname, "global/pg_control") == 0))
+
 /*
  * Define a hash table which we can use to store information about the files
  * mentioned in the backup manifest.
@@ -99,6 +110,9 @@ extern manifest_file *verify_manifest_entry(verifier_context *context,
 extern void verify_checksum(verifier_context *context, manifest_file *m,
 							pg_checksum_context *checksum_ctx,
 							int64 bytes_read);
+extern void verify_control_data(ControlFileData *control_file,
+								const char *controlpath, bool crc_ok,
+								uint64 manifest_system_identifier);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From 61edb2a348d87563aa3fde6b7c6b9e3a6dbeda9f Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 8 Aug 2024 16:01:33 +0530
Subject: [PATCH v12 09/12] Add simple_ptr_list_destroy() and
 simple_ptr_list_destroy_deep() API.

We didn't have any helper function to destroy SimplePtrList, likely
because it wasn't needed before, but it's required in a later patch in
this set.  I've added two functions for this purpose, inspired by
list_free() and list_free_deep().
---
 src/fe_utils/simple_list.c         | 39 ++++++++++++++++++++++++++++++
 src/include/fe_utils/simple_list.h |  2 ++
 2 files changed, 41 insertions(+)

diff --git a/src/fe_utils/simple_list.c b/src/fe_utils/simple_list.c
index 2d88eb54067..9d218911c31 100644
--- a/src/fe_utils/simple_list.c
+++ b/src/fe_utils/simple_list.c
@@ -173,3 +173,42 @@ simple_ptr_list_append(SimplePtrList *list, void *ptr)
 		list->head = cell;
 	list->tail = cell;
 }
+
+/*
+ * Destroy a pointer list and optionally the pointed-to element
+ */
+static void
+simple_ptr_list_destroy_private(SimplePtrList *list, bool deep)
+{
+	SimplePtrListCell *cell;
+
+	cell = list->head;
+	while (cell != NULL)
+	{
+		SimplePtrListCell *next;
+
+		next = cell->next;
+		if (deep)
+			pg_free(cell->ptr);
+		pg_free(cell);
+		cell = next;
+	}
+}
+
+/*
+ * Destroy a pointer list and the pointed-to element
+ */
+void
+simple_ptr_list_destroy_deep(SimplePtrList *list)
+{
+	simple_ptr_list_destroy_private(list, true);
+}
+
+/*
+ * Destroy only pointer list and not the pointed-to element
+ */
+void
+simple_ptr_list_destroy(SimplePtrList *list)
+{
+	simple_ptr_list_destroy_private(list, false);
+}
diff --git a/src/include/fe_utils/simple_list.h b/src/include/fe_utils/simple_list.h
index d42ecded8ed..5b7cbec8a62 100644
--- a/src/include/fe_utils/simple_list.h
+++ b/src/include/fe_utils/simple_list.h
@@ -66,5 +66,7 @@ extern void simple_string_list_destroy(SimpleStringList *list);
 extern const char *simple_string_list_not_touched(SimpleStringList *list);
 
 extern void simple_ptr_list_append(SimplePtrList *list, void *ptr);
+extern void simple_ptr_list_destroy_deep(SimplePtrList *list);
+extern void simple_ptr_list_destroy(SimplePtrList *list);
 
 #endif							/* SIMPLE_LIST_H */
-- 
2.18.0

From 0ba8580c0ee6489f1195bae4d3d5427543479f76 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 21 Aug 2024 11:03:44 +0530
Subject: [PATCH v12 10/12] pg_verifybackup: Add backup format and compression
 option

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the next patch that implements tar format support.
----
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 73 ++++++++++++++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h |  1 +
 2 files changed, 72 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index d04e1d8c8ac..c186678a592 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -62,6 +62,7 @@ static void report_manifest_error(JsonManifestParseContext *context,
 								  const char *fmt,...)
 			pg_attribute_printf(2, 3) pg_attribute_noreturn();
 
+static char find_backup_format(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_plain_backup_file(verifier_context *context,
@@ -97,6 +98,7 @@ main(int argc, char **argv)
 		{"exit-on-error", no_argument, NULL, 'e'},
 		{"ignore", required_argument, NULL, 'i'},
 		{"manifest-path", required_argument, NULL, 'm'},
+		{"format", required_argument, NULL, 'F'},
 		{"no-parse-wal", no_argument, NULL, 'n'},
 		{"progress", no_argument, NULL, 'P'},
 		{"quiet", no_argument, NULL, 'q'},
@@ -118,6 +120,7 @@ main(int argc, char **argv)
 	progname = get_progname(argv[0]);
 
 	memset(&context, 0, sizeof(context));
+	context.format = '\0';
 
 	if (argc > 1)
 	{
@@ -154,7 +157,7 @@ main(int argc, char **argv)
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
 
-	while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1)
+	while ((c = getopt_long(argc, argv, "eF:i:m:nPqsw:", long_options, NULL)) != -1)
 	{
 		switch (c)
 		{
@@ -173,6 +176,15 @@ main(int argc, char **argv)
 				manifest_path = pstrdup(optarg);
 				canonicalize_path(manifest_path);
 				break;
+			case 'F':
+				if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
+					context.format = 'p';
+				else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
+					context.format = 't';
+				else
+					pg_fatal("invalid backup format \"%s\", must be \"plain\" or \"tar\"",
+							 optarg);
+				break;
 			case 'n':
 				no_parse_wal = true;
 				break;
@@ -220,11 +232,26 @@ main(int argc, char **argv)
 		pg_fatal("cannot specify both %s and %s",
 				 "-P/--progress", "-q/--quiet");
 
+	/* Determine the backup format if it hasn't been specified. */
+	if (context.format == '\0')
+		context.format = find_backup_format(&context);
+
 	/* Unless --no-parse-wal was specified, we will need pg_waldump. */
 	if (!no_parse_wal)
 	{
 		int			ret;
 
+		/*
+		 * XXX: In the future, we should consider enhancing pg_waldump to read
+		 * WAL files from the tar archive.
+		 */
+		if (context.format != 'p')
+		{
+			pg_log_error("pg_waldump cannot read from a tar archive");
+			pg_log_error_hint("You must use -n or --no-parse-wal when verifying a tar-format backup.");
+			exit(1);
+		}
+
 		pg_waldump_path = pg_malloc(MAXPGPATH);
 		ret = find_other_exec(argv[0], "pg_waldump",
 							  "pg_waldump (PostgreSQL) " PG_VERSION "\n",
@@ -279,8 +306,13 @@ main(int argc, char **argv)
 	/*
 	 * Now do the expensive work of verifying file checksums, unless we were
 	 * told to skip it.
+	 *
+	 * We were only checking the plain backup here. For the tar backup, file
+	 * checksums verification (if requested) will be done immediately when the
+	 * file is accessed, as we don't have random access to the files like we
+	 * do with plain backups.
 	 */
-	if (!context.skip_checksums)
+	if (!context.skip_checksums && context.format == 'p')
 		verify_backup_checksums(&context);
 
 	/*
@@ -981,6 +1013,42 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
 	return false;
 }
 
+/*
+ * To detect the backup format, it checks for the PG_VERSION file in the backup
+ * directory. If found, it will be considered a plain-format backup; otherwise,
+ * it will be assumed to be a tar-format backup.
+ */
+static char
+find_backup_format(verifier_context *context)
+{
+	char		result;
+	char	   *path;
+	struct stat sb;
+
+	/* Should be here only if the backup format is unknown */
+	Assert(context->format == '\0');
+
+	/* Check PG_VERSION file. */
+	path = psprintf("%s/%s", context->backup_directory, "PG_VERSION");
+	if (stat(path, &sb) == 0)
+		result = 'p';
+	else
+	{
+		if (errno != ENOENT)
+		{
+			pg_log_error("cannot determine backup format : %m");
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+
+		/* Otherwise, it is assumed to be a tar format backup. */
+		result = 't';
+	}
+	pfree(path);
+
+	return result;
+}
+
 /*
  * Print a progress report based on the global variables.
  *
@@ -1038,6 +1106,7 @@ usage(void)
 	printf(_("  -e, --exit-on-error         exit immediately on error\n"));
 	printf(_("  -i, --ignore=RELATIVE_PATH  ignore indicated path\n"));
 	printf(_("  -m, --manifest-path=PATH    use specified path for manifest\n"));
+	printf(_("  -F, --format=p|t            backup format (plain, tar)\n"));
 	printf(_("  -n, --no-parse-wal          do not try to parse WAL files\n"));
 	printf(_("  -P, --progress              show progress information\n"));
 	printf(_("  -q, --quiet                 do not print any output, except for errors\n"));
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 818064c6eed..80031ad4dbc 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -100,6 +100,7 @@ typedef struct verifier_context
 	manifest_data *manifest;
 	char	   *backup_directory;
 	SimpleStringList ignore_list;
+	char		format;			/* backup format:  p(lain)/t(ar) */
 	bool		skip_checksums;
 	bool		exit_on_error;
 	bool		saw_any_error;
-- 
2.18.0

From 34ef4b8f5bad4c232aec86e837bad135449a0f02 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 21 Aug 2024 12:49:04 +0530
Subject: [PATCH v12 11/12] pg_verifybackup: Read tar files and verify its
 contents

This patch implements TAR format backup verification.

For progress reporting support, we perform this verification in two
passes: the first pass calculates total_size, and the second pass
updates done_size as verification progresses.

For the verification, in the first pass, we call verify_tar_backup_file(),
which performs basic verification by expecting only base.tar, pg_wal.tar, or
<tablespaceoid>.tar files and raises an error for any other files.  It
also determines the compression type of the archive file. All this
information is stored in a newly added tarFile struct, which is
appended to a list that will be used in the second pass (by
verify_tar_content()) for the final verification. In the second pass,
the tar archives are read, decompressed, and the required verification
is carried out.

For reading and decompression, fe_utils/astreamer.h is used. For
verification, a new archive streamer has been added in
astreamer_verify.c to handle TAR member files and their contents; see
astreamer_verify_content() for details. The stack of astreamers will
be set up for each TAR file in verify_tar_content(), depending on its
compression type which is detected in the first pass.

When information about a TAR member file (i.e., ASTREAMER_MEMBER_HEADER)
is received, we first verify its entry against the backup manifest. We
then decide if further checks are needed, such as checksum
verification and control data verification (if it is a pg_control
file), once the member file contents are received. Although this
decision could be made when the contents are received, it is more
efficient to make it earlier since the member file contents are
received in multiple iterations. In short, we process
ASTREAMER_MEMBER_CONTENTS multiple times but only once for other
ASTREAMER_MEMBER_* cases. We maintain this information in the
astreamer_verify structure for each member file, which is reset when
the file ends.

Unlike in a plain backup, checksum verification here occurs in two
steps. First, as the contents are received, the checksum is computed
incrementally (see member_compute_checksum). Then, at the end of
processing the member file, the final verification is performed (see
member_verify_checksum).

Similarly, during the content receiving stage, if the file is
pg_control, the data will be copied into a local buffer (see
member_copy_control_data).  The verification will then be carried out
at the end of the member file processing (see member_verify_control_data)
---
 src/bin/pg_verifybackup/Makefile           |   2 +
 src/bin/pg_verifybackup/astreamer_verify.c | 365 +++++++++++++++++++++
 src/bin/pg_verifybackup/meson.build        |   1 +
 src/bin/pg_verifybackup/pg_verifybackup.c  | 287 +++++++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h  |   6 +
 src/tools/pgindent/typedefs.list           |   2 +
 6 files changed, 658 insertions(+), 5 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/astreamer_verify.c

diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile
index 7c045f142e8..374d4a8afd1 100644
--- a/src/bin/pg_verifybackup/Makefile
+++ b/src/bin/pg_verifybackup/Makefile
@@ -17,10 +17,12 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 # We need libpq only because fe_utils does.
+override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 
 OBJS = \
 	$(WIN32RES) \
+	astreamer_verify.o \
 	pg_verifybackup.o
 
 all: pg_verifybackup
diff --git a/src/bin/pg_verifybackup/astreamer_verify.c b/src/bin/pg_verifybackup/astreamer_verify.c
new file mode 100644
index 00000000000..f08868170b4
--- /dev/null
+++ b/src/bin/pg_verifybackup/astreamer_verify.c
@@ -0,0 +1,365 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer_verify.c
+ *
+ * Extend fe_utils/astreamer.h archive streaming facility to verify TAR
+ * format backup.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * src/bin/pg_verifybackup/astreamer_verify.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include "pg_verifybackup.h"
+
+typedef struct astreamer_verify
+{
+	astreamer	base;
+	verifier_context *context;
+	char	   *archive_name;
+	Oid			tblspc_oid;
+	pg_checksum_context *checksum_ctx;
+
+	/* Hold information for a member file verification */
+	manifest_file *mfile;
+	int64		received_bytes;
+	bool		verify_checksum;
+	bool		verify_control_data;
+} astreamer_verify;
+
+static void astreamer_verify_content(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len,
+									 astreamer_archive_context context);
+static void astreamer_verify_finalize(astreamer *streamer);
+static void astreamer_verify_free(astreamer *streamer);
+
+static void member_verify_header(astreamer *streamer, astreamer_member *member);
+static void member_compute_checksum(astreamer *streamer,
+									astreamer_member *member,
+									const char *data, int len);
+static void member_verify_checksum(astreamer *streamer);
+static void member_copy_control_data(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len);
+static void member_verify_control_data(astreamer *streamer);
+static void member_reset_info(astreamer *streamer);
+
+static const astreamer_ops astreamer_verify_ops = {
+	.content = astreamer_verify_content,
+	.finalize = astreamer_verify_finalize,
+	.free = astreamer_verify_free
+};
+
+/*
+ * Create a astreamer that can verifies content of a TAR file.
+ */
+astreamer *
+astreamer_verify_content_new(astreamer *next, verifier_context *context,
+							 char *archive_name, Oid tblspc_oid)
+{
+	astreamer_verify *streamer;
+
+	streamer = palloc0(sizeof(astreamer_verify));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_verify_ops;
+
+	streamer->base.bbs_next = next;
+	streamer->context = context;
+	streamer->archive_name = archive_name;
+	streamer->tblspc_oid = tblspc_oid;
+	initStringInfo(&streamer->base.bbs_buffer);
+
+	if (!context->skip_checksums)
+		streamer->checksum_ctx = pg_malloc(sizeof(pg_checksum_context));
+
+	return &streamer->base;
+}
+
+/*
+ * The main entry point of the archive streamer for verifying tar members.
+ */
+static void
+astreamer_verify_content(astreamer *streamer, astreamer_member *member,
+						 const char *data, int len,
+						 astreamer_archive_context context)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	Assert(context != ASTREAMER_UNKNOWN);
+
+	switch (context)
+	{
+		case ASTREAMER_MEMBER_HEADER:
+
+			/*
+			 * Perform the initial check and setup verification steps.
+			 */
+			member_verify_header(streamer, member);
+			break;
+
+		case ASTREAMER_MEMBER_CONTENTS:
+
+			/*
+			 * Since we are receiving the member content in chunks, it must be
+			 * processed according to the flags set by the member header
+			 * processing routine. This includes performing incremental
+			 * checksum computations and copying control data to the local
+			 * buffer.
+			 */
+			if (mystreamer->verify_checksum)
+				member_compute_checksum(streamer, member, data, len);
+
+			if (mystreamer->verify_control_data)
+				member_copy_control_data(streamer, member, data, len);
+			break;
+
+		case ASTREAMER_MEMBER_TRAILER:
+
+			/*
+			 * We have reached the end of the member file. By this point, we
+			 * should have successfully computed the checksum of the received
+			 * content and copied the entire pg_control file data into our
+			 * local buffer. We can now proceed with the final verification.
+			 */
+			if (mystreamer->verify_checksum)
+				member_verify_checksum(streamer);
+
+			if (mystreamer->verify_control_data)
+				member_verify_control_data(streamer);
+
+			/*
+			 * Reset the temporary information stored for the verification.
+			 */
+			member_reset_info(streamer);
+			break;
+
+		case ASTREAMER_ARCHIVE_TRAILER:
+			break;
+
+		default:
+			/* Shouldn't happen. */
+			pg_fatal("unexpected state while parsing tar file");
+	}
+}
+
+/*
+ * End-of-stream processing for a astreamer_verify stream.
+ */
+static void
+astreamer_verify_finalize(astreamer *streamer)
+{
+	Assert(streamer->bbs_next == NULL);
+}
+
+/*
+ * Free memory associated with a astreamer_verify stream.
+ */
+static void
+astreamer_verify_free(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	if (mystreamer->checksum_ctx)
+		pfree(mystreamer->checksum_ctx);
+
+	pfree(streamer->bbs_buffer.data);
+	pfree(streamer);
+}
+
+/*
+ * Verifies whether the tar member entry exists in the backup manifest.
+ *
+ * If the archive being processed is a tablespace, it prepares the necessary
+ * file path first. If a valid entry is found in the backup manifest, it then
+ * determines whether checksum and control data verification should be
+ * performed during file content processing.
+ */
+static void
+member_verify_header(astreamer *streamer, astreamer_member *member)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_file *m;
+
+	/* We are only interested in files that are not in the ignore list. */
+	if (member->is_directory || member->is_link ||
+		should_ignore_relpath(mystreamer->context, member->pathname))
+		return;
+
+	/*
+	 * The backup manifest stores a relative path to the base directory for
+	 * files belonging to a tablespace, while the tablespace backup tar
+	 * archive does not include this path. Ensure the required path is
+	 * prepared; otherwise, the manifest entry verification will fail.
+	 */
+	if (OidIsValid(mystreamer->tblspc_oid))
+	{
+		char		temp[MAXPGPATH];
+
+		/* Copy original name at temporary space */
+		memcpy(temp, member->pathname, MAXPGPATH);
+
+		snprintf(member->pathname, MAXPGPATH, "%s/%d/%s",
+				 "pg_tblspc", mystreamer->tblspc_oid, temp);
+	}
+
+	/* Check the manifest entry */
+	m = verify_manifest_entry(mystreamer->context, member->pathname,
+							  member->size);
+	mystreamer->mfile = (void *) m;
+
+	/*
+	 * Prepare for checksum and control data verification.
+	 *
+	 * We could have these checks while receiving contents. However, since
+	 * contents are received in multiple iterations, this would result in
+	 * these lengthy checks being performed multiple times. Instead, setting
+	 * flags here and using them before proceeding with verification will be
+	 * more efficient.
+	 */
+	mystreamer->verify_checksum =
+		(!mystreamer->context->skip_checksums && should_verify_checksum(m));
+	mystreamer->verify_control_data =
+		should_verify_control_data(mystreamer->context->manifest, m);
+
+	/* Initialize the context required for checksum verification. */
+	if (mystreamer->verify_checksum &&
+		pg_checksum_init(mystreamer->checksum_ctx, m->checksum_type) < 0)
+	{
+		report_backup_error(mystreamer->context,
+							"%s: could not initialize checksum of file \"%s\"",
+							mystreamer->archive_name, m->pathname);
+
+		/*
+		 * Checksum verification cannot be performed without proper context
+		 * initialization.
+		 */
+		mystreamer->verify_checksum = false;
+	}
+}
+
+/*
+ * Computes the checksum incrementally for the received file content.
+ *
+ * Should have a correctly initialized checksum_ctx, which will be used for
+ * incremental checksum computation.
+ */
+static void
+member_compute_checksum(astreamer *streamer, astreamer_member *member,
+						const char *data, int len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	pg_checksum_context *checksum_ctx = mystreamer->checksum_ctx;
+	manifest_file *m = mystreamer->mfile;
+
+	Assert(mystreamer->verify_checksum);
+
+	/* Should have came for the right file */
+	Assert(strcmp(member->pathname, m->pathname) == 0);
+
+	/*
+	 * The checksum context should match the type noted in the backup
+	 * manifest.
+	 */
+	Assert(checksum_ctx->type == m->checksum_type);
+
+	/*
+	 * Update the total count of computed checksum bytes for cross-checking
+	 * with the file size in the final verification stage.
+	 */
+	mystreamer->received_bytes += len;
+
+	if (pg_checksum_update(checksum_ctx, (uint8 *) data, len) < 0)
+	{
+		report_backup_error(mystreamer->context,
+							"could not update checksum of file \"%s\"",
+							m->pathname);
+		mystreamer->verify_checksum = false;
+	}
+}
+
+/*
+ * Perform the final computation and checksum verification after the entire
+ * file content has been processed.
+ */
+static void
+member_verify_checksum(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	Assert(mystreamer->verify_checksum);
+
+	verify_checksum(mystreamer->context, mystreamer->mfile,
+					mystreamer->checksum_ctx, mystreamer->received_bytes);
+}
+
+/*
+ * Stores the pg_control file contents into a local buffer; we need the entire
+ * control file data for verification.
+ */
+static void
+member_copy_control_data(astreamer *streamer, astreamer_member *member,
+						 const char *data, int len)
+{
+	/* Should be here only for control file */
+	Assert(strcmp(member->pathname, "global/pg_control") == 0);
+	Assert(((astreamer_verify *) streamer)->verify_control_data);
+
+	/* Copy enough control file data needed for verification. */
+	astreamer_buffer_until(streamer, &data, &len, sizeof(ControlFileData));
+}
+
+/*
+ * Performs the CRC calculation of pg_control data and then calls the routines
+ * that execute the final verification of the control file information.
+ */
+static void
+member_verify_control_data(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_data *manifest = mystreamer->context->manifest;
+	ControlFileData control_file;
+	pg_crc32c	crc;
+	bool		crc_ok;
+
+	/* Should be here only for control file */
+	Assert(strcmp(mystreamer->mfile->pathname, "global/pg_control") == 0);
+	Assert(mystreamer->verify_control_data);
+
+	/* Should have enough control file data needed for verification. */
+	if (streamer->bbs_buffer.len != sizeof(ControlFileData))
+		report_fatal_error("%s: unexpected control file size: %d, should be %zu",
+						   mystreamer->archive_name, streamer->bbs_buffer.len,
+						   sizeof(ControlFileData));
+
+	memcpy(&control_file, streamer->bbs_buffer.data, sizeof(ControlFileData));
+
+	/* Check the CRC. */
+	INIT_CRC32C(crc);
+	COMP_CRC32C(crc, (char *) (&control_file), offsetof(ControlFileData, crc));
+	FIN_CRC32C(crc);
+
+	crc_ok = EQ_CRC32C(crc, control_file.crc);
+
+	/* Do the final control data verification. */
+	verify_control_data(&control_file, mystreamer->mfile->pathname, crc_ok,
+						manifest->system_identifier);
+}
+
+/*
+ * Reset flags and free memory allocations for member file verification.
+ */
+static void
+member_reset_info(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	mystreamer->mfile = NULL;
+	mystreamer->received_bytes = 0;
+	mystreamer->verify_checksum = false;
+	mystreamer->verify_control_data = false;
+}
diff --git a/src/bin/pg_verifybackup/meson.build b/src/bin/pg_verifybackup/meson.build
index 7c7d31a0350..0e09d1379d1 100644
--- a/src/bin/pg_verifybackup/meson.build
+++ b/src/bin/pg_verifybackup/meson.build
@@ -1,6 +1,7 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 pg_verifybackup_sources = files(
+  'astreamer_verify.c',
   'pg_verifybackup.c'
 )
 
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index c186678a592..80b1a639803 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -22,6 +22,7 @@
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
+#include "limits.h"
 #include "pg_verifybackup.h"
 #include "pgtime.h"
 
@@ -44,6 +45,16 @@
  */
 #define READ_CHUNK_SIZE				(128 * 1024)
 
+/*
+ * Tar file information needed for content verification.
+ */
+typedef struct tar_file
+{
+	char	   *relpath;
+	Oid			tblspc_oid;
+	pg_compress_algorithm compress_algorithm;
+} tar_file;
+
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -65,8 +76,14 @@ static void report_manifest_error(JsonManifestParseContext *context,
 static char find_backup_format(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
-static void verify_plain_backup_file(verifier_context *context,
-									 char *relpath, char *fullpath);
+static void verify_plain_backup_file(verifier_context *context, char *relpath,
+									 char *fullpath);
+static void verify_tar_backup_file(verifier_context *context, char *relpath,
+								   char *fullpath, SimplePtrList *tarfiles);
+static void verify_tar_file_contents(verifier_context *context,
+									 SimplePtrList *tarfiles);
+static void verify_tar_contents(verifier_context *context, char *relpath,
+								char *fullpath, astreamer *streamer);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -75,6 +92,10 @@ static void verify_file_checksum(verifier_context *context,
 static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
+static astreamer *create_archive_verifier(verifier_context *context,
+										  char *archive_name,
+										  Oid tblspc_oid,
+										  pg_compress_algorithm compress_algo);
 
 static void progress_report(bool finished);
 static void usage(void);
@@ -243,11 +264,11 @@ main(int argc, char **argv)
 
 		/*
 		 * XXX: In the future, we should consider enhancing pg_waldump to read
-		 * WAL files from the tar archive.
+		 * WAL files from the tar file.
 		 */
 		if (context.format != 'p')
 		{
-			pg_log_error("pg_waldump cannot read from a tar archive");
+			pg_log_error("pg_waldump cannot read from a tar");
 			pg_log_error_hint("You must use -n or --no-parse-wal when verifying a tar-format backup.");
 			exit(1);
 		}
@@ -561,6 +582,7 @@ verify_backup_directory(verifier_context *context, char *relpath,
 {
 	DIR		   *dir;
 	struct dirent *dirent;
+	SimplePtrList tarfiles = {NULL, NULL};
 
 	dir = opendir(fullpath);
 	if (dir == NULL)
@@ -600,12 +622,23 @@ verify_backup_directory(verifier_context *context, char *relpath,
 			newrelpath = psprintf("%s/%s", relpath, filename);
 
 		if (!should_ignore_relpath(context, newrelpath))
-			verify_plain_backup_file(context, newrelpath, newfullpath);
+		{
+			if (context->format == 'p')
+				verify_plain_backup_file(context, newrelpath, newfullpath);
+			else
+				verify_tar_backup_file(context, newrelpath, newfullpath,
+									   &tarfiles);
+		}
 
 		pfree(newfullpath);
 		pfree(newrelpath);
 	}
 
+	/* Perform the final verification of the tar contents, if any. */
+	Assert(tarfiles.head == NULL || context->format == 't');
+	if (tarfiles.head != NULL)
+		verify_tar_file_contents(context, &tarfiles);
+
 	if (closedir(dir))
 	{
 		report_backup_error(context,
@@ -682,6 +715,215 @@ verify_plain_backup_file(verifier_context *context, char *relpath,
 		total_size += m->size;
 }
 
+/*
+ * Verify one tar file.
+ *
+ * This function does not perform a complete verification; it only carries out
+ * basic validation of the tar format backup file, detects the compression
+ * type, and appends that information to the tarfiles list. An error will be
+ * reported if the tar file is inaccessible, or if the file type, name, or
+ * compression type is not as expected.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_plain_backup_file. The additional argument outputs a list of valid
+ * tar files.
+ */
+static void
+verify_tar_backup_file(verifier_context *context, char *relpath,
+					   char *fullpath, SimplePtrList *tarfiles)
+{
+	struct stat sb;
+	Oid			tblspc_oid = InvalidOid;
+	pg_compress_algorithm compress_algorithm;
+	tar_file   *tar;
+	char	   *suffix = NULL;
+
+	/* Should be tar format backup */
+	Assert(context->format == 't');
+
+	/* Get file information */
+	if (stat(fullpath, &sb) != 0)
+	{
+		report_backup_error(context,
+							"could not stat file or directory \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	/* In a tar format backup, we expect only plain files. */
+	if (!S_ISREG(sb.st_mode))
+	{
+		report_backup_error(context,
+							"\"%s\" is not a plain file",
+							relpath);
+		return;
+	}
+
+	/*
+	 * We expect tar files for backing up the main directory, tablespace, and
+	 * pg_wal directory.
+	 *
+	 * pg_basebackup writes the main data directory to an archive file named
+	 * base.tar, the pg_wal directory to pg_wal.tar, and the tablespace
+	 * directory to <tablespaceoid>.tar, each followed by a compression type
+	 * extension such as .gz, .lz4, or .zst.
+	 */
+	if (strncmp("base", relpath, 4) == 0)
+		suffix = relpath + 4;
+	else if (strncmp("pg_wal", relpath, 6) == 0)
+		suffix = relpath + 6;
+	else
+	{
+		/* Expected a <tablespaceoid>.tar file here. */
+		uint64		num = strtoul(relpath, &suffix, 10);
+
+		/*
+		 * Report an error if we didn't consume at least one character, if the
+		 * result is 0, or if the value is too large to be a valid OID.
+		 */
+		if (suffix == NULL || num <= 0 || num > OID_MAX)
+			report_backup_error(context,
+								"\"%s\" unexpected file in the tar format backup",
+								relpath);
+		tblspc_oid = (Oid) num;
+	}
+
+	/* Now, check the compression type of the tar */
+	if (strcmp(suffix, ".tar") == 0)
+		compress_algorithm = PG_COMPRESSION_NONE;
+	else if (strcmp(suffix, ".tgz") == 0)
+		compress_algorithm = PG_COMPRESSION_GZIP;
+	else if (strcmp(suffix, ".tar.gz") == 0)
+		compress_algorithm = PG_COMPRESSION_GZIP;
+	else if (strcmp(suffix, ".tar.lz4") == 0)
+		compress_algorithm = PG_COMPRESSION_LZ4;
+	else if (strcmp(suffix, ".tar.zst") == 0)
+		compress_algorithm = PG_COMPRESSION_ZSTD;
+	else
+	{
+		report_backup_error(context,
+							"\"%s\" unexpected file in the tar format backup",
+							relpath);
+		return;
+	}
+
+	/*
+	 * Ignore WALs, as reading and verification will be handled through
+	 * pg_waldump.
+	 */
+	if (strncmp("pg_wal", relpath, 6) == 0)
+		return;
+
+	/*
+	 * Append the information to the list for complete verification at a later
+	 * stage.
+	 */
+	tar = pg_malloc(sizeof(tar_file));
+	tar->relpath = pstrdup(relpath);
+	tar->tblspc_oid = tblspc_oid;
+	tar->compress_algorithm = compress_algorithm;
+
+	simple_ptr_list_append(tarfiles, tar);
+
+	/* Update statistics for progress report, if necessary */
+	if (show_progress)
+		total_size += sb.st_size;
+}
+
+/*
+ * This is the final part of tar file verification, which prepares the
+ * archive streamer stack according to the tar compression format for
+ * each tar file and invokes them for reading, decompressing, and ultimately
+ * verifying the contents.
+ *
+ * The arguments to this function should be a list of valid tar files to
+ * verify, and the allocation will be freed once the verification is complete.
+ */
+static void
+verify_tar_file_contents(verifier_context *context, SimplePtrList *tarfiles)
+{
+	SimplePtrListCell *cell;
+
+	progress_report(false);
+
+	for (cell = tarfiles->head; cell != NULL; cell = cell->next)
+	{
+		tar_file   *tar = (tar_file *) cell->ptr;
+		astreamer  *streamer;
+		char	   *fullpath;
+
+		/* Prepare archive streamer stack */
+		streamer = create_archive_verifier(context,
+										   tar->relpath,
+										   tar->tblspc_oid,
+										   tar->compress_algorithm);
+
+		/* Compute the full pathname to the target file. */
+		fullpath = psprintf("%s/%s", context->backup_directory,
+							tar->relpath);
+
+		/* Invoke the streamer for reading, decompressing, and verifying. */
+		verify_tar_contents(context, tar->relpath, fullpath, streamer);
+
+		/* Cleanup. */
+		pfree(tar->relpath);
+		pfree(tar);
+		pfree(fullpath);
+
+		astreamer_finalize(streamer);
+		astreamer_free(streamer);
+	}
+	simple_ptr_list_destroy(tarfiles);
+
+	progress_report(true);
+}
+
+/*
+ * Performs the actual work for tar content verification. It reads a given tar
+ * archive in predefined chunks and passes it to the streamer, which initiates
+ * routines for decompression (if necessary) and then verifies each member
+ * within the tar file.
+ */
+static void
+verify_tar_contents(verifier_context *context, char *relpath, char *fullpath,
+					astreamer *streamer)
+{
+	int			fd;
+	int			rc;
+	char	   *buffer;
+
+	pg_log_debug("reading \"%s\"", fullpath);
+
+	/* Open the target file. */
+	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
+	{
+		report_backup_error(context, "could not open file \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
+
+	/* Perform the reads */
+	while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
+	{
+		astreamer_content(streamer, NULL, buffer, rc, ASTREAMER_UNKNOWN);
+
+		/* Report progress */
+		done_size += rc;
+		progress_report(false);
+	}
+
+	if (rc < 0)
+		report_backup_error(context, "could not read file \"%s\": %m",
+							relpath);
+
+	/* Close the file. */
+	if (close(fd) != 0)
+		report_backup_error(context, "could not close file \"%s\": %m",
+							relpath);
+}
+
 /*
  * Verify file and its size entry in the manifest.
  */
@@ -1049,6 +1291,41 @@ find_backup_format(verifier_context *context)
 	return result;
 }
 
+/*
+ * Identifies the necessary steps for verifying the contents of the
+ * provided tar file.
+ */
+static astreamer *
+create_archive_verifier(verifier_context *context, char *archive_name,
+						Oid tblspc_oid, pg_compress_algorithm compress_algo)
+{
+	astreamer  *streamer = NULL;
+
+	/* Should be here only for tar backup */
+	Assert(context->format == 't');
+
+	/*
+	 * To verify the contents of the tar, the initial step is to parse its
+	 * content.
+	 */
+	streamer = astreamer_verify_content_new(streamer, context, archive_name,
+											tblspc_oid);
+	streamer = astreamer_tar_parser_new(streamer);
+
+	/*
+	 * If the tar is compressed, we must perform the appropriate decompression
+	 * operation before proceeding with the verification of its contents.
+	 */
+	if (compress_algo == PG_COMPRESSION_GZIP)
+		streamer = astreamer_gzip_decompressor_new(streamer);
+	else if (compress_algo == PG_COMPRESSION_LZ4)
+		streamer = astreamer_lz4_decompressor_new(streamer);
+	else if (compress_algo == PG_COMPRESSION_ZSTD)
+		streamer = astreamer_zstd_decompressor_new(streamer);
+
+	return streamer;
+}
+
 /*
  * Print a progress report based on the global variables.
  *
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 80031ad4dbc..be7438af346 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -18,6 +18,7 @@
 #include "common/hashfn_unstable.h"
 #include "common/logging.h"
 #include "common/parse_manifest.h"
+#include "fe_utils/astreamer.h"
 #include "fe_utils/simple_list.h"
 
 /*
@@ -123,4 +124,9 @@ extern void report_fatal_error(const char *pg_restrict fmt,...)
 extern bool should_ignore_relpath(verifier_context *context,
 								  const char *relpath);
 
+extern astreamer *astreamer_verify_content_new(astreamer *next,
+											   verifier_context *context,
+											   char *archive_name,
+											   Oid tblspc_oid);
+
 #endif							/* PG_VERIFYBACKUP_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 6d424c89186..48f37131e6a 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3330,6 +3330,7 @@ astreamer_plain_writer
 astreamer_recovery_injector
 astreamer_tar_archiver
 astreamer_tar_parser
+astreamer_verify
 astreamer_zstd_frame
 bgworker_main_type
 bh_node_type
@@ -3951,6 +3952,7 @@ substitute_phv_relids_context
 subxids_array_status
 symbol
 tablespaceinfo
+tar_file
 td_entry
 teSection
 temp_tablespaces_extra
-- 
2.18.0

From 00fc4d6018fb6f8c3826bf18144d578fd51a66e8 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 21 Aug 2024 16:04:37 +0530
Subject: [PATCH v12 12/12] pg_verifybackup: Tests and document

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the previous patch that implements tar format support.
----
---
 doc/src/sgml/ref/pg_verifybackup.sgml         | 43 ++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.c     |  6 +-
 src/bin/pg_verifybackup/t/001_basic.pl        |  6 +-
 src/bin/pg_verifybackup/t/004_options.pl      |  2 +-
 src/bin/pg_verifybackup/t/005_bad_manifest.pl |  2 +-
 src/bin/pg_verifybackup/t/008_untar.pl        | 73 ++++++-------------
 src/bin/pg_verifybackup/t/010_client_untar.pl | 48 +-----------
 7 files changed, 76 insertions(+), 104 deletions(-)

diff --git a/doc/src/sgml/ref/pg_verifybackup.sgml b/doc/src/sgml/ref/pg_verifybackup.sgml
index a3f167f9f6e..ea6bc3ccb12 100644
--- a/doc/src/sgml/ref/pg_verifybackup.sgml
+++ b/doc/src/sgml/ref/pg_verifybackup.sgml
@@ -34,8 +34,10 @@ PostgreSQL documentation
    integrity of a database cluster backup taken using
    <command>pg_basebackup</command> against a
    <literal>backup_manifest</literal> generated by the server at the time
-   of the backup.  The backup must be stored in the "plain"
-   format; a "tar" format backup can be checked after extracting it.
+   of the backup.  The backup must be stored in the "plain" or "tar"
+   format.  Verification is supported for <literal>gzip</literal>,
+   <literal>lz4</literal>, and  <literal>zstd</literal> compressed tar backup;
+   any other compressed format backups can be checked after decompressing them.
   </para>
 
   <para>
@@ -168,6 +170,43 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-F <replaceable class="parameter">format</replaceable></option></term>
+      <term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
+      <listitem>
+       <para>
+        Specifies the format of the backup. <replaceable>format</replaceable>
+        can be one of the following:
+
+        <variablelist>
+         <varlistentry>
+          <term><literal>p</literal></term>
+          <term><literal>plain</literal></term>
+          <listitem>
+           <para>
+            Backup consists of plain files with the same layout as the
+            source server's data directory and tablespaces.
+           </para>
+          </listitem>
+         </varlistentry>
+
+         <varlistentry>
+          <term><literal>t</literal></term>
+          <term><literal>tar</literal></term>
+          <listitem>
+           <para>
+            Backup consists of tar files.  A valid backup includes the main data
+            directory in a file named <filename>base.tar</filename>, the WAL
+            files in <filename>pg_wal.tar</filename>, and separate tar files for
+            each tablespace, named after the tablespace's OID, followed by the
+            compression extension.
+           </para>
+           </listitem>
+         </varlistentry>
+        </variablelist></para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-n</option></term>
       <term><option>--no-parse-wal</option></term>
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 80b1a639803..8302f022b22 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -880,9 +880,9 @@ verify_tar_file_contents(verifier_context *context, SimplePtrList *tarfiles)
 
 /*
  * Performs the actual work for tar content verification. It reads a given tar
- * archive in predefined chunks and passes it to the streamer, which initiates
- * routines for decompression (if necessary) and then verifies each member
- * within the tar file.
+ * file in predefined chunks and passes it to the archive streamer, which
+ * initiates routines for decompression (if necessary) and then verifies each
+ * member within the tar file.
  */
 static void
 verify_tar_contents(verifier_context *context, char *relpath, char *fullpath,
diff --git a/src/bin/pg_verifybackup/t/001_basic.pl b/src/bin/pg_verifybackup/t/001_basic.pl
index 2f3e52d296f..ca5b0402b7d 100644
--- a/src/bin/pg_verifybackup/t/001_basic.pl
+++ b/src/bin/pg_verifybackup/t/001_basic.pl
@@ -17,11 +17,11 @@ command_fails_like(
 	qr/no backup directory specified/,
 	'target directory must be specified');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir ],
 	qr/could not open file.*\/backup_manifest\"/,
 	'pg_verifybackup requires a manifest');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir, $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir, $tempdir ],
 	qr/too many command-line arguments/,
 	'multiple target directories not allowed');
 
@@ -31,7 +31,7 @@ close($fh);
 
 # but then try to use an alternate, nonexisting manifest
 command_fails_like(
-	[ 'pg_verifybackup', '-m', "$tempdir/not_the_manifest", $tempdir ],
+	[ 'pg_verifybackup', '-Fp', '-m', "$tempdir/not_the_manifest", $tempdir ],
 	qr/could not open file.*\/not_the_manifest\"/,
 	'pg_verifybackup respects -m flag');
 
diff --git a/src/bin/pg_verifybackup/t/004_options.pl b/src/bin/pg_verifybackup/t/004_options.pl
index 8ed2214408e..2f197648740 100644
--- a/src/bin/pg_verifybackup/t/004_options.pl
+++ b/src/bin/pg_verifybackup/t/004_options.pl
@@ -108,7 +108,7 @@ unlike(
 # Test valid manifest with nonexistent backup directory.
 command_fails_like(
 	[
-		'pg_verifybackup', '-m',
+		'pg_verifybackup', '-Fp', '-m',
 		"$backup_path/backup_manifest", "$backup_path/fake"
 	],
 	qr/could not open directory/,
diff --git a/src/bin/pg_verifybackup/t/005_bad_manifest.pl b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
index c4ed64b62d5..28c51b6feb0 100644
--- a/src/bin/pg_verifybackup/t/005_bad_manifest.pl
+++ b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
@@ -208,7 +208,7 @@ sub test_bad_manifest
 	print $fh $manifest_contents;
 	close($fh);
 
-	command_fails_like([ 'pg_verifybackup', $tempdir ], $regexp, $test_name);
+	command_fails_like([ 'pg_verifybackup', '-Fp', $tempdir ], $regexp, $test_name);
 	return;
 }
 
diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
index 7a09f3b75b2..1c83f38d5b5 100644
--- a/src/bin/pg_verifybackup/t/008_untar.pl
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -16,6 +16,20 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 
+# Create a tablespace directory.
+my $TS1_LOCATION = $primary->backup_dir .'/ts1';
+$TS1_LOCATION =~ s/\/\.\//\//g;    # collapse foo/./bar to foo/bar
+mkdir($TS1_LOCATION);
+
+# Create a tablespace with table in it.
+$primary->safe_psql('postgres', qq(
+		CREATE TABLESPACE regress_ts1 LOCATION '$TS1_LOCATION';
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1';
+		CREATE TABLE regress_tbl1(i int) TABLESPACE regress_ts1;
+		INSERT INTO regress_tbl1 VALUES(generate_series(1,5));));
+my $tsoid = $primary->safe_psql('postgres', qq(
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1'));
+
 my $backup_path = $primary->backup_dir . '/server-backup';
 my $extract_path = $primary->backup_dir . '/extracted-backup';
 
@@ -23,39 +37,31 @@ my @test_configuration = (
 	{
 		'compression_method' => 'none',
 		'backup_flags' => [],
-		'backup_archive' => 'base.tar',
+		'backup_archive' => ['base.tar', "$tsoid.tar"],
 		'enabled' => 1
 	},
 	{
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'server-gzip' ],
-		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.gz', "$tsoid.tar.gz" ],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'server-lz4' ],
-		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => [ '-d', '-m' ],
+		'backup_archive' => ['base.tar.lz4', "$tsoid.tar.lz4" ],
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd:level=1,long' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	});
 
@@ -86,47 +92,16 @@ for my $tc (@test_configuration)
 		my $backup_files = join(',',
 			sort grep { $_ ne '.' && $_ ne '..' } slurp_dir($backup_path));
 		my $expected_backup_files =
-		  join(',', sort ('backup_manifest', $tc->{'backup_archive'}));
+		  join(',', sort ('backup_manifest', @{ $tc->{'backup_archive'} }));
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok(['pg_verifybackup', '-n', '-e', $backup_path],
+			"verify backup, compression $method");
 
 		# Cleanup.
-		unlink($backup_path . '/backup_manifest');
-		unlink($backup_path . '/base.tar');
-		unlink($backup_path . '/' . $tc->{'backup_archive'});
+		rmtree($backup_path);
 		rmtree($extract_path);
 	}
 }
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index 8c076d46dee..6b7d7483f6e 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -29,41 +29,30 @@ my @test_configuration = (
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'client-gzip:5' ],
 		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'client-lz4:5' ],
 		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => ['-d'],
-		'output_file' => 'base.tar',
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:5' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:level=1,long' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'parallel zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:workers=3' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1"),
 		'possibly_unsupported' =>
 		  qr/could not set compression worker count to 3: Unsupported parameter/
@@ -118,40 +107,9 @@ for my $tc (@test_configuration)
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			push @decompress, $backup_path . '/' . $tc->{'output_file'}
-			  if $tc->{'output_file'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok( [ 'pg_verifybackup', '-n', '-e', $backup_path ],
+			"verify backup, compression $method");
 
 		# Cleanup.
 		rmtree($extract_path);
-- 
2.18.0

From b74a6bed49474c4cacdb1a7a3626004ae9ad7f13 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 8 Aug 2024 16:01:33 +0530
Subject: [PATCH v13 09/12] Add simple_ptr_list_destroy() and
 simple_ptr_list_destroy_deep() API.

We didn't have any helper function to destroy SimplePtrList, likely
because it wasn't needed before, but it's required in a later patch in
this set.  I've added two functions for this purpose, inspired by
list_free() and list_free_deep().
---
 src/fe_utils/simple_list.c         | 39 ++++++++++++++++++++++++++++++
 src/include/fe_utils/simple_list.h |  2 ++
 2 files changed, 41 insertions(+)

diff --git a/src/fe_utils/simple_list.c b/src/fe_utils/simple_list.c
index 2d88eb54067..9d218911c31 100644
--- a/src/fe_utils/simple_list.c
+++ b/src/fe_utils/simple_list.c
@@ -173,3 +173,42 @@ simple_ptr_list_append(SimplePtrList *list, void *ptr)
 		list->head = cell;
 	list->tail = cell;
 }
+
+/*
+ * Destroy a pointer list and optionally the pointed-to element
+ */
+static void
+simple_ptr_list_destroy_private(SimplePtrList *list, bool deep)
+{
+	SimplePtrListCell *cell;
+
+	cell = list->head;
+	while (cell != NULL)
+	{
+		SimplePtrListCell *next;
+
+		next = cell->next;
+		if (deep)
+			pg_free(cell->ptr);
+		pg_free(cell);
+		cell = next;
+	}
+}
+
+/*
+ * Destroy a pointer list and the pointed-to element
+ */
+void
+simple_ptr_list_destroy_deep(SimplePtrList *list)
+{
+	simple_ptr_list_destroy_private(list, true);
+}
+
+/*
+ * Destroy only pointer list and not the pointed-to element
+ */
+void
+simple_ptr_list_destroy(SimplePtrList *list)
+{
+	simple_ptr_list_destroy_private(list, false);
+}
diff --git a/src/include/fe_utils/simple_list.h b/src/include/fe_utils/simple_list.h
index d42ecded8ed..5b7cbec8a62 100644
--- a/src/include/fe_utils/simple_list.h
+++ b/src/include/fe_utils/simple_list.h
@@ -66,5 +66,7 @@ extern void simple_string_list_destroy(SimpleStringList *list);
 extern const char *simple_string_list_not_touched(SimpleStringList *list);
 
 extern void simple_ptr_list_append(SimplePtrList *list, void *ptr);
+extern void simple_ptr_list_destroy_deep(SimplePtrList *list);
+extern void simple_ptr_list_destroy(SimplePtrList *list);
 
 #endif							/* SIMPLE_LIST_H */
-- 
2.18.0

From b4fd85d44473e4fb21eee92f06a547f8518ca203 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 21 Aug 2024 11:03:44 +0530
Subject: [PATCH v13 10/12] pg_verifybackup: Add backup format and compression
 option

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the next patch that implements tar format support.
----
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 73 ++++++++++++++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h |  1 +
 2 files changed, 72 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index d04e1d8c8ac..bcae2d2990f 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -62,6 +62,7 @@ static void report_manifest_error(JsonManifestParseContext *context,
 								  const char *fmt,...)
 			pg_attribute_printf(2, 3) pg_attribute_noreturn();
 
+static char find_backup_format(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_plain_backup_file(verifier_context *context,
@@ -97,6 +98,7 @@ main(int argc, char **argv)
 		{"exit-on-error", no_argument, NULL, 'e'},
 		{"ignore", required_argument, NULL, 'i'},
 		{"manifest-path", required_argument, NULL, 'm'},
+		{"format", required_argument, NULL, 'F'},
 		{"no-parse-wal", no_argument, NULL, 'n'},
 		{"progress", no_argument, NULL, 'P'},
 		{"quiet", no_argument, NULL, 'q'},
@@ -118,6 +120,7 @@ main(int argc, char **argv)
 	progname = get_progname(argv[0]);
 
 	memset(&context, 0, sizeof(context));
+	context.format = '\0';
 
 	if (argc > 1)
 	{
@@ -154,7 +157,7 @@ main(int argc, char **argv)
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
 
-	while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1)
+	while ((c = getopt_long(argc, argv, "eF:i:m:nPqsw:", long_options, NULL)) != -1)
 	{
 		switch (c)
 		{
@@ -173,6 +176,15 @@ main(int argc, char **argv)
 				manifest_path = pstrdup(optarg);
 				canonicalize_path(manifest_path);
 				break;
+			case 'F':
+				if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
+					context.format = 'p';
+				else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
+					context.format = 't';
+				else
+					pg_fatal("invalid backup format \"%s\", must be \"plain\" or \"tar\"",
+							 optarg);
+				break;
 			case 'n':
 				no_parse_wal = true;
 				break;
@@ -220,11 +232,26 @@ main(int argc, char **argv)
 		pg_fatal("cannot specify both %s and %s",
 				 "-P/--progress", "-q/--quiet");
 
+	/* Determine the backup format if it hasn't been specified. */
+	if (context.format == '\0')
+		context.format = find_backup_format(&context);
+
 	/* Unless --no-parse-wal was specified, we will need pg_waldump. */
 	if (!no_parse_wal)
 	{
 		int			ret;
 
+		/*
+		 * XXX: In the future, we should consider enhancing pg_waldump to read
+		 * WAL files from the tar file.
+		 */
+		if (context.format != 'p')
+		{
+			pg_log_error("pg_waldump cannot read from a tar");
+			pg_log_error_hint("You must use -n or --no-parse-wal when verifying a tar-format backup.");
+			exit(1);
+		}
+
 		pg_waldump_path = pg_malloc(MAXPGPATH);
 		ret = find_other_exec(argv[0], "pg_waldump",
 							  "pg_waldump (PostgreSQL) " PG_VERSION "\n",
@@ -279,8 +306,13 @@ main(int argc, char **argv)
 	/*
 	 * Now do the expensive work of verifying file checksums, unless we were
 	 * told to skip it.
+	 *
+	 * We were only checking the plain backup here. For the tar backup, file
+	 * checksums verification (if requested) will be done immediately when the
+	 * file is accessed, as we don't have random access to the files like we
+	 * do with plain backups.
 	 */
-	if (!context.skip_checksums)
+	if (!context.skip_checksums && context.format == 'p')
 		verify_backup_checksums(&context);
 
 	/*
@@ -981,6 +1013,42 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
 	return false;
 }
 
+/*
+ * To detect the backup format, it checks for the PG_VERSION file in the backup
+ * directory. If found, it will be considered a plain-format backup; otherwise,
+ * it will be assumed to be a tar-format backup.
+ */
+static char
+find_backup_format(verifier_context *context)
+{
+	char		result;
+	char	   *path;
+	struct stat sb;
+
+	/* Should be here only if the backup format is unknown */
+	Assert(context->format == '\0');
+
+	/* Check PG_VERSION file. */
+	path = psprintf("%s/%s", context->backup_directory, "PG_VERSION");
+	if (stat(path, &sb) == 0)
+		result = 'p';
+	else
+	{
+		if (errno != ENOENT)
+		{
+			pg_log_error("cannot determine backup format : %m");
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+
+		/* Otherwise, it is assumed to be a tar format backup. */
+		result = 't';
+	}
+	pfree(path);
+
+	return result;
+}
+
 /*
  * Print a progress report based on the global variables.
  *
@@ -1038,6 +1106,7 @@ usage(void)
 	printf(_("  -e, --exit-on-error         exit immediately on error\n"));
 	printf(_("  -i, --ignore=RELATIVE_PATH  ignore indicated path\n"));
 	printf(_("  -m, --manifest-path=PATH    use specified path for manifest\n"));
+	printf(_("  -F, --format=p|t            backup format (plain, tar)\n"));
 	printf(_("  -n, --no-parse-wal          do not try to parse WAL files\n"));
 	printf(_("  -P, --progress              show progress information\n"));
 	printf(_("  -q, --quiet                 do not print any output, except for errors\n"));
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 818064c6eed..80031ad4dbc 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -100,6 +100,7 @@ typedef struct verifier_context
 	manifest_data *manifest;
 	char	   *backup_directory;
 	SimpleStringList ignore_list;
+	char		format;			/* backup format:  p(lain)/t(ar) */
 	bool		skip_checksums;
 	bool		exit_on_error;
 	bool		saw_any_error;
-- 
2.18.0

From 402a301f203cebac028d1aef5ba8db2cb87890b7 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 21 Aug 2024 12:49:04 +0530
Subject: [PATCH v13 11/12] pg_verifybackup: Read tar files and verify its
 contents

This patch implements TAR format backup verification.

For progress reporting support, we perform this verification in two
passes: the first pass calculates total_size, and the second pass
updates done_size as verification progresses.

For the verification, in the first pass, we call verify_tar_backup_file(),
which performs basic verification by expecting only base.tar, pg_wal.tar, or
<tablespaceoid>.tar files and raises an error for any other files.  It
also determines the compression type of the archive file. All this
information is stored in a newly added tarFile struct, which is
appended to a list that will be used in the second pass (by
verify_tar_content()) for the final verification. In the second pass,
the tar archives are read, decompressed, and the required verification
is carried out.

For reading and decompression, fe_utils/astreamer.h is used. For
verification, a new archive streamer has been added in
astreamer_verify.c to handle TAR member files and their contents; see
astreamer_verify_content() for details. The stack of astreamers will
be set up for each TAR file in verify_tar_content(), depending on its
compression type which is detected in the first pass.

When information about a TAR member file (i.e., ASTREAMER_MEMBER_HEADER)
is received, we first verify its entry against the backup manifest. We
then decide if further checks are needed, such as checksum
verification and control data verification (if it is a pg_control
file), once the member file contents are received. Although this
decision could be made when the contents are received, it is more
efficient to make it earlier since the member file contents are
received in multiple iterations. In short, we process
ASTREAMER_MEMBER_CONTENTS multiple times but only once for other
ASTREAMER_MEMBER_* cases. We maintain this information in the
astreamer_verify structure for each member file, which is reset when
the file ends.

Unlike in a plain backup, checksum verification here occurs in two
steps. First, as the contents are received, the checksum is computed
incrementally (see member_compute_checksum). Then, at the end of
processing the member file, the final verification is performed (see
member_verify_checksum).

Similarly, during the content receiving stage, if the file is
pg_control, the data will be copied into a local buffer (see
member_copy_control_data).  The verification will then be carried out
at the end of the member file processing (see member_verify_control_data)
---
 src/bin/pg_verifybackup/Makefile           |   2 +
 src/bin/pg_verifybackup/astreamer_verify.c | 365 +++++++++++++++++++++
 src/bin/pg_verifybackup/meson.build        |   1 +
 src/bin/pg_verifybackup/pg_verifybackup.c  | 341 ++++++++++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h  |   6 +
 src/tools/pgindent/typedefs.list           |   2 +
 6 files changed, 714 insertions(+), 3 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/astreamer_verify.c

diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile
index 7c045f142e8..374d4a8afd1 100644
--- a/src/bin/pg_verifybackup/Makefile
+++ b/src/bin/pg_verifybackup/Makefile
@@ -17,10 +17,12 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 # We need libpq only because fe_utils does.
+override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 
 OBJS = \
 	$(WIN32RES) \
+	astreamer_verify.o \
 	pg_verifybackup.o
 
 all: pg_verifybackup
diff --git a/src/bin/pg_verifybackup/astreamer_verify.c b/src/bin/pg_verifybackup/astreamer_verify.c
new file mode 100644
index 00000000000..f08868170b4
--- /dev/null
+++ b/src/bin/pg_verifybackup/astreamer_verify.c
@@ -0,0 +1,365 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer_verify.c
+ *
+ * Extend fe_utils/astreamer.h archive streaming facility to verify TAR
+ * format backup.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * src/bin/pg_verifybackup/astreamer_verify.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include "pg_verifybackup.h"
+
+typedef struct astreamer_verify
+{
+	astreamer	base;
+	verifier_context *context;
+	char	   *archive_name;
+	Oid			tblspc_oid;
+	pg_checksum_context *checksum_ctx;
+
+	/* Hold information for a member file verification */
+	manifest_file *mfile;
+	int64		received_bytes;
+	bool		verify_checksum;
+	bool		verify_control_data;
+} astreamer_verify;
+
+static void astreamer_verify_content(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len,
+									 astreamer_archive_context context);
+static void astreamer_verify_finalize(astreamer *streamer);
+static void astreamer_verify_free(astreamer *streamer);
+
+static void member_verify_header(astreamer *streamer, astreamer_member *member);
+static void member_compute_checksum(astreamer *streamer,
+									astreamer_member *member,
+									const char *data, int len);
+static void member_verify_checksum(astreamer *streamer);
+static void member_copy_control_data(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len);
+static void member_verify_control_data(astreamer *streamer);
+static void member_reset_info(astreamer *streamer);
+
+static const astreamer_ops astreamer_verify_ops = {
+	.content = astreamer_verify_content,
+	.finalize = astreamer_verify_finalize,
+	.free = astreamer_verify_free
+};
+
+/*
+ * Create a astreamer that can verifies content of a TAR file.
+ */
+astreamer *
+astreamer_verify_content_new(astreamer *next, verifier_context *context,
+							 char *archive_name, Oid tblspc_oid)
+{
+	astreamer_verify *streamer;
+
+	streamer = palloc0(sizeof(astreamer_verify));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_verify_ops;
+
+	streamer->base.bbs_next = next;
+	streamer->context = context;
+	streamer->archive_name = archive_name;
+	streamer->tblspc_oid = tblspc_oid;
+	initStringInfo(&streamer->base.bbs_buffer);
+
+	if (!context->skip_checksums)
+		streamer->checksum_ctx = pg_malloc(sizeof(pg_checksum_context));
+
+	return &streamer->base;
+}
+
+/*
+ * The main entry point of the archive streamer for verifying tar members.
+ */
+static void
+astreamer_verify_content(astreamer *streamer, astreamer_member *member,
+						 const char *data, int len,
+						 astreamer_archive_context context)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	Assert(context != ASTREAMER_UNKNOWN);
+
+	switch (context)
+	{
+		case ASTREAMER_MEMBER_HEADER:
+
+			/*
+			 * Perform the initial check and setup verification steps.
+			 */
+			member_verify_header(streamer, member);
+			break;
+
+		case ASTREAMER_MEMBER_CONTENTS:
+
+			/*
+			 * Since we are receiving the member content in chunks, it must be
+			 * processed according to the flags set by the member header
+			 * processing routine. This includes performing incremental
+			 * checksum computations and copying control data to the local
+			 * buffer.
+			 */
+			if (mystreamer->verify_checksum)
+				member_compute_checksum(streamer, member, data, len);
+
+			if (mystreamer->verify_control_data)
+				member_copy_control_data(streamer, member, data, len);
+			break;
+
+		case ASTREAMER_MEMBER_TRAILER:
+
+			/*
+			 * We have reached the end of the member file. By this point, we
+			 * should have successfully computed the checksum of the received
+			 * content and copied the entire pg_control file data into our
+			 * local buffer. We can now proceed with the final verification.
+			 */
+			if (mystreamer->verify_checksum)
+				member_verify_checksum(streamer);
+
+			if (mystreamer->verify_control_data)
+				member_verify_control_data(streamer);
+
+			/*
+			 * Reset the temporary information stored for the verification.
+			 */
+			member_reset_info(streamer);
+			break;
+
+		case ASTREAMER_ARCHIVE_TRAILER:
+			break;
+
+		default:
+			/* Shouldn't happen. */
+			pg_fatal("unexpected state while parsing tar file");
+	}
+}
+
+/*
+ * End-of-stream processing for a astreamer_verify stream.
+ */
+static void
+astreamer_verify_finalize(astreamer *streamer)
+{
+	Assert(streamer->bbs_next == NULL);
+}
+
+/*
+ * Free memory associated with a astreamer_verify stream.
+ */
+static void
+astreamer_verify_free(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	if (mystreamer->checksum_ctx)
+		pfree(mystreamer->checksum_ctx);
+
+	pfree(streamer->bbs_buffer.data);
+	pfree(streamer);
+}
+
+/*
+ * Verifies whether the tar member entry exists in the backup manifest.
+ *
+ * If the archive being processed is a tablespace, it prepares the necessary
+ * file path first. If a valid entry is found in the backup manifest, it then
+ * determines whether checksum and control data verification should be
+ * performed during file content processing.
+ */
+static void
+member_verify_header(astreamer *streamer, astreamer_member *member)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_file *m;
+
+	/* We are only interested in files that are not in the ignore list. */
+	if (member->is_directory || member->is_link ||
+		should_ignore_relpath(mystreamer->context, member->pathname))
+		return;
+
+	/*
+	 * The backup manifest stores a relative path to the base directory for
+	 * files belonging to a tablespace, while the tablespace backup tar
+	 * archive does not include this path. Ensure the required path is
+	 * prepared; otherwise, the manifest entry verification will fail.
+	 */
+	if (OidIsValid(mystreamer->tblspc_oid))
+	{
+		char		temp[MAXPGPATH];
+
+		/* Copy original name at temporary space */
+		memcpy(temp, member->pathname, MAXPGPATH);
+
+		snprintf(member->pathname, MAXPGPATH, "%s/%d/%s",
+				 "pg_tblspc", mystreamer->tblspc_oid, temp);
+	}
+
+	/* Check the manifest entry */
+	m = verify_manifest_entry(mystreamer->context, member->pathname,
+							  member->size);
+	mystreamer->mfile = (void *) m;
+
+	/*
+	 * Prepare for checksum and control data verification.
+	 *
+	 * We could have these checks while receiving contents. However, since
+	 * contents are received in multiple iterations, this would result in
+	 * these lengthy checks being performed multiple times. Instead, setting
+	 * flags here and using them before proceeding with verification will be
+	 * more efficient.
+	 */
+	mystreamer->verify_checksum =
+		(!mystreamer->context->skip_checksums && should_verify_checksum(m));
+	mystreamer->verify_control_data =
+		should_verify_control_data(mystreamer->context->manifest, m);
+
+	/* Initialize the context required for checksum verification. */
+	if (mystreamer->verify_checksum &&
+		pg_checksum_init(mystreamer->checksum_ctx, m->checksum_type) < 0)
+	{
+		report_backup_error(mystreamer->context,
+							"%s: could not initialize checksum of file \"%s\"",
+							mystreamer->archive_name, m->pathname);
+
+		/*
+		 * Checksum verification cannot be performed without proper context
+		 * initialization.
+		 */
+		mystreamer->verify_checksum = false;
+	}
+}
+
+/*
+ * Computes the checksum incrementally for the received file content.
+ *
+ * Should have a correctly initialized checksum_ctx, which will be used for
+ * incremental checksum computation.
+ */
+static void
+member_compute_checksum(astreamer *streamer, astreamer_member *member,
+						const char *data, int len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	pg_checksum_context *checksum_ctx = mystreamer->checksum_ctx;
+	manifest_file *m = mystreamer->mfile;
+
+	Assert(mystreamer->verify_checksum);
+
+	/* Should have came for the right file */
+	Assert(strcmp(member->pathname, m->pathname) == 0);
+
+	/*
+	 * The checksum context should match the type noted in the backup
+	 * manifest.
+	 */
+	Assert(checksum_ctx->type == m->checksum_type);
+
+	/*
+	 * Update the total count of computed checksum bytes for cross-checking
+	 * with the file size in the final verification stage.
+	 */
+	mystreamer->received_bytes += len;
+
+	if (pg_checksum_update(checksum_ctx, (uint8 *) data, len) < 0)
+	{
+		report_backup_error(mystreamer->context,
+							"could not update checksum of file \"%s\"",
+							m->pathname);
+		mystreamer->verify_checksum = false;
+	}
+}
+
+/*
+ * Perform the final computation and checksum verification after the entire
+ * file content has been processed.
+ */
+static void
+member_verify_checksum(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	Assert(mystreamer->verify_checksum);
+
+	verify_checksum(mystreamer->context, mystreamer->mfile,
+					mystreamer->checksum_ctx, mystreamer->received_bytes);
+}
+
+/*
+ * Stores the pg_control file contents into a local buffer; we need the entire
+ * control file data for verification.
+ */
+static void
+member_copy_control_data(astreamer *streamer, astreamer_member *member,
+						 const char *data, int len)
+{
+	/* Should be here only for control file */
+	Assert(strcmp(member->pathname, "global/pg_control") == 0);
+	Assert(((astreamer_verify *) streamer)->verify_control_data);
+
+	/* Copy enough control file data needed for verification. */
+	astreamer_buffer_until(streamer, &data, &len, sizeof(ControlFileData));
+}
+
+/*
+ * Performs the CRC calculation of pg_control data and then calls the routines
+ * that execute the final verification of the control file information.
+ */
+static void
+member_verify_control_data(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_data *manifest = mystreamer->context->manifest;
+	ControlFileData control_file;
+	pg_crc32c	crc;
+	bool		crc_ok;
+
+	/* Should be here only for control file */
+	Assert(strcmp(mystreamer->mfile->pathname, "global/pg_control") == 0);
+	Assert(mystreamer->verify_control_data);
+
+	/* Should have enough control file data needed for verification. */
+	if (streamer->bbs_buffer.len != sizeof(ControlFileData))
+		report_fatal_error("%s: unexpected control file size: %d, should be %zu",
+						   mystreamer->archive_name, streamer->bbs_buffer.len,
+						   sizeof(ControlFileData));
+
+	memcpy(&control_file, streamer->bbs_buffer.data, sizeof(ControlFileData));
+
+	/* Check the CRC. */
+	INIT_CRC32C(crc);
+	COMP_CRC32C(crc, (char *) (&control_file), offsetof(ControlFileData, crc));
+	FIN_CRC32C(crc);
+
+	crc_ok = EQ_CRC32C(crc, control_file.crc);
+
+	/* Do the final control data verification. */
+	verify_control_data(&control_file, mystreamer->mfile->pathname, crc_ok,
+						manifest->system_identifier);
+}
+
+/*
+ * Reset flags and free memory allocations for member file verification.
+ */
+static void
+member_reset_info(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	mystreamer->mfile = NULL;
+	mystreamer->received_bytes = 0;
+	mystreamer->verify_checksum = false;
+	mystreamer->verify_control_data = false;
+}
diff --git a/src/bin/pg_verifybackup/meson.build b/src/bin/pg_verifybackup/meson.build
index 7c7d31a0350..0e09d1379d1 100644
--- a/src/bin/pg_verifybackup/meson.build
+++ b/src/bin/pg_verifybackup/meson.build
@@ -1,6 +1,7 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 pg_verifybackup_sources = files(
+  'astreamer_verify.c',
   'pg_verifybackup.c'
 )
 
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index bcae2d2990f..9cbfc07a7ba 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -22,6 +22,7 @@
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
+#include "limits.h"
 #include "pg_verifybackup.h"
 #include "pgtime.h"
 
@@ -44,6 +45,16 @@
  */
 #define READ_CHUNK_SIZE				(128 * 1024)
 
+/*
+ * Tar file information needed for content verification.
+ */
+typedef struct tar_file
+{
+	char	   *relpath;
+	Oid			tblspc_oid;
+	pg_compress_algorithm compress_algorithm;
+} tar_file;
+
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -63,10 +74,18 @@ static void report_manifest_error(JsonManifestParseContext *context,
 			pg_attribute_printf(2, 3) pg_attribute_noreturn();
 
 static char find_backup_format(verifier_context *context);
+static void verify_plain_backup(verifier_context *context);
+static void verify_tar_backup(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
-static void verify_plain_backup_file(verifier_context *context,
-									 char *relpath, char *fullpath);
+static void verify_plain_backup_file(verifier_context *context, char *relpath,
+									 char *fullpath);
+static void precheck_tar_backup_file(verifier_context *context, char *relpath,
+									 char *fullpath, SimplePtrList *tarfiles);
+static void verify_all_tar_files(verifier_context *context,
+								 SimplePtrList *tarfiles);
+static void verify_one_tar_file(verifier_context *context, char *relpath,
+								char *fullpath, astreamer *streamer);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -75,6 +94,10 @@ static void verify_file_checksum(verifier_context *context,
 static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
+static astreamer *create_archive_verifier(verifier_context *context,
+										  char *archive_name,
+										  Oid tblspc_oid,
+										  pg_compress_algorithm compress_algo);
 
 static void progress_report(bool finished);
 static void usage(void);
@@ -294,7 +317,10 @@ main(int argc, char **argv)
 	 * match. We also set the "matched" flag on every manifest entry that
 	 * corresponds to a file on disk.
 	 */
-	verify_backup_directory(&context, NULL, context.backup_directory);
+	if (context.format == 'p')
+		verify_plain_backup(&context);
+	else
+		verify_tar_backup(&context);
 
 	/*
 	 * The "matched" flag should now be set on every entry in the hash table.
@@ -546,6 +572,16 @@ verifybackup_per_wal_range_cb(JsonManifestParseContext *context,
 	manifest->last_wal_range = range;
 }
 
+/*
+ * Verify plain backup.
+ */
+static void
+verify_plain_backup(verifier_context *context)
+{
+	Assert(context->format == 'p');
+	verify_backup_directory(context, NULL, context->backup_directory);
+}
+
 /*
  * Verify one directory.
  *
@@ -682,6 +718,270 @@ verify_plain_backup_file(verifier_context *context, char *relpath,
 		total_size += m->size;
 }
 
+/*
+ * Verify tar backup.
+ *
+ * Unlike plan backup verification, tar backup verification carried out in two
+ * steps; in the first step this would simply sanity check on expected tar file
+ * to be present in the backup directory and it's compression type and collect
+ * these information is list. In the second pass, the tar archives are read,
+ * decompressed, and the required verification is carried out.
+ */
+static void
+verify_tar_backup(verifier_context *context)
+{
+	DIR		   *dir;
+	struct dirent *dirent;
+	SimplePtrList tarfiles = {NULL, NULL};
+	char	   *fullpath = context->backup_directory;
+
+	Assert(context->format == 't');
+
+	/*
+	 * If the backup directory cannot be found, treat this as a fatal error.
+	 */
+	dir = opendir(fullpath);
+	if (dir == NULL)
+		report_fatal_error("could not open directory \"%s\": %m", fullpath);
+
+	while (errno = 0, (dirent = readdir(dir)) != NULL)
+	{
+		char	   *filename = dirent->d_name;
+		char	   *newfullpath = psprintf("%s/%s", fullpath, filename);
+
+		/* Skip "." and ".." */
+		if (filename[0] == '.' && (filename[1] == '\0'
+								   || strcmp(filename, "..") == 0))
+			continue;
+
+		if (!should_ignore_relpath(context, filename))
+			precheck_tar_backup_file(context, filename, newfullpath,
+									 &tarfiles);
+
+		pfree(newfullpath);
+	}
+
+	/* Perform the final verification of the tar contents. */
+	verify_all_tar_files(context, &tarfiles);
+
+	if (closedir(dir))
+	{
+		report_backup_error(context,
+							"could not close directory \"%s\": %m", fullpath);
+		return;
+	}
+}
+
+/*
+ * Preparatory steps for verifying files in tar format backups.
+ *
+ * Carries out basic validation of the tar format backup file, detects the
+ * compression type, and appends that information to the tarfiles list. An
+ * error will be reported if the tar file is inaccessible, or if the file type,
+ * name, or compression type is not as expected.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_plain_backup_file. The additional argument outputs a list of valid
+ * tar files.
+ */
+static void
+precheck_tar_backup_file(verifier_context *context, char *relpath,
+						 char *fullpath, SimplePtrList *tarfiles)
+{
+	struct stat sb;
+	Oid			tblspc_oid = InvalidOid;
+	pg_compress_algorithm compress_algorithm;
+	tar_file   *tar;
+	char	   *suffix = NULL;
+
+	/* Should be tar format backup */
+	Assert(context->format == 't');
+
+	/* Get file information */
+	if (stat(fullpath, &sb) != 0)
+	{
+		report_backup_error(context,
+							"could not stat file or directory \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	/* In a tar format backup, we expect only plain files. */
+	if (!S_ISREG(sb.st_mode))
+	{
+		report_backup_error(context,
+							"\"%s\" is not a plain file",
+							relpath);
+		return;
+	}
+
+	/*
+	 * We expect tar files for backing up the main directory, tablespace, and
+	 * pg_wal directory.
+	 *
+	 * pg_basebackup writes the main data directory to an archive file named
+	 * base.tar, the pg_wal directory to pg_wal.tar, and the tablespace
+	 * directory to <tablespaceoid>.tar, each followed by a compression type
+	 * extension such as .gz, .lz4, or .zst.
+	 */
+	if (strncmp("base", relpath, 4) == 0)
+		suffix = relpath + 4;
+	else if (strncmp("pg_wal", relpath, 6) == 0)
+		suffix = relpath + 6;
+	else
+	{
+		/* Expected a <tablespaceoid>.tar file here. */
+		uint64		num = strtoul(relpath, &suffix, 10);
+
+		/*
+		 * Report an error if we didn't consume at least one character, if the
+		 * result is 0, or if the value is too large to be a valid OID.
+		 */
+		if (suffix == NULL || num <= 0 || num > OID_MAX)
+			report_backup_error(context,
+								"\"%s\" unexpected file in the tar format backup",
+								relpath);
+		tblspc_oid = (Oid) num;
+	}
+
+	/* Now, check the compression type of the tar */
+	if (strcmp(suffix, ".tar") == 0)
+		compress_algorithm = PG_COMPRESSION_NONE;
+	else if (strcmp(suffix, ".tgz") == 0)
+		compress_algorithm = PG_COMPRESSION_GZIP;
+	else if (strcmp(suffix, ".tar.gz") == 0)
+		compress_algorithm = PG_COMPRESSION_GZIP;
+	else if (strcmp(suffix, ".tar.lz4") == 0)
+		compress_algorithm = PG_COMPRESSION_LZ4;
+	else if (strcmp(suffix, ".tar.zst") == 0)
+		compress_algorithm = PG_COMPRESSION_ZSTD;
+	else
+	{
+		report_backup_error(context,
+							"\"%s\" unexpected file in the tar format backup",
+							relpath);
+		return;
+	}
+
+	/*
+	 * Ignore WALs, as reading and verification will be handled through
+	 * pg_waldump.
+	 */
+	if (strncmp("pg_wal", relpath, 6) == 0)
+		return;
+
+	/*
+	 * Append the information to the list for complete verification at a later
+	 * stage.
+	 */
+	tar = pg_malloc(sizeof(tar_file));
+	tar->relpath = pstrdup(relpath);
+	tar->tblspc_oid = tblspc_oid;
+	tar->compress_algorithm = compress_algorithm;
+
+	simple_ptr_list_append(tarfiles, tar);
+
+	/* Update statistics for progress report, if necessary */
+	if (show_progress)
+		total_size += sb.st_size;
+}
+
+/*
+ * The final steps for verifying files in tar format backups.
+ *
+ * Prepares the archive streamer stack according to the tar compression format
+ * for each tar file and invokes them for reading, decompressing, and
+ * ultimately verifying the contents.
+ *
+ * The arguments to this function should be a list of valid tar files to
+ * verify, and the allocation will be freed once the verification is complete.
+ */
+static void
+verify_all_tar_files(verifier_context *context, SimplePtrList *tarfiles)
+{
+	SimplePtrListCell *cell;
+
+	progress_report(false);
+
+	for (cell = tarfiles->head; cell != NULL; cell = cell->next)
+	{
+		tar_file   *tar = (tar_file *) cell->ptr;
+		astreamer  *streamer;
+		char	   *fullpath;
+
+		/* Prepare archive streamer stack */
+		streamer = create_archive_verifier(context,
+										   tar->relpath,
+										   tar->tblspc_oid,
+										   tar->compress_algorithm);
+
+		/* Compute the full pathname to the target file. */
+		fullpath = psprintf("%s/%s", context->backup_directory,
+							tar->relpath);
+
+		/* Invoke the streamer for reading, decompressing, and verifying. */
+		verify_one_tar_file(context, tar->relpath, fullpath, streamer);
+
+		/* Cleanup. */
+		pfree(tar->relpath);
+		pfree(tar);
+		pfree(fullpath);
+
+		astreamer_finalize(streamer);
+		astreamer_free(streamer);
+	}
+	simple_ptr_list_destroy(tarfiles);
+
+	progress_report(true);
+}
+
+/*
+ * Verification of a single tar file content.
+ *
+ * It reads a given tar archive in predefined chunks and passes it to the
+ * streamer, which initiates routines for decompression (if necessary) and then
+ * verifies each member within the tar file.
+ */
+static void
+verify_one_tar_file(verifier_context *context, char *relpath, char *fullpath,
+					astreamer *streamer)
+{
+	int			fd;
+	int			rc;
+	char	   *buffer;
+
+	pg_log_debug("reading \"%s\"", fullpath);
+
+	/* Open the target file. */
+	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
+	{
+		report_backup_error(context, "could not open file \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
+
+	/* Perform the reads */
+	while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
+	{
+		astreamer_content(streamer, NULL, buffer, rc, ASTREAMER_UNKNOWN);
+
+		/* Report progress */
+		done_size += rc;
+		progress_report(false);
+	}
+
+	if (rc < 0)
+		report_backup_error(context, "could not read file \"%s\": %m",
+							relpath);
+
+	/* Close the file. */
+	if (close(fd) != 0)
+		report_backup_error(context, "could not close file \"%s\": %m",
+							relpath);
+}
+
 /*
  * Verify file and its size entry in the manifest.
  */
@@ -1049,6 +1349,41 @@ find_backup_format(verifier_context *context)
 	return result;
 }
 
+/*
+ * Identifies the necessary steps for verifying the contents of the
+ * provided tar file.
+ */
+static astreamer *
+create_archive_verifier(verifier_context *context, char *archive_name,
+						Oid tblspc_oid, pg_compress_algorithm compress_algo)
+{
+	astreamer  *streamer = NULL;
+
+	/* Should be here only for tar backup */
+	Assert(context->format == 't');
+
+	/*
+	 * To verify the contents of the tar, the initial step is to parse its
+	 * content.
+	 */
+	streamer = astreamer_verify_content_new(streamer, context, archive_name,
+											tblspc_oid);
+	streamer = astreamer_tar_parser_new(streamer);
+
+	/*
+	 * If the tar is compressed, we must perform the appropriate decompression
+	 * operation before proceeding with the verification of its contents.
+	 */
+	if (compress_algo == PG_COMPRESSION_GZIP)
+		streamer = astreamer_gzip_decompressor_new(streamer);
+	else if (compress_algo == PG_COMPRESSION_LZ4)
+		streamer = astreamer_lz4_decompressor_new(streamer);
+	else if (compress_algo == PG_COMPRESSION_ZSTD)
+		streamer = astreamer_zstd_decompressor_new(streamer);
+
+	return streamer;
+}
+
 /*
  * Print a progress report based on the global variables.
  *
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 80031ad4dbc..be7438af346 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -18,6 +18,7 @@
 #include "common/hashfn_unstable.h"
 #include "common/logging.h"
 #include "common/parse_manifest.h"
+#include "fe_utils/astreamer.h"
 #include "fe_utils/simple_list.h"
 
 /*
@@ -123,4 +124,9 @@ extern void report_fatal_error(const char *pg_restrict fmt,...)
 extern bool should_ignore_relpath(verifier_context *context,
 								  const char *relpath);
 
+extern astreamer *astreamer_verify_content_new(astreamer *next,
+											   verifier_context *context,
+											   char *archive_name,
+											   Oid tblspc_oid);
+
 #endif							/* PG_VERIFYBACKUP_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 9e951a9e6f3..be3d32ef68a 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3329,6 +3329,7 @@ astreamer_plain_writer
 astreamer_recovery_injector
 astreamer_tar_archiver
 astreamer_tar_parser
+astreamer_verify
 astreamer_zstd_frame
 bgworker_main_type
 bh_node_type
@@ -3950,6 +3951,7 @@ substitute_phv_relids_context
 subxids_array_status
 symbol
 tablespaceinfo
+tar_file
 td_entry
 teSection
 temp_tablespaces_extra
-- 
2.18.0

From 500d9bc69d7d786fae637315261dca79916b91ab Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 29 Aug 2024 19:01:22 +0530
Subject: [PATCH v13 12/12] pg_verifybackup: Tests and document

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the previous patch that implements tar format support.
----
---
 doc/src/sgml/ref/pg_verifybackup.sgml         |  43 +++-
 src/bin/pg_verifybackup/t/001_basic.pl        |   6 +-
 src/bin/pg_verifybackup/t/002_algorithm.pl    |  32 ++-
 src/bin/pg_verifybackup/t/003_corruption.pl   | 187 ++++++++++++++++--
 src/bin/pg_verifybackup/t/004_options.pl      |   2 +-
 src/bin/pg_verifybackup/t/005_bad_manifest.pl |   2 +-
 src/bin/pg_verifybackup/t/008_untar.pl        |  73 +++----
 src/bin/pg_verifybackup/t/010_client_untar.pl |  48 +----
 8 files changed, 274 insertions(+), 119 deletions(-)

diff --git a/doc/src/sgml/ref/pg_verifybackup.sgml b/doc/src/sgml/ref/pg_verifybackup.sgml
index a3f167f9f6e..ea6bc3ccb12 100644
--- a/doc/src/sgml/ref/pg_verifybackup.sgml
+++ b/doc/src/sgml/ref/pg_verifybackup.sgml
@@ -34,8 +34,10 @@ PostgreSQL documentation
    integrity of a database cluster backup taken using
    <command>pg_basebackup</command> against a
    <literal>backup_manifest</literal> generated by the server at the time
-   of the backup.  The backup must be stored in the "plain"
-   format; a "tar" format backup can be checked after extracting it.
+   of the backup.  The backup must be stored in the "plain" or "tar"
+   format.  Verification is supported for <literal>gzip</literal>,
+   <literal>lz4</literal>, and  <literal>zstd</literal> compressed tar backup;
+   any other compressed format backups can be checked after decompressing them.
   </para>
 
   <para>
@@ -168,6 +170,43 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-F <replaceable class="parameter">format</replaceable></option></term>
+      <term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
+      <listitem>
+       <para>
+        Specifies the format of the backup. <replaceable>format</replaceable>
+        can be one of the following:
+
+        <variablelist>
+         <varlistentry>
+          <term><literal>p</literal></term>
+          <term><literal>plain</literal></term>
+          <listitem>
+           <para>
+            Backup consists of plain files with the same layout as the
+            source server's data directory and tablespaces.
+           </para>
+          </listitem>
+         </varlistentry>
+
+         <varlistentry>
+          <term><literal>t</literal></term>
+          <term><literal>tar</literal></term>
+          <listitem>
+           <para>
+            Backup consists of tar files.  A valid backup includes the main data
+            directory in a file named <filename>base.tar</filename>, the WAL
+            files in <filename>pg_wal.tar</filename>, and separate tar files for
+            each tablespace, named after the tablespace's OID, followed by the
+            compression extension.
+           </para>
+           </listitem>
+         </varlistentry>
+        </variablelist></para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-n</option></term>
       <term><option>--no-parse-wal</option></term>
diff --git a/src/bin/pg_verifybackup/t/001_basic.pl b/src/bin/pg_verifybackup/t/001_basic.pl
index 2f3e52d296f..ca5b0402b7d 100644
--- a/src/bin/pg_verifybackup/t/001_basic.pl
+++ b/src/bin/pg_verifybackup/t/001_basic.pl
@@ -17,11 +17,11 @@ command_fails_like(
 	qr/no backup directory specified/,
 	'target directory must be specified');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir ],
 	qr/could not open file.*\/backup_manifest\"/,
 	'pg_verifybackup requires a manifest');
 command_fails_like(
-	[ 'pg_verifybackup', $tempdir, $tempdir ],
+	[ 'pg_verifybackup', '-Fp', $tempdir, $tempdir ],
 	qr/too many command-line arguments/,
 	'multiple target directories not allowed');
 
@@ -31,7 +31,7 @@ close($fh);
 
 # but then try to use an alternate, nonexisting manifest
 command_fails_like(
-	[ 'pg_verifybackup', '-m', "$tempdir/not_the_manifest", $tempdir ],
+	[ 'pg_verifybackup', '-Fp', '-m', "$tempdir/not_the_manifest", $tempdir ],
 	qr/could not open file.*\/not_the_manifest\"/,
 	'pg_verifybackup respects -m flag');
 
diff --git a/src/bin/pg_verifybackup/t/002_algorithm.pl b/src/bin/pg_verifybackup/t/002_algorithm.pl
index fb2a1fd7c4e..ac276f3da6b 100644
--- a/src/bin/pg_verifybackup/t/002_algorithm.pl
+++ b/src/bin/pg_verifybackup/t/002_algorithm.pl
@@ -14,24 +14,33 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 
-for my $algorithm (qw(bogus none crc32c sha224 sha256 sha384 sha512))
+sub test_checksums
 {
-	my $backup_path = $primary->backup_dir . '/' . $algorithm;
+	my ($format, $algorithm) = @_;
+	my $backup_path = $primary->backup_dir . '/' . $format . '/' . $algorithm;
 	my @backup = (
 		'pg_basebackup', '-D', $backup_path,
 		'--manifest-checksums', $algorithm, '--no-sync', '-cfast');
 	my @verify = ('pg_verifybackup', '-e', $backup_path);
 
+	if ($format eq 'tar')
+	{
+		# Add tar backup format option
+		push @backup, ('-F', 't');
+		# Add switch to skip WAL verification.
+		push @verify, ('-n');
+	}
+
 	# A backup with a bogus algorithm should fail.
 	if ($algorithm eq 'bogus')
 	{
 		$primary->command_fails(\@backup,
-			"backup fails with algorithm \"$algorithm\"");
-		next;
+			"$format backup fails with algorithm \"$algorithm\"");
+		return;
 	}
 
 	# A backup with a valid algorithm should work.
-	$primary->command_ok(\@backup, "backup ok with algorithm \"$algorithm\"");
+	$primary->command_ok(\@backup, "$format backup ok with algorithm \"$algorithm\"");
 
 	# We expect each real checksum algorithm to be mentioned on every line of
 	# the backup manifest file except the first and last; for simplicity, we
@@ -39,7 +48,7 @@ for my $algorithm (qw(bogus none crc32c sha224 sha256 sha384 sha512))
 	# is none, we just check that the manifest exists.
 	if ($algorithm eq 'none')
 	{
-		ok(-f "$backup_path/backup_manifest", "backup manifest exists");
+		ok(-f "$backup_path/backup_manifest", "$format backup manifest exists");
 	}
 	else
 	{
@@ -52,10 +61,19 @@ for my $algorithm (qw(bogus none crc32c sha224 sha256 sha384 sha512))
 
 	# Make sure that it verifies OK.
 	$primary->command_ok(\@verify,
-		"verify backup with algorithm \"$algorithm\"");
+		"verify $format backup with algorithm \"$algorithm\"");
 
 	# Remove backup immediately to save disk space.
 	rmtree($backup_path);
 }
 
+# Do the check
+for my $format (qw(plain tar))
+{
+	for my $algorithm (qw(bogus none crc32c sha224 sha256 sha384 sha512))
+	{
+		test_checksums($format, $algorithm);
+	}
+}
+
 done_testing();
diff --git a/src/bin/pg_verifybackup/t/003_corruption.pl b/src/bin/pg_verifybackup/t/003_corruption.pl
index ae91e043384..d0c3ffedd14 100644
--- a/src/bin/pg_verifybackup/t/003_corruption.pl
+++ b/src/bin/pg_verifybackup/t/003_corruption.pl
@@ -11,6 +11,8 @@ use PostgreSQL::Test::Cluster;
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+my $tar = $ENV{TAR};
+
 my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
@@ -32,62 +34,73 @@ EOM
 my @scenario = (
 	{
 		'name' => 'extra_file',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_extra_file,
 		'fails_like' =>
 		  qr/extra_file.*present on disk but not in the manifest/
 	},
 	{
 		'name' => 'extra_tablespace_file',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_extra_tablespace_file,
 		'fails_like' =>
 		  qr/extra_ts_file.*present on disk but not in the manifest/
 	},
 	{
 		'name' => 'missing_file',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_missing_file,
 		'fails_like' =>
 		  qr/pg_xact\/0000.*present in the manifest but not on disk/
 	},
 	{
 		'name' => 'missing_tablespace',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_missing_tablespace,
 		'fails_like' =>
 		  qr/pg_tblspc.*present in the manifest but not on disk/
 	},
 	{
 		'name' => 'append_to_file',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_append_to_file,
 		'fails_like' => qr/has size \d+ on disk but size \d+ in the manifest/
 	},
 	{
 		'name' => 'truncate_file',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_truncate_file,
 		'fails_like' => qr/has size 0 on disk but size \d+ in the manifest/
 	},
 	{
 		'name' => 'replace_file',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_replace_file,
 		'fails_like' => qr/checksum mismatch for file/
 	},
 	{
 		'name' => 'system_identifier',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_system_identifier,
 		'fails_like' =>
 		  qr/manifest system identifier is .*, but control file has/
 	},
 	{
 		'name' => 'bad_manifest',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_bad_manifest,
 		'fails_like' => qr/manifest checksum mismatch/
 	},
 	{
 		'name' => 'open_file_fails',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_open_file_fails,
 		'fails_like' => qr/could not open file/,
 		'skip_on_windows' => 1
 	},
 	{
 		'name' => 'open_directory_fails',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_open_directory_fails,
 		'cleanup' => \&cleanup_open_directory_fails,
 		'fails_like' => qr/could not open directory/,
@@ -95,10 +108,61 @@ my @scenario = (
 	},
 	{
 		'name' => 'search_directory_fails',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_search_directory_fails,
 		'cleanup' => \&cleanup_search_directory_fails,
 		'fails_like' => qr/could not stat file or directory/,
 		'skip_on_windows' => 1
+	},
+	{
+		'name' => 'tar_backup_unexpected_file',
+		'backup_format' => 't',
+		'mutilate' => \&mutilate_extra_file,
+		'fails_like' =>
+		  qr/extra_file.*unexpected file in the tar format backup/
+	},
+	{
+		'name' => 'tar_backup_extra_file',
+		'backup_format' => 't',
+		'mutilate' => \&mutilate_tar_backup_extra_file,
+		'fails_like' =>
+		  qr/extra_tar_member_file.*present on disk but not in the manifest/
+	},
+	{
+		'name' => 'tar_backup_missing_file',
+		'backup_format' => 't',
+		'mutilate' => \&mutilate_tar_backup_missing_file,
+		'fails_like' =>
+		  qr/pg_xact\/0000.*present in the manifest but not on disk/,
+		'skip_on_windows' => 1
+	},
+	{
+		'name' => 'tar_backup_append_to_file',
+		'backup_format' => 't',
+		'mutilate' => \&mutilate_tar_backup_append_to_file,
+		'fails_like' => qr/has size \d+ on disk but size \d+ in the manifest/,
+		'skip_on_windows' => 1
+	},
+	{
+		'name' => 'tar_backup_truncate_file',
+		'backup_format' => 't',
+		'mutilate' => \&mutilate_tar_backup_truncate_file,
+		'fails_like' => qr/has size 0 on disk but size \d+ in the manifest/,
+		'skip_on_windows' => 1
+	},
+	{
+		'name' => 'tar_backup_replace_file',
+		'backup_format' => 't',
+		'mutilate' => \&mutilate_tar_backup_replace_file,
+		'fails_like' => qr/checksum mismatch for file/,
+		'skip_on_windows' => 1
+	},
+	{
+		'name' => 'tar_backup_system_identifier',
+		'backup_format' => 't',
+		'mutilate' => \&mutilate_system_identifier,
+		'fails_like' =>
+		  qr/manifest system identifier is .*, but control file has/
 	});
 
 for my $scenario (@scenario)
@@ -111,29 +175,40 @@ for my $scenario (@scenario)
 		  if ($scenario->{'skip_on_windows'}
 			&& ($windows_os || $Config::Config{osname} eq 'cygwin'));
 
+		# Skip tests for tar format-backup if tar is not available.
+		skip "no tar program available", 4
+		if ($scenario->{'backup_format'} eq 't' && (!defined $tar || $tar eq ''));
+
 		# Take a backup and check that it verifies OK.
 		my $backup_path = $primary->backup_dir . '/' . $name;
 		my $backup_ts_path = PostgreSQL::Test::Utils::tempdir_short();
+
+		my @backup = (
+				'pg_basebackup', '-D', $backup_path, '--no-sync', '-cfast',
+				'-T', "${source_ts_path}=${backup_ts_path}");
+		my @verify = ('pg_verifybackup', $backup_path);
+
+		if ($scenario->{'backup_format'} eq 't')
+		{
+			# Add tar backup format option
+			push @backup, ('-F', 't');
+			# Add switch to skip WAL verification.
+			push @verify, ('-n');
+		}
+
 		# The tablespace map parameter confuses Msys2, which tries to mangle
 		# it. Tell it not to.
 		# See https://www.msys2.org/wiki/Porting/#filesystem-namespaces
 		local $ENV{MSYS2_ARG_CONV_EXCL} = $source_ts_prefix;
-		$primary->command_ok(
-			[
-				'pg_basebackup', '-D', $backup_path, '--no-sync', '-cfast',
-				'-T', "${source_ts_path}=${backup_ts_path}"
-			],
-			"base backup ok");
-		command_ok([ 'pg_verifybackup', $backup_path ],
-			"intact backup verified");
+
+		$primary->command_ok( \@backup, "base backup ok");
+		command_ok(\@verify, "intact backup verified");
 
 		# Mutilate the backup in some way.
 		$scenario->{'mutilate'}->($backup_path);
 
 		# Now check that the backup no longer verifies.
-		command_fails_like(
-			[ 'pg_verifybackup', $backup_path ],
-			$scenario->{'fails_like'},
+		command_fails_like(\@verify, $scenario->{'fails_like'},
 			"corrupt backup fails verification: $name");
 
 		# Run cleanup hook, if provided.
@@ -260,6 +335,7 @@ sub mutilate_system_identifier
 		$backup_path . '/backup_manifest')
 	  or BAIL_OUT "could not copy manifest to $backup_path";
 	$node->teardown_node(fail_ok => 1);
+	$node->clean_node();
 	return;
 }
 
@@ -316,4 +392,93 @@ sub cleanup_search_directory_fails
 	return;
 }
 
+# Unpack base.tar, perform the specified file operation, and then repack the
+# modified content into base.tar at the same location.
+sub mutilate_base_tar
+{
+	my ($backup_path, $op) = @_;
+
+	my $archive = 'base.tar';
+	my $tmpdir = "$backup_path/tmpdir";
+	mkdir($tmpdir) || die "$!";
+
+	# Extract the archive
+	system_or_bail($tar, '-xf', "$backup_path/$archive", '-C', "$tmpdir");
+	unlink("$backup_path/$archive") || die "$!";
+
+	if ($op eq 'add')
+	{
+		create_extra_file($tmpdir, 'extra_tar_member_file');
+	}
+	elsif ($op eq 'delete')
+	{
+		mutilate_missing_file($tmpdir);
+	}
+	elsif ($op eq 'append')
+	{
+		mutilate_append_to_file($tmpdir);
+	}
+	elsif ($op eq 'truncate')
+	{
+		mutilate_truncate_file($tmpdir);
+	}
+	elsif ($op eq 'replace')
+	{
+		mutilate_replace_file($tmpdir);
+	}
+	else
+	{
+		die "mutilate_tar_backup: \"$op\" invalid operation";
+	}
+
+
+	# Navigate to the extracted location and list the files.
+	chdir("$tmpdir") || die "$!";
+	my @files = glob("*");
+	# Repack the extracted content
+	system_or_bail($tar, '-cf', "$backup_path/$archive", @files);
+	chdir($backup_path) || die "$!";
+	rmtree("$tmpdir") || die "$!";
+}
+
+# Add a file into the base.tar of the backup.
+sub mutilate_tar_backup_extra_file
+{
+	my ($backup_path) = @_;
+	mutilate_base_tar($backup_path, 'add');
+	return;
+}
+
+# Remove a file.
+sub mutilate_tar_backup_missing_file
+{
+	my ($backup_path) = @_;
+	mutilate_base_tar($backup_path, 'delete');
+	return;
+}
+
+# Append an additional bytes to a file.
+sub mutilate_tar_backup_append_to_file
+{
+	my ($backup_path) = @_;
+	mutilate_base_tar($backup_path, 'append');
+	return;
+}
+
+# Truncate a file to zero length.
+sub mutilate_tar_backup_truncate_file
+{
+	my ($backup_path) = @_;
+	mutilate_base_tar($backup_path, 'truncate');
+	return;
+}
+
+# Replace a file's contents
+sub mutilate_tar_backup_replace_file
+{
+	my ($backup_path) = @_;
+	mutilate_base_tar($backup_path, 'replace');
+	return;
+}
+
 done_testing();
diff --git a/src/bin/pg_verifybackup/t/004_options.pl b/src/bin/pg_verifybackup/t/004_options.pl
index 8ed2214408e..2f197648740 100644
--- a/src/bin/pg_verifybackup/t/004_options.pl
+++ b/src/bin/pg_verifybackup/t/004_options.pl
@@ -108,7 +108,7 @@ unlike(
 # Test valid manifest with nonexistent backup directory.
 command_fails_like(
 	[
-		'pg_verifybackup', '-m',
+		'pg_verifybackup', '-Fp', '-m',
 		"$backup_path/backup_manifest", "$backup_path/fake"
 	],
 	qr/could not open directory/,
diff --git a/src/bin/pg_verifybackup/t/005_bad_manifest.pl b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
index c4ed64b62d5..28c51b6feb0 100644
--- a/src/bin/pg_verifybackup/t/005_bad_manifest.pl
+++ b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
@@ -208,7 +208,7 @@ sub test_bad_manifest
 	print $fh $manifest_contents;
 	close($fh);
 
-	command_fails_like([ 'pg_verifybackup', $tempdir ], $regexp, $test_name);
+	command_fails_like([ 'pg_verifybackup', '-Fp', $tempdir ], $regexp, $test_name);
 	return;
 }
 
diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
index 7a09f3b75b2..1c83f38d5b5 100644
--- a/src/bin/pg_verifybackup/t/008_untar.pl
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -16,6 +16,20 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 
+# Create a tablespace directory.
+my $TS1_LOCATION = $primary->backup_dir .'/ts1';
+$TS1_LOCATION =~ s/\/\.\//\//g;    # collapse foo/./bar to foo/bar
+mkdir($TS1_LOCATION);
+
+# Create a tablespace with table in it.
+$primary->safe_psql('postgres', qq(
+		CREATE TABLESPACE regress_ts1 LOCATION '$TS1_LOCATION';
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1';
+		CREATE TABLE regress_tbl1(i int) TABLESPACE regress_ts1;
+		INSERT INTO regress_tbl1 VALUES(generate_series(1,5));));
+my $tsoid = $primary->safe_psql('postgres', qq(
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1'));
+
 my $backup_path = $primary->backup_dir . '/server-backup';
 my $extract_path = $primary->backup_dir . '/extracted-backup';
 
@@ -23,39 +37,31 @@ my @test_configuration = (
 	{
 		'compression_method' => 'none',
 		'backup_flags' => [],
-		'backup_archive' => 'base.tar',
+		'backup_archive' => ['base.tar', "$tsoid.tar"],
 		'enabled' => 1
 	},
 	{
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'server-gzip' ],
-		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.gz', "$tsoid.tar.gz" ],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'server-lz4' ],
-		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => [ '-d', '-m' ],
+		'backup_archive' => ['base.tar.lz4', "$tsoid.tar.lz4" ],
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd:level=1,long' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	});
 
@@ -86,47 +92,16 @@ for my $tc (@test_configuration)
 		my $backup_files = join(',',
 			sort grep { $_ ne '.' && $_ ne '..' } slurp_dir($backup_path));
 		my $expected_backup_files =
-		  join(',', sort ('backup_manifest', $tc->{'backup_archive'}));
+		  join(',', sort ('backup_manifest', @{ $tc->{'backup_archive'} }));
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok(['pg_verifybackup', '-n', '-e', $backup_path],
+			"verify backup, compression $method");
 
 		# Cleanup.
-		unlink($backup_path . '/backup_manifest');
-		unlink($backup_path . '/base.tar');
-		unlink($backup_path . '/' . $tc->{'backup_archive'});
+		rmtree($backup_path);
 		rmtree($extract_path);
 	}
 }
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index 8c076d46dee..6b7d7483f6e 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -29,41 +29,30 @@ my @test_configuration = (
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'client-gzip:5' ],
 		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'client-lz4:5' ],
 		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => ['-d'],
-		'output_file' => 'base.tar',
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:5' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:level=1,long' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'parallel zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:workers=3' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1"),
 		'possibly_unsupported' =>
 		  qr/could not set compression worker count to 3: Unsupported parameter/
@@ -118,40 +107,9 @@ for my $tc (@test_configuration)
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			push @decompress, $backup_path . '/' . $tc->{'output_file'}
-			  if $tc->{'output_file'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok( [ 'pg_verifybackup', '-n', '-e', $backup_path ],
+			"verify backup, compression $method");
 
 		# Cleanup.
 		rmtree($extract_path);
-- 
2.18.0

From 4f98ffa42916fe179e9a87b9043393b6449f1705 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 14 Aug 2024 10:42:37 +0530
Subject: [PATCH v13 06/12] Refactor: split verify_backup_file() function and
 rename it.

The function verify_backup_file() has now been renamed to
verify_plain_backup_file() to make it clearer that it is specifically
used for verifying files in a plain backup. Similarly, in a future
patch, we would have a verify_tar_backup_file() function for
verifying TAR backup files.

In addition to that, moved the manifest entry verification code into a
new function called verify_manifest_entry() so that it can be reused
for tar backup verification. If verify_manifest_entry() doesn't find
an entry, it reports an error as before and returns NULL to the
caller. This is why a NULL check is added to should_verify_checksum().
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 58 +++++++++++++++--------
 src/bin/pg_verifybackup/pg_verifybackup.h |  6 ++-
 2 files changed, 42 insertions(+), 22 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 3fcfb167217..5bfc98e7874 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -64,8 +64,8 @@ static void report_manifest_error(JsonManifestParseContext *context,
 
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
-static void verify_backup_file(verifier_context *context,
-							   char *relpath, char *fullpath);
+static void verify_plain_backup_file(verifier_context *context,
+									 char *relpath, char *fullpath);
 static void verify_control_file(const char *controlpath,
 								uint64 manifest_system_identifier);
 static void report_extra_backup_files(verifier_context *context);
@@ -570,7 +570,7 @@ verify_backup_directory(verifier_context *context, char *relpath,
 			newrelpath = psprintf("%s/%s", relpath, filename);
 
 		if (!should_ignore_relpath(context, newrelpath))
-			verify_backup_file(context, newrelpath, newfullpath);
+			verify_plain_backup_file(context, newrelpath, newfullpath);
 
 		pfree(newfullpath);
 		pfree(newrelpath);
@@ -591,7 +591,8 @@ verify_backup_directory(verifier_context *context, char *relpath,
  * verify_backup_directory.
  */
 static void
-verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
+verify_plain_backup_file(verifier_context *context, char *relpath,
+						 char *fullpath)
 {
 	struct stat sb;
 	manifest_file *m;
@@ -627,6 +628,32 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		return;
 	}
 
+	/* Check the backup manifest entry for this file. */
+	m = verify_manifest_entry(context, relpath, sb.st_size);
+
+	/*
+	 * Validate the manifest system identifier, not available in manifest
+	 * version 1.
+	 */
+	if (context->manifest->version != 1 &&
+		strcmp(relpath, "global/pg_control") == 0 &&
+		m != NULL && m->matched && !m->bad)
+		verify_control_file(fullpath, context->manifest->system_identifier);
+
+	/* Update statistics for progress report, if necessary */
+	if (show_progress && !context->skip_checksums &&
+		should_verify_checksum(m))
+		total_size += m->size;
+}
+
+/*
+ * Verify file and its size entry in the manifest.
+ */
+manifest_file *
+verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
+{
+	manifest_file *m;
+
 	/* Check whether there's an entry in the manifest hash. */
 	m = manifest_files_lookup(context->manifest->files, relpath);
 	if (m == NULL)
@@ -634,40 +661,29 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		report_backup_error(context,
 							"\"%s\" is present on disk but not in the manifest",
 							relpath);
-		return;
+		return NULL;
 	}
 
 	/* Flag this entry as having been encountered in the filesystem. */
 	m->matched = true;
 
 	/* Check that the size matches. */
-	if (m->size != sb.st_size)
+	if (m->size != filesize)
 	{
 		report_backup_error(context,
 							"\"%s\" has size %lld on disk but size %zu in the manifest",
-							relpath, (long long int) sb.st_size, m->size);
+							relpath, (long long int) filesize, m->size);
 		m->bad = true;
 	}
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0)
-		verify_control_file(fullpath, context->manifest->system_identifier);
-
-	/* Update statistics for progress report, if necessary */
-	if (show_progress && !context->skip_checksums &&
-		should_verify_checksum(m))
-		total_size += m->size;
-
 	/*
 	 * We don't verify checksums at this stage. We first finish verifying that
 	 * we have the expected set of files with the expected sizes, and only
 	 * afterwards verify the checksums. That's because computing checksums may
 	 * take a while, and we'd like to report more obvious problems quickly.
 	 */
+
+	return m;
 }
 
 /*
@@ -830,7 +846,7 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 
 	/*
 	 * Double-check that we read the expected number of bytes from the file.
-	 * Normally, a file size mismatch would be caught in verify_backup_file
+	 * Normally, a file size mismatch would be caught in verify_manifest_entry
 	 * and this check would never be reached, but this provides additional
 	 * safety and clarity in the event of concurrent modifications or
 	 * filesystem misbehavior.
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index d8c566ed587..ff9476e356e 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -37,7 +37,8 @@ typedef struct manifest_file
 } manifest_file;
 
 #define should_verify_checksum(m) \
-	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+	(((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
 
 /*
  * Define a hash table which we can use to store information about the files
@@ -93,6 +94,9 @@ typedef struct verifier_context
 	bool		saw_any_error;
 } verifier_context;
 
+extern manifest_file *verify_manifest_entry(verifier_context *context,
+											char *relpath, int64 filesize);
+
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
 			pg_attribute_printf(2, 3);
-- 
2.18.0

From ab254802604c8c31b5ec05784d938593c6efd9b2 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 14 Aug 2024 15:14:15 +0530
Subject: [PATCH v13 07/12] Refactor: split verify_file_checksum() function.

Move the core functionality of verify_file_checksum to a new function
to reuse it instead of duplicating the code.

The verify_file_checksum() function is designed to take a file path,
open and read the file contents, and then calculate the checksum.
However, for TAR backups, instead of a file path, we receive the file
content in chunks, and the checksum needs to be calculated
incrementally. While the initial operations for plain and TAR backup
checksum calculations differ, the final checks and error handling are
the same. By moving the shared logic to a separate function, we can
reuse the code for both types of backups.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 20 +++++++++++++++++---
 src/bin/pg_verifybackup/pg_verifybackup.h |  3 +++
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 5bfc98e7874..e44d0377cd5 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -792,8 +792,6 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	int			fd;
 	int			rc;
 	size_t		bytes_read = 0;
-	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
-	int			checksumlen;
 
 	/* Open the target file. */
 	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
@@ -844,6 +842,22 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	if (rc < 0)
 		return;
 
+	/* Do the final computation and verification. */
+	verify_checksum(context, m, &checksum_ctx, bytes_read);
+}
+
+/*
+ * A helper function to finalize checksum computation and verify it against the
+ * backup manifest information.
+ */
+void
+verify_checksum(verifier_context *context, manifest_file *m,
+				pg_checksum_context *checksum_ctx, int64 bytes_read)
+{
+	const char *relpath = m->pathname;
+	int			checksumlen;
+	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
+
 	/*
 	 * Double-check that we read the expected number of bytes from the file.
 	 * Normally, a file size mismatch would be caught in verify_manifest_entry
@@ -860,7 +874,7 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	}
 
 	/* Get the final checksum. */
-	checksumlen = pg_checksum_final(&checksum_ctx, checksumbuf);
+	checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
 	if (checksumlen < 0)
 	{
 		report_backup_error(context,
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index ff9476e356e..fe0ce8a89aa 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -96,6 +96,9 @@ typedef struct verifier_context
 
 extern manifest_file *verify_manifest_entry(verifier_context *context,
 											char *relpath, int64 filesize);
+extern void verify_checksum(verifier_context *context, manifest_file *m,
+							pg_checksum_context *checksum_ctx,
+							int64 bytes_read);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From 744f359f0782d590cac0f29bde76430561c038b5 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 21 Aug 2024 10:51:23 +0530
Subject: [PATCH v13 08/12] Refactor: split verify_control_file.

Separated the manifest entry verification code into a new function and
introduced the should_verify_control_data() macro, similar to
should_verify_checksum().

Like verify_file_checksum(), verify_control_file() is too design to
accept the pg_control file patch which will be opened and respective
information will be verified. But, in case of tar backup we would be
having pg_control file contents instead, that needs to be verified in
the same way.  For that reason the code that doing the verification is
separated into separate function to so that can be reused for the tar
backup verification as well.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 42 ++++++++++-------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 14 ++++++++
 2 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index e44d0377cd5..d04e1d8c8ac 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -66,8 +66,6 @@ static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_plain_backup_file(verifier_context *context,
 									 char *relpath, char *fullpath);
-static void verify_control_file(const char *controlpath,
-								uint64 manifest_system_identifier);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -631,14 +629,20 @@ verify_plain_backup_file(verifier_context *context, char *relpath,
 	/* Check the backup manifest entry for this file. */
 	m = verify_manifest_entry(context, relpath, sb.st_size);
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0 &&
-		m != NULL && m->matched && !m->bad)
-		verify_control_file(fullpath, context->manifest->system_identifier);
+	/* Validate the pg_control information */
+	if (should_verify_control_data(context->manifest, m))
+	{
+		ControlFileData *control_file;
+		bool		crc_ok;
+
+		pg_log_debug("reading \"%s\"", fullpath);
+		control_file = get_controlfile_by_exact_path(fullpath, &crc_ok);
+
+		verify_control_data(control_file, fullpath, crc_ok,
+							context->manifest->system_identifier);
+		/* Release memory. */
+		pfree(control_file);
+	}
 
 	/* Update statistics for progress report, if necessary */
 	if (show_progress && !context->skip_checksums &&
@@ -687,18 +691,13 @@ verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
 }
 
 /*
- * Sanity check control file and validate system identifier against manifest
- * system identifier.
+ * Sanity check control file data and validate system identifier against
+ * manifest system identifier.
  */
-static void
-verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
+void
+verify_control_data(ControlFileData *control_file, const char *controlpath,
+					bool crc_ok, uint64 manifest_system_identifier)
 {
-	ControlFileData *control_file;
-	bool		crc_ok;
-
-	pg_log_debug("reading \"%s\"", controlpath);
-	control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
-
 	/* Control file contents not meaningful if CRC is bad. */
 	if (!crc_ok)
 		report_fatal_error("%s: CRC is incorrect", controlpath);
@@ -714,9 +713,6 @@ verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
 						   controlpath,
 						   (unsigned long long) manifest_system_identifier,
 						   (unsigned long long) control_file->system_identifier);
-
-	/* Release memory. */
-	pfree(control_file);
 }
 
 /*
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index fe0ce8a89aa..818064c6eed 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -40,6 +40,17 @@ typedef struct manifest_file
 	(((m) != NULL) && ((m)->matched) && !((m)->bad) && \
 	 (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
 
+/*
+ * Validate the control file and its system identifier against the manifest
+ * system identifier. Note that this feature is not available in manifest
+ * version 1. This validation should only be performed after the manifest entry
+ * validation for the pg_control file has been completed without errors.
+ */
+#define should_verify_control_data(manifest, m) \
+	(((manifest)->version != 1) && \
+	 ((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (strcmp((m)->pathname, "global/pg_control") == 0))
+
 /*
  * Define a hash table which we can use to store information about the files
  * mentioned in the backup manifest.
@@ -99,6 +110,9 @@ extern manifest_file *verify_manifest_entry(verifier_context *context,
 extern void verify_checksum(verifier_context *context, manifest_file *m,
 							pg_checksum_context *checksum_ctx,
 							int64 bytes_read);
+extern void verify_control_data(ControlFileData *control_file,
+								const char *controlpath, bool crc_ok,
+								uint64 manifest_system_identifier);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From e02e69b65e70553313ea1287e65d77cd34a00688 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 14 Aug 2024 10:42:37 +0530
Subject: [PATCH v14 06/12] Refactor: split verify_backup_file() function and
 rename it.

The function verify_backup_file() has now been renamed to
verify_plain_backup_file() to make it clearer that it is specifically
used for verifying files in a plain backup. Similarly, in a future
patch, we would have a verify_tar_backup_file() function for
verifying TAR backup files.

In addition to that, moved the manifest entry verification code into a
new function called verify_manifest_entry() so that it can be reused
for tar backup verification. If verify_manifest_entry() doesn't find
an entry, it reports an error as before and returns NULL to the
caller. This is why a NULL check is added to should_verify_checksum().
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 58 +++++++++++++++--------
 src/bin/pg_verifybackup/pg_verifybackup.h |  6 ++-
 2 files changed, 42 insertions(+), 22 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 3fcfb167217..5bfc98e7874 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -64,8 +64,8 @@ static void report_manifest_error(JsonManifestParseContext *context,
 
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
-static void verify_backup_file(verifier_context *context,
-							   char *relpath, char *fullpath);
+static void verify_plain_backup_file(verifier_context *context,
+									 char *relpath, char *fullpath);
 static void verify_control_file(const char *controlpath,
 								uint64 manifest_system_identifier);
 static void report_extra_backup_files(verifier_context *context);
@@ -570,7 +570,7 @@ verify_backup_directory(verifier_context *context, char *relpath,
 			newrelpath = psprintf("%s/%s", relpath, filename);
 
 		if (!should_ignore_relpath(context, newrelpath))
-			verify_backup_file(context, newrelpath, newfullpath);
+			verify_plain_backup_file(context, newrelpath, newfullpath);
 
 		pfree(newfullpath);
 		pfree(newrelpath);
@@ -591,7 +591,8 @@ verify_backup_directory(verifier_context *context, char *relpath,
  * verify_backup_directory.
  */
 static void
-verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
+verify_plain_backup_file(verifier_context *context, char *relpath,
+						 char *fullpath)
 {
 	struct stat sb;
 	manifest_file *m;
@@ -627,6 +628,32 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		return;
 	}
 
+	/* Check the backup manifest entry for this file. */
+	m = verify_manifest_entry(context, relpath, sb.st_size);
+
+	/*
+	 * Validate the manifest system identifier, not available in manifest
+	 * version 1.
+	 */
+	if (context->manifest->version != 1 &&
+		strcmp(relpath, "global/pg_control") == 0 &&
+		m != NULL && m->matched && !m->bad)
+		verify_control_file(fullpath, context->manifest->system_identifier);
+
+	/* Update statistics for progress report, if necessary */
+	if (show_progress && !context->skip_checksums &&
+		should_verify_checksum(m))
+		total_size += m->size;
+}
+
+/*
+ * Verify file and its size entry in the manifest.
+ */
+manifest_file *
+verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
+{
+	manifest_file *m;
+
 	/* Check whether there's an entry in the manifest hash. */
 	m = manifest_files_lookup(context->manifest->files, relpath);
 	if (m == NULL)
@@ -634,40 +661,29 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		report_backup_error(context,
 							"\"%s\" is present on disk but not in the manifest",
 							relpath);
-		return;
+		return NULL;
 	}
 
 	/* Flag this entry as having been encountered in the filesystem. */
 	m->matched = true;
 
 	/* Check that the size matches. */
-	if (m->size != sb.st_size)
+	if (m->size != filesize)
 	{
 		report_backup_error(context,
 							"\"%s\" has size %lld on disk but size %zu in the manifest",
-							relpath, (long long int) sb.st_size, m->size);
+							relpath, (long long int) filesize, m->size);
 		m->bad = true;
 	}
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0)
-		verify_control_file(fullpath, context->manifest->system_identifier);
-
-	/* Update statistics for progress report, if necessary */
-	if (show_progress && !context->skip_checksums &&
-		should_verify_checksum(m))
-		total_size += m->size;
-
 	/*
 	 * We don't verify checksums at this stage. We first finish verifying that
 	 * we have the expected set of files with the expected sizes, and only
 	 * afterwards verify the checksums. That's because computing checksums may
 	 * take a while, and we'd like to report more obvious problems quickly.
 	 */
+
+	return m;
 }
 
 /*
@@ -830,7 +846,7 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 
 	/*
 	 * Double-check that we read the expected number of bytes from the file.
-	 * Normally, a file size mismatch would be caught in verify_backup_file
+	 * Normally, a file size mismatch would be caught in verify_manifest_entry
 	 * and this check would never be reached, but this provides additional
 	 * safety and clarity in the event of concurrent modifications or
 	 * filesystem misbehavior.
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index d8c566ed587..ff9476e356e 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -37,7 +37,8 @@ typedef struct manifest_file
 } manifest_file;
 
 #define should_verify_checksum(m) \
-	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
+	(((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
 
 /*
  * Define a hash table which we can use to store information about the files
@@ -93,6 +94,9 @@ typedef struct verifier_context
 	bool		saw_any_error;
 } verifier_context;
 
+extern manifest_file *verify_manifest_entry(verifier_context *context,
+											char *relpath, int64 filesize);
+
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
 			pg_attribute_printf(2, 3);
-- 
2.18.0

From 01286f76085dcaf72238eb6ad0a80f38d9ee05a3 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 14 Aug 2024 15:14:15 +0530
Subject: [PATCH v14 07/12] Refactor: split verify_file_checksum() function.

Move the core functionality of verify_file_checksum to a new function
to reuse it instead of duplicating the code.

The verify_file_checksum() function is designed to take a file path,
open and read the file contents, and then calculate the checksum.
However, for TAR backups, instead of a file path, we receive the file
content in chunks, and the checksum needs to be calculated
incrementally. While the initial operations for plain and TAR backup
checksum calculations differ, the final checks and error handling are
the same. By moving the shared logic to a separate function, we can
reuse the code for both types of backups.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 20 +++++++++++++++++---
 src/bin/pg_verifybackup/pg_verifybackup.h |  3 +++
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 5bfc98e7874..e44d0377cd5 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -792,8 +792,6 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	int			fd;
 	int			rc;
 	size_t		bytes_read = 0;
-	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
-	int			checksumlen;
 
 	/* Open the target file. */
 	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
@@ -844,6 +842,22 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	if (rc < 0)
 		return;
 
+	/* Do the final computation and verification. */
+	verify_checksum(context, m, &checksum_ctx, bytes_read);
+}
+
+/*
+ * A helper function to finalize checksum computation and verify it against the
+ * backup manifest information.
+ */
+void
+verify_checksum(verifier_context *context, manifest_file *m,
+				pg_checksum_context *checksum_ctx, int64 bytes_read)
+{
+	const char *relpath = m->pathname;
+	int			checksumlen;
+	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
+
 	/*
 	 * Double-check that we read the expected number of bytes from the file.
 	 * Normally, a file size mismatch would be caught in verify_manifest_entry
@@ -860,7 +874,7 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	}
 
 	/* Get the final checksum. */
-	checksumlen = pg_checksum_final(&checksum_ctx, checksumbuf);
+	checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
 	if (checksumlen < 0)
 	{
 		report_backup_error(context,
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index ff9476e356e..fe0ce8a89aa 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -96,6 +96,9 @@ typedef struct verifier_context
 
 extern manifest_file *verify_manifest_entry(verifier_context *context,
 											char *relpath, int64 filesize);
+extern void verify_checksum(verifier_context *context, manifest_file *m,
+							pg_checksum_context *checksum_ctx,
+							int64 bytes_read);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From 616f5a911dbc4c82e999d3b64a98bec7c9f557a8 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 21 Aug 2024 10:51:23 +0530
Subject: [PATCH v14 08/12] Refactor: split verify_control_file.

Separated the manifest entry verification code into a new function and
introduced the should_verify_control_data() macro, similar to
should_verify_checksum().

Like verify_file_checksum(), verify_control_file() is too design to
accept the pg_control file patch which will be opened and respective
information will be verified. But, in case of tar backup we would be
having pg_control file contents instead, that needs to be verified in
the same way.  For that reason the code that doing the verification is
separated into separate function to so that can be reused for the tar
backup verification as well.
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 42 ++++++++++-------------
 src/bin/pg_verifybackup/pg_verifybackup.h | 14 ++++++++
 2 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index e44d0377cd5..d04e1d8c8ac 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -66,8 +66,6 @@ static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_plain_backup_file(verifier_context *context,
 									 char *relpath, char *fullpath);
-static void verify_control_file(const char *controlpath,
-								uint64 manifest_system_identifier);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -631,14 +629,20 @@ verify_plain_backup_file(verifier_context *context, char *relpath,
 	/* Check the backup manifest entry for this file. */
 	m = verify_manifest_entry(context, relpath, sb.st_size);
 
-	/*
-	 * Validate the manifest system identifier, not available in manifest
-	 * version 1.
-	 */
-	if (context->manifest->version != 1 &&
-		strcmp(relpath, "global/pg_control") == 0 &&
-		m != NULL && m->matched && !m->bad)
-		verify_control_file(fullpath, context->manifest->system_identifier);
+	/* Validate the pg_control information */
+	if (should_verify_control_data(context->manifest, m))
+	{
+		ControlFileData *control_file;
+		bool		crc_ok;
+
+		pg_log_debug("reading \"%s\"", fullpath);
+		control_file = get_controlfile_by_exact_path(fullpath, &crc_ok);
+
+		verify_control_data(control_file, fullpath, crc_ok,
+							context->manifest->system_identifier);
+		/* Release memory. */
+		pfree(control_file);
+	}
 
 	/* Update statistics for progress report, if necessary */
 	if (show_progress && !context->skip_checksums &&
@@ -687,18 +691,13 @@ verify_manifest_entry(verifier_context *context, char *relpath, int64 filesize)
 }
 
 /*
- * Sanity check control file and validate system identifier against manifest
- * system identifier.
+ * Sanity check control file data and validate system identifier against
+ * manifest system identifier.
  */
-static void
-verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
+void
+verify_control_data(ControlFileData *control_file, const char *controlpath,
+					bool crc_ok, uint64 manifest_system_identifier)
 {
-	ControlFileData *control_file;
-	bool		crc_ok;
-
-	pg_log_debug("reading \"%s\"", controlpath);
-	control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
-
 	/* Control file contents not meaningful if CRC is bad. */
 	if (!crc_ok)
 		report_fatal_error("%s: CRC is incorrect", controlpath);
@@ -714,9 +713,6 @@ verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
 						   controlpath,
 						   (unsigned long long) manifest_system_identifier,
 						   (unsigned long long) control_file->system_identifier);
-
-	/* Release memory. */
-	pfree(control_file);
 }
 
 /*
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index fe0ce8a89aa..818064c6eed 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -40,6 +40,17 @@ typedef struct manifest_file
 	(((m) != NULL) && ((m)->matched) && !((m)->bad) && \
 	 (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
 
+/*
+ * Validate the control file and its system identifier against the manifest
+ * system identifier. Note that this feature is not available in manifest
+ * version 1. This validation should only be performed after the manifest entry
+ * validation for the pg_control file has been completed without errors.
+ */
+#define should_verify_control_data(manifest, m) \
+	(((manifest)->version != 1) && \
+	 ((m) != NULL) && ((m)->matched) && !((m)->bad) && \
+	 (strcmp((m)->pathname, "global/pg_control") == 0))
+
 /*
  * Define a hash table which we can use to store information about the files
  * mentioned in the backup manifest.
@@ -99,6 +110,9 @@ extern manifest_file *verify_manifest_entry(verifier_context *context,
 extern void verify_checksum(verifier_context *context, manifest_file *m,
 							pg_checksum_context *checksum_ctx,
 							int64 bytes_read);
+extern void verify_control_data(ControlFileData *control_file,
+								const char *controlpath, bool crc_ok,
+								uint64 manifest_system_identifier);
 
 extern void report_backup_error(verifier_context *context,
 								const char *pg_restrict fmt,...)
-- 
2.18.0

From 8b60229b2ae59e7c01a2fdb83614b1aa362d52ab Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 8 Aug 2024 16:01:33 +0530
Subject: [PATCH v14 09/12] Add simple_ptr_list_destroy() and
 simple_ptr_list_destroy_deep() API.

We didn't have any helper function to destroy SimplePtrList, likely
because it wasn't needed before, but it's required in a later patch in
this set.  I've added two functions for this purpose, inspired by
list_free() and list_free_deep().
---
 src/fe_utils/simple_list.c         | 19 +++++++++++++++++++
 src/include/fe_utils/simple_list.h |  1 +
 2 files changed, 20 insertions(+)

diff --git a/src/fe_utils/simple_list.c b/src/fe_utils/simple_list.c
index 2d88eb54067..c07e6bd9180 100644
--- a/src/fe_utils/simple_list.c
+++ b/src/fe_utils/simple_list.c
@@ -173,3 +173,22 @@ simple_ptr_list_append(SimplePtrList *list, void *ptr)
 		list->head = cell;
 	list->tail = cell;
 }
+
+/*
+ * Destroy only pointer list and not the pointed-to element
+ */
+void
+simple_ptr_list_destroy(SimplePtrList *list)
+{
+	SimplePtrListCell *cell;
+
+	cell = list->head;
+	while (cell != NULL)
+	{
+		SimplePtrListCell *next;
+
+		next = cell->next;
+		pg_free(cell);
+		cell = next;
+	}
+}
diff --git a/src/include/fe_utils/simple_list.h b/src/include/fe_utils/simple_list.h
index d42ecded8ed..c83ab6f77e4 100644
--- a/src/include/fe_utils/simple_list.h
+++ b/src/include/fe_utils/simple_list.h
@@ -66,5 +66,6 @@ extern void simple_string_list_destroy(SimpleStringList *list);
 extern const char *simple_string_list_not_touched(SimpleStringList *list);
 
 extern void simple_ptr_list_append(SimplePtrList *list, void *ptr);
+extern void simple_ptr_list_destroy(SimplePtrList *list);
 
 #endif							/* SIMPLE_LIST_H */
-- 
2.18.0

From d253d21df5f3137461e6410b2ee590ca052a3aea Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 21 Aug 2024 11:03:44 +0530
Subject: [PATCH v14 10/12] pg_verifybackup: Add backup format and compression
 option

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the next patch that implements tar format support.
----
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 68 ++++++++++++++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h |  1 +
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index d04e1d8c8ac..c1542983b93 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -62,6 +62,7 @@ static void report_manifest_error(JsonManifestParseContext *context,
 								  const char *fmt,...)
 			pg_attribute_printf(2, 3) pg_attribute_noreturn();
 
+static char find_backup_format(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_plain_backup_file(verifier_context *context,
@@ -97,6 +98,7 @@ main(int argc, char **argv)
 		{"exit-on-error", no_argument, NULL, 'e'},
 		{"ignore", required_argument, NULL, 'i'},
 		{"manifest-path", required_argument, NULL, 'm'},
+		{"format", required_argument, NULL, 'F'},
 		{"no-parse-wal", no_argument, NULL, 'n'},
 		{"progress", no_argument, NULL, 'P'},
 		{"quiet", no_argument, NULL, 'q'},
@@ -118,6 +120,7 @@ main(int argc, char **argv)
 	progname = get_progname(argv[0]);
 
 	memset(&context, 0, sizeof(context));
+	context.format = '\0';
 
 	if (argc > 1)
 	{
@@ -154,7 +157,7 @@ main(int argc, char **argv)
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
 
-	while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1)
+	while ((c = getopt_long(argc, argv, "eF:i:m:nPqsw:", long_options, NULL)) != -1)
 	{
 		switch (c)
 		{
@@ -173,6 +176,15 @@ main(int argc, char **argv)
 				manifest_path = pstrdup(optarg);
 				canonicalize_path(manifest_path);
 				break;
+			case 'F':
+				if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
+					context.format = 'p';
+				else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
+					context.format = 't';
+				else
+					pg_fatal("invalid backup format \"%s\", must be \"plain\" or \"tar\"",
+							 optarg);
+				break;
 			case 'n':
 				no_parse_wal = true;
 				break;
@@ -261,6 +273,21 @@ main(int argc, char **argv)
 	 */
 	context.manifest = parse_manifest_file(manifest_path);
 
+	/* Determine the backup format if it hasn't been specified. */
+	if (context.format == '\0')
+		context.format = find_backup_format(&context);
+
+	/*
+	 * XXX: In the future, we should consider enhancing pg_waldump to read
+	 * WAL files from the tar file.
+	 */
+	if (!no_parse_wal && context.format == 't')
+	{
+		pg_log_error("pg_waldump cannot read tar files");
+		pg_log_error_hint("You must use -n or --no-parse-wal when verifying a tar-format backup.");
+		exit(1);
+	}
+
 	/*
 	 * Now scan the files in the backup directory. At this stage, we verify
 	 * that every file on disk is present in the manifest and that the sizes
@@ -279,8 +306,13 @@ main(int argc, char **argv)
 	/*
 	 * Now do the expensive work of verifying file checksums, unless we were
 	 * told to skip it.
+	 *
+	 * We were only checking the plain backup here. For the tar backup, file
+	 * checksums verification (if requested) will be done immediately when the
+	 * file is accessed, as we don't have random access to the files like we
+	 * do with plain backups.
 	 */
-	if (!context.skip_checksums)
+	if (!context.skip_checksums && context.format == 'p')
 		verify_backup_checksums(&context);
 
 	/*
@@ -981,6 +1013,37 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
 	return false;
 }
 
+/*
+ * To detect the backup format, it checks for the PG_VERSION file in the backup
+ * directory. If found, it will be considered a plain-format backup; otherwise,
+ * it will be assumed to be a tar-format backup.
+ */
+static char
+find_backup_format(verifier_context *context)
+{
+	char		result;
+	char	   *path;
+	struct stat sb;
+
+	/* Should be here only if the backup format is unknown */
+	Assert(context->format == '\0');
+
+	/* Check PG_VERSION file. */
+	path = psprintf("%s/%s", context->backup_directory, "PG_VERSION");
+	if (stat(path, &sb) == 0)
+		result = 'p';
+	else if (errno == ENOENT)
+		result = 't';
+	else
+	{
+		pg_log_error("could not stat file \"%s\": %m", path);
+		exit(1);
+	}
+	pfree(path);
+
+	return result;
+}
+
 /*
  * Print a progress report based on the global variables.
  *
@@ -1038,6 +1101,7 @@ usage(void)
 	printf(_("  -e, --exit-on-error         exit immediately on error\n"));
 	printf(_("  -i, --ignore=RELATIVE_PATH  ignore indicated path\n"));
 	printf(_("  -m, --manifest-path=PATH    use specified path for manifest\n"));
+	printf(_("  -F, --format=p|t            backup format (plain, tar)\n"));
 	printf(_("  -n, --no-parse-wal          do not try to parse WAL files\n"));
 	printf(_("  -P, --progress              show progress information\n"));
 	printf(_("  -q, --quiet                 do not print any output, except for errors\n"));
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 818064c6eed..80031ad4dbc 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -100,6 +100,7 @@ typedef struct verifier_context
 	manifest_data *manifest;
 	char	   *backup_directory;
 	SimpleStringList ignore_list;
+	char		format;			/* backup format:  p(lain)/t(ar) */
 	bool		skip_checksums;
 	bool		exit_on_error;
 	bool		saw_any_error;
-- 
2.18.0

From 68a85869b9a8c3b872035661df85976c890116f4 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 21 Aug 2024 12:49:04 +0530
Subject: [PATCH v14 11/12] pg_verifybackup: Read tar files and verify its
 contents

This patch implements TAR format backup verification.

For progress reporting support, we perform this verification in two
passes: the first pass calculates total_size, and the second pass
updates done_size as verification progresses.

For the verification, in the first pass, we call precheck_tar_backup_file(),
which performs basic verification by expecting only base.tar, pg_wal.tar, or
<tablespaceoid>.tar files and raises an error for any other files.  It
also determines the compression type of the archive file. All this
information is stored in a newly added tarFile struct, which is
appended to a list that will be used in the second pass for the final
verification. In the second pass, the tar archives are read,
decompressed, and the required verification is carried out.

For reading and decompression, fe_utils/astreamer.h is used. For
verification, a new archive streamer has been added in
astreamer_verify.c to handle TAR member files and their contents; see
astreamer_verify_content() for details. The stack of astreamers will
be set up for each TAR file in verify_tar_content(), depending on its
compression type which is detected in the first pass.

When information about a TAR member file (i.e., ASTREAMER_MEMBER_HEADER)
is received, we first verify its entry against the backup manifest. We
then decide if further checks are needed, such as checksum
verification and control data verification (if it is a pg_control
file), once the member file contents are received. Although this
decision could be made when the contents are received, it is more
efficient to make it earlier since the member file contents are
received in multiple iterations. In short, we process
ASTREAMER_MEMBER_CONTENTS multiple times but only once for other
ASTREAMER_MEMBER_* cases. We maintain this information in the
astreamer_verify structure for each member file, which is reset when
the file ends.

Unlike in a plain backup, checksum verification here occurs in two
steps. First, as the contents are received, the checksum is computed
incrementally (see member_compute_checksum). Then, at the end of
processing the member file, the final verification is performed (see
member_verify_checksum).

Similarly, during the content receiving stage, if the file is
pg_control, the data will be copied into a local buffer (see
member_copy_control_data).  The verification will then be carried out
at the end of the member file processing (see member_verify_control_data)
---
 src/bin/pg_verifybackup/Makefile           |   2 +
 src/bin/pg_verifybackup/astreamer_verify.c | 358 +++++++++++++++++++++
 src/bin/pg_verifybackup/meson.build        |   1 +
 src/bin/pg_verifybackup/pg_verifybackup.c  | 326 ++++++++++++++++++-
 src/bin/pg_verifybackup/pg_verifybackup.h  |   6 +
 src/tools/pgindent/typedefs.list           |   2 +
 6 files changed, 692 insertions(+), 3 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/astreamer_verify.c

diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile
index 7c045f142e8..374d4a8afd1 100644
--- a/src/bin/pg_verifybackup/Makefile
+++ b/src/bin/pg_verifybackup/Makefile
@@ -17,10 +17,12 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 # We need libpq only because fe_utils does.
+override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 
 OBJS = \
 	$(WIN32RES) \
+	astreamer_verify.o \
 	pg_verifybackup.o
 
 all: pg_verifybackup
diff --git a/src/bin/pg_verifybackup/astreamer_verify.c b/src/bin/pg_verifybackup/astreamer_verify.c
new file mode 100644
index 00000000000..b496e9320ea
--- /dev/null
+++ b/src/bin/pg_verifybackup/astreamer_verify.c
@@ -0,0 +1,358 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer_verify.c
+ *
+ * Extend fe_utils/astreamer.h archive streaming facility to verify TAR
+ * format backup.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * src/bin/pg_verifybackup/astreamer_verify.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include "pg_verifybackup.h"
+
+typedef struct astreamer_verify
+{
+	astreamer	base;
+	verifier_context *context;
+	char	   *archive_name;
+	Oid			tblspc_oid;
+	pg_checksum_context *checksum_ctx;
+
+	/* Hold information for a member file verification */
+	manifest_file *mfile;
+	int64		received_bytes;
+	bool		verify_checksum;
+	bool		verify_control_data;
+} astreamer_verify;
+
+static void astreamer_verify_content(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len,
+									 astreamer_archive_context context);
+static void astreamer_verify_finalize(astreamer *streamer);
+static void astreamer_verify_free(astreamer *streamer);
+
+static void member_verify_header(astreamer *streamer, astreamer_member *member);
+static void member_compute_checksum(astreamer *streamer,
+									astreamer_member *member,
+									const char *data, int len);
+static void member_verify_checksum(astreamer *streamer);
+static void member_copy_control_data(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len);
+static void member_verify_control_data(astreamer *streamer);
+static void member_reset_info(astreamer *streamer);
+
+static const astreamer_ops astreamer_verify_ops = {
+	.content = astreamer_verify_content,
+	.finalize = astreamer_verify_finalize,
+	.free = astreamer_verify_free
+};
+
+/*
+ * Create a astreamer that can verifies content of a TAR file.
+ */
+astreamer *
+astreamer_verify_content_new(astreamer *next, verifier_context *context,
+							 char *archive_name, Oid tblspc_oid)
+{
+	astreamer_verify *streamer;
+
+	streamer = palloc0(sizeof(astreamer_verify));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_verify_ops;
+
+	streamer->base.bbs_next = next;
+	streamer->context = context;
+	streamer->archive_name = archive_name;
+	streamer->tblspc_oid = tblspc_oid;
+	initStringInfo(&streamer->base.bbs_buffer);
+
+	if (!context->skip_checksums)
+		streamer->checksum_ctx = pg_malloc(sizeof(pg_checksum_context));
+
+	return &streamer->base;
+}
+
+/*
+ * The main entry point of the archive streamer for verifying tar members.
+ */
+static void
+astreamer_verify_content(astreamer *streamer, astreamer_member *member,
+						 const char *data, int len,
+						 astreamer_archive_context context)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	Assert(context != ASTREAMER_UNKNOWN);
+
+	switch (context)
+	{
+		case ASTREAMER_MEMBER_HEADER:
+
+			/*
+			 * Perform the initial check and setup verification steps.
+			 */
+			member_verify_header(streamer, member);
+			break;
+
+		case ASTREAMER_MEMBER_CONTENTS:
+
+			/*
+			 * Since we are receiving the member content in chunks, it must be
+			 * processed according to the flags set by the member header
+			 * processing routine. This includes performing incremental
+			 * checksum computations and copying control data to the local
+			 * buffer.
+			 */
+			if (mystreamer->verify_checksum)
+				member_compute_checksum(streamer, member, data, len);
+
+			if (mystreamer->verify_control_data)
+				member_copy_control_data(streamer, member, data, len);
+			break;
+
+		case ASTREAMER_MEMBER_TRAILER:
+
+			/*
+			 * We have reached the end of the member file. By this point, we
+			 * should have successfully computed the checksum of the received
+			 * content and copied the entire pg_control file data into our
+			 * local buffer. We can now proceed with the final verification.
+			 */
+			if (mystreamer->verify_checksum)
+				member_verify_checksum(streamer);
+
+			if (mystreamer->verify_control_data)
+				member_verify_control_data(streamer);
+
+			/*
+			 * Reset the temporary information stored for the verification.
+			 */
+			member_reset_info(streamer);
+			break;
+
+		case ASTREAMER_ARCHIVE_TRAILER:
+			break;
+
+		default:
+			/* Shouldn't happen. */
+			pg_fatal("unexpected state while parsing tar file");
+	}
+}
+
+/*
+ * End-of-stream processing for a astreamer_verify stream.
+ */
+static void
+astreamer_verify_finalize(astreamer *streamer)
+{
+	Assert(streamer->bbs_next == NULL);
+}
+
+/*
+ * Free memory associated with a astreamer_verify stream.
+ */
+static void
+astreamer_verify_free(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	if (mystreamer->checksum_ctx)
+		pfree(mystreamer->checksum_ctx);
+
+	pfree(streamer->bbs_buffer.data);
+	pfree(streamer);
+}
+
+/*
+ * Verifies whether the tar member entry exists in the backup manifest.
+ *
+ * If the archive being processed is a tablespace, it prepares the necessary
+ * file path first. If a valid entry is found in the backup manifest, it then
+ * determines whether checksum and control data verification should be
+ * performed during file content processing.
+ */
+static void
+member_verify_header(astreamer *streamer, astreamer_member *member)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_file *m;
+	char		pathname[MAXPGPATH];
+
+	/* We are only interested in normal files. */
+	if (member->is_directory || member->is_link)
+		return;
+
+	/*
+	 * The backup manifest stores a relative path to the base directory for
+	 * files belonging to a tablespace, while the tablespace backup tar
+	 * archive does not include this path. Ensure the required path is
+	 * prepared; otherwise, the manifest entry verification will fail.
+	 */
+	if (OidIsValid(mystreamer->tblspc_oid))
+		snprintf(pathname, MAXPGPATH, "%s/%u/%s",
+				 "pg_tblspc", mystreamer->tblspc_oid, member->pathname);
+	else
+		memcpy(pathname, member->pathname, MAXPGPATH);
+
+
+	/* Ignore any files that are listed in the ignore list. */
+	if (should_ignore_relpath(mystreamer->context, pathname))
+		return;
+
+	/* Check the manifest entry */
+	m = verify_manifest_entry(mystreamer->context, pathname,
+							  member->size);
+	mystreamer->mfile = m;
+
+	/* Prepare for checksum and control data verification. */
+	mystreamer->verify_checksum =
+		(!mystreamer->context->skip_checksums && should_verify_checksum(m));
+	mystreamer->verify_control_data =
+		should_verify_control_data(mystreamer->context->manifest, m);
+
+	/* Initialize the context required for checksum verification. */
+	if (mystreamer->verify_checksum &&
+		pg_checksum_init(mystreamer->checksum_ctx, m->checksum_type) < 0)
+	{
+		report_backup_error(mystreamer->context,
+							"%s: could not initialize checksum of file \"%s\"",
+							mystreamer->archive_name, m->pathname);
+
+		/*
+		 * Checksum verification cannot be performed without proper context
+		 * initialization.
+		 */
+		mystreamer->verify_checksum = false;
+	}
+}
+
+/*
+ * Computes the checksum incrementally for the received file content.
+ *
+ * Should have a correctly initialized checksum_ctx, which will be used for
+ * incremental checksum computation.
+ */
+static void
+member_compute_checksum(astreamer *streamer, astreamer_member *member,
+						const char *data, int len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	pg_checksum_context *checksum_ctx = mystreamer->checksum_ctx;
+	manifest_file *m = mystreamer->mfile;
+
+	Assert(mystreamer->verify_checksum);
+
+	/*
+	 * Should have been applied to the correct file. Note that strcmp() cannot
+	 * be used because the member pathname (if it belongs to a tablespace) is
+	 * not relative to the base directory, unlike the backup manifest. For
+	 * more details, see member_verify_header().
+	 */
+	Assert(should_verify_checksum(m));
+	Assert(m->checksum_type == checksum_ctx->type);
+	Assert(strstr(m->pathname, member->pathname));
+
+	/*
+	 * Update the total count of computed checksum bytes for cross-checking
+	 * with the file size in the final verification stage.
+	 */
+	mystreamer->received_bytes += len;
+
+	if (pg_checksum_update(checksum_ctx, (uint8 *) data, len) < 0)
+	{
+		report_backup_error(mystreamer->context,
+							"could not update checksum of file \"%s\"",
+							m->pathname);
+		mystreamer->verify_checksum = false;
+	}
+}
+
+/*
+ * Perform the final computation and checksum verification after the entire
+ * file content has been processed.
+ */
+static void
+member_verify_checksum(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	Assert(mystreamer->verify_checksum);
+
+	verify_checksum(mystreamer->context, mystreamer->mfile,
+					mystreamer->checksum_ctx, mystreamer->received_bytes);
+}
+
+/*
+ * Stores the pg_control file contents into a local buffer; we need the entire
+ * control file data for verification.
+ */
+static void
+member_copy_control_data(astreamer *streamer, astreamer_member *member,
+						 const char *data, int len)
+{
+	/* Should be here only for control file */
+	Assert(strcmp(member->pathname, "global/pg_control") == 0);
+	Assert(((astreamer_verify *) streamer)->verify_control_data);
+
+	/* Copy enough control file data needed for verification. */
+	astreamer_buffer_until(streamer, &data, &len, sizeof(ControlFileData));
+}
+
+/*
+ * Performs the CRC calculation of pg_control data and then calls the routines
+ * that execute the final verification of the control file information.
+ */
+static void
+member_verify_control_data(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_data *manifest = mystreamer->context->manifest;
+	ControlFileData *control_file;
+	pg_crc32c	crc;
+	bool		crc_ok;
+
+	/* Should be here only for control file */
+	Assert(strcmp(mystreamer->mfile->pathname, "global/pg_control") == 0);
+	Assert(mystreamer->verify_control_data);
+
+	/* Should have enough control file data needed for verification. */
+	if (streamer->bbs_buffer.len != sizeof(ControlFileData))
+		report_fatal_error("%s: unexpected control file size: %d, should be %zu",
+						   mystreamer->archive_name, streamer->bbs_buffer.len,
+						   sizeof(ControlFileData));
+
+	control_file = (ControlFileData *) streamer->bbs_buffer.data;
+
+	/* Check the CRC. */
+	INIT_CRC32C(crc);
+	COMP_CRC32C(crc, (char *) (control_file), offsetof(ControlFileData, crc));
+	FIN_CRC32C(crc);
+
+	crc_ok = EQ_CRC32C(crc, control_file->crc);
+
+	/* Do the final control data verification. */
+	verify_control_data(control_file, mystreamer->mfile->pathname, crc_ok,
+						manifest->system_identifier);
+}
+
+/*
+ * Reset flags and free memory allocations for member file verification.
+ */
+static void
+member_reset_info(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	mystreamer->mfile = NULL;
+	mystreamer->received_bytes = 0;
+	mystreamer->verify_checksum = false;
+	mystreamer->verify_control_data = false;
+}
diff --git a/src/bin/pg_verifybackup/meson.build b/src/bin/pg_verifybackup/meson.build
index 7c7d31a0350..0e09d1379d1 100644
--- a/src/bin/pg_verifybackup/meson.build
+++ b/src/bin/pg_verifybackup/meson.build
@@ -1,6 +1,7 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 pg_verifybackup_sources = files(
+  'astreamer_verify.c',
   'pg_verifybackup.c'
 )
 
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index c1542983b93..e63a0ed0798 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -22,6 +22,7 @@
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
+#include "limits.h"
 #include "pg_verifybackup.h"
 #include "pgtime.h"
 
@@ -44,6 +45,16 @@
  */
 #define READ_CHUNK_SIZE				(128 * 1024)
 
+/*
+ * Tar file information needed for content verification.
+ */
+typedef struct tar_file
+{
+	char	   *relpath;
+	Oid			tblspc_oid;
+	pg_compress_algorithm compress_algorithm;
+} tar_file;
+
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -63,10 +74,16 @@ static void report_manifest_error(JsonManifestParseContext *context,
 			pg_attribute_printf(2, 3) pg_attribute_noreturn();
 
 static char find_backup_format(verifier_context *context);
+static void verify_plain_backup(verifier_context *context);
+static void verify_tar_backup(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
-static void verify_plain_backup_file(verifier_context *context,
-									 char *relpath, char *fullpath);
+static void verify_plain_backup_file(verifier_context *context, char *relpath,
+									 char *fullpath);
+static void precheck_tar_backup_file(verifier_context *context, char *relpath,
+									 char *fullpath, SimplePtrList *tarfiles);
+static void verify_tar_file(verifier_context *context, char *relpath,
+							char *fullpath, astreamer *streamer);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -75,6 +92,10 @@ static void verify_file_checksum(verifier_context *context,
 static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
+static astreamer *create_archive_verifier(verifier_context *context,
+										  char *archive_name,
+										  Oid tblspc_oid,
+										  pg_compress_algorithm compress_algo);
 
 static void progress_report(bool finished);
 static void usage(void);
@@ -294,7 +315,10 @@ main(int argc, char **argv)
 	 * match. We also set the "matched" flag on every manifest entry that
 	 * corresponds to a file on disk.
 	 */
-	verify_backup_directory(&context, NULL, context.backup_directory);
+	if (context.format == 'p')
+		verify_plain_backup(&context);
+	else
+		verify_tar_backup(&context);
 
 	/*
 	 * The "matched" flag should now be set on every entry in the hash table.
@@ -546,6 +570,16 @@ verifybackup_per_wal_range_cb(JsonManifestParseContext *context,
 	manifest->last_wal_range = range;
 }
 
+/*
+ * Verify plain backup.
+ */
+static void
+verify_plain_backup(verifier_context *context)
+{
+	Assert(context->format == 'p');
+	verify_backup_directory(context, NULL, context->backup_directory);
+}
+
 /*
  * Verify one directory.
  *
@@ -682,6 +716,257 @@ verify_plain_backup_file(verifier_context *context, char *relpath,
 		total_size += m->size;
 }
 
+/*
+ * Verify tar backup.
+ *
+ * Unlike plan backup verification, tar backup verification carried out in two
+ * passes; in the first pass this would simply sanity check on expected tar file
+ * to be present in the backup directory and it's compression type and collect
+ * these information is list. In the second pass, the tar archives are read,
+ * decompressed, and the required verification is carried out.
+ */
+static void
+verify_tar_backup(verifier_context *context)
+{
+	DIR		   *dir;
+	struct dirent *dirent;
+	SimplePtrList tarfiles = {NULL, NULL};
+	SimplePtrListCell *cell;
+	char	   *fullpath;
+
+	Assert(context->format == 't');
+
+	progress_report(false);
+
+	/*
+	 * If the backup directory cannot be found, treat this as a fatal error.
+	 */
+	fullpath = context->backup_directory;
+	dir = opendir(fullpath);
+	if (dir == NULL)
+		report_fatal_error("could not open directory \"%s\": %m", fullpath);
+
+	while (errno = 0, (dirent = readdir(dir)) != NULL)
+	{
+		char	   *filename = dirent->d_name;
+		char	   *newfullpath = psprintf("%s/%s", fullpath, filename);
+
+		/* Skip "." and ".." */
+		if (filename[0] == '.' && (filename[1] == '\0'
+								   || strcmp(filename, "..") == 0))
+			continue;
+
+		/* First pass: Collect valid tar files from the backup. */
+		if (!should_ignore_relpath(context, filename))
+			precheck_tar_backup_file(context, filename, newfullpath,
+									 &tarfiles);
+
+		pfree(newfullpath);
+	}
+
+	if (closedir(dir))
+	{
+		report_backup_error(context,
+							"could not close directory \"%s\": %m", fullpath);
+		return;
+	}
+
+	/* Second pass: Perform the final verification of the tar contents. */
+	for (cell = tarfiles.head; cell != NULL; cell = cell->next)
+	{
+		tar_file   *tar = (tar_file *) cell->ptr;
+		astreamer  *streamer;
+
+		/*
+		 * Prepares the archive streamer stack according to the tar
+		 * compression format.
+		 */
+		streamer = create_archive_verifier(context,
+										   tar->relpath,
+										   tar->tblspc_oid,
+										   tar->compress_algorithm);
+
+		/* Compute the full pathname to the target file. */
+		fullpath = psprintf("%s/%s", context->backup_directory,
+							tar->relpath);
+
+		/* Invoke the streamer for reading, decompressing, and verifying. */
+		verify_tar_file(context, tar->relpath, fullpath, streamer);
+
+		/* Cleanup. */
+		pfree(tar->relpath);
+		pfree(tar);
+		pfree(fullpath);
+
+		astreamer_finalize(streamer);
+		astreamer_free(streamer);
+	}
+	simple_ptr_list_destroy(&tarfiles);
+
+	progress_report(true);
+}
+
+/*
+ * Preparatory steps for verifying files in tar format backups.
+ *
+ * Carries out basic validation of the tar format backup file, detects the
+ * compression type, and appends that information to the tarfiles list. An
+ * error will be reported if the tar file is inaccessible, or if the file type,
+ * name, or compression type is not as expected.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_plain_backup_file. The additional argument outputs a list of valid
+ * tar files.
+ */
+static void
+precheck_tar_backup_file(verifier_context *context, char *relpath,
+						 char *fullpath, SimplePtrList *tarfiles)
+{
+	struct stat sb;
+	Oid			tblspc_oid = InvalidOid;
+	pg_compress_algorithm compress_algorithm;
+	tar_file   *tar;
+	char	   *suffix = NULL;
+
+	/* Should be tar format backup */
+	Assert(context->format == 't');
+
+	/* Get file information */
+	if (stat(fullpath, &sb) != 0)
+	{
+		report_backup_error(context,
+							"could not stat file or directory \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	/* In a tar format backup, we expect only plain files. */
+	if (!S_ISREG(sb.st_mode))
+	{
+		report_backup_error(context,
+							"\"%s\" is not a plain file",
+							relpath);
+		return;
+	}
+
+	/*
+	 * We expect tar files for backing up the main directory, tablespace, and
+	 * pg_wal directory.
+	 *
+	 * pg_basebackup writes the main data directory to an archive file named
+	 * base.tar, the pg_wal directory to pg_wal.tar, and the tablespace
+	 * directory to <tablespaceoid>.tar, each followed by a compression type
+	 * extension such as .gz, .lz4, or .zst.
+	 */
+	if (strncmp("base", relpath, 4) == 0)
+		suffix = relpath + 4;
+	else if (strncmp("pg_wal", relpath, 6) == 0)
+		suffix = relpath + 6;
+	else
+	{
+		/* Expected a <tablespaceoid>.tar file here. */
+		uint64		num = strtoul(relpath, &suffix, 10);
+
+		/*
+		 * Report an error if we didn't consume at least one character, if the
+		 * result is 0, or if the value is too large to be a valid OID.
+		 */
+		if (suffix == NULL || num <= 0 || num > OID_MAX)
+			report_backup_error(context,
+								"file \"%s\" is not expected in a tar format backup",
+								relpath);
+		tblspc_oid = (Oid) num;
+	}
+
+	/* Now, check the compression type of the tar */
+	if (strcmp(suffix, ".tar") == 0)
+		compress_algorithm = PG_COMPRESSION_NONE;
+	else if (strcmp(suffix, ".tgz") == 0)
+		compress_algorithm = PG_COMPRESSION_GZIP;
+	else if (strcmp(suffix, ".tar.gz") == 0)
+		compress_algorithm = PG_COMPRESSION_GZIP;
+	else if (strcmp(suffix, ".tar.lz4") == 0)
+		compress_algorithm = PG_COMPRESSION_LZ4;
+	else if (strcmp(suffix, ".tar.zst") == 0)
+		compress_algorithm = PG_COMPRESSION_ZSTD;
+	else
+	{
+		report_backup_error(context,
+							"file \"%s\" is not expected in a tar format backup",
+							relpath);
+		return;
+	}
+
+	/*
+	 * Ignore WALs, as reading and verification will be handled through
+	 * pg_waldump.
+	 */
+	if (strncmp("pg_wal", relpath, 6) == 0)
+		return;
+
+	/*
+	 * Append the information to the list for complete verification at a later
+	 * stage.
+	 */
+	tar = pg_malloc(sizeof(tar_file));
+	tar->relpath = pstrdup(relpath);
+	tar->tblspc_oid = tblspc_oid;
+	tar->compress_algorithm = compress_algorithm;
+
+	simple_ptr_list_append(tarfiles, tar);
+
+	/* Update statistics for progress report, if necessary */
+	if (show_progress)
+		total_size += sb.st_size;
+}
+
+/*
+ * Verification of a single tar file content.
+ *
+ * It reads a given tar archive in predefined chunks and passes it to the
+ * streamer, which initiates routines for decompression (if necessary) and then
+ * verifies each member within the tar file.
+ */
+static void
+verify_tar_file(verifier_context *context, char *relpath, char *fullpath,
+				astreamer *streamer)
+{
+	int			fd;
+	int			rc;
+	char	   *buffer;
+
+	pg_log_debug("reading \"%s\"", fullpath);
+
+	/* Open the target file. */
+	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
+	{
+		report_backup_error(context, "could not open file \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
+
+	/* Perform the reads */
+	while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
+	{
+		astreamer_content(streamer, NULL, buffer, rc, ASTREAMER_UNKNOWN);
+
+		/* Report progress */
+		done_size += rc;
+		progress_report(false);
+	}
+
+	if (rc < 0)
+		report_backup_error(context, "could not read file \"%s\": %m",
+							relpath);
+
+	/* Close the file. */
+	if (close(fd) != 0)
+		report_backup_error(context, "could not close file \"%s\": %m",
+							relpath);
+}
+
 /*
  * Verify file and its size entry in the manifest.
  */
@@ -1044,6 +1329,41 @@ find_backup_format(verifier_context *context)
 	return result;
 }
 
+/*
+ * Identifies the necessary steps for verifying the contents of the
+ * provided tar file.
+ */
+static astreamer *
+create_archive_verifier(verifier_context *context, char *archive_name,
+						Oid tblspc_oid, pg_compress_algorithm compress_algo)
+{
+	astreamer  *streamer = NULL;
+
+	/* Should be here only for tar backup */
+	Assert(context->format == 't');
+
+	/*
+	 * To verify the contents of the tar, the initial step is to parse its
+	 * content.
+	 */
+	streamer = astreamer_verify_content_new(streamer, context, archive_name,
+											tblspc_oid);
+	streamer = astreamer_tar_parser_new(streamer);
+
+	/*
+	 * If the tar is compressed, we must perform the appropriate decompression
+	 * operation before proceeding with the verification of its contents.
+	 */
+	if (compress_algo == PG_COMPRESSION_GZIP)
+		streamer = astreamer_gzip_decompressor_new(streamer);
+	else if (compress_algo == PG_COMPRESSION_LZ4)
+		streamer = astreamer_lz4_decompressor_new(streamer);
+	else if (compress_algo == PG_COMPRESSION_ZSTD)
+		streamer = astreamer_zstd_decompressor_new(streamer);
+
+	return streamer;
+}
+
 /*
  * Print a progress report based on the global variables.
  *
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 80031ad4dbc..be7438af346 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -18,6 +18,7 @@
 #include "common/hashfn_unstable.h"
 #include "common/logging.h"
 #include "common/parse_manifest.h"
+#include "fe_utils/astreamer.h"
 #include "fe_utils/simple_list.h"
 
 /*
@@ -123,4 +124,9 @@ extern void report_fatal_error(const char *pg_restrict fmt,...)
 extern bool should_ignore_relpath(verifier_context *context,
 								  const char *relpath);
 
+extern astreamer *astreamer_verify_content_new(astreamer *next,
+											   verifier_context *context,
+											   char *archive_name,
+											   Oid tblspc_oid);
+
 #endif							/* PG_VERIFYBACKUP_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index e9ebddde24d..2b155586f8c 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3330,6 +3330,7 @@ astreamer_plain_writer
 astreamer_recovery_injector
 astreamer_tar_archiver
 astreamer_tar_parser
+astreamer_verify
 astreamer_zstd_frame
 bgworker_main_type
 bh_node_type
@@ -3951,6 +3952,7 @@ substitute_phv_relids_context
 subxids_array_status
 symbol
 tablespaceinfo
+tar_file
 td_entry
 teSection
 temp_tablespaces_extra
-- 
2.18.0

From aeed7ab45f6c15abae8e4935b61c3659a9ba139e Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Thu, 12 Sep 2024 15:35:49 +0530
Subject: [PATCH v14 12/12] pg_verifybackup: Tests and document

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the previous patch that implements tar format support.
----
---
 doc/src/sgml/ref/pg_verifybackup.sgml         |  45 ++-
 src/bin/pg_verifybackup/t/002_algorithm.pl    |  34 ++-
 src/bin/pg_verifybackup/t/003_corruption.pl   | 267 +++++++++++++++++-
 src/bin/pg_verifybackup/t/004_options.pl      |   2 +-
 src/bin/pg_verifybackup/t/008_untar.pl        |  71 ++---
 src/bin/pg_verifybackup/t/010_client_untar.pl |  48 +---
 6 files changed, 352 insertions(+), 115 deletions(-)

diff --git a/doc/src/sgml/ref/pg_verifybackup.sgml b/doc/src/sgml/ref/pg_verifybackup.sgml
index a3f167f9f6e..8380178ca49 100644
--- a/doc/src/sgml/ref/pg_verifybackup.sgml
+++ b/doc/src/sgml/ref/pg_verifybackup.sgml
@@ -34,8 +34,12 @@ PostgreSQL documentation
    integrity of a database cluster backup taken using
    <command>pg_basebackup</command> against a
    <literal>backup_manifest</literal> generated by the server at the time
-   of the backup.  The backup must be stored in the "plain"
-   format; a "tar" format backup can be checked after extracting it.
+   of the backup. The backup may be stored either in the "plain" or the "tar"
+   format; this includes tar-format backups compressed with any algorithm
+   supported by <application>pg_basebackup</application>. However, at present,
+   <literal>WAL</literal> verification is supported only for plain-format
+   backups. Therefore, if the backup is stored in tar-format, the
+   <literal>-n, --no-parse-wal</literal> option should be used.
   </para>
 
   <para>
@@ -168,6 +172,43 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-F <replaceable class="parameter">format</replaceable></option></term>
+      <term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
+      <listitem>
+       <para>
+        Specifies the format of the backup. <replaceable>format</replaceable>
+        can be one of the following:
+
+        <variablelist>
+         <varlistentry>
+          <term><literal>p</literal></term>
+          <term><literal>plain</literal></term>
+          <listitem>
+           <para>
+            Backup consists of plain files with the same layout as the
+            source server's data directory and tablespaces.
+           </para>
+          </listitem>
+         </varlistentry>
+
+         <varlistentry>
+          <term><literal>t</literal></term>
+          <term><literal>tar</literal></term>
+          <listitem>
+           <para>
+            Backup consists of tar files.  A valid backup includes the main data
+            directory in a file named <filename>base.tar</filename>, the WAL
+            files in <filename>pg_wal.tar</filename>, and separate tar files for
+            each tablespace, named after the tablespace's OID, followed by the
+            compression extension.
+           </para>
+           </listitem>
+         </varlistentry>
+        </variablelist></para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-n</option></term>
       <term><option>--no-parse-wal</option></term>
diff --git a/src/bin/pg_verifybackup/t/002_algorithm.pl b/src/bin/pg_verifybackup/t/002_algorithm.pl
index fb2a1fd7c4e..4959d5bd0b9 100644
--- a/src/bin/pg_verifybackup/t/002_algorithm.pl
+++ b/src/bin/pg_verifybackup/t/002_algorithm.pl
@@ -14,24 +14,35 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 
-for my $algorithm (qw(bogus none crc32c sha224 sha256 sha384 sha512))
+sub test_checksums
 {
-	my $backup_path = $primary->backup_dir . '/' . $algorithm;
+	my ($format, $algorithm) = @_;
+	my $backup_path = $primary->backup_dir . '/' . $format . '/' . $algorithm;
 	my @backup = (
 		'pg_basebackup', '-D', $backup_path,
 		'--manifest-checksums', $algorithm, '--no-sync', '-cfast');
 	my @verify = ('pg_verifybackup', '-e', $backup_path);
 
+	if ($format eq 'tar')
+	{
+		# Add switch to get a tar-format backup
+		push @backup, ('-F', 't');
+
+		# Add switch to skip WAL verification, which is not yet supported for
+		# tar-format backups
+		push @verify, ('-n');
+	}
+
 	# A backup with a bogus algorithm should fail.
 	if ($algorithm eq 'bogus')
 	{
 		$primary->command_fails(\@backup,
-			"backup fails with algorithm \"$algorithm\"");
-		next;
+			"$format format backup fails with algorithm \"$algorithm\"");
+		return;
 	}
 
 	# A backup with a valid algorithm should work.
-	$primary->command_ok(\@backup, "backup ok with algorithm \"$algorithm\"");
+	$primary->command_ok(\@backup, "$format format backup ok with algorithm \"$algorithm\"");
 
 	# We expect each real checksum algorithm to be mentioned on every line of
 	# the backup manifest file except the first and last; for simplicity, we
@@ -39,7 +50,7 @@ for my $algorithm (qw(bogus none crc32c sha224 sha256 sha384 sha512))
 	# is none, we just check that the manifest exists.
 	if ($algorithm eq 'none')
 	{
-		ok(-f "$backup_path/backup_manifest", "backup manifest exists");
+		ok(-f "$backup_path/backup_manifest", "$format format backup manifest exists");
 	}
 	else
 	{
@@ -52,10 +63,19 @@ for my $algorithm (qw(bogus none crc32c sha224 sha256 sha384 sha512))
 
 	# Make sure that it verifies OK.
 	$primary->command_ok(\@verify,
-		"verify backup with algorithm \"$algorithm\"");
+		"verify $format format backup with algorithm \"$algorithm\"");
 
 	# Remove backup immediately to save disk space.
 	rmtree($backup_path);
 }
 
+# Do the check
+for my $format (qw(plain tar))
+{
+	for my $algorithm (qw(bogus none crc32c sha224 sha256 sha384 sha512))
+	{
+		test_checksums($format, $algorithm);
+	}
+}
+
 done_testing();
diff --git a/src/bin/pg_verifybackup/t/003_corruption.pl b/src/bin/pg_verifybackup/t/003_corruption.pl
index ae91e043384..c953bbc20d8 100644
--- a/src/bin/pg_verifybackup/t/003_corruption.pl
+++ b/src/bin/pg_verifybackup/t/003_corruption.pl
@@ -11,6 +11,8 @@ use PostgreSQL::Test::Cluster;
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+my $tar = $ENV{TAR};
+
 my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
@@ -32,62 +34,73 @@ EOM
 my @scenario = (
 	{
 		'name' => 'extra_file',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_extra_file,
 		'fails_like' =>
 		  qr/extra_file.*present on disk but not in the manifest/
 	},
 	{
 		'name' => 'extra_tablespace_file',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_extra_tablespace_file,
 		'fails_like' =>
 		  qr/extra_ts_file.*present on disk but not in the manifest/
 	},
 	{
 		'name' => 'missing_file',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_missing_file,
 		'fails_like' =>
 		  qr/pg_xact\/0000.*present in the manifest but not on disk/
 	},
 	{
 		'name' => 'missing_tablespace',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_missing_tablespace,
 		'fails_like' =>
 		  qr/pg_tblspc.*present in the manifest but not on disk/
 	},
 	{
 		'name' => 'append_to_file',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_append_to_file,
 		'fails_like' => qr/has size \d+ on disk but size \d+ in the manifest/
 	},
 	{
 		'name' => 'truncate_file',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_truncate_file,
 		'fails_like' => qr/has size 0 on disk but size \d+ in the manifest/
 	},
 	{
 		'name' => 'replace_file',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_replace_file,
 		'fails_like' => qr/checksum mismatch for file/
 	},
 	{
 		'name' => 'system_identifier',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_system_identifier,
 		'fails_like' =>
 		  qr/manifest system identifier is .*, but control file has/
 	},
 	{
 		'name' => 'bad_manifest',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_bad_manifest,
 		'fails_like' => qr/manifest checksum mismatch/
 	},
 	{
 		'name' => 'open_file_fails',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_open_file_fails,
 		'fails_like' => qr/could not open file/,
 		'skip_on_windows' => 1
 	},
 	{
 		'name' => 'open_directory_fails',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_open_directory_fails,
 		'cleanup' => \&cleanup_open_directory_fails,
 		'fails_like' => qr/could not open directory/,
@@ -95,10 +108,78 @@ my @scenario = (
 	},
 	{
 		'name' => 'search_directory_fails',
+		'backup_format' => 'p',
 		'mutilate' => \&mutilate_search_directory_fails,
 		'cleanup' => \&cleanup_search_directory_fails,
 		'fails_like' => qr/could not stat file or directory/,
 		'skip_on_windows' => 1
+	},
+	{
+		'name' => 'tar_backup_unexpected_file',
+		'backup_format' => 't',
+		'mutilate' => \&mutilate_extra_file,
+		'fails_like' =>
+		  qr/file "extra_file" is not expected in a tar format backup/
+	},
+	{
+		'name' => 'tar_backup_extra_file',
+		'backup_format' => 't',
+		'mutilate' => \&mutilate_tar_backup_extra_file,
+		'fails_like' =>
+		  qr/extra_tar_member_file.*present on disk but not in the manifest/
+	},
+	{
+		'name' => 'tar_extra_tablespace_file',
+		'backup_format' => 't',
+		'mutilate' => \&mutilate_tar_extra_tablespace_file,
+		'fails_like' =>
+		  qr/extra_ts_member_file.*present on disk but not in the manifest/
+	},
+	{
+		'name' => 'tar_backup_missing_file',
+		'backup_format' => 't',
+		'mutilate' => \&mutilate_tar_backup_missing_file,
+		'fails_like' =>
+		  qr/pg_xact\/0000.*present in the manifest but not on disk/,
+	},
+	{
+		'name' => 'tar_missing_tablespace',
+		'backup_format' => 't',
+		'mutilate' => \&mutilate_tar_missing_tablespace,
+		'fails_like' =>
+		  qr/pg_tblspc.*present in the manifest but not on disk/
+	},
+	{
+		'name' => 'tar_missing_tablespace_file',
+		'backup_format' => 't',
+		'mutilate' => \&mutilate_tar_missing_tablespace_file,
+		'fails_like' =>
+		  qr/pg_tblspc.*present in the manifest but not on disk/
+	},
+	{
+		'name' => 'tar_backup_append_to_file',
+		'backup_format' => 't',
+		'mutilate' => \&mutilate_tar_backup_append_to_file,
+		'fails_like' => qr/has size \d+ on disk but size \d+ in the manifest/,
+	},
+	{
+		'name' => 'tar_backup_truncate_file',
+		'backup_format' => 't',
+		'mutilate' => \&mutilate_tar_backup_truncate_file,
+		'fails_like' => qr/has size 0 on disk but size \d+ in the manifest/,
+	},
+	{
+		'name' => 'tar_backup_replace_file',
+		'backup_format' => 't',
+		'mutilate' => \&mutilate_tar_backup_replace_file,
+		'fails_like' => qr/checksum mismatch for file/,
+	},
+	{
+		'name' => 'tar_backup_system_identifier',
+		'backup_format' => 't',
+		'mutilate' => \&mutilate_system_identifier,
+		'fails_like' =>
+		  qr/manifest system identifier is .*, but control file has/
 	});
 
 for my $scenario (@scenario)
@@ -111,29 +192,42 @@ for my $scenario (@scenario)
 		  if ($scenario->{'skip_on_windows'}
 			&& ($windows_os || $Config::Config{osname} eq 'cygwin'));
 
+		# Skip tests for tar format-backup if tar is not available.
+		skip "no tar program available", 4
+		if ($scenario->{'backup_format'} eq 't' && (!defined $tar || $tar eq ''));
+
 		# Take a backup and check that it verifies OK.
 		my $backup_path = $primary->backup_dir . '/' . $name;
 		my $backup_ts_path = PostgreSQL::Test::Utils::tempdir_short();
+
+		my @backup = (
+				'pg_basebackup', '-D', $backup_path, '--no-sync', '-cfast',
+				'-T', "${source_ts_path}=${backup_ts_path}");
+		my @verify = ('pg_verifybackup', $backup_path);
+
+		if ($scenario->{'backup_format'} eq 't')
+		{
+			# Add switch to get a tar-format backup
+			push @backup, ('-F', 't');
+
+			# Add switch to skip WAL verification, which is not yet supported
+			# for tar-format backups
+			push @verify, ('-n');
+		}
+
 		# The tablespace map parameter confuses Msys2, which tries to mangle
 		# it. Tell it not to.
 		# See https://www.msys2.org/wiki/Porting/#filesystem-namespaces
 		local $ENV{MSYS2_ARG_CONV_EXCL} = $source_ts_prefix;
-		$primary->command_ok(
-			[
-				'pg_basebackup', '-D', $backup_path, '--no-sync', '-cfast',
-				'-T', "${source_ts_path}=${backup_ts_path}"
-			],
-			"base backup ok");
-		command_ok([ 'pg_verifybackup', $backup_path ],
-			"intact backup verified");
+
+		$primary->command_ok( \@backup, "base backup ok");
+		command_ok(\@verify, "intact backup verified");
 
 		# Mutilate the backup in some way.
 		$scenario->{'mutilate'}->($backup_path);
 
 		# Now check that the backup no longer verifies.
-		command_fails_like(
-			[ 'pg_verifybackup', $backup_path ],
-			$scenario->{'fails_like'},
+		command_fails_like(\@verify, $scenario->{'fails_like'},
 			"corrupt backup fails verification: $name");
 
 		# Run cleanup hook, if provided.
@@ -260,6 +354,7 @@ sub mutilate_system_identifier
 		$backup_path . '/backup_manifest')
 	  or BAIL_OUT "could not copy manifest to $backup_path";
 	$node->teardown_node(fail_ok => 1);
+	$node->clean_node();
 	return;
 }
 
@@ -316,4 +411,154 @@ sub cleanup_search_directory_fails
 	return;
 }
 
+# Unpack tar file, perform the specified file operation, and then repack the
+# modified content into same tar file at the same location.
+sub mutilate_base_tar
+{
+	my ($backup_path, $archive, $op) = @_;
+
+	my $tmpdir = "$backup_path/tmpdir";
+	mkdir($tmpdir) || die "$!";
+
+	# Extract the archive
+	system_or_bail($tar, '-xf', "$backup_path/$archive", '-C', "$tmpdir");
+	unlink("$backup_path/$archive") || die "$!";
+
+	if ($op eq 'add')
+	{
+		if ($archive eq 'base.tar')
+		{
+			create_extra_file($tmpdir, 'extra_tar_member_file');
+		}
+		else
+		{
+			# must be a tablespace archive
+			my ($catvdir) = grep { $_ ne '.' && $_ ne '..' }
+			slurp_dir("$tmpdir");
+			my ($tsdboid) = grep { $_ ne '.' && $_ ne '..' }
+			slurp_dir("$tmpdir/$catvdir");
+			create_extra_file($tmpdir,
+				"$catvdir/$tsdboid/extra_ts_member_file");
+		}
+	}
+	elsif ($op eq 'delete')
+	{
+		if ($archive eq 'base.tar')
+		{
+			mutilate_missing_file($tmpdir);
+		}
+		else
+		{
+			# must be a tablespace archive
+			my ($catvdir) = grep { $_ ne '.' && $_ ne '..' }
+			  slurp_dir("$tmpdir");
+			my ($tsdboid) = grep { $_ ne '.' && $_ ne '..' }
+			  slurp_dir("$tmpdir/$catvdir");
+			my ($reloid) = grep { $_ ne '.' && $_ ne '..' }
+			  slurp_dir("$tmpdir/$catvdir/$tsdboid");
+			my $pathname = "$tmpdir/$catvdir/$tsdboid/$reloid";
+			unlink($pathname) || die "$pathname: $!";
+		}
+	}
+	elsif ($op eq 'append')
+	{
+		mutilate_append_to_file($tmpdir);
+	}
+	elsif ($op eq 'truncate')
+	{
+		mutilate_truncate_file($tmpdir);
+	}
+	elsif ($op eq 'replace')
+	{
+		mutilate_replace_file($tmpdir);
+	}
+	else
+	{
+		die "mutilate_tar_backup: \"$op\" invalid operation";
+	}
+
+
+	# Navigate to the extracted location and list the files.
+	chdir("$tmpdir") || die "$!";
+	my @files = glob("*");
+	# Repack the extracted content
+	system_or_bail($tar, '-cf', "$backup_path/$archive", @files);
+	chdir($backup_path) || die "$!";
+	rmtree("$tmpdir") || die "$!";
+}
+
+# Add a file to the main directory archive.
+sub mutilate_tar_backup_extra_file
+{
+	my ($backup_path) = @_;
+	mutilate_base_tar($backup_path, 'base.tar', 'add');
+	return;
+}
+
+# Add a file to the user-defined tablespace archive.
+sub mutilate_tar_extra_tablespace_file
+{
+	my ($backup_path) = @_;
+	my ($archive) =
+	  grep { $_ =~ qr/\d+\.tar/ } slurp_dir("$backup_path");
+	die "tablespace tar backup not found." unless defined $archive;
+	mutilate_base_tar($backup_path, $archive, 'add');
+	return;
+}
+
+# Remove a file from main directory archive.
+sub mutilate_tar_backup_missing_file
+{
+	my ($backup_path) = @_;
+	mutilate_base_tar($backup_path, 'base.tar', 'delete');
+	return;
+}
+
+# Remove the user-defined tablespace archive.
+sub mutilate_tar_missing_tablespace
+{
+	my ($backup_path) = @_;
+	my ($archive) =
+	  grep { $_ =~ qr/\d+\.tar/ } slurp_dir("$backup_path");
+	die "tablespace tar backup not found." unless defined $archive;
+	my $pathname = "$backup_path/$archive";
+	unlink($pathname) || die "$pathname: $!";
+	return;
+}
+
+# Remove the files from the user-defined tablespace archive.
+sub mutilate_tar_missing_tablespace_file
+{
+	my ($backup_path) = @_;
+	my ($archive) =
+	  grep { $_ =~ qr/\d+\.tar/ } slurp_dir("$backup_path");
+	die "tablespace tar backup not found." unless defined $archive;
+	mutilate_base_tar($backup_path, $archive, 'delete');
+	return;
+}
+
+# Append an additional bytes to a file.
+sub mutilate_tar_backup_append_to_file
+{
+	my ($backup_path) = @_;
+	mutilate_base_tar($backup_path, 'base.tar', 'append');
+	return;
+}
+
+# Truncate a file to zero length.
+sub mutilate_tar_backup_truncate_file
+{
+	my ($backup_path) = @_;
+	mutilate_base_tar($backup_path, 'base.tar', 'truncate');
+	return;
+}
+
+# Replace a file's contents
+sub mutilate_tar_backup_replace_file
+{
+	my ($backup_path) = @_;
+	mutilate_base_tar($backup_path, 'base.tar', 'replace');
+	return;
+}
+
 done_testing();
diff --git a/src/bin/pg_verifybackup/t/004_options.pl b/src/bin/pg_verifybackup/t/004_options.pl
index 8ed2214408e..2f197648740 100644
--- a/src/bin/pg_verifybackup/t/004_options.pl
+++ b/src/bin/pg_verifybackup/t/004_options.pl
@@ -108,7 +108,7 @@ unlike(
 # Test valid manifest with nonexistent backup directory.
 command_fails_like(
 	[
-		'pg_verifybackup', '-m',
+		'pg_verifybackup', '-Fp', '-m',
 		"$backup_path/backup_manifest", "$backup_path/fake"
 	],
 	qr/could not open directory/,
diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
index 7a09f3b75b2..e7ec8369362 100644
--- a/src/bin/pg_verifybackup/t/008_untar.pl
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -16,6 +16,18 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 
+# Create a tablespace directory.
+my $source_ts_path = PostgreSQL::Test::Utils::tempdir_short();
+
+# Create a tablespace with table in it.
+$primary->safe_psql('postgres', qq(
+		CREATE TABLESPACE regress_ts1 LOCATION '$source_ts_path';
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1';
+		CREATE TABLE regress_tbl1(i int) TABLESPACE regress_ts1;
+		INSERT INTO regress_tbl1 VALUES(generate_series(1,5));));
+my $tsoid = $primary->safe_psql('postgres', qq(
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1'));
+
 my $backup_path = $primary->backup_dir . '/server-backup';
 my $extract_path = $primary->backup_dir . '/extracted-backup';
 
@@ -23,39 +35,31 @@ my @test_configuration = (
 	{
 		'compression_method' => 'none',
 		'backup_flags' => [],
-		'backup_archive' => 'base.tar',
+		'backup_archive' => ['base.tar', "$tsoid.tar"],
 		'enabled' => 1
 	},
 	{
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'server-gzip' ],
-		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.gz', "$tsoid.tar.gz" ],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'server-lz4' ],
-		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => [ '-d', '-m' ],
+		'backup_archive' => ['base.tar.lz4', "$tsoid.tar.lz4" ],
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd:level=1,long' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	});
 
@@ -86,47 +90,16 @@ for my $tc (@test_configuration)
 		my $backup_files = join(',',
 			sort grep { $_ ne '.' && $_ ne '..' } slurp_dir($backup_path));
 		my $expected_backup_files =
-		  join(',', sort ('backup_manifest', $tc->{'backup_archive'}));
+		  join(',', sort ('backup_manifest', @{ $tc->{'backup_archive'} }));
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok(['pg_verifybackup', '-n', '-e', $backup_path],
+			"verify backup, compression $method");
 
 		# Cleanup.
-		unlink($backup_path . '/backup_manifest');
-		unlink($backup_path . '/base.tar');
-		unlink($backup_path . '/' . $tc->{'backup_archive'});
+		rmtree($backup_path);
 		rmtree($extract_path);
 	}
 }
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index 8c076d46dee..6b7d7483f6e 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -29,41 +29,30 @@ my @test_configuration = (
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'client-gzip:5' ],
 		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'client-lz4:5' ],
 		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => ['-d'],
-		'output_file' => 'base.tar',
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:5' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:level=1,long' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'parallel zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:workers=3' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1"),
 		'possibly_unsupported' =>
 		  qr/could not set compression worker count to 3: Unsupported parameter/
@@ -118,40 +107,9 @@ for my $tc (@test_configuration)
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			push @decompress, $backup_path . '/' . $tc->{'output_file'}
-			  if $tc->{'output_file'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok( [ 'pg_verifybackup', '-n', '-e', $backup_path ],
+			"verify backup, compression $method");
 
 		# Cleanup.
 		rmtree($extract_path);
-- 
2.18.0

From e8bf51858c4151b957775e0b17b91e3f42b02589 Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Wed, 14 Aug 2024 10:42:37 +0530
Subject: [PATCH v15] pg_verifybackup: Verify tar-format backups.

This also works for compressed tar-format backups. However, -n must be
used, because we use pg_waldump to verify WAL, and it doesn't yet know
how to verify WAL that is stored inside of a tarfile.

Amul Sul, reviewed by Sravan Kumar and by me, and substantially
revised by me.
---
 doc/src/sgml/ref/pg_verifybackup.sgml         |  47 +-
 src/bin/pg_verifybackup/Makefile              |   2 +
 src/bin/pg_verifybackup/astreamer_verify.c    | 428 +++++++++++++++++
 src/bin/pg_verifybackup/meson.build           |   1 +
 src/bin/pg_verifybackup/pg_verifybackup.c     | 433 ++++++++++++++++--
 src/bin/pg_verifybackup/pg_verifybackup.h     |   7 +
 src/bin/pg_verifybackup/t/002_algorithm.pl    |  34 +-
 src/bin/pg_verifybackup/t/003_corruption.pl   |  77 +++-
 src/bin/pg_verifybackup/t/004_options.pl      |  17 +
 src/bin/pg_verifybackup/t/008_untar.pl        |  71 +--
 src/bin/pg_verifybackup/t/010_client_untar.pl |  48 +-
 src/fe_utils/simple_list.c                    |  19 +
 src/include/fe_utils/simple_list.h            |   1 +
 src/tools/pgindent/typedefs.list              |   2 +
 14 files changed, 1033 insertions(+), 154 deletions(-)
 create mode 100644 src/bin/pg_verifybackup/astreamer_verify.c

diff --git a/doc/src/sgml/ref/pg_verifybackup.sgml b/doc/src/sgml/ref/pg_verifybackup.sgml
index a3f167f9f6e..53341024cd2 100644
--- a/doc/src/sgml/ref/pg_verifybackup.sgml
+++ b/doc/src/sgml/ref/pg_verifybackup.sgml
@@ -34,8 +34,12 @@ PostgreSQL documentation
    integrity of a database cluster backup taken using
    <command>pg_basebackup</command> against a
    <literal>backup_manifest</literal> generated by the server at the time
-   of the backup.  The backup must be stored in the "plain"
-   format; a "tar" format backup can be checked after extracting it.
+   of the backup. The backup may be stored either in the "plain" or the "tar"
+   format; this includes tar-format backups compressed with any algorithm
+   supported by <application>pg_basebackup</application>. However, at present,
+   <literal>WAL</literal> verification is supported only for plain-format
+   backups. Therefore, if the backup is stored in tar-format, the
+   <literal>-n, --no-parse-wal</literal> option should be used.
   </para>
 
   <para>
@@ -168,6 +172,45 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-F <replaceable class="parameter">format</replaceable></option></term>
+      <term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
+      <listitem>
+       <para>
+        Specifies the format of the backup. <replaceable>format</replaceable>
+        can be one of the following:
+
+        <variablelist>
+         <varlistentry>
+          <term><literal>p</literal></term>
+          <term><literal>plain</literal></term>
+          <listitem>
+           <para>
+            Backup consists of plain files with the same layout as the
+            source server's data directory and tablespaces.
+           </para>
+          </listitem>
+         </varlistentry>
+
+         <varlistentry>
+          <term><literal>t</literal></term>
+          <term><literal>tar</literal></term>
+          <listitem>
+           <para>
+            Backup consists of tar files, which  may be compressed.  A valid
+            backup includes the main data directory in a file named
+            <filename>base.tar</filename>, the WAL files in
+            <filename>pg_wal.tar</filename>, and separate tar files for
+            each tablespace, named after the tablespace's OID. If the backup
+            is compressed, the relevant compression extension is added to the
+            end of each file name.
+           </para>
+           </listitem>
+         </varlistentry>
+        </variablelist></para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-n</option></term>
       <term><option>--no-parse-wal</option></term>
diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile
index 7c045f142e8..374d4a8afd1 100644
--- a/src/bin/pg_verifybackup/Makefile
+++ b/src/bin/pg_verifybackup/Makefile
@@ -17,10 +17,12 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 # We need libpq only because fe_utils does.
+override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 
 OBJS = \
 	$(WIN32RES) \
+	astreamer_verify.o \
 	pg_verifybackup.o
 
 all: pg_verifybackup
diff --git a/src/bin/pg_verifybackup/astreamer_verify.c b/src/bin/pg_verifybackup/astreamer_verify.c
new file mode 100644
index 00000000000..57072fdfe04
--- /dev/null
+++ b/src/bin/pg_verifybackup/astreamer_verify.c
@@ -0,0 +1,428 @@
+/*-------------------------------------------------------------------------
+ *
+ * astreamer_verify.c
+ *
+ * Archive streamer for verification of a tar format backup (including
+ * compressed tar format backups).
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ *
+ * src/bin/pg_verifybackup/astreamer_verify.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include "catalog/pg_control.h"
+#include "pg_verifybackup.h"
+
+typedef struct astreamer_verify
+{
+	/* These fields don't change once initialized. */
+	astreamer	base;
+	verifier_context *context;
+	char	   *archive_name;
+	Oid			tblspc_oid;
+
+	/* These fields change for each archive member. */
+	manifest_file *mfile;
+	bool		verify_checksum;
+	bool		verify_control_data;
+	pg_checksum_context *checksum_ctx;
+	uint64		checksum_bytes;
+	ControlFileData control_file;
+	uint64		control_file_bytes;
+} astreamer_verify;
+
+static void astreamer_verify_content(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len,
+									 astreamer_archive_context context);
+static void astreamer_verify_finalize(astreamer *streamer);
+static void astreamer_verify_free(astreamer *streamer);
+
+static void member_verify_header(astreamer *streamer, astreamer_member *member);
+static void member_compute_checksum(astreamer *streamer,
+									astreamer_member *member,
+									const char *data, int len);
+static void member_verify_checksum(astreamer *streamer);
+static void member_copy_control_data(astreamer *streamer,
+									 astreamer_member *member,
+									 const char *data, int len);
+static void member_verify_control_data(astreamer *streamer);
+static void member_reset_info(astreamer *streamer);
+
+static const astreamer_ops astreamer_verify_ops = {
+	.content = astreamer_verify_content,
+	.finalize = astreamer_verify_finalize,
+	.free = astreamer_verify_free
+};
+
+/*
+ * Create an astreamer that can verify a tar file.
+ */
+astreamer *
+astreamer_verify_content_new(astreamer *next, verifier_context *context,
+							 char *archive_name, Oid tblspc_oid)
+{
+	astreamer_verify *streamer;
+
+	streamer = palloc0(sizeof(astreamer_verify));
+	*((const astreamer_ops **) &streamer->base.bbs_ops) =
+		&astreamer_verify_ops;
+
+	streamer->base.bbs_next = next;
+	streamer->context = context;
+	streamer->archive_name = archive_name;
+	streamer->tblspc_oid = tblspc_oid;
+
+	if (!context->skip_checksums)
+		streamer->checksum_ctx = pg_malloc(sizeof(pg_checksum_context));
+
+	return &streamer->base;
+}
+
+/*
+ * Main entry point of the archive streamer for verifying tar members.
+ */
+static void
+astreamer_verify_content(astreamer *streamer, astreamer_member *member,
+						 const char *data, int len,
+						 astreamer_archive_context context)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	Assert(context != ASTREAMER_UNKNOWN);
+
+	switch (context)
+	{
+		case ASTREAMER_MEMBER_HEADER:
+			/* Initial setup plus decide which checks to perform. */
+			member_verify_header(streamer, member);
+			break;
+
+		case ASTREAMER_MEMBER_CONTENTS:
+			/* Incremental work required to verify file contents. */
+			if (mystreamer->verify_checksum)
+				member_compute_checksum(streamer, member, data, len);
+			if (mystreamer->verify_control_data)
+				member_copy_control_data(streamer, member, data, len);
+			break;
+
+		case ASTREAMER_MEMBER_TRAILER:
+			/* Now we've got all the file data. */
+			if (mystreamer->verify_checksum)
+				member_verify_checksum(streamer);
+			if (mystreamer->verify_control_data)
+				member_verify_control_data(streamer);
+
+			/* Reset for next archive member. */
+			member_reset_info(streamer);
+			break;
+
+		case ASTREAMER_ARCHIVE_TRAILER:
+			break;
+
+		default:
+			/* Shouldn't happen. */
+			pg_fatal("unexpected state while parsing tar file");
+	}
+}
+
+/*
+ * End-of-stream processing for a astreamer_verify stream.
+ */
+static void
+astreamer_verify_finalize(astreamer *streamer)
+{
+	Assert(streamer->bbs_next == NULL);
+}
+
+/*
+ * Free memory associated with a astreamer_verify stream.
+ */
+static void
+astreamer_verify_free(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	if (mystreamer->checksum_ctx)
+		pfree(mystreamer->checksum_ctx);
+
+	pfree(streamer);
+}
+
+/*
+ * Prepare to validate the next archive member.
+ */
+static void
+member_verify_header(astreamer *streamer, astreamer_member *member)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_file *m;
+	char		pathname[MAXPGPATH];
+
+	/* We are only interested in normal files. */
+	if (member->is_directory || member->is_link)
+		return;
+
+	/*
+	 * The backup manifest stores a relative path to the base directory for
+	 * files belonging to a tablespace, while the tablespace backup tar
+	 * archive does not include this path.
+	 *
+	 * The pathname taken from the tar file could contain '.' or '..'
+	 * references, which we want to remove, so apply canonicalize_path(). It
+	 * could also be an absolute pathname, which we want to treat as a
+	 * relative path, so prepend "./" if we're not adding a tablespace prefix
+	 * to make sure that canonicalize_path() does what we want.
+	 */
+	if (OidIsValid(mystreamer->tblspc_oid))
+		snprintf(pathname, MAXPGPATH, "%s/%u/%s",
+				 "pg_tblspc", mystreamer->tblspc_oid, member->pathname);
+	else
+		snprintf(pathname, MAXPGPATH, "./%s", member->pathname);
+	canonicalize_path(pathname);
+
+	/* Ignore any files that are listed in the ignore list. */
+	if (should_ignore_relpath(mystreamer->context, pathname))
+		return;
+
+	/* Check whether there's an entry in the manifest hash. */
+	m = manifest_files_lookup(mystreamer->context->manifest->files, pathname);
+	if (m == NULL)
+	{
+		report_backup_error(mystreamer->context,
+							"\"%s\" is present in \"%s\" but not in the manifest",
+							member->pathname, mystreamer->archive_name);
+		return;
+	}
+	mystreamer->mfile = m;
+
+	/* Flag this entry as having been encountered in a tar archive. */
+	m->matched = true;
+
+	/* Check that the size matches. */
+	if (m->size != member->size)
+	{
+		report_backup_error(mystreamer->context,
+							"\"%s\" has size %lld in \"%s\" but size %zu in the manifest",
+							member->pathname, (long long int) member->size,
+							mystreamer->archive_name, m->size);
+		m->bad = true;
+		return;
+	}
+
+	/*
+	 * Decide whether we're going to verify the checksum for this file, and
+	 * whether we're going to perform the additional validation that we do
+	 * only for the control file.
+	 */
+	mystreamer->verify_checksum =
+		(!mystreamer->context->skip_checksums && should_verify_checksum(m));
+	mystreamer->verify_control_data =
+		mystreamer->context->manifest->version != 1 &&
+		!m->bad && strcmp(m->pathname, "global/pg_control") == 0;
+
+	/* If we're going to verify the checksum, initial a checksum context. */
+	if (mystreamer->verify_checksum &&
+		pg_checksum_init(mystreamer->checksum_ctx, m->checksum_type) < 0)
+	{
+		report_backup_error(mystreamer->context,
+							"%s: could not initialize checksum of file \"%s\"",
+							mystreamer->archive_name, m->pathname);
+
+		/*
+		 * Checksum verification cannot be performed without proper context
+		 * initialization.
+		 */
+		mystreamer->verify_checksum = false;
+	}
+}
+
+/*
+ * Computes the checksum incrementally for the received file content.
+ *
+ * Should have a correctly initialized checksum_ctx, which will be used for
+ * incremental checksum computation.
+ */
+static void
+member_compute_checksum(astreamer *streamer, astreamer_member *member,
+						const char *data, int len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	pg_checksum_context *checksum_ctx = mystreamer->checksum_ctx;
+	manifest_file *m = mystreamer->mfile;
+
+	Assert(mystreamer->verify_checksum);
+	Assert(m->checksum_type == checksum_ctx->type);
+
+	/*
+	 * Update the total count of computed checksum bytes so that we can
+	 * cross-check against the file size.
+	 */
+	mystreamer->checksum_bytes += len;
+
+	/* Feed these bytes to the checksum calculation. */
+	if (pg_checksum_update(checksum_ctx, (uint8 *) data, len) < 0)
+	{
+		report_backup_error(mystreamer->context,
+							"could not update checksum of file \"%s\"",
+							m->pathname);
+		mystreamer->verify_checksum = false;
+	}
+}
+
+/*
+ * Perform the final computation and checksum verification after the entire
+ * file content has been processed.
+ */
+static void
+member_verify_checksum(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_file *m = mystreamer->mfile;
+	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
+	int			checksumlen;
+
+	Assert(mystreamer->verify_checksum);
+
+	/*
+	 * It's unclear how this could fail, but let's check anyway to be safe.
+	 */
+	if (mystreamer->checksum_bytes != m->size)
+	{
+		report_backup_error(mystreamer->context,
+							"file \"%s\" in \"%s\" should contain %zu bytes, but read %zu bytes",
+							m->pathname, mystreamer->archive_name,
+							m->size, mystreamer->checksum_bytes);
+		return;
+	}
+
+	/* Get the final checksum. */
+	checksumlen = pg_checksum_final(mystreamer->checksum_ctx, checksumbuf);
+	if (checksumlen < 0)
+	{
+		report_backup_error(mystreamer->context,
+							"could not finalize checksum of file \"%s\"",
+							m->pathname);
+		return;
+	}
+
+	/* And check it against the manifest. */
+	if (checksumlen != m->checksum_length)
+		report_backup_error(mystreamer->context,
+							"file \"%s\" in \"%s\" has checksum of length %d, but expected %d",
+							m->pathname, mystreamer->archive_name,
+							m->checksum_length, checksumlen);
+	else if (memcmp(checksumbuf, m->checksum_payload, checksumlen) != 0)
+		report_backup_error(mystreamer->context,
+							"checksum mismatch for file \"%s\" in \"%s\"",
+							m->pathname, mystreamer->archive_name);
+}
+
+/*
+ * Stores the pg_control file contents into a local buffer; we need the entire
+ * control file data for verification.
+ */
+static void
+member_copy_control_data(astreamer *streamer, astreamer_member *member,
+						 const char *data, int len)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	/* Should be here only for control file */
+	Assert(mystreamer->verify_control_data);
+
+	/*
+	 * Copy the new data into the control file buffer, but do not overrun the
+	 * buffer. Note that the on-disk length of the control file is expected to
+	 * be PG_CONTROL_FILE_SIZE, but the part that fits in our buffer is
+	 * shorter, just sizeof(ControlFileData).
+	 */
+	if (mystreamer->control_file_bytes <= sizeof(ControlFileData))
+	{
+		int			remaining;
+
+		remaining = sizeof(ControlFileData) - mystreamer->control_file_bytes;
+		memcpy(((char *) &mystreamer->control_file)
+			   + mystreamer->control_file_bytes,
+			   data, Min(len, remaining));
+	}
+
+	/* Remember how many bytes we saw, even if we didn't buffer them. */
+	mystreamer->control_file_bytes += len;
+}
+
+/*
+ * Performs the CRC calculation of pg_control data and then calls the routines
+ * that execute the final verification of the control file information.
+ */
+static void
+member_verify_control_data(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+	manifest_data *manifest = mystreamer->context->manifest;
+	pg_crc32c	crc;
+
+	/* Should be here only for control file */
+	Assert(strcmp(mystreamer->mfile->pathname, "global/pg_control") == 0);
+	Assert(mystreamer->verify_control_data);
+
+	/*
+	 * If the control file is not the right length, that's a big problem.
+	 *
+	 * NB: There is a theoretical overflow risk here from casting to int, but
+	 * it isn't likely to be a real problem and this enables us to match the
+	 * same format string that pg_rewind uses for this case. Perhaps both this
+	 * and pg_rewind should use an unsigned 64-bit value, but for now we don't
+	 * worry about it.
+	 */
+	if (mystreamer->control_file_bytes != PG_CONTROL_FILE_SIZE)
+		report_fatal_error("unexpected control file size %d, expected %d",
+						   (int) mystreamer->control_file_bytes,
+						   PG_CONTROL_FILE_SIZE);
+
+	/* Compute the CRC. */
+	INIT_CRC32C(crc);
+	COMP_CRC32C(crc, &mystreamer->control_file,
+				offsetof(ControlFileData, crc));
+	FIN_CRC32C(crc);
+
+	/* Control file contents not meaningful if CRC is bad. */
+	if (!EQ_CRC32C(crc, mystreamer->control_file.crc))
+		report_fatal_error("%s: %s: CRC is incorrect",
+						   mystreamer->archive_name,
+						   mystreamer->mfile->pathname);
+
+	/* Can't interpret control file if not current version. */
+	if (mystreamer->control_file.pg_control_version != PG_CONTROL_VERSION)
+		report_fatal_error("%s: %s: unexpected control file version",
+						   mystreamer->archive_name,
+						   mystreamer->mfile->pathname);
+
+	/* System identifiers should match. */
+	if (manifest->system_identifier !=
+		mystreamer->control_file.system_identifier)
+		report_fatal_error("%s: %s: manifest system identifier is %llu, but control file has %llu",
+						   mystreamer->archive_name,
+						   mystreamer->mfile->pathname,
+						   (unsigned long long) manifest->system_identifier,
+						   (unsigned long long) mystreamer->control_file.system_identifier);
+}
+
+/*
+ * Reset flags and free memory allocations for member file verification.
+ */
+static void
+member_reset_info(astreamer *streamer)
+{
+	astreamer_verify *mystreamer = (astreamer_verify *) streamer;
+
+	mystreamer->mfile = NULL;
+	mystreamer->verify_checksum = false;
+	mystreamer->verify_control_data = false;
+	mystreamer->checksum_bytes = 0;
+	mystreamer->control_file_bytes = 0;
+}
diff --git a/src/bin/pg_verifybackup/meson.build b/src/bin/pg_verifybackup/meson.build
index 7c7d31a0350..0e09d1379d1 100644
--- a/src/bin/pg_verifybackup/meson.build
+++ b/src/bin/pg_verifybackup/meson.build
@@ -1,6 +1,7 @@
 # Copyright (c) 2022-2024, PostgreSQL Global Development Group
 
 pg_verifybackup_sources = files(
+  'astreamer_verify.c',
   'pg_verifybackup.c'
 )
 
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 3fcfb167217..1504576303b 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -22,6 +22,7 @@
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
+#include "limits.h"
 #include "pg_verifybackup.h"
 #include "pgtime.h"
 
@@ -44,6 +45,16 @@
  */
 #define READ_CHUNK_SIZE				(128 * 1024)
 
+/*
+ * Tar file information needed for content verification.
+ */
+typedef struct tar_file
+{
+	char	   *relpath;
+	Oid			tblspc_oid;
+	pg_compress_algorithm compress_algorithm;
+} tar_file;
+
 static manifest_data *parse_manifest_file(char *manifest_path);
 static void verifybackup_version_cb(JsonManifestParseContext *context,
 									int manifest_version);
@@ -62,12 +73,18 @@ static void report_manifest_error(JsonManifestParseContext *context,
 								  const char *fmt,...)
 			pg_attribute_printf(2, 3) pg_attribute_noreturn();
 
-static void verify_backup_directory(verifier_context *context,
-									char *relpath, char *fullpath);
-static void verify_backup_file(verifier_context *context,
-							   char *relpath, char *fullpath);
+static void verify_tar_backup(verifier_context *context, DIR *dir);
+static void verify_plain_backup_directory(verifier_context *context,
+										  char *relpath, char *fullpath,
+										  DIR *dir);
+static void verify_plain_backup_file(verifier_context *context, char *relpath,
+									 char *fullpath);
 static void verify_control_file(const char *controlpath,
 								uint64 manifest_system_identifier);
+static void precheck_tar_backup_file(verifier_context *context, char *relpath,
+									 char *fullpath, SimplePtrList *tarfiles);
+static void verify_tar_file(verifier_context *context, char *relpath,
+							char *fullpath, astreamer *streamer);
 static void report_extra_backup_files(verifier_context *context);
 static void verify_backup_checksums(verifier_context *context);
 static void verify_file_checksum(verifier_context *context,
@@ -76,6 +93,10 @@ static void verify_file_checksum(verifier_context *context,
 static void parse_required_wal(verifier_context *context,
 							   char *pg_waldump_path,
 							   char *wal_directory);
+static astreamer *create_archive_verifier(verifier_context *context,
+										  char *archive_name,
+										  Oid tblspc_oid,
+										  pg_compress_algorithm compress_algo);
 
 static void progress_report(bool finished);
 static void usage(void);
@@ -99,6 +120,7 @@ main(int argc, char **argv)
 		{"exit-on-error", no_argument, NULL, 'e'},
 		{"ignore", required_argument, NULL, 'i'},
 		{"manifest-path", required_argument, NULL, 'm'},
+		{"format", required_argument, NULL, 'F'},
 		{"no-parse-wal", no_argument, NULL, 'n'},
 		{"progress", no_argument, NULL, 'P'},
 		{"quiet", no_argument, NULL, 'q'},
@@ -114,6 +136,7 @@ main(int argc, char **argv)
 	bool		quiet = false;
 	char	   *wal_directory = NULL;
 	char	   *pg_waldump_path = NULL;
+	DIR		   *dir;
 
 	pg_logging_init(argv[0]);
 	set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_verifybackup"));
@@ -156,7 +179,7 @@ main(int argc, char **argv)
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
 
-	while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1)
+	while ((c = getopt_long(argc, argv, "eF:i:m:nPqsw:", long_options, NULL)) != -1)
 	{
 		switch (c)
 		{
@@ -175,6 +198,15 @@ main(int argc, char **argv)
 				manifest_path = pstrdup(optarg);
 				canonicalize_path(manifest_path);
 				break;
+			case 'F':
+				if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
+					context.format = 'p';
+				else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
+					context.format = 't';
+				else
+					pg_fatal("invalid backup format \"%s\", must be \"plain\" or \"tar\"",
+							 optarg);
+				break;
 			case 'n':
 				no_parse_wal = true;
 				break;
@@ -264,25 +296,75 @@ main(int argc, char **argv)
 	context.manifest = parse_manifest_file(manifest_path);
 
 	/*
-	 * Now scan the files in the backup directory. At this stage, we verify
-	 * that every file on disk is present in the manifest and that the sizes
-	 * match. We also set the "matched" flag on every manifest entry that
-	 * corresponds to a file on disk.
+	 * If the backup directory cannot be found, treat this as a fatal error.
+	 */
+	dir = opendir(context.backup_directory);
+	if (dir == NULL)
+		report_fatal_error("could not open directory \"%s\": %m",
+						   context.backup_directory);
+
+	/*
+	 * At this point, we know that the backup directory exists, so it's now
+	 * reasonable to check for files immediately inside it. Thus, before going
+	 * futher, if the user did not specify the backup format, check for
+	 * PG_VERSION to distinguish between tar and plain format.
 	 */
-	verify_backup_directory(&context, NULL, context.backup_directory);
+	if (context.format == '\0')
+	{
+		struct stat sb;
+		char	   *path;
+
+		path = psprintf("%s/%s", context.backup_directory, "PG_VERSION");
+		if (stat(path, &sb) == 0)
+			context.format = 'p';
+		else if (errno != ENOENT)
+		{
+			pg_log_error("could not stat file \"%s\": %m", path);
+			exit(1);
+		}
+		else
+		{
+			/* No PG_VERSION, so assume tar format. */
+			context.format = 't';
+		}
+		pfree(path);
+	}
+
+	/*
+	 * XXX: In the future, we should consider enhancing pg_waldump to read
+	 * WAL files from an archive.
+	 */
+	if (!no_parse_wal && context.format == 't')
+	{
+		pg_log_error("pg_waldump cannot read tar files");
+		pg_log_error_hint("You must use -n or --no-parse-wal when verifying a tar-format backup.");
+		exit(1);
+	}
+
+	/*
+	 * Perform the appropriate type of verification appropriate based on the
+	 * backup format. This will close 'dir'.
+	 */
+	if (context.format == 'p')
+		verify_plain_backup_directory(&context, NULL, context.backup_directory,
+									  dir);
+	else
+		verify_tar_backup(&context, dir);
 
 	/*
 	 * The "matched" flag should now be set on every entry in the hash table.
 	 * Any entries for which the bit is not set are files mentioned in the
-	 * manifest that don't exist on disk.
+	 * manifest that don't exist on disk (or in the relevant tar files).
 	 */
 	report_extra_backup_files(&context);
 
 	/*
-	 * Now do the expensive work of verifying file checksums, unless we were
-	 * told to skip it.
+	 * If this is a tar-format backup, checksums were already verified above;
+	 * but if it's a plain-format backup, we postpone it until this point,
+	 * since the earlier checks can be performed just by knowing which files
+	 * are present, without needing to read all of them.
 	 */
-	if (!context.skip_checksums)
+	if (context.format == 'p' && !context.skip_checksums)
 		verify_backup_checksums(&context);
 
 	/*
@@ -517,35 +599,27 @@ verifybackup_per_wal_range_cb(JsonManifestParseContext *context,
 }
 
 /*
- * Verify one directory.
+ * Verify one directory of a plain-format backup.
  *
  * 'relpath' is NULL if we are to verify the top-level backup directory,
  * and otherwise the relative path to the directory that is to be verified.
  *
  * 'fullpath' is the backup directory with 'relpath' appended; i.e. the actual
  * filesystem path at which it can be found.
+ *
+ * 'dir' is an open directory handle, or NULL if the caller wants us to
+ * open it. If the caller chooses to pass a handle, we'll close it when
+ * we're done with it.
  */
 static void
-verify_backup_directory(verifier_context *context, char *relpath,
-						char *fullpath)
+verify_plain_backup_directory(verifier_context *context, char *relpath,
+							  char *fullpath, DIR *dir)
 {
-	DIR		   *dir;
 	struct dirent *dirent;
 
-	dir = opendir(fullpath);
-	if (dir == NULL)
+	/* Open the directory unless the caller did it. */
+	if (dir == NULL && ((dir = opendir(fullpath)) == NULL))
 	{
-		/*
-		 * If even the toplevel backup directory cannot be found, treat this
-		 * as a fatal error.
-		 */
-		if (relpath == NULL)
-			report_fatal_error("could not open directory \"%s\": %m", fullpath);
-
-		/*
-		 * Otherwise, treat this as a non-fatal error, but ignore any further
-		 * errors related to this path and anything beneath it.
-		 */
 		report_backup_error(context,
 							"could not open directory \"%s\": %m", fullpath);
 		simple_string_list_append(&context->ignore_list, relpath);
@@ -570,7 +644,7 @@ verify_backup_directory(verifier_context *context, char *relpath,
 			newrelpath = psprintf("%s/%s", relpath, filename);
 
 		if (!should_ignore_relpath(context, newrelpath))
-			verify_backup_file(context, newrelpath, newfullpath);
+			verify_plain_backup_file(context, newrelpath, newfullpath);
 
 		pfree(newfullpath);
 		pfree(newrelpath);
@@ -587,11 +661,12 @@ verify_backup_directory(verifier_context *context, char *relpath,
 /*
  * Verify one file (which might actually be a directory or a symlink).
  *
- * The arguments to this function have the same meaning as the arguments to
- * verify_backup_directory.
+ * The arguments to this function have the same meaning as the similarly named
+ * arguments to verify_plain_backup_directory.
  */
 static void
-verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
+verify_plain_backup_file(verifier_context *context, char *relpath,
+						 char *fullpath)
 {
 	struct stat sb;
 	manifest_file *m;
@@ -614,7 +689,7 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 	/* If it's a directory, just recurse. */
 	if (S_ISDIR(sb.st_mode))
 	{
-		verify_backup_directory(context, relpath, fullpath);
+		verify_plain_backup_directory(context, relpath, fullpath, NULL);
 		return;
 	}
 
@@ -703,6 +778,252 @@ verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
 	pfree(control_file);
 }
 
+/*
+ * Verify tar backup.
+ *
+ * The caller should pass a handle to the target directory, which we will
+ * close when we're done with it.
+ */
+static void
+verify_tar_backup(verifier_context *context, DIR *dir)
+{
+	struct dirent *dirent;
+	SimplePtrList tarfiles = {NULL, NULL};
+	SimplePtrListCell *cell;
+
+	Assert(context->format != 'p');
+
+	progress_report(false);
+
+	/* First pass: scan the directory for tar files. */
+	while (errno = 0, (dirent = readdir(dir)) != NULL)
+	{
+		char	   *filename = dirent->d_name;
+
+		/* Skip "." and ".." */
+		if (filename[0] == '.' && (filename[1] == '\0'
+								   || strcmp(filename, "..") == 0))
+			continue;
+
+		/*
+		 * Unless it's something we should ignore, perform prechecks and add
+		 * it to the list.
+		 */
+		if (!should_ignore_relpath(context, filename))
+		{
+			char	   *fullpath;
+
+			fullpath = psprintf("%s/%s", context->backup_directory, filename);
+			precheck_tar_backup_file(context, filename, fullpath, &tarfiles);
+			pfree(fullpath);
+		}
+	}
+
+	if (closedir(dir))
+	{
+		report_backup_error(context,
+							"could not close directory \"%s\": %m",
+							context->backup_directory);
+		return;
+	}
+
+	/* Second pass: Perform the final verification of the tar contents. */
+	for (cell = tarfiles.head; cell != NULL; cell = cell->next)
+	{
+		tar_file   *tar = (tar_file *) cell->ptr;
+		astreamer  *streamer;
+		char	   *fullpath;
+
+		/*
+		 * Prepares the archive streamer stack according to the tar
+		 * compression format.
+		 */
+		streamer = create_archive_verifier(context,
+										   tar->relpath,
+										   tar->tblspc_oid,
+										   tar->compress_algorithm);
+
+		/* Compute the full pathname to the target file. */
+		fullpath = psprintf("%s/%s", context->backup_directory,
+							tar->relpath);
+
+		/* Invoke the streamer for reading, decompressing, and verifying. */
+		verify_tar_file(context, tar->relpath, fullpath, streamer);
+
+		/* Cleanup. */
+		pfree(tar->relpath);
+		pfree(tar);
+		pfree(fullpath);
+
+		astreamer_finalize(streamer);
+		astreamer_free(streamer);
+	}
+	simple_ptr_list_destroy(&tarfiles);
+
+	progress_report(true);
+}
+
+/*
+ * Preparatory steps for verifying files in tar format backups.
+ *
+ * Carries out basic validation of the tar format backup file, detects the
+ * compression type, and appends that information to the tarfiles list. An
+ * error will be reported if the tar file is inaccessible, or if the file type,
+ * name, or compression type is not as expected.
+ *
+ * The arguments to this function are mostly the same as the
+ * verify_plain_backup_file. The additional argument outputs a list of valid
+ * tar files.
+ */
+static void
+precheck_tar_backup_file(verifier_context *context, char *relpath,
+						 char *fullpath, SimplePtrList *tarfiles)
+{
+	struct stat sb;
+	Oid			tblspc_oid = InvalidOid;
+	pg_compress_algorithm compress_algorithm;
+	tar_file   *tar;
+	char	   *suffix = NULL;
+
+	/* Should be tar format backup */
+	Assert(context->format == 't');
+
+	/* Get file information */
+	if (stat(fullpath, &sb) != 0)
+	{
+		report_backup_error(context,
+							"could not stat file or directory \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	/* In a tar format backup, we expect only plain files. */
+	if (!S_ISREG(sb.st_mode))
+	{
+		report_backup_error(context,
+							"\"%s\" is not a plain file",
+							relpath);
+		return;
+	}
+
+	/*
+	 * We expect tar files for backing up the main directory, tablespace, and
+	 * pg_wal directory.
+	 *
+	 * pg_basebackup writes the main data directory to an archive file named
+	 * base.tar, the pg_wal directory to pg_wal.tar, and the tablespace
+	 * directory to <tablespaceoid>.tar, each followed by a compression type
+	 * extension such as .gz, .lz4, or .zst.
+	 */
+	if (strncmp("base", relpath, 4) == 0)
+		suffix = relpath + 4;
+	else if (strncmp("pg_wal", relpath, 6) == 0)
+		suffix = relpath + 6;
+	else
+	{
+		/* Expected a <tablespaceoid>.tar file here. */
+		uint64		num = strtoul(relpath, &suffix, 10);
+
+		/*
+		 * Report an error if we didn't consume at least one character, if the
+		 * result is 0, or if the value is too large to be a valid OID.
+		 */
+		if (suffix == NULL || num <= 0 || num > OID_MAX)
+			report_backup_error(context,
+								"file \"%s\" is not expected in a tar format backup",
+								relpath);
+		tblspc_oid = (Oid) num;
+	}
+
+	/* Now, check the compression type of the tar */
+	if (strcmp(suffix, ".tar") == 0)
+		compress_algorithm = PG_COMPRESSION_NONE;
+	else if (strcmp(suffix, ".tgz") == 0)
+		compress_algorithm = PG_COMPRESSION_GZIP;
+	else if (strcmp(suffix, ".tar.gz") == 0)
+		compress_algorithm = PG_COMPRESSION_GZIP;
+	else if (strcmp(suffix, ".tar.lz4") == 0)
+		compress_algorithm = PG_COMPRESSION_LZ4;
+	else if (strcmp(suffix, ".tar.zst") == 0)
+		compress_algorithm = PG_COMPRESSION_ZSTD;
+	else
+	{
+		report_backup_error(context,
+							"file \"%s\" is not expected in a tar format backup",
+							relpath);
+		return;
+	}
+
+	/*
+	 * Ignore WALs, as reading and verification will be handled through
+	 * pg_waldump.
+	 */
+	if (strncmp("pg_wal", relpath, 6) == 0)
+		return;
+
+	/*
+	 * Append the information to the list for complete verification at a later
+	 * stage.
+	 */
+	tar = pg_malloc(sizeof(tar_file));
+	tar->relpath = pstrdup(relpath);
+	tar->tblspc_oid = tblspc_oid;
+	tar->compress_algorithm = compress_algorithm;
+
+	simple_ptr_list_append(tarfiles, tar);
+
+	/* Update statistics for progress report, if necessary */
+	if (show_progress)
+		total_size += sb.st_size;
+}
+
+/*
+ * Verification of a single tar file content.
+ *
+ * It reads a given tar archive in predefined chunks and passes it to the
+ * streamer, which initiates routines for decompression (if necessary) and then
+ * verifies each member within the tar file.
+ */
+static void
+verify_tar_file(verifier_context *context, char *relpath, char *fullpath,
+				astreamer *streamer)
+{
+	int			fd;
+	int			rc;
+	char	   *buffer;
+
+	pg_log_debug("reading \"%s\"", fullpath);
+
+	/* Open the target file. */
+	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
+	{
+		report_backup_error(context, "could not open file \"%s\": %m",
+							relpath);
+		return;
+	}
+
+	buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
+
+	/* Perform the reads */
+	while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
+	{
+		astreamer_content(streamer, NULL, buffer, rc, ASTREAMER_UNKNOWN);
+
+		/* Report progress */
+		done_size += rc;
+		progress_report(false);
+	}
+
+	if (rc < 0)
+		report_backup_error(context, "could not read file \"%s\": %m",
+							relpath);
+
+	/* Close the file. */
+	if (close(fd) != 0)
+		report_backup_error(context, "could not close file \"%s\": %m",
+							relpath);
+}
+
 /*
  * Scan the hash table for entries where the 'matched' flag is not set; report
  * that such files are present in the manifest but not on disk.
@@ -830,10 +1151,10 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 
 	/*
 	 * Double-check that we read the expected number of bytes from the file.
-	 * Normally, a file size mismatch would be caught in verify_backup_file
-	 * and this check would never be reached, but this provides additional
-	 * safety and clarity in the event of concurrent modifications or
-	 * filesystem misbehavior.
+	 * Normally, mismatches would be caught in verify_plain_backup_file and
+	 * this check would never be reached, but this provides additional safety
+	 * and clarity in the event of concurrent modifications or filesystem
+	 * misbehavior.
 	 */
 	if (bytes_read != m->size)
 	{
@@ -955,6 +1276,37 @@ should_ignore_relpath(verifier_context *context, const char *relpath)
 	return false;
 }
 
+/*
+ * Create a chain of archive streamers appropriate for verifying a given
+ * archive.
+ */
+static astreamer *
+create_archive_verifier(verifier_context *context, char *archive_name,
+						Oid tblspc_oid, pg_compress_algorithm compress_algo)
+{
+	astreamer  *streamer = NULL;
+
+	/* Should be here only for tar backup */
+	Assert(context->format == 't');
+
+	/* Last step is the actual verification. */
+	streamer = astreamer_verify_content_new(streamer, context, archive_name,
+											tblspc_oid);
+
+	/* Before that we must parse the tar file. */
+	streamer = astreamer_tar_parser_new(streamer);
+
+	/* Before that we must decompress, if archive is compressed. */
+	if (compress_algo == PG_COMPRESSION_GZIP)
+		streamer = astreamer_gzip_decompressor_new(streamer);
+	else if (compress_algo == PG_COMPRESSION_LZ4)
+		streamer = astreamer_lz4_decompressor_new(streamer);
+	else if (compress_algo == PG_COMPRESSION_ZSTD)
+		streamer = astreamer_zstd_decompressor_new(streamer);
+
+	return streamer;
+}
+
 /*
  * Print a progress report based on the global variables.
  *
@@ -1010,6 +1362,7 @@ usage(void)
 	printf(_("Usage:\n  %s [OPTION]... BACKUPDIR\n\n"), progname);
 	printf(_("Options:\n"));
 	printf(_("  -e, --exit-on-error         exit immediately on error\n"));
+	printf(_("  -F, --format=p|t            backup format (plain, tar)\n"));
 	printf(_("  -i, --ignore=RELATIVE_PATH  ignore indicated path\n"));
 	printf(_("  -m, --manifest-path=PATH    use specified path for manifest\n"));
 	printf(_("  -n, --no-parse-wal          do not try to parse WAL files\n"));
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index d8c566ed587..183b1d5111b 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -18,6 +18,7 @@
 #include "common/hashfn_unstable.h"
 #include "common/logging.h"
 #include "common/parse_manifest.h"
+#include "fe_utils/astreamer.h"
 #include "fe_utils/simple_list.h"
 
 /*
@@ -88,6 +89,7 @@ typedef struct verifier_context
 	manifest_data *manifest;
 	char	   *backup_directory;
 	SimpleStringList ignore_list;
+	char		format;			/* backup format:  p(lain)/t(ar) */
 	bool		skip_checksums;
 	bool		exit_on_error;
 	bool		saw_any_error;
@@ -101,4 +103,9 @@ extern void report_fatal_error(const char *pg_restrict fmt,...)
 extern bool should_ignore_relpath(verifier_context *context,
 								  const char *relpath);
 
+extern astreamer *astreamer_verify_content_new(astreamer *next,
+											   verifier_context *context,
+											   char *archive_name,
+											   Oid tblspc_oid);
+
 #endif							/* PG_VERIFYBACKUP_H */
diff --git a/src/bin/pg_verifybackup/t/002_algorithm.pl b/src/bin/pg_verifybackup/t/002_algorithm.pl
index fb2a1fd7c4e..4959d5bd0b9 100644
--- a/src/bin/pg_verifybackup/t/002_algorithm.pl
+++ b/src/bin/pg_verifybackup/t/002_algorithm.pl
@@ -14,24 +14,35 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 
-for my $algorithm (qw(bogus none crc32c sha224 sha256 sha384 sha512))
+sub test_checksums
 {
-	my $backup_path = $primary->backup_dir . '/' . $algorithm;
+	my ($format, $algorithm) = @_;
+	my $backup_path = $primary->backup_dir . '/' . $format . '/' . $algorithm;
 	my @backup = (
 		'pg_basebackup', '-D', $backup_path,
 		'--manifest-checksums', $algorithm, '--no-sync', '-cfast');
 	my @verify = ('pg_verifybackup', '-e', $backup_path);
 
+	if ($format eq 'tar')
+	{
+		# Add switch to get a tar-format backup
+		push @backup, ('-F', 't');
+
+		# Add switch to skip WAL verification, which is not yet supported for
+		# tar-format backups
+		push @verify, ('-n');
+	}
+
 	# A backup with a bogus algorithm should fail.
 	if ($algorithm eq 'bogus')
 	{
 		$primary->command_fails(\@backup,
-			"backup fails with algorithm \"$algorithm\"");
-		next;
+			"$format format backup fails with algorithm \"$algorithm\"");
+		return;
 	}
 
 	# A backup with a valid algorithm should work.
-	$primary->command_ok(\@backup, "backup ok with algorithm \"$algorithm\"");
+	$primary->command_ok(\@backup, "$format format backup ok with algorithm \"$algorithm\"");
 
 	# We expect each real checksum algorithm to be mentioned on every line of
 	# the backup manifest file except the first and last; for simplicity, we
@@ -39,7 +50,7 @@ for my $algorithm (qw(bogus none crc32c sha224 sha256 sha384 sha512))
 	# is none, we just check that the manifest exists.
 	if ($algorithm eq 'none')
 	{
-		ok(-f "$backup_path/backup_manifest", "backup manifest exists");
+		ok(-f "$backup_path/backup_manifest", "$format format backup manifest exists");
 	}
 	else
 	{
@@ -52,10 +63,19 @@ for my $algorithm (qw(bogus none crc32c sha224 sha256 sha384 sha512))
 
 	# Make sure that it verifies OK.
 	$primary->command_ok(\@verify,
-		"verify backup with algorithm \"$algorithm\"");
+		"verify $format format backup with algorithm \"$algorithm\"");
 
 	# Remove backup immediately to save disk space.
 	rmtree($backup_path);
 }
 
+# Do the check
+for my $format (qw(plain tar))
+{
+	for my $algorithm (qw(bogus none crc32c sha224 sha256 sha384 sha512))
+	{
+		test_checksums($format, $algorithm);
+	}
+}
+
 done_testing();
diff --git a/src/bin/pg_verifybackup/t/003_corruption.pl b/src/bin/pg_verifybackup/t/003_corruption.pl
index ae91e043384..8fe911a3aec 100644
--- a/src/bin/pg_verifybackup/t/003_corruption.pl
+++ b/src/bin/pg_verifybackup/t/003_corruption.pl
@@ -5,12 +5,15 @@
 
 use strict;
 use warnings FATAL => 'all';
+use Cwd;
 use File::Path qw(rmtree);
 use File::Copy;
 use PostgreSQL::Test::Cluster;
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+my $tar = $ENV{TAR};
+
 my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
@@ -34,35 +37,35 @@ my @scenario = (
 		'name' => 'extra_file',
 		'mutilate' => \&mutilate_extra_file,
 		'fails_like' =>
-		  qr/extra_file.*present on disk but not in the manifest/
+		  qr/extra_file.*present (on disk|in "[^"]+") but not in the manifest/
 	},
 	{
 		'name' => 'extra_tablespace_file',
 		'mutilate' => \&mutilate_extra_tablespace_file,
 		'fails_like' =>
-		  qr/extra_ts_file.*present on disk but not in the manifest/
+		  qr/extra_ts_file.*present (on disk|in "[^"]+") but not in the manifest/
 	},
 	{
 		'name' => 'missing_file',
 		'mutilate' => \&mutilate_missing_file,
 		'fails_like' =>
-		  qr/pg_xact\/0000.*present in the manifest but not on disk/
+		  qr/pg_xact\/0000.*present in the manifest but not (on disk|in "[^"]+")/
 	},
 	{
 		'name' => 'missing_tablespace',
 		'mutilate' => \&mutilate_missing_tablespace,
 		'fails_like' =>
-		  qr/pg_tblspc.*present in the manifest but not on disk/
+		  qr/pg_tblspc.*present in the manifest but not (on disk|in "[^"]+")/
 	},
 	{
 		'name' => 'append_to_file',
 		'mutilate' => \&mutilate_append_to_file,
-		'fails_like' => qr/has size \d+ on disk but size \d+ in the manifest/
+		'fails_like' => qr/has size \d+ (on disk|in "[^"]+") but size \d+ in the manifest/
 	},
 	{
 		'name' => 'truncate_file',
 		'mutilate' => \&mutilate_truncate_file,
-		'fails_like' => qr/has size 0 on disk but size \d+ in the manifest/
+		'fails_like' => qr/has size 0 (on disk|in "[^"]+") but size \d+ in the manifest/
 	},
 	{
 		'name' => 'replace_file',
@@ -84,21 +87,21 @@ my @scenario = (
 		'name' => 'open_file_fails',
 		'mutilate' => \&mutilate_open_file_fails,
 		'fails_like' => qr/could not open file/,
-		'skip_on_windows' => 1
+		'needs_unix_permissions' => 1
 	},
 	{
 		'name' => 'open_directory_fails',
 		'mutilate' => \&mutilate_open_directory_fails,
 		'cleanup' => \&cleanup_open_directory_fails,
 		'fails_like' => qr/could not open directory/,
-		'skip_on_windows' => 1
+		'needs_unix_permissions' => 1
 	},
 	{
 		'name' => 'search_directory_fails',
 		'mutilate' => \&mutilate_search_directory_fails,
 		'cleanup' => \&cleanup_search_directory_fails,
 		'fails_like' => qr/could not stat file or directory/,
-		'skip_on_windows' => 1
+		'needs_unix_permissions' => 1
 	});
 
 for my $scenario (@scenario)
@@ -108,7 +111,7 @@ for my $scenario (@scenario)
   SKIP:
 	{
 		skip "unix-style permissions not supported on Windows", 4
-		  if ($scenario->{'skip_on_windows'}
+		  if ($scenario->{'needs_unix_permissions'}
 			&& ($windows_os || $Config::Config{osname} eq 'cygwin'));
 
 		# Take a backup and check that it verifies OK.
@@ -140,7 +143,59 @@ for my $scenario (@scenario)
 		$scenario->{'cleanup'}->($backup_path)
 		  if exists $scenario->{'cleanup'};
 
-		# Finally, use rmtree to reclaim space.
+		# Turn it into a tar-format backup and see if we can still detect the
+		# same problem, unless the scenario needs UNIX permissions or we don't
+		# have a TAR program available. Note that this destructively modifies
+		# the backup directory.
+		if (! $scenario->{'needs_unix_permissions'} ||
+			!defined $tar || $tar eq '')
+		{
+			my $tar_backup_path = $primary->backup_dir . '/tar_' . $name;
+			mkdir($tar_backup_path) || die "mkdir $tar_backup_path: $!";
+
+			# tar and then remove each tablespace. We remove the original files
+			# so that they don't also end up in base.tar.
+			my @tsoid = grep { $_ ne '.' && $_ ne '..' }
+				slurp_dir("$backup_path/pg_tblspc");
+			my $cwd = getcwd;
+			for my $tsoid (@tsoid)
+			{
+				my $tspath = $backup_path . '/pg_tblspc/' . $tsoid;
+
+				chdir($tspath) || die "chdir: $!";
+				command_ok([ $tar, '-cf', "$tar_backup_path/$tsoid.tar", '.' ]);
+				chdir($cwd) || die "chdir: $!";
+				rmtree($tspath);
+			}
+
+			# tar and remove pg_wal
+			chdir($backup_path . '/pg_wal') || die "chdir: $!";
+			command_ok([ $tar, '-cf', "$tar_backup_path/pg_wal.tar", '.' ]);
+			chdir($cwd) || die "chdir: $!";
+			rmtree($backup_path . '/pg_wal');
+
+			# move the backup manifest
+			move($backup_path . '/backup_manifest',
+				$tar_backup_path . '/backup_manifest')
+				or die "could not copy manifest to $tar_backup_path";
+
+			# Construct base.tar with what's left.
+			chdir($backup_path) || die "chdir: $!";
+			command_ok([ $tar, '-cf', "$tar_backup_path/base.tar", '.' ]);
+			chdir($cwd) || die "chdir: $!";
+
+			# Now check that the backup no longer verifies. We must use -n
+			# here, because pg_waldump can't yet read WAL from a tarfile.
+			command_fails_like(
+				[ 'pg_verifybackup', '-n', $tar_backup_path ],
+				$scenario->{'fails_like'},
+				"corrupt backup fails verification: $name");
+
+			# Use rmtree to reclaim space.
+			rmtree($tar_backup_path);
+		}
+
+		# Use rmtree to reclaim space.
 		rmtree($backup_path);
 	}
 }
diff --git a/src/bin/pg_verifybackup/t/004_options.pl b/src/bin/pg_verifybackup/t/004_options.pl
index 8ed2214408e..9dbb8c1a0ac 100644
--- a/src/bin/pg_verifybackup/t/004_options.pl
+++ b/src/bin/pg_verifybackup/t/004_options.pl
@@ -28,6 +28,23 @@ ok($result, "-q succeeds: exit code 0");
 is($stdout, '', "-q succeeds: no stdout");
 is($stderr, '', "-q succeeds: no stderr");
 
+# Should still work if we specify -Fp.
+$primary->command_ok(
+	[ 'pg_verifybackup', '-Fp', $backup_path ],
+	"verifies with -Fp");
+
+# Should not work if we specify -Fy because that's invalid.
+$primary->command_fails_like(
+	[ 'pg_verifybackup', '-Fy', $backup_path ],
+	qr(invalid backup format "y", must be "plain" or "tar"),
+	"does not verify with -Fy");
+
+# Should produce a lengthy list of errors; we test for just one of those.
+$primary->command_fails_like(
+	[ 'pg_verifybackup', '-Ft', '-n', $backup_path ],
+	qr("pg_multixact" is not a plain file),
+	"does not verify with -Ft -n");
+
 # Test invalid options
 command_fails_like(
 	[ 'pg_verifybackup', '--progress', '--quiet', $backup_path ],
diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
index 7a09f3b75b2..e7ec8369362 100644
--- a/src/bin/pg_verifybackup/t/008_untar.pl
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -16,6 +16,18 @@ my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;
 
+# Create a tablespace directory.
+my $source_ts_path = PostgreSQL::Test::Utils::tempdir_short();
+
+# Create a tablespace with table in it.
+$primary->safe_psql('postgres', qq(
+		CREATE TABLESPACE regress_ts1 LOCATION '$source_ts_path';
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1';
+		CREATE TABLE regress_tbl1(i int) TABLESPACE regress_ts1;
+		INSERT INTO regress_tbl1 VALUES(generate_series(1,5));));
+my $tsoid = $primary->safe_psql('postgres', qq(
+		SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1'));
+
 my $backup_path = $primary->backup_dir . '/server-backup';
 my $extract_path = $primary->backup_dir . '/extracted-backup';
 
@@ -23,39 +35,31 @@ my @test_configuration = (
 	{
 		'compression_method' => 'none',
 		'backup_flags' => [],
-		'backup_archive' => 'base.tar',
+		'backup_archive' => ['base.tar', "$tsoid.tar"],
 		'enabled' => 1
 	},
 	{
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'server-gzip' ],
-		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.gz', "$tsoid.tar.gz" ],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'server-lz4' ],
-		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => [ '-d', '-m' ],
+		'backup_archive' => ['base.tar.lz4', "$tsoid.tar.lz4" ],
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'server-zstd:level=1,long' ],
-		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
+		'backup_archive' => [ 'base.tar.zst', "$tsoid.tar.zst" ],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	});
 
@@ -86,47 +90,16 @@ for my $tc (@test_configuration)
 		my $backup_files = join(',',
 			sort grep { $_ ne '.' && $_ ne '..' } slurp_dir($backup_path));
 		my $expected_backup_files =
-		  join(',', sort ('backup_manifest', $tc->{'backup_archive'}));
+		  join(',', sort ('backup_manifest', @{ $tc->{'backup_archive'} }));
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok(['pg_verifybackup', '-n', '-e', $backup_path],
+			"verify backup, compression $method");
 
 		# Cleanup.
-		unlink($backup_path . '/backup_manifest');
-		unlink($backup_path . '/base.tar');
-		unlink($backup_path . '/' . $tc->{'backup_archive'});
+		rmtree($backup_path);
 		rmtree($extract_path);
 	}
 }
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index 8c076d46dee..6b7d7483f6e 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -29,41 +29,30 @@ my @test_configuration = (
 		'compression_method' => 'gzip',
 		'backup_flags' => [ '--compress', 'client-gzip:5' ],
 		'backup_archive' => 'base.tar.gz',
-		'decompress_program' => $ENV{'GZIP_PROGRAM'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
 	},
 	{
 		'compression_method' => 'lz4',
 		'backup_flags' => [ '--compress', 'client-lz4:5' ],
 		'backup_archive' => 'base.tar.lz4',
-		'decompress_program' => $ENV{'LZ4'},
-		'decompress_flags' => ['-d'],
-		'output_file' => 'base.tar',
 		'enabled' => check_pg_config("#define USE_LZ4 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:5' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:level=1,long' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1")
 	},
 	{
 		'compression_method' => 'parallel zstd',
 		'backup_flags' => [ '--compress', 'client-zstd:workers=3' ],
 		'backup_archive' => 'base.tar.zst',
-		'decompress_program' => $ENV{'ZSTD'},
-		'decompress_flags' => ['-d'],
 		'enabled' => check_pg_config("#define USE_ZSTD 1"),
 		'possibly_unsupported' =>
 		  qr/could not set compression worker count to 3: Unsupported parameter/
@@ -118,40 +107,9 @@ for my $tc (@test_configuration)
 		is($backup_files, $expected_backup_files,
 			"found expected backup files, compression $method");
 
-		# Decompress.
-		if (exists $tc->{'decompress_program'})
-		{
-			my @decompress = ($tc->{'decompress_program'});
-			push @decompress, @{ $tc->{'decompress_flags'} }
-			  if $tc->{'decompress_flags'};
-			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
-			push @decompress, $backup_path . '/' . $tc->{'output_file'}
-			  if $tc->{'output_file'};
-			system_or_bail(@decompress);
-		}
-
-	  SKIP:
-		{
-			my $tar = $ENV{TAR};
-			# don't check for a working tar here, to accommodate various odd
-			# cases. If tar doesn't work the init_from_backup below will fail.
-			skip "no tar program available", 1
-			  if (!defined $tar || $tar eq '');
-
-			# Untar.
-			mkdir($extract_path);
-			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
-				'-C', $extract_path);
-
-			# Verify.
-			$primary->command_ok(
-				[
-					'pg_verifybackup', '-n',
-					'-m', "$backup_path/backup_manifest",
-					'-e', $extract_path
-				],
-				"verify backup, compression $method");
-		}
+		# Verify tar backup.
+		$primary->command_ok( [ 'pg_verifybackup', '-n', '-e', $backup_path ],
+			"verify backup, compression $method");
 
 		# Cleanup.
 		rmtree($extract_path);
diff --git a/src/fe_utils/simple_list.c b/src/fe_utils/simple_list.c
index 2d88eb54067..c07e6bd9180 100644
--- a/src/fe_utils/simple_list.c
+++ b/src/fe_utils/simple_list.c
@@ -173,3 +173,22 @@ simple_ptr_list_append(SimplePtrList *list, void *ptr)
 		list->head = cell;
 	list->tail = cell;
 }
+
+/*
+ * Destroy only pointer list and not the pointed-to element
+ */
+void
+simple_ptr_list_destroy(SimplePtrList *list)
+{
+	SimplePtrListCell *cell;
+
+	cell = list->head;
+	while (cell != NULL)
+	{
+		SimplePtrListCell *next;
+
+		next = cell->next;
+		pg_free(cell);
+		cell = next;
+	}
+}
diff --git a/src/include/fe_utils/simple_list.h b/src/include/fe_utils/simple_list.h
index d42ecded8ed..c83ab6f77e4 100644
--- a/src/include/fe_utils/simple_list.h
+++ b/src/include/fe_utils/simple_list.h
@@ -66,5 +66,6 @@ extern void simple_string_list_destroy(SimpleStringList *list);
 extern const char *simple_string_list_not_touched(SimpleStringList *list);
 
 extern void simple_ptr_list_append(SimplePtrList *list, void *ptr);
+extern void simple_ptr_list_destroy(SimplePtrList *list);
 
 #endif							/* SIMPLE_LIST_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index b6135f03479..5fabb127d7e 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3336,6 +3336,7 @@ astreamer_plain_writer
 astreamer_recovery_injector
 astreamer_tar_archiver
 astreamer_tar_parser
+astreamer_verify
 astreamer_zstd_frame
 bgworker_main_type
 bh_node_type
@@ -3957,6 +3958,7 @@ substitute_phv_relids_context
 subxids_array_status
 symbol
 tablespaceinfo
+tar_file
 td_entry
 teSection
 temp_tablespaces_extra
-- 
2.39.3 (Apple Git-145)

From d1a925bf02d1f4cb4d4f736ef32b4c821dec976d Mon Sep 17 00:00:00 2001
From: Robert Haas <rhaas@postgresql.org>
Date: Mon, 30 Sep 2024 11:16:15 -0400
Subject: [PATCH] Fix issues reported by Coverity.

---
 src/bin/pg_verifybackup/astreamer_verify.c | 2 +-
 src/bin/pg_verifybackup/pg_verifybackup.c  | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/bin/pg_verifybackup/astreamer_verify.c b/src/bin/pg_verifybackup/astreamer_verify.c
index 57072fdfe04..eb17dfbd95f 100644
--- a/src/bin/pg_verifybackup/astreamer_verify.c
+++ b/src/bin/pg_verifybackup/astreamer_verify.c
@@ -341,7 +341,7 @@ member_copy_control_data(astreamer *streamer, astreamer_member *member,
 	 * be PG_CONTROL_FILE_SIZE, but the part that fits in our buffer is
 	 * shorter, just sizeof(ControlFileData).
 	 */
-	if (mystreamer->control_file_bytes <= sizeof(ControlFileData))
+	if (mystreamer->control_file_bytes < sizeof(ControlFileData))
 	{
 		int			remaining;
 
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index a9d41a6b838..32467a1ba09 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -929,9 +929,12 @@ precheck_tar_backup_file(verifier_context *context, char *relpath,
 		 * result is 0, or if the value is too large to be a valid OID.
 		 */
 		if (suffix == NULL || num <= 0 || num > OID_MAX)
+		{
 			report_backup_error(context,
 								"file \"%s\" is not expected in a tar format backup",
 								relpath);
+			return;
+		}
 		tblspc_oid = (Oid) num;
 	}
 
@@ -1014,6 +1017,8 @@ verify_tar_file(verifier_context *context, char *relpath, char *fullpath,
 		progress_report(false);
 	}
 
+	pg_free(buffer);
+
 	if (rc < 0)
 		report_backup_error(context, "could not read file \"%s\": %m",
 							relpath);
-- 
2.39.3 (Apple Git-145)

From d0f79c4a9cf980a564a68cbc61a4100cab8ad28c Mon Sep 17 00:00:00 2001
From: Robert Haas <rhaas@postgresql.org>
Date: Tue, 1 Oct 2024 10:23:27 -0400
Subject: [PATCH v1] Try to use uint64 to size_t for file sizes.

---
 src/bin/pg_combinebackup/load_manifest.c   |  4 ++--
 src/bin/pg_combinebackup/load_manifest.h   |  2 +-
 src/bin/pg_combinebackup/write_manifest.c  |  5 +++--
 src/bin/pg_combinebackup/write_manifest.h  |  2 +-
 src/bin/pg_verifybackup/astreamer_verify.c | 13 ++++++++-----
 src/bin/pg_verifybackup/pg_verifybackup.c  | 16 +++++++++-------
 src/bin/pg_verifybackup/pg_verifybackup.h  |  2 +-
 src/common/parse_manifest.c                |  4 ++--
 src/include/common/parse_manifest.h        |  2 +-
 9 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/src/bin/pg_combinebackup/load_manifest.c b/src/bin/pg_combinebackup/load_manifest.c
index be8e6273fcb..3a3ad6c2474 100644
--- a/src/bin/pg_combinebackup/load_manifest.c
+++ b/src/bin/pg_combinebackup/load_manifest.c
@@ -60,7 +60,7 @@ static void combinebackup_version_cb(JsonManifestParseContext *context,
 static void combinebackup_system_identifier_cb(JsonManifestParseContext *context,
 											   uint64 manifest_system_identifier);
 static void combinebackup_per_file_cb(JsonManifestParseContext *context,
-									  const char *pathname, size_t size,
+									  const char *pathname, uint64 size,
 									  pg_checksum_type checksum_type,
 									  int checksum_length,
 									  uint8 *checksum_payload);
@@ -267,7 +267,7 @@ combinebackup_system_identifier_cb(JsonManifestParseContext *context,
  */
 static void
 combinebackup_per_file_cb(JsonManifestParseContext *context,
-						  const char *pathname, size_t size,
+						  const char *pathname, uint64 size,
 						  pg_checksum_type checksum_type,
 						  int checksum_length, uint8 *checksum_payload)
 {
diff --git a/src/bin/pg_combinebackup/load_manifest.h b/src/bin/pg_combinebackup/load_manifest.h
index a96ae12eb8e..8e657179af0 100644
--- a/src/bin/pg_combinebackup/load_manifest.h
+++ b/src/bin/pg_combinebackup/load_manifest.h
@@ -23,7 +23,7 @@ typedef struct manifest_file
 {
 	uint32		status;			/* hash status */
 	const char *pathname;
-	size_t		size;
+	uint64		size;
 	pg_checksum_type checksum_type;
 	int			checksum_length;
 	uint8	   *checksum_payload;
diff --git a/src/bin/pg_combinebackup/write_manifest.c b/src/bin/pg_combinebackup/write_manifest.c
index 369d6d2071c..6fea07e7c64 100644
--- a/src/bin/pg_combinebackup/write_manifest.c
+++ b/src/bin/pg_combinebackup/write_manifest.c
@@ -74,7 +74,7 @@ create_manifest_writer(char *directory, uint64 system_identifier)
  */
 void
 add_file_to_manifest(manifest_writer *mwriter, const char *manifest_path,
-					 size_t size, time_t mtime,
+					 uint64 size, time_t mtime,
 					 pg_checksum_type checksum_type,
 					 int checksum_length,
 					 uint8 *checksum_payload)
@@ -104,7 +104,8 @@ add_file_to_manifest(manifest_writer *mwriter, const char *manifest_path,
 		appendStringInfoString(&mwriter->buf, "\", ");
 	}
 
-	appendStringInfo(&mwriter->buf, "\"Size\": %zu, ", size);
+	appendStringInfo(&mwriter->buf, "\"Size\": %llu, ",
+					 (unsigned long long) size);
 
 	appendStringInfoString(&mwriter->buf, "\"Last-Modified\": \"");
 	enlargeStringInfo(&mwriter->buf, 128);
diff --git a/src/bin/pg_combinebackup/write_manifest.h b/src/bin/pg_combinebackup/write_manifest.h
index ebc4f9441ad..d2becaba1f9 100644
--- a/src/bin/pg_combinebackup/write_manifest.h
+++ b/src/bin/pg_combinebackup/write_manifest.h
@@ -23,7 +23,7 @@ extern manifest_writer *create_manifest_writer(char *directory,
 											   uint64 system_identifier);
 extern void add_file_to_manifest(manifest_writer *mwriter,
 								 const char *manifest_path,
-								 size_t size, time_t mtime,
+								 uint64 size, time_t mtime,
 								 pg_checksum_type checksum_type,
 								 int checksum_length,
 								 uint8 *checksum_payload);
diff --git a/src/bin/pg_verifybackup/astreamer_verify.c b/src/bin/pg_verifybackup/astreamer_verify.c
index f7ecdc1f655..a442b2849fc 100644
--- a/src/bin/pg_verifybackup/astreamer_verify.c
+++ b/src/bin/pg_verifybackup/astreamer_verify.c
@@ -207,9 +207,11 @@ member_verify_header(astreamer *streamer, astreamer_member *member)
 	if (m->size != member->size)
 	{
 		report_backup_error(mystreamer->context,
-							"\"%s\" has size %lld in \"%s\" but size %zu in the manifest",
-							member->pathname, (long long int) member->size,
-							mystreamer->archive_name, m->size);
+							"\"%s\" has size %llu in \"%s\" but size %llu in the manifest",
+							member->pathname,
+							(unsigned long long) member->size,
+							mystreamer->archive_name,
+							(unsigned long long) m->size);
 		m->bad = true;
 		return;
 	}
@@ -294,9 +296,10 @@ member_verify_checksum(astreamer *streamer)
 	if (mystreamer->checksum_bytes != m->size)
 	{
 		report_backup_error(mystreamer->context,
-							"file \"%s\" in \"%s\" should contain %zu bytes, but read %zu bytes",
+							"file \"%s\" in \"%s\" should contain %llu bytes, but read %llu bytes",
 							m->pathname, mystreamer->archive_name,
-							m->size, mystreamer->checksum_bytes);
+							(unsigned long long) m->size,
+							(unsigned long long) mystreamer->checksum_bytes);
 		return;
 	}
 
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 32467a1ba09..0719cb89783 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -61,7 +61,7 @@ static void verifybackup_version_cb(JsonManifestParseContext *context,
 static void verifybackup_system_identifier(JsonManifestParseContext *context,
 										   uint64 manifest_system_identifier);
 static void verifybackup_per_file_cb(JsonManifestParseContext *context,
-									 const char *pathname, size_t size,
+									 const char *pathname, uint64 size,
 									 pg_checksum_type checksum_type,
 									 int checksum_length,
 									 uint8 *checksum_payload);
@@ -547,7 +547,7 @@ verifybackup_system_identifier(JsonManifestParseContext *context,
  */
 static void
 verifybackup_per_file_cb(JsonManifestParseContext *context,
-						 const char *pathname, size_t size,
+						 const char *pathname, uint64 size,
 						 pg_checksum_type checksum_type,
 						 int checksum_length, uint8 *checksum_payload)
 {
@@ -719,8 +719,9 @@ verify_plain_backup_file(verifier_context *context, char *relpath,
 	if (m->size != sb.st_size)
 	{
 		report_backup_error(context,
-							"\"%s\" has size %lld on disk but size %zu in the manifest",
-							relpath, (long long int) sb.st_size, m->size);
+							"\"%s\" has size %llu on disk but size %llu in the manifest",
+							relpath, (unsigned long long) sb.st_size,
+							(unsigned long long) m->size);
 		m->bad = true;
 	}
 
@@ -1101,7 +1102,7 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	const char *relpath = m->pathname;
 	int			fd;
 	int			rc;
-	size_t		bytes_read = 0;
+	uint64		bytes_read = 0;
 	uint8		checksumbuf[PG_CHECKSUM_MAX_LENGTH];
 	int			checksumlen;
 
@@ -1164,8 +1165,9 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 	if (bytes_read != m->size)
 	{
 		report_backup_error(context,
-							"file \"%s\" should contain %zu bytes, but read %zu bytes",
-							relpath, m->size, bytes_read);
+							"file \"%s\" should contain %llu bytes, but read %llu bytes",
+							relpath, (unsigned long long) m->size,
+							(unsigned long long) bytes_read);
 		return;
 	}
 
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.h b/src/bin/pg_verifybackup/pg_verifybackup.h
index 183b1d5111b..2f864fb0f3f 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.h
+++ b/src/bin/pg_verifybackup/pg_verifybackup.h
@@ -29,7 +29,7 @@ typedef struct manifest_file
 {
 	uint32		status;			/* hash status */
 	const char *pathname;
-	size_t		size;
+	uint64		size;
 	pg_checksum_type checksum_type;
 	int			checksum_length;
 	uint8	   *checksum_payload;
diff --git a/src/common/parse_manifest.c b/src/common/parse_manifest.c
index 5a7b491e9a9..ad2d0fd808f 100644
--- a/src/common/parse_manifest.c
+++ b/src/common/parse_manifest.c
@@ -650,7 +650,7 @@ static void
 json_manifest_finalize_file(JsonManifestParseState *parse)
 {
 	JsonManifestParseContext *context = parse->context;
-	size_t		size;
+	uint64		size;
 	char	   *ep;
 	int			checksum_string_length;
 	pg_checksum_type checksum_type;
@@ -688,7 +688,7 @@ json_manifest_finalize_file(JsonManifestParseState *parse)
 	}
 
 	/* Parse size. */
-	size = strtoul(parse->size, &ep, 10);
+	size = strtou64(parse->size, &ep, 10);
 	if (*ep)
 		json_manifest_parse_failure(parse->context,
 									"file size is not an integer");
diff --git a/src/include/common/parse_manifest.h b/src/include/common/parse_manifest.h
index ee571a568a1..1b8bc447e44 100644
--- a/src/include/common/parse_manifest.h
+++ b/src/include/common/parse_manifest.h
@@ -28,7 +28,7 @@ typedef void (*json_manifest_system_identifier_callback) (JsonManifestParseConte
 														  uint64 manifest_system_identifier);
 typedef void (*json_manifest_per_file_callback) (JsonManifestParseContext *,
 												 const char *pathname,
-												 size_t size, pg_checksum_type checksum_type,
+												 uint64 size, pg_checksum_type checksum_type,
 												 int checksum_length, uint8 *checksum_payload);
 typedef void (*json_manifest_per_wal_range_callback) (JsonManifestParseContext *,
 													  TimeLineID tli,
-- 
2.39.3 (Apple Git-145)