From 7769392066efcd9c00a5c2be722eb91af10787a4 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Sun, 5 Apr 2026 02:40:56 +1200 Subject: [PATCH 4/4] pg_waldump: Use astreamer_libarchive.c. If this build supports libarchive, use astreamer_libarchive_reader instead of astreamer_tar_parser to read WAL archives. This allows modern tar formats with more types of compression to be used. (Proof-of-concept) --- src/bin/pg_waldump/Makefile | 5 +++++ src/bin/pg_waldump/archive_waldump.c | 31 +++++++++++++++++++++++++++- src/bin/pg_waldump/meson.build | 2 +- src/bin/pg_waldump/t/001_basic.pl | 9 +++++++- 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/bin/pg_waldump/Makefile b/src/bin/pg_waldump/Makefile index aabb87566a2..09005bf4ba4 100644 --- a/src/bin/pg_waldump/Makefile +++ b/src/bin/pg_waldump/Makefile @@ -23,6 +23,11 @@ OBJS = \ override CPPFLAGS := -DFRONTEND -I$(libpq_srcdir) $(CPPFLAGS) LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils +ifeq ($(with_libarchive), yes) +# XXX figure out where this should go +LDFLAGS_INTERNAL += $(LIBARCHIVE_LIBS) +endif + RMGRDESCSOURCES = $(sort $(notdir $(wildcard $(top_srcdir)/src/backend/access/rmgrdesc/*desc*.c))) RMGRDESCOBJS = $(patsubst %.c,%.o,$(RMGRDESCSOURCES)) diff --git a/src/bin/pg_waldump/archive_waldump.c b/src/bin/pg_waldump/archive_waldump.c index e4a4bf44a7e..938a253ecab 100644 --- a/src/bin/pg_waldump/archive_waldump.c +++ b/src/bin/pg_waldump/archive_waldump.c @@ -129,12 +129,30 @@ void init_archive_reader(XLogDumpPrivate *privateInfo, pg_compress_algorithm compression) { - int fd; astreamer *streamer; ArchivedWALFile *entry = NULL; XLogLongPageHeader longhdr; ArchivedWAL_iterator iter; +#ifdef USE_LIBARCHIVE + char *pathname = NULL; + + /* Open tar archive with libarchive. */ + streamer = astreamer_waldump_new(privateInfo); + if (privateInfo->archive_dir) + pathname = psprintf("%s/%s", + privateInfo->archive_dir, + privateInfo->archive_name); + streamer = + astreamer_libarchive_reader_new_pathname(streamer, + pathname ? + pathname : + privateInfo->archive_name); + if (pathname) + pfree(pathname); +#else + int fd; + /* Open tar archive and store its file descriptor */ fd = open_file_in_directory(privateInfo->archive_dir, privateInfo->archive_name); @@ -157,6 +175,7 @@ init_archive_reader(XLogDumpPrivate *privateInfo, streamer = astreamer_lz4_decompressor_new(streamer); else if (compression == PG_COMPRESSION_ZSTD) streamer = astreamer_zstd_decompressor_new(streamer); +#endif privateInfo->archive_streamer = streamer; @@ -286,10 +305,12 @@ free_archive_reader(XLogDumpPrivate *privateInfo) privateInfo->archive_read_buf = NULL; } +#ifndef USE_LIBARCHIVE /* Close the file. */ if (close(privateInfo->archive_fd) != 0) pg_log_error("could not close file \"%s\": %m", privateInfo->archive_name); +#endif } /* @@ -537,12 +558,19 @@ get_archive_wal_entry(const char *fname, XLogDumpPrivate *privateInfo) static bool read_archive_file(XLogDumpPrivate *privateInfo) { +#ifndef USE_LIBARCHIVE int rc; +#endif /* Fail if we already reached EOF in a prior call. */ if (privateInfo->archive_fd_eof) return false; +#ifdef USE_LIBARCHIVE + /* Tell libarchive to read more data. */ + astreamer_pull(privateInfo->archive_streamer); +#else + /* Try to read some more data. */ rc = read(privateInfo->archive_fd, privateInfo->archive_read_buf, privateInfo->archive_read_buf_size); @@ -569,6 +597,7 @@ read_archive_file(XLogDumpPrivate *privateInfo) /* Set flag to ensure we don't finalize more than once. */ privateInfo->archive_fd_eof = true; } +#endif return true; } diff --git a/src/bin/pg_waldump/meson.build b/src/bin/pg_waldump/meson.build index 5296f21b82c..0b2c4021107 100644 --- a/src/bin/pg_waldump/meson.build +++ b/src/bin/pg_waldump/meson.build @@ -19,7 +19,7 @@ endif pg_waldump = executable('pg_waldump', pg_waldump_sources, - dependencies: [frontend_code, libpq, lz4, zstd], + dependencies: [frontend_code, libarchive, libpq, lz4, zstd], c_args: ['-DFRONTEND'], # needed for xlogreader et al kwargs: default_bin_args, ) diff --git a/src/bin/pg_waldump/t/001_basic.pl b/src/bin/pg_waldump/t/001_basic.pl index 7dd1c3dd63e..62a15228b38 100644 --- a/src/bin/pg_waldump/t/001_basic.pl +++ b/src/bin/pg_waldump/t/001_basic.pl @@ -11,7 +11,13 @@ use Test::More; use List::Util qw(shuffle); my $tar = $ENV{TAR}; -my @tar_p_flags = tar_portability_options($tar); +my @tar_p_flags; + +# If we don't have libarchive, then we tell tar to stick to ustar format that +# astreamer_tar.c can decode. Otherwise we should be able to accept anything +# that any current tar produces. +@tar_p_flags = tar_portability_options($tar) + if !check_pg_config("#define USE_LIBARCHIVE"); program_help_ok('pg_waldump'); program_version_ok('pg_waldump'); @@ -373,6 +379,7 @@ my @scenarios = ( 'compression_flags' => '-czf', 'is_archive' => 1, 'enabled' => check_pg_config("#define HAVE_LIBZ 1") + || check_pg_config("#define USE_LIBARCHIVE") }); for my $scenario (@scenarios) -- 2.53.0