From bc66e2b035ebd24de71ea41c909c62ca1aae2e2d Mon Sep 17 00:00:00 2001 From: Justin Pryzby Date: Sat, 14 Jan 2023 10:45:02 -0600 Subject: [PATCH 7/8] f! --- src/bin/pg_dump/compress_io.c | 42 ++++++++++++++------------- src/bin/pg_dump/compress_io.h | 3 +- src/bin/pg_dump/compress_lz4.c | 34 +++++++++++----------- src/bin/pg_dump/compress_lz4.h | 4 +-- src/bin/pg_dump/pg_backup_directory.c | 10 +++++-- src/bin/pg_dump/t/002_pg_dump.pl | 2 +- 6 files changed, 51 insertions(+), 44 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 95b18843080..01bf3df0db0 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -39,13 +39,14 @@ * * The compressed stream API is a wrapper around the C standard fopen() and * libz's gzopen() APIs and custom LZ4 calls which provide similar - * libz's gzopen() APIs. It allows you to use the same functions for - * compressed and uncompressed streams. cfopen_read() first tries to open - * the file with given name, and if it fails, it tries to open the same - * file with the .gz suffix. cfopen_write() opens a file for writing, an - * extra argument specifies if the file should be compressed, and adds the - * .gz suffix to the filename if so. This allows you to easily handle both - * compressed and uncompressed files. + * functionality. It allows you to use the same functions for compressed and + * uncompressed streams. cfopen_read() first tries to open the file with given + * name, and if it fails, it tries to open the same file with the .gz suffix, + * failing that it tries to open the same file with the .lz4 suffix. + * cfopen_write() opens a file for writing, an extra argument specifies the + * method to use should the file be compressed, and adds the appropriate + * suffix, .gz or .lz4, to the filename if so. This allows you to easily handle + * both compressed and uncompressed files. * * IDENTIFICATION * src/bin/pg_dump/compress_io.c @@ -335,6 +336,14 @@ InitCompressFileHandle(const pg_compress_specification *compression_spec) return CFH; } +static bool +check_compressed_file(const char *path, char **fname, char *ext) +{ + free_keep_errno(*fname); + *fname = psprintf("%s.%s", path, ext); + return (access(*fname, F_OK) == 0); +} + /* * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". @@ -348,7 +357,7 @@ InitCompressFileHandle(const pg_compress_specification *compression_spec) * */ CompressFileHandle * -InitDiscoverCompressFileHandle(const char *path, const char *mode) +InitDiscoverCompressFileHandle(const char *path, const char *mode, pg_compress_algorithm alg) { CompressFileHandle *CFH = NULL; struct stat st; @@ -366,20 +375,13 @@ InitDiscoverCompressFileHandle(const char *path, const char *mode) bool exists; exists = (stat(path, &st) == 0); - /* avoid unused warning if it is not build with compression */ + /* avoid unused warning if it is not built with compression */ if (exists) compression_spec.algorithm = PG_COMPRESSION_NONE; -#ifdef HAVE_LIBZ - if (!exists) - { - free_keep_errno(fname); - fname = psprintf("%s.gz", path); - exists = (stat(fname, &st) == 0); - - if (exists) - compression_spec.algorithm = PG_COMPRESSION_GZIP; - } -#endif + else if (alg == PG_COMPRESSION_GZIP && check_compressed_file(path, &fname, "gz")) + compression_spec.algorithm = alg; + else if (alg == PG_COMPRESSION_LZ4 && check_compressed_file(path, &fname, "lz4")) + compression_spec.algorithm = alg; } CFH = InitCompressFileHandle(&compression_spec); diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index afe6b22efaf..2600182c469 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -93,7 +93,8 @@ extern CompressFileHandle *InitCompressFileHandle( const pg_compress_specification *compression_spec); extern CompressFileHandle *InitDiscoverCompressFileHandle(const char *path, - const char *mode); + const char *mode, + pg_compress_algorithm alg); extern int DestroyCompressFileHandle(CompressFileHandle *CFH); #endif diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c index c97e16187a0..0e259a6251a 100644 --- a/src/bin/pg_dump/compress_lz4.c +++ b/src/bin/pg_dump/compress_lz4.c @@ -117,13 +117,13 @@ EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs) /* Public routines that support LZ4 compressed data I/O */ void -InitCompressorLZ4(CompressorState *cs, const pg_compress_specification compression_spec) +InitCompressorLZ4(CompressorState *cs, const pg_compress_specification *compression_spec) { cs->readData = ReadDataFromArchiveLZ4; cs->writeData = WriteDataToArchiveLZ4; cs->end = EndCompressorLZ4; - cs->compression_spec = compression_spec; + cs->compression_spec = *compression_spec; /* Will be lazy init'd */ cs->private_data = pg_malloc0(sizeof(LZ4CompressorState)); @@ -189,7 +189,7 @@ LZ4File_get_error(CompressFileHandle *CFH) /* * Prepare an already alloc'ed LZ4File struct for subsequent calls. * - * It creates the nessary contexts for the operations. When compressing, + * It creates the necessary contexts for the operations. When compressing, * it additionally writes the LZ4 header in the output stream. */ static int @@ -228,7 +228,7 @@ LZ4File_init(LZ4File * fs, int size, bool compressing) if (fwrite(fs->buffer, 1, status, fs->fp) != status) { - errno = errno ? : ENOSPC; + errno = errno ? errno : ENOSPC; return 1; } } @@ -255,7 +255,7 @@ LZ4File_init(LZ4File * fs, int size, bool compressing) /* * Read already decompressed content from the overflow buffer into 'ptr' up to * 'size' bytes, if available. If the eol_flag is set, then stop at the first - * occurance of the new line char prior to 'size' bytes. + * occurrence of the new line char prior to 'size' bytes. * * Any unread content in the overflow buffer, is moved to the beginning. */ @@ -309,10 +309,10 @@ LZ4File_read_internal(LZ4File * fs, void *ptr, int ptrsize, bool eol_flag) void *readbuf; /* Lazy init */ - if (!fs->inited && LZ4File_init(fs, size, false /* decompressing */ )) + if (LZ4File_init(fs, size, false /* decompressing */ )) return -1; - /* Verfiy that there is enough space in the outbuf */ + /* Verify that there is enough space in the outbuf */ if (size > fs->buflen) { fs->buflen = size; @@ -363,10 +363,10 @@ LZ4File_read_internal(LZ4File * fs, void *ptr, int ptrsize, bool eol_flag) if (outlen > 0 && dsize < size && eol_found == false) { char *p; - size_t lib = (eol_flag == 0) ? size - dsize : size - 1 - dsize; + size_t lib = eol_flag ? size - 1 - dsize : size - dsize ; size_t len = outlen < lib ? outlen : lib; - if (eol_flag == true && + if (eol_flag && (p = memchr(fs->buffer, '\n', outlen)) && (size_t) (p - fs->buffer + 1) <= len) { @@ -377,7 +377,7 @@ LZ4File_read_internal(LZ4File * fs, void *ptr, int ptrsize, bool eol_flag) memcpy((char *) ptr + dsize, fs->buffer, len); dsize += len; - /* move what did not fit, if any, at the begining of the buf */ + /* move what did not fit, if any, at the beginning of the buf */ if (len < outlen) memmove(fs->buffer, fs->buffer + len, outlen - len); outlen -= len; @@ -414,7 +414,7 @@ LZ4File_write(const void *ptr, size_t size, CompressFileHandle *CFH) size_t status; int remaining = size; - if (!fs->inited && LZ4File_init(fs, size, true)) + if (LZ4File_init(fs, size, true)) return -1; while (remaining > 0) @@ -433,7 +433,7 @@ LZ4File_write(const void *ptr, size_t size, CompressFileHandle *CFH) if (fwrite(fs->buffer, 1, status, fs->fp) != status) { - errno = errno ? : ENOSPC; + errno = errno ? errno : ENOSPC; return 1; } } @@ -520,7 +520,7 @@ LZ4File_close(CompressFileHandle *CFH) LZ4F_getErrorName(status)); else if ((ret = fwrite(fs->buffer, 1, status, fs->fp)) != status) { - errno = errno ? : ENOSPC; + errno = errno ? errno : ENOSPC; WRITE_ERROR_EXIT; } @@ -582,7 +582,7 @@ LZ4File_open_write(const char *path, const char *mode, CompressFileHandle *CFH) } void -InitCompressLZ4(CompressFileHandle *CFH, const pg_compress_specification compression_spec) +InitCompressLZ4(CompressFileHandle *CFH, const pg_compress_specification *compression_spec) { LZ4File *lz4fp; @@ -596,7 +596,7 @@ InitCompressLZ4(CompressFileHandle *CFH, const pg_compress_specification compres CFH->close_func = LZ4File_close; CFH->get_error_func = LZ4File_get_error; - CFH->compression_spec = compression_spec; + CFH->compression_spec = *compression_spec; lz4fp = pg_malloc0(sizeof(*lz4fp)); if (CFH->compression_spec.level >= 0) lz4fp->prefs.compressionLevel = CFH->compression_spec.level; @@ -605,13 +605,13 @@ InitCompressLZ4(CompressFileHandle *CFH, const pg_compress_specification compres } #else /* USE_LZ4 */ void -InitCompressorLZ4(CompressorState *cs, const pg_compress_specification compression_spec) +InitCompressorLZ4(CompressorState *cs, const pg_compress_specification *compression_spec) { pg_fatal("this build does not support compression with %s", "LZ4"); } void -InitCompressLZ4(CompressFileHandle *CFH, const pg_compress_specification compression_spec) +InitCompressLZ4(CompressFileHandle *CFH, const pg_compress_specification *compression_spec) { pg_fatal("this build does not support compression with %s", "LZ4"); } diff --git a/src/bin/pg_dump/compress_lz4.h b/src/bin/pg_dump/compress_lz4.h index 74595db1b98..69a3d9c171f 100644 --- a/src/bin/pg_dump/compress_lz4.h +++ b/src/bin/pg_dump/compress_lz4.h @@ -16,7 +16,7 @@ #include "compress_io.h" -extern void InitCompressorLZ4(CompressorState *cs, const pg_compress_specification compression_spec); -extern void InitCompressLZ4(CompressFileHandle *CFH, const pg_compress_specification compression_spec); +extern void InitCompressorLZ4(CompressorState *cs, const pg_compress_specification *compression_spec); +extern void InitCompressLZ4(CompressFileHandle *CFH, const pg_compress_specification *compression_spec); #endif /* _COMPRESS_LZ4_H_ */ diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 2d4baf58c22..4845fd9368c 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -201,7 +201,8 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = InitDiscoverCompressFileHandle(fname, PG_BINARY_R); + tocFH = InitDiscoverCompressFileHandle(fname, PG_BINARY_R, + PG_COMPRESSION_NONE); if (tocFH == NULL) pg_fatal("could not open input file \"%s\": %m", fname); @@ -390,7 +391,8 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - CFH = InitDiscoverCompressFileHandle(filename, PG_BINARY_R); + CFH = InitDiscoverCompressFileHandle(filename, PG_BINARY_R, + AH->compression_spec.algorithm); if (!CFH) pg_fatal("could not open input file \"%s\": %m", filename); @@ -442,7 +444,8 @@ _LoadLOs(ArchiveHandle *AH) setFilePath(AH, tocfname, "blobs.toc"); - CFH = ctx->LOsTocFH = InitDiscoverCompressFileHandle(tocfname, PG_BINARY_R); + CFH = ctx->LOsTocFH = InitDiscoverCompressFileHandle(tocfname, PG_BINARY_R, + AH->compression_spec.algorithm); if (ctx->LOsTocFH == NULL) pg_fatal("could not open large object TOC file \"%s\" for input: %m", @@ -783,6 +786,7 @@ _PrepParallelRestore(ArchiveHandle *AH) { /* It might be compressed */ strlcat(fname, ".gz", sizeof(fname)); + // XXX if (stat(fname, &st) == 0) te->dataLength = st.st_size; } diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index f497ec60407..263995a2b7a 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -183,7 +183,7 @@ my %pgdump_runs = ( "$tempdir/compression_lz4_dir/blobs.toc.lz4", ], }, - # Verify that data files where compressed + # Verify that data files were compressed glob_patterns => [ "$tempdir/compression_lz4_dir/toc.dat", "$tempdir/compression_lz4_dir/*.dat.lz4", -- 2.25.1