From 1669812c3177a0c54e36d892676a45a51232096a Mon Sep 17 00:00:00 2001 From: Georgios Kokolatos Date: Wed, 30 Nov 2022 12:58:03 +0000 Subject: [PATCH v11 1/4] Teach pg_dump about compress_spec and use it throughout. Align pg_dump with the rest of the binaries which use common compression. It is teaching pg_dump.c about the common compression definitions and interfaces. Then it propagates those throughout the code. --- doc/src/sgml/ref/pg_dump.sgml | 30 ++++++-- src/bin/pg_dump/compress_io.c | 102 ++++++++++---------------- src/bin/pg_dump/compress_io.h | 20 ++--- src/bin/pg_dump/pg_backup.h | 7 +- src/bin/pg_dump/pg_backup_archiver.c | 71 ++++++++++++------ src/bin/pg_dump/pg_backup_archiver.h | 10 +-- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 13 +++- src/bin/pg_dump/pg_backup_tar.c | 11 ++- src/bin/pg_dump/pg_dump.c | 97 ++++++++++++++++++------ src/bin/pg_dump/t/001_basic.pl | 27 ++++++- src/bin/pg_dump/t/002_pg_dump.pl | 2 +- 12 files changed, 243 insertions(+), 153 deletions(-) diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 8b9d9f4cad..3fb8fdce81 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -644,17 +644,31 @@ PostgreSQL documentation - - + + [:level] + + [:level] - Specify the compression level to use. Zero means no compression. + Specify the compression method and/or the compression level to use. + The compression method can be set to gzip or + none for no compression. A compression level can + be optionally specified, by appending the level number after a colon + (:). If no level is specified, the default compression + level will be used for the specified method. If only a level is + specified without mentioning a method, gzip compression + will be used. + + + For the custom and directory archive formats, this specifies compression of - individual table-data segments, and the default is to compress - at a moderate level. - For plain text output, setting a nonzero compression level causes - the entire output file to be compressed, as though it had been - fed through gzip; but the default is not to compress. + individual table-data segments, and the default is to compress using + gzip at a moderate level. For plain text output, + setting a nonzero compression level causes the entire output file to be compressed, + as though it had been fed through gzip; but the default + is not to compress. + + The tar archive format currently does not support compression at all. diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 62f940ff7a..2c9d730fce 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -64,7 +64,7 @@ /* typedef appears in compress_io.h */ struct CompressorState { - CompressionAlgorithm comprAlg; + pg_compress_specification compress_spec; WriteFunc writeF; #ifdef HAVE_LIBZ @@ -74,9 +74,6 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ static void InitCompressorZlib(CompressorState *cs, int level); @@ -93,57 +90,30 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - pg_fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(const pg_compress_specification compress_spec, + WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); #ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) + if (compress_spec.algorithm == PG_COMPRESSION_GZIP) pg_fatal("not built with zlib support"); #endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->compress_spec = compress_spec; /* * Perform compression algorithm specific initialization. */ #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + if (cs->compress_spec.algorithm == PG_COMPRESSION_GZIP) + InitCompressorZlib(cs, cs->compress_spec.level); #endif return cs; @@ -154,15 +124,12 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compress_spec, + ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) + if (compress_spec.algorithm == PG_COMPRESSION_NONE) ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + if (compress_spec.algorithm == PG_COMPRESSION_GZIP) { #ifdef HAVE_LIBZ ReadDataFromArchiveZlib(AH, readF); @@ -179,18 +146,23 @@ void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen) { - switch (cs->comprAlg) + switch (cs->compress_spec.algorithm) { - case COMPR_ALG_LIBZ: + case PG_COMPRESSION_GZIP: #ifdef HAVE_LIBZ WriteDataToArchiveZlib(AH, cs, data, dLen); #else pg_fatal("not built with zlib support"); #endif break; - case COMPR_ALG_NONE: + case PG_COMPRESSION_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + case PG_COMPRESSION_LZ4: + /* fallthrough */ + case PG_COMPRESSION_ZSTD: + pg_fatal("invalid compression method"); + break; } } @@ -201,7 +173,7 @@ void EndCompressor(ArchiveHandle *AH, CompressorState *cs) { #ifdef HAVE_LIBZ - if (cs->comprAlg == COMPR_ALG_LIBZ) + if (cs->compress_spec.algorithm == PG_COMPRESSION_GZIP) EndCompressorZlib(AH, cs); #endif free(cs); @@ -452,21 +424,25 @@ cfp * cfopen_read(const char *path, const char *mode) { cfp *fp; + pg_compress_specification compress_spec = {0}; + compress_spec.algorithm = PG_COMPRESSION_GZIP; #ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compress_spec); else #endif { - fp = cfopen(path, mode, 0); + compress_spec.algorithm = PG_COMPRESSION_NONE; + fp = cfopen(path, mode, compress_spec); #ifdef HAVE_LIBZ if (fp == NULL) { char *fname; + compress_spec.algorithm = PG_COMPRESSION_GZIP; fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fp = cfopen(fname, mode, compress_spec); free_keep_errno(fname); } #endif @@ -479,26 +455,27 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * If 'compress_spec.algorithm' is GZIP, a gzip compressed stream is opened, + * and 'compress_spec.level' used. The ".gz" suffix is automatically added to + * 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, + const pg_compress_specification compress_spec) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compress_spec.algorithm == PG_COMPRESSION_NONE) + fp = cfopen(path, mode, compress_spec); else { #ifdef HAVE_LIBZ char *fname; fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, compression); + fp = cfopen(fname, mode, compress_spec); free_keep_errno(fname); #else pg_fatal("not built with zlib support"); @@ -515,20 +492,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, + const pg_compress_specification compress_spec) { cfp *fp = pg_malloc(sizeof(cfp)); - if (compression != 0) + if (compress_spec.algorithm == PG_COMPRESSION_GZIP) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + if (compress_spec.level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compress_spec.level); fp->compressedfp = gzopen(path, mode_compression); } else diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index f635787692..d6335fff02 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,10 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(const pg_compress_specification compress_spec, + WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, + const pg_compress_specification compress_spec, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +52,13 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, + const pg_compress_specification compress_spec); +extern cfp *cfdopen(int fd, const char *mode, + pg_compress_specification compress_spec); extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen_write(const char *path, const char *mode, + const pg_compress_specification compress_spec); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index e8b7898297..61c412c8cb 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -23,6 +23,7 @@ #ifndef PG_BACKUP_H #define PG_BACKUP_H +#include "common/compression.h" #include "fe_utils/simple_list.h" #include "libpq-fe.h" @@ -143,7 +144,8 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + pg_compress_specification compress_spec; /* Specification for + * compression */ int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -303,7 +305,8 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + const pg_compress_specification compress_spec, + bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index f39c0fa36f..79347d387b 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,8 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + const pg_compress_specification compress_spec, + bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, const TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +99,8 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, + const pg_compress_specification compress_spec); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -239,12 +241,13 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + const pg_compress_specification compress_spec, + bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, dosync, - mode, setupDumpWorker); + ArchiveHandle *AH = _allocAH(FileSpec, fmt, compress_spec, + dosync, mode, setupDumpWorker); return (Archive *) AH; } @@ -254,7 +257,12 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + ArchiveHandle *AH; + pg_compress_specification compress_spec = {0}; + + compress_spec.algorithm = PG_COMPRESSION_NONE; + AH = _allocAH(FileSpec, fmt, compress_spec, true, + archModeRead, setupRestoreWorker); return (Archive *) AH; } @@ -384,7 +392,8 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP && + AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -459,8 +468,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compress_spec.algorithm != PG_COMPRESSION_NONE) + SetOutput(AH, ropt->filename, ropt->compress_spec); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -739,7 +748,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compress_spec.algorithm != PG_COMPRESSION_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -969,6 +978,8 @@ NewRestoreOptions(void) opts->format = archUnknown; opts->cparams.promptPassword = TRI_DEFAULT; opts->dumpSections = DUMP_UNSECTIONED; + opts->compress_spec.algorithm = PG_COMPRESSION_NONE; + opts->compress_spec.level = 0; return opts; } @@ -1115,23 +1126,28 @@ PrintTOCSummary(Archive *AHX) ArchiveHandle *AH = (ArchiveHandle *) AHX; RestoreOptions *ropt = AH->public.ropt; TocEntry *te; + pg_compress_specification out_compress_spec = {0}; teSection curSection; OutputContext sav; const char *fmtName; char stamp_str[64]; + /* TOC is always uncompressed */ + out_compress_spec.algorithm = PG_COMPRESSION_NONE; + sav = SaveOutput(AH); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, out_compress_spec); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) strcpy(stamp_str, "[unknown]"); ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); - ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", + ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %s\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, + get_compress_algorithm_name(AH->compress_spec.algorithm)); switch (AH->format) { @@ -1485,7 +1501,8 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, + const pg_compress_specification compress_spec) { int fn; @@ -1508,12 +1525,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compress_spec.algorithm == PG_COMPRESSION_GZIP) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compress_spec.level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2198,7 +2215,8 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + const pg_compress_specification compress_spec, + bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2249,7 +2267,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compress_spec = compress_spec; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2264,7 +2282,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compress_spec.algorithm != PG_COMPRESSION_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3669,7 +3687,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compress_spec.level); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3740,21 +3758,26 @@ ReadHead(ArchiveHandle *AH) pg_fatal("expected format (%d) differs from format found in file (%d)", AH->format, fmt); + AH->compress_spec.algorithm = PG_COMPRESSION_NONE; if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compress_spec.level = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compress_spec.level = ReadInt(AH); + + if (AH->compress_spec.level != 0) + AH->compress_spec.algorithm = PG_COMPRESSION_GZIP; } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compress_spec.algorithm = PG_COMPRESSION_GZIP; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif + if (AH->version >= K_VERS_1_4) { struct tm crtm; diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index 42687c4ec8..d58b96b2dc 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -331,14 +331,8 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + pg_compress_specification compress_spec; /* Requested specification for + * compression */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index a0a55a1edd..6a2112c45f 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, AH->compress_spec, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 798182b6f7..7d2cddbb2c 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -327,7 +327,8 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, + AH->compress_spec); if (ctx->dataFH == NULL) pg_fatal("could not open output file \"%s\": %m", fname); } @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) if (AH->mode == archModeWrite) { cfp *tocFH; + pg_compress_specification compress_spec = {0}; char fname[MAXPGPATH]; setFilePath(AH, fname, "toc.dat"); @@ -581,7 +583,8 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + compress_spec.algorithm = PG_COMPRESSION_NONE; + tocFH = cfopen_write(fname, PG_BINARY_W, compress_spec); if (tocFH == NULL) pg_fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,12 +642,14 @@ static void _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; + pg_compress_specification compress_spec = {0}; char fname[MAXPGPATH]; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + compress_spec.algorithm = PG_COMPRESSION_NONE; + ctx->blobsTocFH = cfopen_write(fname, "ab", compress_spec); if (ctx->blobsTocFH == NULL) pg_fatal("could not open output file \"%s\": %m", fname); } @@ -662,7 +667,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compress_spec); if (ctx->dataFH == NULL) pg_fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 402b93c610..87bfed76fd 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -194,7 +194,7 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) * possible since gzdopen uses buffered IO which totally screws file * positioning. */ - if (AH->compression != 0) + if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE) pg_fatal("compression is not supported by tar archive format"); } else @@ -328,7 +328,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) } } - if (AH->compression == 0) + if (AH->compress_spec.algorithm == PG_COMPRESSION_NONE) tm->nFH = ctx->tarFH; else pg_fatal("compression is not supported by tar archive format"); @@ -383,7 +383,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) umask(old_umask); - if (AH->compression == 0) + if (AH->compress_spec.algorithm == PG_COMPRESSION_NONE) tm->nFH = tm->tmpFH; else pg_fatal("compression is not supported by tar archive format"); @@ -401,7 +401,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) static void tarClose(ArchiveHandle *AH, TAR_MEMBER *th) { - if (AH->compression != 0) + if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE) pg_fatal("compression is not supported by tar archive format"); if (th->mode == 'w') @@ -800,7 +800,6 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -888,7 +887,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) if (oid == 0) pg_fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) + if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE) pg_fatal("compression is not supported by tar archive format"); sprintf(fname, "blob_%u.dat", oid); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index da427f4d4a..4830983c86 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -164,6 +164,8 @@ static void setup_connection(Archive *AH, const char *dumpencoding, const char *dumpsnapshot, char *use_role); static ArchiveFormat parseArchiveFormat(const char *format, ArchiveMode *mode); +static bool parse_compression(const char *opt, + pg_compress_specification *compress_spec); static void expand_schema_name_patterns(Archive *fout, SimpleStringList *patterns, SimpleOidList *oids, @@ -340,8 +342,9 @@ main(int argc, char **argv) const char *dumpsnapshot = NULL; char *use_role = NULL; int numWorkers = 1; - int compressLevel = -1; int plainText = 0; + pg_compress_specification compress_spec = {0}; + bool user_compression_defined = false; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -561,10 +564,10 @@ main(int argc, char **argv) dopt.aclsSkip = true; break; - case 'Z': /* Compression Level */ - if (!option_parse_int(optarg, "-Z/--compress", 0, 9, - &compressLevel)) + case 'Z': /* Compression */ + if (!parse_compression(optarg, &compress_spec)) exit_nicely(1); + user_compression_defined = true; break; case 0: @@ -687,23 +690,20 @@ main(int argc, char **argv) if (archiveFormat == archNull) plainText = 1; - /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + /* + * Custom and directory formats are compressed by default (zlib), others + * not + */ + if (user_compression_defined == false) { + parse_compress_specification(PG_COMPRESSION_NONE, NULL, &compress_spec); #ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + parse_compress_specification(PG_COMPRESSION_GZIP, NULL, + &compress_spec); #endif - compressLevel = 0; } -#ifndef HAVE_LIBZ - if (compressLevel != 0) - pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; -#endif - /* * If emitting an archive format, we always want to emit a DATABASE item, * in case --create is specified at pg_restore time. @@ -716,8 +716,8 @@ main(int argc, char **argv) pg_fatal("parallel backup only supported by the directory format"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, - archiveMode, setupDumpWorker); + fout = CreateArchive(filename, archiveFormat, compress_spec, + dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ SetArchiveOptions(fout, &dopt, NULL); @@ -948,10 +948,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compress_spec = compress_spec; ropt->suppressDumpWarnings = true; /* We've already shown them */ @@ -998,7 +995,8 @@ help(const char *progname) printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n")); printf(_(" -v, --verbose verbose mode\n")); printf(_(" -V, --version output version information, then exit\n")); - printf(_(" -Z, --compress=0-9 compression level for compressed formats\n")); + printf(_(" -Z, --compress=METHOD[:LEVEL]\n" + " compress as specified\n")); printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n")); printf(_(" --no-sync do not wait for changes to be written safely to disk\n")); printf(_(" -?, --help show this help, then exit\n")); @@ -1258,6 +1256,61 @@ get_synchronized_snapshot(Archive *fout) return result; } +/* + * Interprets and validates a compression option using the common compression + * parsing functions. If the requested compression is not available then the + * archives are uncompressed. + */ +static bool +parse_compression(const char *opt, pg_compress_specification *compress_spec) +{ + char *algorithm_str = NULL; + char *level_str = NULL; + char *validation_error = NULL; + bool supports_compression = true; + + parse_compress_options(opt, &algorithm_str, &level_str); + if (!parse_compress_algorithm(algorithm_str, &(compress_spec->algorithm))) + { + pg_log_error("invalid compression method: \"%s\" (gzip, none)", + algorithm_str); + return false; + } + + /* Switch off unimplemented or unavailable compressions. */ + if (compress_spec->algorithm != PG_COMPRESSION_NONE && + compress_spec->algorithm != PG_COMPRESSION_GZIP) + supports_compression = false; + +#ifndef HAVE_LIBZ + if (compress_spec->algorithm == PG_COMPRESSION_GZIP) + supports_compression = false; +#endif + + if (!supports_compression) + { + pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); + parse_compress_specification(PG_COMPRESSION_NONE, NULL, compress_spec); + + pg_free(algorithm_str); + pg_free(level_str); + + return true; + } + + /* Parse and validate the rest of the options */ + parse_compress_specification(compress_spec->algorithm, level_str, + compress_spec); + validation_error = validate_compress_specification(compress_spec); + if (validation_error) + { + pg_log_error("invalid compression specification: %s", validation_error); + return false; + } + + return true; +} + static ArchiveFormat parseArchiveFormat(const char *format, ArchiveMode *mode) { diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl index a583c8a6d2..f8d0b2fce5 100644 --- a/src/bin/pg_dump/t/001_basic.pl +++ b/src/bin/pg_dump/t/001_basic.pl @@ -121,16 +121,32 @@ command_fails_like( 'pg_restore: cannot specify both --single-transaction and multiple jobs'); command_fails_like( - [ 'pg_dump', '-Z', '-1' ], - qr/\Qpg_dump: error: -Z\/--compress must be in range 0..9\E/, - 'pg_dump: -Z/--compress must be in range'); + [ 'pg_dump', '--compress', 'garbage' ], + qr/\Qpg_dump: error: invalid compression method: "garbage" (gzip, none)\E/, + 'pg_dump: invalid --compress'); + +command_fails_like( + [ 'pg_dump', '--compress', 'none:1' ], + qr/\Qpg_dump: error: invalid compression specification: compression algorithm "none" does not accept a compression level\E/, + 'pg_dump: invalid compression specification: compression algorithm "none" does not accept a compression level'); + if (check_pg_config("#define HAVE_LIBZ 1")) { + command_fails_like( + [ 'pg_dump', '-Z', '15' ], + qr/\Qpg_dump: error: invalid compression specification: compression algorithm "gzip" expects a compression level between 1 and 9 (default at -1)\E/, + 'pg_dump: invalid compression specification: must be in range'); + command_fails_like( [ 'pg_dump', '--compress', '1', '--format', 'tar' ], qr/\Qpg_dump: error: compression is not supported by tar archive format\E/, 'pg_dump: compression is not supported by tar archive format'); + + command_fails_like( + [ 'pg_dump', '-Z', 'gzip:nonInt' ], + qr/\Qpg_dump: error: invalid compression specification: unrecognized compression option: "nonInt"\E/, + 'pg_dump: invalid compression specification: must be an integer'); } else { @@ -139,6 +155,11 @@ else [ 'pg_dump', '--compress', '1', '--format', 'tar', '-j3' ], qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/, 'pg_dump: warning: compression not available in this installation'); + + command_fails_like( + [ 'pg_dump', '-Z', 'gzip:nonInt', '--format', 'tar', '-j2' ], + qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/, + 'pg_dump: invalid compression specification: must be an integer'); } command_fails_like( diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index fe53ed0f89..d604558f03 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -87,7 +87,7 @@ my %pgdump_runs = ( compile_option => 'gzip', dump_cmd => [ 'pg_dump', '--jobs=2', - '--format=directory', '--compress=1', + '--format=directory', '--compress=gzip:1', "--file=$tempdir/compression_gzip_dir", 'postgres', ], # Give coverage for manually compressed blob.toc files during -- 2.34.1