From bf5deb8e3a770e26da876b2268832ab1a07bbb84 Mon Sep 17 00:00:00 2001 From: Georgios Kokolatos Date: Tue, 5 Jul 2022 12:33:28 +0000 Subject: [PATCH v7 1/3] Prepare pg_dump for additional compression methods This commmit does some of the heavy lifting required for additional compression methods. First it is teaching pg_dump.c about the definitions and interfaces found in common/compression.h. Then it is propagating those throughout the code. Commit bf9aa490db introduced cfp in compress_io.{c,h} with the intent of unifying compression related code and allow for the introduction of additional archive formats. However, pg_backup_archiver.c was not using that API. This commit teaches pg_backup_archiver.c about cfp and is using it through out. --- doc/src/sgml/ref/pg_dump.sgml | 30 +- src/bin/pg_dump/compress_io.c | 427 ++++++++++++++++---------- src/bin/pg_dump/compress_io.h | 35 ++- src/bin/pg_dump/pg_backup.h | 7 +- src/bin/pg_dump/pg_backup_archiver.c | 178 +++++------ src/bin/pg_dump/pg_backup_archiver.h | 37 +-- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 13 +- src/bin/pg_dump/pg_backup_tar.c | 12 +- src/bin/pg_dump/pg_dump.c | 151 +++++++-- src/bin/pg_dump/t/001_basic.pl | 10 + src/bin/pg_dump/t/002_pg_dump.pl | 2 +- 12 files changed, 552 insertions(+), 356 deletions(-) diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 5efb442b44..94885d0812 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -644,17 +644,31 @@ PostgreSQL documentation - - + + [:level] + + [:level] - Specify the compression level to use. Zero means no compression. + Specify the compression method and/or the compression level to use. + The compression method can be set to gzip or + none for no compression. A compression level can + be optionally specified, by appending the level number after a colon + (:). If no level is specified, the default compression + level will be used for the specified method. If only a level is + specified without mentioning a method, gzip compression + will be used. + + + For the custom and directory archive formats, this specifies compression of - individual table-data segments, and the default is to compress - at a moderate level. - For plain text output, setting a nonzero compression level causes - the entire output file to be compressed, as though it had been - fed through gzip; but the default is not to compress. + individual table-data segments, and the default is to compress using + gzip at a moderate level. For plain text output, + setting a nonzero compression level causes the entire output file to be compressed, + as though it had been fed through gzip; but the default + is not to compress. + + The tar archive format currently does not support compression at all. diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 62f940ff7a..146396172b 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -64,7 +64,7 @@ /* typedef appears in compress_io.h */ struct CompressorState { - CompressionAlgorithm comprAlg; + pg_compress_algorithm compress_algorithm; WriteFunc writeF; #ifdef HAVE_LIBZ @@ -74,9 +74,6 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ static void InitCompressorZlib(CompressorState *cs, int level); @@ -93,57 +90,30 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - pg_fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(const pg_compress_specification compress_spec, + WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); #ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) + if (compress_spec.algorithm == PG_COMPRESSION_GZIP) pg_fatal("not built with zlib support"); #endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->compress_algorithm = compress_spec.algorithm; /* * Perform compression algorithm specific initialization. */ #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + if (cs->compress_algorithm == PG_COMPRESSION_GZIP) + InitCompressorZlib(cs, compress_spec.level); #endif return cs; @@ -154,21 +124,24 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compress_spec, + ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (compress_spec.algorithm) { + case PG_COMPRESSION_NONE: + ReadDataFromArchiveNone(AH, readF); + break; + case PG_COMPRESSION_GZIP: #ifdef HAVE_LIBZ - ReadDataFromArchiveZlib(AH, readF); + ReadDataFromArchiveZlib(AH, readF); #else - pg_fatal("not built with zlib support"); + pg_fatal("not built with zlib support"); #endif + break; + default: + pg_fatal("invalid compression method"); + break; } } @@ -179,18 +152,21 @@ void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen) { - switch (cs->comprAlg) + switch (cs->compress_algorithm) { - case COMPR_ALG_LIBZ: + case PG_COMPRESSION_GZIP: #ifdef HAVE_LIBZ WriteDataToArchiveZlib(AH, cs, data, dLen); #else pg_fatal("not built with zlib support"); #endif break; - case COMPR_ALG_NONE: + case PG_COMPRESSION_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + default: + pg_fatal("invalid compression method"); + break; } } @@ -200,11 +176,23 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, void EndCompressor(ArchiveHandle *AH, CompressorState *cs) { + switch (cs->compress_algorithm) + { + case PG_COMPRESSION_GZIP: #ifdef HAVE_LIBZ - if (cs->comprAlg == COMPR_ALG_LIBZ) - EndCompressorZlib(AH, cs); + EndCompressorZlib(AH, cs); +#else + pg_fatal("not built with zlib support"); #endif - free(cs); + break; + case PG_COMPRESSION_NONE: + free(cs); + break; + + default: + pg_fatal("invalid compression method"); + break; + } } /* Private routines, specific to each compression method. */ @@ -418,10 +406,8 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, */ struct cfp { - FILE *uncompressedfp; -#ifdef HAVE_LIBZ - gzFile compressedfp; -#endif + pg_compress_algorithm compress_algorithm; + void *fp; }; #ifdef HAVE_LIBZ @@ -452,21 +438,25 @@ cfp * cfopen_read(const char *path, const char *mode) { cfp *fp; + pg_compress_specification compress_spec = {0}; + compress_spec.algorithm = PG_COMPRESSION_GZIP; #ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compress_spec); else #endif { - fp = cfopen(path, mode, 0); + compress_spec.algorithm = PG_COMPRESSION_NONE; + fp = cfopen(path, mode, compress_spec); #ifdef HAVE_LIBZ if (fp == NULL) { char *fname; + compress_spec.algorithm = PG_COMPRESSION_GZIP; fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fp = cfopen(fname, mode, compress_spec); free_keep_errno(fname); } #endif @@ -479,26 +469,27 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * If 'compress_spec.algorithm' is GZIP, a gzip compressed stream is opened, + * and 'compress_spec.level' used. The ".gz" suffix is automatically added to + * 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, + const pg_compress_specification compress_spec) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compress_spec.algorithm == PG_COMPRESSION_NONE) + fp = cfopen(path, mode, compress_spec); else { #ifdef HAVE_LIBZ char *fname; fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, compression); + fp = cfopen(fname, mode, compress_spec); free_keep_errno(fname); #else pg_fatal("not built with zlib support"); @@ -509,60 +500,96 @@ cfopen_write(const char *path, const char *mode, int compression) } /* - * Opens file 'path' in 'mode'. If 'compression' is non-zero, the file - * is opened with libz gzopen(), otherwise with plain fopen(). + * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or + * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The + * descriptor is not dup'ed and it is the caller's responsibility to do so. + * The caller must verify that the 'compress_algorithm' is supported by the + * current build. * * On failure, return NULL with an error code in errno. */ -cfp * -cfopen(const char *path, const char *mode, int compression) +static cfp * +cfopen_internal(const char *path, int fd, const char *mode, + pg_compress_algorithm compress_algorithm, int compressionLevel) { cfp *fp = pg_malloc(sizeof(cfp)); - if (compression != 0) + fp->compress_algorithm = compress_algorithm; + + switch (compress_algorithm) { -#ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) - { - /* user has specified a compression level, so tell zlib to use it */ - char mode_compression[32]; + case PG_COMPRESSION_NONE: + if (fd >= 0) + fp->fp = fdopen(fd, mode); + else + fp->fp = fopen(path, mode); + if (fp->fp == NULL) + { + free_keep_errno(fp); + fp = NULL; + } - snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); - fp->compressedfp = gzopen(path, mode_compression); - } - else - { - /* don't specify a level, just use the zlib default */ - fp->compressedfp = gzopen(path, mode); - } + break; + case PG_COMPRESSION_GZIP: +#ifdef HAVE_LIBZ + if (compressionLevel != Z_DEFAULT_COMPRESSION) + { + /* + * user has specified a compression level, so tell zlib to use + * it + */ + char mode_compression[32]; + + snprintf(mode_compression, sizeof(mode_compression), "%s%d", + mode, compressionLevel); + if (fd >= 0) + fp->fp = gzdopen(fd, mode_compression); + else + fp->fp = gzopen(path, mode_compression); + } + else + { + /* don't specify a level, just use the zlib default */ + if (fd >= 0) + fp->fp = gzdopen(fd, mode); + else + fp->fp = gzopen(path, mode); + } - fp->uncompressedfp = NULL; - if (fp->compressedfp == NULL) - { - free_keep_errno(fp); - fp = NULL; - } + if (fp->fp == NULL) + { + free_keep_errno(fp); + fp = NULL; + } #else - pg_fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + pg_fatal("not built with zlib support"); #endif - fp->uncompressedfp = fopen(path, mode); - if (fp->uncompressedfp == NULL) - { - free_keep_errno(fp); - fp = NULL; - } + break; + default: + pg_fatal("invalid compression method"); + break; } return fp; } +cfp * +cfopen(const char *path, const char *mode, + const pg_compress_specification compress_spec) +{ + return cfopen_internal(path, -1, mode, + compress_spec.algorithm, + compress_spec.level); +} + +cfp * +cfdopen(int fd, const char *mode, + const pg_compress_specification compress_spec) +{ + return cfopen_internal(NULL, fd, mode, + compress_spec.algorithm, + compress_spec.level); +} int cfread(void *ptr, int size, cfp *fp) @@ -572,38 +599,61 @@ cfread(void *ptr, int size, cfp *fp) if (size == 0) return 0; -#ifdef HAVE_LIBZ - if (fp->compressedfp) + switch (fp->compress_algorithm) { - ret = gzread(fp->compressedfp, ptr, size); - if (ret != size && !gzeof(fp->compressedfp)) - { - int errnum; - const char *errmsg = gzerror(fp->compressedfp, &errnum); + case PG_COMPRESSION_NONE: + ret = fread(ptr, 1, size, fp->fp); + if (ret != size && !feof(fp->fp)) + READ_ERROR_EXIT(fp->fp); - pg_fatal("could not read from input file: %s", - errnum == Z_ERRNO ? strerror(errno) : errmsg); - } - } - else + break; + case PG_COMPRESSION_GZIP: +#ifdef HAVE_LIBZ + ret = gzread(fp->fp, ptr, size); + if (ret != size && !gzeof(fp->fp)) + { + int errnum; + const char *errmsg = gzerror(fp->fp, &errnum); + + pg_fatal("could not read from input file: %s", + errnum == Z_ERRNO ? strerror(errno) : errmsg); + } +#else + pg_fatal("not built with zlib support"); #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); + break; + + default: + pg_fatal("invalid compression method"); + break; } + return ret; } int cfwrite(const void *ptr, int size, cfp *fp) { + int ret = 0; + + switch (fp->compress_algorithm) + { + case PG_COMPRESSION_NONE: + ret = fwrite(ptr, 1, size, fp->fp); + break; + case PG_COMPRESSION_GZIP: #ifdef HAVE_LIBZ - if (fp->compressedfp) - return gzwrite(fp->compressedfp, ptr, size); - else + ret = gzwrite(fp->fp, ptr, size); +#else + pg_fatal("not built with zlib support"); #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + break; + default: + pg_fatal("invalid compression method"); + break; + } + + return ret; } int @@ -611,24 +661,31 @@ cfgetc(cfp *fp) { int ret; -#ifdef HAVE_LIBZ - if (fp->compressedfp) + switch (fp->compress_algorithm) { - ret = gzgetc(fp->compressedfp); - if (ret == EOF) - { - if (!gzeof(fp->compressedfp)) - pg_fatal("could not read from input file: %s", strerror(errno)); - else - pg_fatal("could not read from input file: end of file"); - } - } - else + case PG_COMPRESSION_NONE: + ret = fgetc(fp->fp); + if (ret == EOF) + READ_ERROR_EXIT(fp->fp); + + break; + case PG_COMPRESSION_GZIP: +#ifdef HAVE_LIBZ + ret = gzgetc((gzFile) fp->fp); + if (ret == EOF) + { + if (!gzeof(fp->fp)) + pg_fatal("could not read from input file: %s", strerror(errno)); + else + pg_fatal("could not read from input file: end of file"); + } +#else + pg_fatal("not built with zlib support"); #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); + break; + default: + pg_fatal("invalid compression method"); + break; } return ret; @@ -637,65 +694,107 @@ cfgetc(cfp *fp) char * cfgets(cfp *fp, char *buf, int len) { + char *ret; + + switch (fp->compress_algorithm) + { + case PG_COMPRESSION_NONE: + ret = fgets(buf, len, fp->fp); + + break; + case PG_COMPRESSION_GZIP: #ifdef HAVE_LIBZ - if (fp->compressedfp) - return gzgets(fp->compressedfp, buf, len); - else + ret = gzgets(fp->fp, buf, len); +#else + pg_fatal("not built with zlib support"); #endif - return fgets(buf, len, fp->uncompressedfp); + break; + default: + pg_fatal("invalid compression method"); + break; + } + + return ret; } int cfclose(cfp *fp) { - int result; + int ret; if (fp == NULL) { errno = EBADF; return EOF; } -#ifdef HAVE_LIBZ - if (fp->compressedfp) + + switch (fp->compress_algorithm) { - result = gzclose(fp->compressedfp); - fp->compressedfp = NULL; - } - else + case PG_COMPRESSION_NONE: + ret = fclose(fp->fp); + fp->fp = NULL; + + break; + case PG_COMPRESSION_GZIP: +#ifdef HAVE_LIBZ + ret = gzclose(fp->fp); + fp->fp = NULL; +#else + pg_fatal("not built with zlib support"); #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; + break; + default: + pg_fatal("invalid compression method"); + break; } + free_keep_errno(fp); - return result; + return ret; } int cfeof(cfp *fp) { + int ret; + + switch (fp->compress_algorithm) + { + case PG_COMPRESSION_NONE: + ret = feof(fp->fp); + + break; + case PG_COMPRESSION_GZIP: #ifdef HAVE_LIBZ - if (fp->compressedfp) - return gzeof(fp->compressedfp); - else + ret = gzeof(fp->fp); +#else + pg_fatal("not built with zlib support"); #endif - return feof(fp->uncompressedfp); + break; + default: + pg_fatal("invalid compression method"); + break; + } + + return ret; } const char * get_cfp_error(cfp *fp) { -#ifdef HAVE_LIBZ - if (fp->compressedfp) + if (fp->compress_algorithm == PG_COMPRESSION_GZIP) { +#ifdef HAVE_LIBZ int errnum; - const char *errmsg = gzerror(fp->compressedfp, &errnum); + const char *errmsg = gzerror(fp->fp, &errnum); if (errnum != Z_ERRNO) return errmsg; - } +#else + pg_fatal("not built with zlib support"); #endif + } + return strerror(errno); } diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index f635787692..2b42f030a8 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -17,16 +17,25 @@ #include "pg_backup_archiver.h" +#ifdef HAVE_LIBZ +#include +#define GZCLOSE(fh) gzclose(fh) +#define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s)) +#define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s)) +#define GZEOF(fh) gzeof(fh) +#else +#define GZCLOSE(fh) fclose(fh) +#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) +#define GZREAD(p, s, n, fh) fread(p, s, n, fh) +#define GZEOF(fh) feof(fh) +/* this is just the redefinition of a libz constant */ +#define Z_DEFAULT_COMPRESSION (-1) +#endif + /* Initial buffer sizes used in zlib compression. */ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +55,10 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(const pg_compress_specification compress_spec, + WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, + const pg_compress_specification compress_spec, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +67,13 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, + const pg_compress_specification compress_spec); +extern cfp *cfdopen(int fd, const char *mode, + pg_compress_specification compress_spec); extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen_write(const char *path, const char *mode, + const pg_compress_specification compress_spec); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index fcc5f6bd05..cf9fad4706 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -23,6 +23,7 @@ #ifndef PG_BACKUP_H #define PG_BACKUP_H +#include "common/compression.h" #include "fe_utils/simple_list.h" #include "libpq-fe.h" @@ -143,7 +144,8 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + pg_compress_specification compress_spec; /* Specification for + * compression */ int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -303,7 +305,8 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + const pg_compress_specification compress_spec, + bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 233198afc0..fa84c31ecb 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -31,6 +31,7 @@ #endif #include "common/string.h" +#include "compress_io.h" #include "dumputils.h" #include "fe_utils/string_utils.h" #include "lib/stringinfo.h" @@ -43,13 +44,6 @@ #define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n" #define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n" -/* state needed to save/restore an archive's output target */ -typedef struct _outputContext -{ - void *OF; - int gzOut; -} OutputContext; - /* * State for tracking TocEntrys that are ready to process during a parallel * restore. (This used to be a list, and we still call it that, though now @@ -70,7 +64,8 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + const pg_compress_specification compress_spec, + bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,9 +93,10 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); -static OutputContext SaveOutput(ArchiveHandle *AH); -static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); +static void SetOutput(ArchiveHandle *AH, const char *filename, + const pg_compress_specification compress_spec); +static cfp *SaveOutput(ArchiveHandle *AH); +static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput); static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel); static void restore_toc_entries_prefork(ArchiveHandle *AH, @@ -239,12 +235,13 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + const pg_compress_specification compress_spec, + bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, dosync, - mode, setupDumpWorker); + ArchiveHandle *AH = _allocAH(FileSpec, fmt, compress_spec, + dosync, mode, setupDumpWorker); return (Archive *) AH; } @@ -254,7 +251,12 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + ArchiveHandle *AH; + pg_compress_specification compress_spec; + + compress_spec.algorithm = PG_COMPRESSION_NONE; + AH = _allocAH(FileSpec, fmt, compress_spec, true, + archModeRead, setupRestoreWorker); return (Archive *) AH; } @@ -269,11 +271,8 @@ CloseArchive(Archive *AHX) AH->ClosePtr(AH); /* Close the output */ - errno = 0; /* in case gzclose() doesn't set it */ - if (AH->gzOut) - res = GZCLOSE(AH->OF); - else if (AH->OF != stdout) - res = fclose(AH->OF); + errno = 0; + res = cfclose(AH->OF); if (res != 0) pg_fatal("could not close output file: %m"); @@ -355,7 +354,7 @@ RestoreArchive(Archive *AHX) RestoreOptions *ropt = AH->public.ropt; bool parallel_mode; TocEntry *te; - OutputContext sav; + cfp *sav; AH->stage = STAGE_INITIALIZING; @@ -384,7 +383,8 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP && + AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -459,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compress_spec.algorithm != PG_COMPRESSION_NONE) + SetOutput(AH, ropt->filename, ropt->compress_spec); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -739,7 +739,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compress_spec.algorithm != PG_COMPRESSION_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -969,6 +969,8 @@ NewRestoreOptions(void) opts->format = archUnknown; opts->cparams.promptPassword = TRI_DEFAULT; opts->dumpSections = DUMP_UNSECTIONED; + opts->compress_spec.algorithm = PG_COMPRESSION_NONE; + opts->compress_spec.level = INT_MIN; return opts; } @@ -1115,23 +1117,28 @@ PrintTOCSummary(Archive *AHX) ArchiveHandle *AH = (ArchiveHandle *) AHX; RestoreOptions *ropt = AH->public.ropt; TocEntry *te; + pg_compress_specification out_compress_spec; teSection curSection; - OutputContext sav; + cfp *sav; const char *fmtName; char stamp_str[64]; + /* TOC is always uncompressed */ + out_compress_spec.algorithm = PG_COMPRESSION_NONE; + sav = SaveOutput(AH); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, out_compress_spec); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) strcpy(stamp_str, "[unknown]"); ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); - ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", + ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %s\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, + get_compress_algorithm_name(AH->compress_spec.algorithm)); switch (AH->format) { @@ -1485,60 +1492,35 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, + const pg_compress_specification compress_spec) { - int fn; + const char *mode; + int fn = -1; if (filename) { if (strcmp(filename, "-") == 0) fn = fileno(stdout); - else - fn = -1; } else if (AH->FH) fn = fileno(AH->FH); else if (AH->fSpec) { - fn = -1; filename = AH->fSpec; } else fn = fileno(stdout); - /* If compression explicitly requested, use gzopen */ -#ifdef HAVE_LIBZ - if (compression != 0) - { - char fmode[14]; + if (AH->mode == archModeAppend) + mode = PG_BINARY_A; + else + mode = PG_BINARY_W; - /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); - if (fn >= 0) - AH->OF = gzdopen(dup(fn), fmode); - else - AH->OF = gzopen(filename, fmode); - AH->gzOut = 1; - } + if (fn >= 0) + AH->OF = cfdopen(dup(fn), mode, compress_spec); else -#endif - { /* Use fopen */ - if (AH->mode == archModeAppend) - { - if (fn >= 0) - AH->OF = fdopen(dup(fn), PG_BINARY_A); - else - AH->OF = fopen(filename, PG_BINARY_A); - } - else - { - if (fn >= 0) - AH->OF = fdopen(dup(fn), PG_BINARY_W); - else - AH->OF = fopen(filename, PG_BINARY_W); - } - AH->gzOut = 0; - } + AH->OF = cfopen(filename, mode, compress_spec); if (!AH->OF) { @@ -1549,33 +1531,24 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) } } -static OutputContext +static cfp * SaveOutput(ArchiveHandle *AH) { - OutputContext sav; - - sav.OF = AH->OF; - sav.gzOut = AH->gzOut; - - return sav; + return (cfp *) AH->OF; } static void -RestoreOutput(ArchiveHandle *AH, OutputContext savedContext) +RestoreOutput(ArchiveHandle *AH, cfp *savedOutput) { int res; - errno = 0; /* in case gzclose() doesn't set it */ - if (AH->gzOut) - res = GZCLOSE(AH->OF); - else - res = fclose(AH->OF); + errno = 0; + res = cfclose(AH->OF); if (res != 0) pg_fatal("could not close output file: %m"); - AH->gzOut = savedContext.gzOut; - AH->OF = savedContext.OF; + AH->OF = savedOutput; } @@ -1699,22 +1672,17 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH) bytes_written = size * nmemb; } - else if (AH->gzOut) - bytes_written = GZWRITE(ptr, size, nmemb, AH->OF); else if (AH->CustomOutPtr) bytes_written = AH->CustomOutPtr(AH, ptr, size * nmemb); + /* + * If we're doing a restore, and it's direct to DB, and we're connected + * then send it to the DB. + */ + else if (RestoringToDB(AH)) + bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb); else - { - /* - * If we're doing a restore, and it's direct to DB, and we're - * connected then send it to the DB. - */ - if (RestoringToDB(AH)) - bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb); - else - bytes_written = fwrite(ptr, size, nmemb, AH->OF) * size; - } + bytes_written = cfwrite(ptr, size * nmemb, AH->OF); if (bytes_written != size * nmemb) WRITE_ERROR_EXIT; @@ -2198,10 +2166,12 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + const pg_compress_specification compress_spec, + bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; + pg_compress_specification out_compress_spec = {0}; pg_log_debug("allocating AH for %s, format %d", FileSpec ? FileSpec : "(stdio)", fmt); @@ -2249,14 +2219,14 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compress_spec = compress_spec; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); /* Open stdout with no compression for AH output handle */ - AH->gzOut = 0; - AH->OF = stdout; + out_compress_spec.algorithm = PG_COMPRESSION_NONE; + AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec); /* * On Windows, we need to use binary mode to read/write non-text files, @@ -2264,7 +2234,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compress_spec.algorithm != PG_COMPRESSION_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3705,7 +3675,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compress_spec.level); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3776,21 +3746,25 @@ ReadHead(ArchiveHandle *AH) pg_fatal("expected format (%d) differs from format found in file (%d)", AH->format, fmt); + AH->compress_spec.algorithm = PG_COMPRESSION_NONE; if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compress_spec.level = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compress_spec.level = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compress_spec.level = Z_DEFAULT_COMPRESSION; + if (AH->compress_spec.level != INT_MIN) #ifndef HAVE_LIBZ - if (AH->compression != 0) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); +#else + AH->compress_spec.algorithm = PG_COMPRESSION_GZIP; #endif + if (AH->version >= K_VERS_1_4) { struct tm crtm; diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index 084cd87e8d..341d406515 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -32,30 +32,6 @@ #define LOBBUFSIZE 16384 -#ifdef HAVE_LIBZ -#include -#define GZCLOSE(fh) gzclose(fh) -#define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s)) -#define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s)) -#define GZEOF(fh) gzeof(fh) -#else -#define GZCLOSE(fh) fclose(fh) -#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) -#define GZREAD(p, s, n, fh) fread(p, s, n, fh) -#define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ -#define Z_DEFAULT_COMPRESSION (-1) - -typedef struct _z_stream -{ - void *next_in; - void *next_out; - size_t avail_in; - size_t avail_out; -} z_stream; -typedef z_stream *z_streamp; -#endif - /* Data block types */ #define BLK_DATA 1 #define BLK_BLOBS 3 @@ -319,8 +295,7 @@ struct _archiveHandle char *fSpec; /* Archive File Spec */ FILE *FH; /* General purpose file handle */ - void *OF; - int gzOut; /* Output file */ + void *OF; /* Output file */ struct _tocEntry *toc; /* Header of circular list of TOC entries */ int tocCount; /* Number of TOC entries */ @@ -331,14 +306,8 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + pg_compress_specification compress_spec; /* Requested specification for + * compression */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 1023fea01b..43ccbe1339 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, AH->compress_spec, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 3f46f7988a..17c7130c75 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -327,7 +327,8 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, + AH->compress_spec); if (ctx->dataFH == NULL) pg_fatal("could not open output file \"%s\": %m", fname); } @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) if (AH->mode == archModeWrite) { cfp *tocFH; + pg_compress_specification compress_spec; char fname[MAXPGPATH]; setFilePath(AH, fname, "toc.dat"); @@ -581,7 +583,8 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + compress_spec.algorithm = PG_COMPRESSION_NONE; + tocFH = cfopen_write(fname, PG_BINARY_W, compress_spec); if (tocFH == NULL) pg_fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,12 +642,14 @@ static void _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; + pg_compress_specification compress_spec; char fname[MAXPGPATH]; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + compress_spec.algorithm = PG_COMPRESSION_NONE; + ctx->blobsTocFH = cfopen_write(fname, "ab", compress_spec); if (ctx->blobsTocFH == NULL) pg_fatal("could not open output file \"%s\": %m", fname); } @@ -662,7 +667,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compress_spec); if (ctx->dataFH == NULL) pg_fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index bfc49b66d2..085f5c7f20 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -35,6 +35,7 @@ #include #include "common/file_utils.h" +#include "compress_io.h" #include "fe_utils/string_utils.h" #include "pg_backup_archiver.h" #include "pg_backup_tar.h" @@ -194,7 +195,7 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) * possible since gzdopen uses buffered IO which totally screws file * positioning. */ - if (AH->compression != 0) + if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE) pg_fatal("compression is not supported by tar archive format"); } else @@ -328,7 +329,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) } } - if (AH->compression == 0) + if (AH->compress_spec.algorithm == PG_COMPRESSION_NONE) tm->nFH = ctx->tarFH; else pg_fatal("compression is not supported by tar archive format"); @@ -383,7 +384,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) umask(old_umask); - if (AH->compression == 0) + if (AH->compress_spec.algorithm == PG_COMPRESSION_NONE) tm->nFH = tm->tmpFH; else pg_fatal("compression is not supported by tar archive format"); @@ -401,7 +402,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) static void tarClose(ArchiveHandle *AH, TAR_MEMBER *th) { - if (AH->compression != 0) + if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE) pg_fatal("compression is not supported by tar archive format"); if (th->mode == 'w') @@ -800,7 +801,6 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -888,7 +888,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) if (oid == 0) pg_fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) + if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE) pg_fatal("compression is not supported by tar archive format"); sprintf(fname, "blob_%u.dat", oid); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index c871cb727d..dd214e7670 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -54,7 +54,9 @@ #include "catalog/pg_subscription.h" #include "catalog/pg_trigger_d.h" #include "catalog/pg_type_d.h" +#include "common/compression.h" #include "common/connect.h" +#include "compress_io.h" #include "dumputils.h" #include "fe_utils/option_utils.h" #include "fe_utils/string_utils.h" @@ -163,6 +165,8 @@ static void setup_connection(Archive *AH, const char *dumpencoding, const char *dumpsnapshot, char *use_role); static ArchiveFormat parseArchiveFormat(const char *format, ArchiveMode *mode); +static bool parse_compression_option(const char *opt, + pg_compress_specification *compress_spec); static void expand_schema_name_patterns(Archive *fout, SimpleStringList *patterns, SimpleOidList *oids, @@ -339,8 +343,9 @@ main(int argc, char **argv) const char *dumpsnapshot = NULL; char *use_role = NULL; int numWorkers = 1; - int compressLevel = -1; int plainText = 0; + pg_compress_specification compress_spec = {0}; + bool user_compression_defined = false; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -560,10 +565,10 @@ main(int argc, char **argv) dopt.aclsSkip = true; break; - case 'Z': /* Compression Level */ - if (!option_parse_int(optarg, "-Z/--compress", 0, 9, - &compressLevel)) + case 'Z': /* Compression */ + if (!parse_compression_option(optarg, &compress_spec)) exit_nicely(1); + user_compression_defined = true; break; case 0: @@ -686,23 +691,23 @@ main(int argc, char **argv) if (archiveFormat == archNull) plainText = 1; - /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + /* + * Custom and directory formats are compressed by default (zlib), others + * not + */ + if (user_compression_defined == false) { + compress_spec.algorithm = PG_COMPRESSION_NONE; + compress_spec.level = INT_MIN; #ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { + compress_spec.algorithm = PG_COMPRESSION_GZIP; + compress_spec.level = Z_DEFAULT_COMPRESSION; + } #endif - compressLevel = 0; } -#ifndef HAVE_LIBZ - if (compressLevel != 0) - pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; -#endif - /* * If emitting an archive format, we always want to emit a DATABASE item, * in case --create is specified at pg_restore time. @@ -715,8 +720,8 @@ main(int argc, char **argv) pg_fatal("parallel backup only supported by the directory format"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, - archiveMode, setupDumpWorker); + fout = CreateArchive(filename, archiveFormat, compress_spec, + dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ SetArchiveOptions(fout, &dopt, NULL); @@ -947,10 +952,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compress_spec = compress_spec; ropt->suppressDumpWarnings = true; /* We've already shown them */ @@ -997,7 +999,8 @@ help(const char *progname) printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n")); printf(_(" -v, --verbose verbose mode\n")); printf(_(" -V, --version output version information, then exit\n")); - printf(_(" -Z, --compress=0-9 compression level for compressed formats\n")); + printf(_(" -Z, --compress=METHOD[:LEVEL]\n" + " compress as specified\n")); printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n")); printf(_(" --no-sync do not wait for changes to be written safely to disk\n")); printf(_(" -?, --help show this help, then exit\n")); @@ -1257,6 +1260,110 @@ get_synchronized_snapshot(Archive *fout) return result; } +/* + * Interprets a compression option of the format 'method[:LEVEL]' of legacy just + * '[LEVEL]'. In the later format, gzip is implied. The parsed method and level + * are returned in pg_compress_specification. In case of error, the function + * returns false. + */ +static bool +parse_compression_option(const char *opt, + pg_compress_specification *compress_spec) +{ + char *method; + const char *sep; + int methodlen; + bool supports_compression = true; + bool res = true; + + /* set defaults */ + compress_spec->algorithm = PG_COMPRESSION_NONE; + compress_spec->level = INT_MIN; + + /* find the separator if exists */ + sep = strchr(opt, ':'); + + /* + * If there is no separator, then it is either a legacy format, or only + * the method has been passed. + */ + if (!sep) + { + if (strspn(opt, "-0123456789") == strlen(opt)) + { + res = option_parse_int(opt, "-Z/--compress", 0, 9, + &(compress_spec->level)); + compress_spec->algorithm = (compress_spec->level > 0) ? + PG_COMPRESSION_GZIP : + PG_COMPRESSION_NONE; + } + else + { + method = pg_strdup(opt); + res = parse_compress_algorithm(method, + &compress_spec->algorithm); + if (!res) + pg_log_error("invalid compression method \"%s\" (gzip, none)", + method); + + pg_free(method); + } + } + else + { + /* otherwise, it should be method:LEVEL */ + methodlen = sep - opt + 1; + method = pg_malloc0(methodlen); + snprintf(method, methodlen, "%.*s", methodlen - 1, opt); + + res = parse_compress_algorithm(method, &compress_spec->algorithm); + if (res) + { + char *error_detail = NULL; + char *detail; + + sep++; + detail = pg_strdup(sep); + parse_compress_specification(compress_spec->algorithm, + detail, + compress_spec); + error_detail = validate_compress_specification(compress_spec); + if (error_detail != NULL) + { + pg_log_error("invalid compression specification: %s", error_detail); + res = false; + } + + pg_free(detail); + } + + pg_free(method); + } + + /* there is no need to check further when an error is already detected */ + if (!res) + return false; + + /* verify that the requested compression is supported */ + + if (compress_spec->algorithm != PG_COMPRESSION_NONE && + compress_spec->algorithm != PG_COMPRESSION_GZIP) + supports_compression = false; + +#ifndef HAVE_LIBZ + if (compress_spec->algorithm == PG_COMPRESSION_GZIP) + supports_compression = false; +#endif + + if (!supports_compression) + { + pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); + compress_spec->algorithm = PG_COMPRESSION_NONE; + } + + return true; +} + static ArchiveFormat parseArchiveFormat(const char *format, ArchiveMode *mode) { diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl index a583c8a6d2..e1f1a8801c 100644 --- a/src/bin/pg_dump/t/001_basic.pl +++ b/src/bin/pg_dump/t/001_basic.pl @@ -120,6 +120,16 @@ command_fails_like( qr/\Qpg_restore: error: cannot specify both --single-transaction and multiple jobs\E/, 'pg_restore: cannot specify both --single-transaction and multiple jobs'); +command_fails_like( + [ 'pg_dump', '--compress', 'garbage' ], + qr/\Qpg_dump: error: invalid compression method "garbage" (gzip, none)\E/, + 'pg_dump: invalid --compress'); + +command_fails_like( + [ 'pg_dump', '--compress', 'none:1' ], + qr/\Qpg_dump: error: invalid compression specification: compression algorithm "none" does not accept a compression level\E/, + 'pg_dump: invalid compression specification: compression algorithm "none" does not accept a compression level'); + command_fails_like( [ 'pg_dump', '-Z', '-1' ], qr/\Qpg_dump: error: -Z\/--compress must be in range 0..9\E/, diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 1f08716f69..a02b578750 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -87,7 +87,7 @@ my %pgdump_runs = ( compile_option => 'gzip', dump_cmd => [ 'pg_dump', '--jobs=2', - '--format=directory', '--compress=1', + '--format=directory', '--compress=gzip:1', "--file=$tempdir/compression_gzip_dir", 'postgres', ], # Give coverage for manually compressed blob.toc files during -- 2.34.1