diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile index 0367466..efb031a 100644 *** a/src/bin/pg_dump/Makefile --- b/src/bin/pg_dump/Makefile *************** override CPPFLAGS := -I$(libpq_srcdir) $ *** 20,26 **** OBJS= pg_backup_archiver.o pg_backup_db.o pg_backup_custom.o \ pg_backup_files.o pg_backup_null.o pg_backup_tar.o \ ! dumputils.o $(WIN32RES) KEYWRDOBJS = keywords.o kwlookup.o --- 20,26 ---- OBJS= pg_backup_archiver.o pg_backup_db.o pg_backup_custom.o \ pg_backup_files.o pg_backup_null.o pg_backup_tar.o \ ! dumputils.o compress_io.o $(WIN32RES) KEYWRDOBJS = keywords.o kwlookup.o diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index ...c31f3a9 . *** a/src/bin/pg_dump/compress_io.c --- b/src/bin/pg_dump/compress_io.c *************** *** 0 **** --- 1,381 ---- + /*------------------------------------------------------------------------- + * + * compress_io.c + * Routines for archivers to write an uncompressed or compressed data + * stream. + * + * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * pg_dump will read the system catalogs in a database and dump out a + * script that reproduces the schema in terms of SQL that is understood + * by PostgreSQL + * + * IDENTIFICATION + * XXX + * + *------------------------------------------------------------------------- + */ + + #include "compress_io.h" + + static const char *modulename = gettext_noop("compress_io"); + + #ifdef HAVE_LIBZ + static void _DoInflate(ArchiveHandle *AH, CompressorState *cs, ReadFunc readF); + static void _DoDeflate(ArchiveHandle *AH, CompressorState *cs, int flush, WriteFunc writeF); + static void _DoInflateZlib(ArchiveHandle *AH, CompressorState *cs, ReadFunc readF); + static void _DoDeflateZlib(ArchiveHandle *AH, CompressorState *cs, int flush, WriteFunc writeF); + #endif + + /* + * If a compression library is in use, then startit up. This is called from + * StartData & StartBlob. The buffers are setup in the Init routine. + */ + void + InitCompressorState(ArchiveHandle *AH, CompressorState *cs, CompressorAction action) + { + if (AH->compression == 0 && cs->comprAlg != COMPR_ALG_NONE) + AH->compression = -1; + + Assert(AH->compression == 0 ? + (cs->comprAlg == COMPR_ALG_NONE) : + (cs->comprAlg != COMPR_ALG_NONE)); + + if (cs->comprAlg == COMPR_ALG_LIBZ) + { + #ifdef HAVE_LIBZ + z_streamp zp = cs->zp; + + if (AH->compression < 0 || AH->compression > 9) + AH->compression = Z_DEFAULT_COMPRESSION; + + zp->zalloc = Z_NULL; + zp->zfree = Z_NULL; + zp->opaque = Z_NULL; + + if (action == COMPRESSOR_DEFLATE) + if (deflateInit(zp, AH->compression) != Z_OK) + die_horribly(AH, modulename, "could not initialize compression library: %s\n", zp->msg); + if (action == COMPRESSOR_INFLATE) + if (inflateInit(zp) != Z_OK) + die_horribly(AH, modulename, "could not initialize compression library: %s\n", zp->msg); + + /* Just be paranoid - maybe End is called after Start, with no Write */ + zp->next_out = (void *) cs->comprOut; + zp->avail_out = comprOutInitSize; + #endif + } + + /* Nothing to be done for COMPR_ALG_NONE */ + } + + /* + * Terminate compression library context and flush its buffers. If no compression + * library is in use then just return. + */ + void + FlushCompressorState(ArchiveHandle *AH, CompressorState *cs, WriteFunc writeF) + { + Assert(AH->compression == 0 ? + (cs->comprAlg == COMPR_ALG_NONE) : + (cs->comprAlg != COMPR_ALG_NONE)); + + #ifdef HAVE_LIBZ + if (cs->comprAlg == COMPR_ALG_LIBZ) + { + z_streamp zp = cs->zp; + + zp->next_in = NULL; + zp->avail_in = 0; + + _DoDeflate(AH, cs, Z_FINISH, writeF); + + if (deflateEnd(zp) != Z_OK) + die_horribly(AH, modulename, "could not close compression stream: %s\n", zp->msg); + } + #endif + /* Nothing to be done for COMPR_ALG_NONE */ + } + + #ifdef HAVE_LIBZ + void + _DoDeflate(ArchiveHandle *AH, CompressorState *cs, int flush, WriteFunc writeF) + { + switch(cs->comprAlg) + { + case COMPR_ALG_LIBZ: + _DoDeflateZlib(AH, cs, flush, writeF); + break; + case COMPR_ALG_NONE: + Assert(false); + break; + } + } + + /* + * Send compressed data to the output stream (via writeF). + */ + void + _DoDeflateZlib(ArchiveHandle *AH, CompressorState *cs, int flush, WriteFunc writeF) + { + z_streamp zp = cs->zp; + char *out = cs->comprOut; + int res = Z_OK; + + Assert(AH->compression != 0); + + while (cs->zp->avail_in != 0 || flush) + { + res = deflate(zp, flush); + if (res == Z_STREAM_ERROR) + die_horribly(AH, modulename, "could not compress data: %s\n", zp->msg); + if (((flush == Z_FINISH) && (zp->avail_out < comprOutInitSize)) + || (zp->avail_out == 0) + || (zp->avail_in != 0) + ) + { + /* + * Extra paranoia: avoid zero-length chunks, since a zero length + * chunk is the EOF marker in the custom format. This should never + * happen but... + */ + if (zp->avail_out < comprOutInitSize) + { + /* + * Any write function shoud do its own error checking but + * to make sure we do a check here as well... + */ + size_t len = comprOutInitSize - zp->avail_out; + if (writeF(AH, out, len) != len) + die_horribly(AH, modulename, "could not write to output file: %s\n", strerror(errno)); + } + zp->next_out = (void *) out; + zp->avail_out = comprOutInitSize; + } + + if (res == Z_STREAM_END) + break; + } + } + + static void + _DoInflate(ArchiveHandle *AH, CompressorState *cs, ReadFunc readF) + { + switch(cs->comprAlg) + { + case COMPR_ALG_LIBZ: + _DoInflateZlib(AH, cs, readF); + break; + case COMPR_ALG_NONE: + Assert(false); + break; + } + } + + /* + * This function is void as it either returns successfully or fails via + * die_horribly(). + */ + static void + _DoInflateZlib(ArchiveHandle *AH, CompressorState *cs, ReadFunc readF) + { + z_streamp zp = cs->zp; + char *out = cs->comprOut; + int res = Z_OK; + size_t cnt; + void *in; + + Assert(AH->compression != 0); + + /* no minimal chunk size for zlib */ + while ((cnt = readF(AH, &in, 0))) + { + zp->next_in = (void *) in; + zp->avail_in = cnt; + + while (zp->avail_in > 0) + { + zp->next_out = (void *) out; + zp->avail_out = comprOutInitSize; + + res = inflate(zp, 0); + if (res != Z_OK && res != Z_STREAM_END) + die_horribly(AH, modulename, "could not uncompress data: %s\n", zp->msg); + + out[comprOutInitSize - zp->avail_out] = '\0'; + ahwrite(out, 1, comprOutInitSize - zp->avail_out, AH); + } + } + + zp->next_in = NULL; + zp->avail_in = 0; + while (res != Z_STREAM_END) + { + zp->next_out = (void *) out; + zp->avail_out = comprOutInitSize; + res = inflate(zp, 0); + if (res != Z_OK && res != Z_STREAM_END) + die_horribly(AH, modulename, "could not uncompress data: %s\n", zp->msg); + + out[comprOutInitSize - zp->avail_out] = '\0'; + ahwrite(out, 1, comprOutInitSize - zp->avail_out, AH); + } + + if (inflateEnd(zp) != Z_OK) + die_horribly(AH, modulename, "could not close compression library: %s\n", zp->msg); + } + #endif + + void + ReadDataFromArchive(ArchiveHandle *AH, CompressorState *cs, ReadFunc readF) + { + Assert(AH->compression == 0 ? + (cs->comprAlg == COMPR_ALG_NONE) : + (cs->comprAlg != COMPR_ALG_NONE)); + + switch(cs->comprAlg) + { + case COMPR_ALG_LIBZ: + #ifdef HAVE_LIBZ + _DoInflate(AH, cs, readF); + #endif + break; + case COMPR_ALG_NONE: + { + size_t cnt; + void *in; + + /* no minimal chunk size for uncompressed data */ + while ((cnt = readF(AH, &in, 0))) + { + ahwrite(in, 1, cnt, AH); + } + } + } + } + + size_t + WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, WriteFunc writeF, + const void *data, size_t dLen) + { + Assert(AH->compression == 0 ? + (cs->comprAlg == COMPR_ALG_NONE) : + (cs->comprAlg != COMPR_ALG_NONE)); + + switch(cs->comprAlg) + { + case COMPR_ALG_LIBZ: + #ifdef HAVE_LIBZ + cs->zp->next_in = (void *) data; + cs->zp->avail_in = dLen; + _DoDeflate(AH, cs, Z_NO_FLUSH, writeF); + #endif + break; + case COMPR_ALG_NONE: + /* + * Any write function shoud do its own error checking but to make sure + * we do a check here as well... + */ + if (writeF(AH, data, dLen) != dLen) + die_horribly(AH, modulename, "could not write to output file: %s\n", strerror(errno)); + } + /* we have either succeeded in writing dLen bytes or we have called die_horribly() */ + return dLen; + } + + CompressorState * + AllocateCompressorState(ArchiveHandle *AH) + { + CompressorAlgorithm alg = COMPR_ALG_NONE; + CompressorState *cs; + + /* + * AH->compression is set either on the commandline when creating an archive + * or by ReadHead() when restoring an archive. + */ + + if (AH->compression == Z_DEFAULT_COMPRESSION || + (AH->compression > 0 && AH->compression <= 9)) + alg = COMPR_ALG_LIBZ; + else if (AH->compression == COMPRESSION_NONE) + alg = COMPR_ALG_NONE; + else + die_horribly(AH, modulename, "Invalid compression code: %d\n", + AH->compression); + + #ifndef HAVE_LIBZ + /* if no compression was specified and we are not built with libz support + * anyway, fall back to no compression */ + if (AH->compression == Z_DEFAULT_COMPRESSION) + { + AH->compression = COMPRESSION_NONE; + alg = COMPR_ALG_NONE; + } + else if (alg == COMPR_ALG_LIBZ) + die_horribly(AH, modulename, "not built with zlib support\n"); + #endif + + cs = (CompressorState *) malloc(sizeof(CompressorState)); + if (cs == NULL) + die_horribly(AH, modulename, "out of memory\n"); + + cs->comprAlg = alg; + + switch(alg) + { + case COMPR_ALG_LIBZ: + #ifdef HAVE_LIBZ + cs->zp = (z_streamp) malloc(sizeof(z_stream)); + if (cs->zp == NULL) + die_horribly(AH, modulename, "out of memory\n"); + + /* + * comprOutInitSize is the buffer size we tell zlib it can output + * to. We actually allocate one extra byte because some routines + * want to append a trailing zero byte to the zlib output. The + * input buffer is expansible and is always of size + * cs->comprInSize; comprInInitSize is just the initial default + * size for it. + */ + cs->comprOut = (char *) malloc(comprOutInitSize + 1); + cs->comprIn = (char *) malloc(comprInInitSize); + cs->comprInSize = comprInInitSize; + cs->comprOutSize = comprOutInitSize; + + if (cs->comprOut == NULL || cs->comprIn == NULL) + die_horribly(AH, modulename, "out of memory\n"); + #endif + break; + case COMPR_ALG_NONE: + cs->comprOut = (char *) malloc(comprOutInitSize + 1); + cs->comprIn = (char *) malloc(comprInInitSize); + cs->comprInSize = comprInInitSize; + cs->comprOutSize = comprOutInitSize; + + if (cs->comprOut == NULL || cs->comprIn == NULL) + die_horribly(AH, modulename, "out of memory\n"); + break; + } + + return cs; + } + + void + FreeCompressorState(CompressorState *cs) + { + free(cs->comprOut); + free(cs->comprIn); + switch(cs->comprAlg) + { + case COMPR_ALG_LIBZ: + #ifdef HAVE_LIBZ + free(cs->zp); + #endif + break; + case COMPR_ALG_NONE: + break; + } + free(cs); + } + diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index ...b6b94d3 . *** a/src/bin/pg_dump/compress_io.h --- b/src/bin/pg_dump/compress_io.h *************** *** 0 **** --- 1,73 ---- + /*------------------------------------------------------------------------- + * + * compress_io.h + * Routines for archivers to write an uncompressed or compressed data + * stream. + * + * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * pg_dump will read the system catalogs in a database and dump out a + * script that reproduces the schema in terms of SQL that is understood + * by PostgreSQL + * + * IDENTIFICATION + * XXX + * + *------------------------------------------------------------------------- + */ + + #include "pg_backup_archiver.h" + + #define comprOutInitSize 65536 + #define comprInInitSize 65536 + + + typedef enum + { + COMPRESSOR_INFLATE, + COMPRESSOR_DEFLATE + } CompressorAction; + + typedef enum + { + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, + } CompressorAlgorithm; + + typedef struct + { + CompressorAlgorithm comprAlg; + #ifdef HAVE_LIBZ + z_streamp zp; + #endif + char *comprOut; + char *comprIn; + size_t comprInSize; + size_t comprOutSize; + } CompressorState; + + typedef size_t (*WriteFunc)(ArchiveHandle *AH, const void *buf, size_t len); + /* + * The sizeHint parameter tells the format which size is required for the algorithm. + * If the format doesn't know better it should send back that many bytes of input. + * If the format was written by blocks however, then the format already knows the block + * size and can deliver exactly the size of the next block. + * + * The custom archive is written in such blocks. + * The directory archive however is just a continuous stream of data. Other + * compressed formats than libz however deal with blocks on the algorithm level + * and then the algorithm is able to tell the format the amount of data that it + * is ready to consume next. + */ + typedef size_t (*ReadFunc)(ArchiveHandle *AH, void **buf, size_t sizeHint); + + void ReadDataFromArchive(ArchiveHandle *AH, CompressorState *cs, ReadFunc readF); + size_t WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, WriteFunc writeF, const void *data, size_t dLen); + + void InitCompressorState(ArchiveHandle *AH, CompressorState *cs, CompressorAction action); + void FlushCompressorState(ArchiveHandle *AH, CompressorState *cs, WriteFunc writeF); + + void FreeCompressorState(CompressorState *cs); + CompressorState *AllocateCompressorState(ArchiveHandle *AH); + diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index d1a9c54..a28956b 100644 *** a/src/bin/pg_dump/pg_backup_archiver.c --- b/src/bin/pg_dump/pg_backup_archiver.c *************** *** 22,27 **** --- 22,28 ---- #include "pg_backup_db.h" #include "dumputils.h" + #include "compress_io.h" #include #include diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index ae0c6e0..705b2e4 100644 *** a/src/bin/pg_dump/pg_backup_archiver.h --- b/src/bin/pg_dump/pg_backup_archiver.h *************** *** 49,54 **** --- 49,55 ---- #define GZCLOSE(fh) fclose(fh) #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) + /* this is just the redefinition of a libz constant */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream *************** typedef struct _z_stream *** 61,66 **** --- 62,76 ---- typedef z_stream *z_streamp; #endif + /* XXX eventually this should be an enum. However if we want something + * pluggable in the long run it can get hard to add values to a central + * enum from the plugins... */ + #define COMPRESSION_UNKNOWN (-2) + #define COMPRESSION_NONE 0 + + /* XXX should we change the archive version for pg_dump with directory support? + * XXX We are not actually modifying the existing formats, but on the other hand + * XXX a file could now be compressed with liblzf. */ /* Current archive version number (the format we can output) */ #define K_VERS_MAJOR 1 #define K_VERS_MINOR 12 *************** typedef struct _archiveHandle *** 267,272 **** --- 277,288 ---- struct _tocEntry *currToc; /* Used when dumping data */ int compression; /* Compression requested on open */ + /* Possible values for compression: + -2 COMPRESSION_UNKNOWN + -1 Z_DEFAULT_COMPRESSION + 0 COMPRESSION_NONE + 1-9 levels for gzip compression + */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ *************** int ahprintf(ArchiveHandle *AH, const *** 381,384 **** --- 397,411 ---- void ahlog(ArchiveHandle *AH, int level, const char *fmt,...) __attribute__((format(printf, 3, 4))); + #ifdef USE_ASSERT_CHECKING + #define Assert(condition) \ + if (!(condition)) \ + { \ + write_msg(NULL, "Failed assertion in %s, line %d\n", \ + __FILE__, __LINE__); \ + abort();\ + } + #else + #define Assert(condition) + #endif #endif diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 2bc7e8f..95135b7 100644 *** a/src/bin/pg_dump/pg_backup_custom.c --- b/src/bin/pg_dump/pg_backup_custom.c *************** *** 25,30 **** --- 25,31 ---- */ #include "pg_backup_archiver.h" + #include "compress_io.h" /*-------- * Routines in the format interface *************** static void _LoadBlobs(ArchiveHandle *AH *** 58,77 **** static void _Clone(ArchiveHandle *AH); static void _DeClone(ArchiveHandle *AH); ! /*------------ ! * Buffers used in zlib compression and extra data stored in archive and ! * in TOC entries. ! *------------ ! */ ! #define zlibOutSize 4096 ! #define zlibInSize 4096 typedef struct { ! z_streamp zp; ! char *zlibOut; ! char *zlibIn; ! size_t inSize; int hasSeek; pgoff_t filePos; pgoff_t dataStart; --- 59,70 ---- static void _Clone(ArchiveHandle *AH); static void _DeClone(ArchiveHandle *AH); ! static size_t _CustomWriteFunc(ArchiveHandle *AH, const void *buf, size_t len); ! static size_t _CustomReadFunction(ArchiveHandle *AH, void **buf, size_t sizeHint); typedef struct { ! CompressorState *cs; int hasSeek; pgoff_t filePos; pgoff_t dataStart; *************** static void _readBlockHeader(ArchiveHand *** 92,98 **** static void _StartDataCompressor(ArchiveHandle *AH, TocEntry *te); static void _EndDataCompressor(ArchiveHandle *AH, TocEntry *te); static pgoff_t _getFilePos(ArchiveHandle *AH, lclContext *ctx); - static int _DoDeflate(ArchiveHandle *AH, lclContext *ctx, int flush); static const char *modulename = gettext_noop("custom archiver"); --- 85,90 ---- *************** InitArchiveFmt_Custom(ArchiveHandle *AH) *** 144,179 **** die_horribly(AH, modulename, "out of memory\n"); AH->formatData = (void *) ctx; - ctx->zp = (z_streamp) malloc(sizeof(z_stream)); - if (ctx->zp == NULL) - die_horribly(AH, modulename, "out of memory\n"); - /* Initialize LO buffering */ AH->lo_buf_size = LOBBUFSIZE; AH->lo_buf = (void *) malloc(LOBBUFSIZE); if (AH->lo_buf == NULL) die_horribly(AH, modulename, "out of memory\n"); - /* - * zlibOutSize is the buffer size we tell zlib it can output to. We - * actually allocate one extra byte because some routines want to append a - * trailing zero byte to the zlib output. The input buffer is expansible - * and is always of size ctx->inSize; zlibInSize is just the initial - * default size for it. - */ - ctx->zlibOut = (char *) malloc(zlibOutSize + 1); - ctx->zlibIn = (char *) malloc(zlibInSize); - ctx->inSize = zlibInSize; ctx->filePos = 0; - if (ctx->zlibOut == NULL || ctx->zlibIn == NULL) - die_horribly(AH, modulename, "out of memory\n"); - /* * Now open the file */ if (AH->mode == archModeWrite) { if (AH->fSpec && strcmp(AH->fSpec, "") != 0) { AH->FH = fopen(AH->fSpec, PG_BINARY_W); --- 136,156 ---- die_horribly(AH, modulename, "out of memory\n"); AH->formatData = (void *) ctx; /* Initialize LO buffering */ AH->lo_buf_size = LOBBUFSIZE; AH->lo_buf = (void *) malloc(LOBBUFSIZE); if (AH->lo_buf == NULL) die_horribly(AH, modulename, "out of memory\n"); ctx->filePos = 0; /* * Now open the file */ if (AH->mode == archModeWrite) { + ctx->cs = AllocateCompressorState(AH); + if (AH->fSpec && strcmp(AH->fSpec, "") != 0) { AH->FH = fopen(AH->fSpec, PG_BINARY_W); *************** InitArchiveFmt_Custom(ArchiveHandle *AH) *** 211,216 **** --- 188,195 ---- ctx->hasSeek = checkSeek(AH->FH); ReadHead(AH); + ctx->cs = AllocateCompressorState(AH); + ReadToc(AH); ctx->dataStart = _getFilePos(AH, ctx); } *************** static size_t *** 340,356 **** _WriteData(ArchiveHandle *AH, const void *data, size_t dLen) { lclContext *ctx = (lclContext *) AH->formatData; ! z_streamp zp = ctx->zp; ! ! zp->next_in = (void *) data; ! zp->avail_in = dLen; ! while (zp->avail_in != 0) ! { ! /* printf("Deflating %lu bytes\n", (unsigned long) dLen); */ ! _DoDeflate(AH, ctx, 0); ! } ! return dLen; } /* --- 319,327 ---- _WriteData(ArchiveHandle *AH, const void *data, size_t dLen) { lclContext *ctx = (lclContext *) AH->formatData; ! CompressorState *cs = ctx->cs; ! return WriteDataToArchive(AH, cs, _CustomWriteFunc, data, dLen); } /* *************** static void *** 533,639 **** _PrintData(ArchiveHandle *AH) { lclContext *ctx = (lclContext *) AH->formatData; ! z_streamp zp = ctx->zp; ! size_t blkLen; ! char *in = ctx->zlibIn; ! size_t cnt; ! ! #ifdef HAVE_LIBZ ! int res; ! char *out = ctx->zlibOut; ! #endif ! ! #ifdef HAVE_LIBZ ! ! res = Z_OK; ! ! if (AH->compression != 0) ! { ! zp->zalloc = Z_NULL; ! zp->zfree = Z_NULL; ! zp->opaque = Z_NULL; ! ! if (inflateInit(zp) != Z_OK) ! die_horribly(AH, modulename, "could not initialize compression library: %s\n", zp->msg); ! } ! #endif ! ! blkLen = ReadInt(AH); ! while (blkLen != 0) ! { ! if (blkLen + 1 > ctx->inSize) ! { ! free(ctx->zlibIn); ! ctx->zlibIn = NULL; ! ctx->zlibIn = (char *) malloc(blkLen + 1); ! if (!ctx->zlibIn) ! die_horribly(AH, modulename, "out of memory\n"); ! ! ctx->inSize = blkLen + 1; ! in = ctx->zlibIn; ! } ! ! cnt = fread(in, 1, blkLen, AH->FH); ! if (cnt != blkLen) ! { ! if (feof(AH->FH)) ! die_horribly(AH, modulename, ! "could not read from input file: end of file\n"); ! else ! die_horribly(AH, modulename, ! "could not read from input file: %s\n", strerror(errno)); ! } ! ! ctx->filePos += blkLen; ! ! zp->next_in = (void *) in; ! zp->avail_in = blkLen; ! ! #ifdef HAVE_LIBZ ! if (AH->compression != 0) ! { ! while (zp->avail_in != 0) ! { ! zp->next_out = (void *) out; ! zp->avail_out = zlibOutSize; ! res = inflate(zp, 0); ! if (res != Z_OK && res != Z_STREAM_END) ! die_horribly(AH, modulename, "could not uncompress data: %s\n", zp->msg); ! ! out[zlibOutSize - zp->avail_out] = '\0'; ! ahwrite(out, 1, zlibOutSize - zp->avail_out, AH); ! } ! } ! else ! #endif ! { ! in[zp->avail_in] = '\0'; ! ahwrite(in, 1, zp->avail_in, AH); ! zp->avail_in = 0; ! } ! blkLen = ReadInt(AH); ! } ! ! #ifdef HAVE_LIBZ ! if (AH->compression != 0) ! { ! zp->next_in = NULL; ! zp->avail_in = 0; ! while (res != Z_STREAM_END) ! { ! zp->next_out = (void *) out; ! zp->avail_out = zlibOutSize; ! res = inflate(zp, 0); ! if (res != Z_OK && res != Z_STREAM_END) ! die_horribly(AH, modulename, "could not uncompress data: %s\n", zp->msg); ! out[zlibOutSize - zp->avail_out] = '\0'; ! ahwrite(out, 1, zlibOutSize - zp->avail_out, AH); ! } ! if (inflateEnd(zp) != Z_OK) ! die_horribly(AH, modulename, "could not close compression library: %s\n", zp->msg); ! } ! #endif } static void --- 504,513 ---- _PrintData(ArchiveHandle *AH) { lclContext *ctx = (lclContext *) AH->formatData; ! CompressorState *cs = ctx->cs; ! InitCompressorState(AH, cs, COMPRESSOR_INFLATE); ! ReadDataFromArchive(AH, cs, _CustomReadFunction); } static void *************** static void *** 683,701 **** _skipData(ArchiveHandle *AH) { lclContext *ctx = (lclContext *) AH->formatData; size_t blkLen; ! char *in = ctx->zlibIn; size_t cnt; blkLen = ReadInt(AH); while (blkLen != 0) { ! if (blkLen > ctx->inSize) { ! free(ctx->zlibIn); ! ctx->zlibIn = (char *) malloc(blkLen); ! ctx->inSize = blkLen; ! in = ctx->zlibIn; } cnt = fread(in, 1, blkLen, AH->FH); if (cnt != blkLen) --- 557,576 ---- _skipData(ArchiveHandle *AH) { lclContext *ctx = (lclContext *) AH->formatData; + CompressorState *cs = ctx->cs; size_t blkLen; ! char *in = cs->comprIn; size_t cnt; blkLen = ReadInt(AH); while (blkLen != 0) { ! if (blkLen > cs->comprInSize) { ! free(cs->comprIn); ! cs->comprIn = (char *) malloc(blkLen); ! cs->comprInSize = blkLen; ! in = cs->comprIn; } cnt = fread(in, 1, blkLen, AH->FH); if (cnt != blkLen) *************** _readBlockHeader(ArchiveHandle *AH, int *** 961,1099 **** } /* ! * If zlib is available, then startit up. This is called from ! * StartData & StartBlob. The buffers are setup in the Init routine. */ static void _StartDataCompressor(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; ! z_streamp zp = ctx->zp; ! ! #ifdef HAVE_LIBZ ! ! if (AH->compression < 0 || AH->compression > 9) ! AH->compression = Z_DEFAULT_COMPRESSION; ! if (AH->compression != 0) ! { ! zp->zalloc = Z_NULL; ! zp->zfree = Z_NULL; ! zp->opaque = Z_NULL; ! if (deflateInit(zp, AH->compression) != Z_OK) ! die_horribly(AH, modulename, "could not initialize compression library: %s\n", zp->msg); ! } ! #else ! AH->compression = 0; ! #endif ! /* Just be paranoid - maybe End is called after Start, with no Write */ ! zp->next_out = (void *) ctx->zlibOut; ! zp->avail_out = zlibOutSize; } ! /* ! * Send compressed data to the output stream (via ahwrite). ! * Each data chunk is preceded by it's length. ! * In the case of Z0, or no zlib, just write the raw data. ! * ! */ ! static int ! _DoDeflate(ArchiveHandle *AH, lclContext *ctx, int flush) { ! z_streamp zp = ctx->zp; ! #ifdef HAVE_LIBZ ! char *out = ctx->zlibOut; ! int res = Z_OK; ! if (AH->compression != 0) { ! res = deflate(zp, flush); ! if (res == Z_STREAM_ERROR) ! die_horribly(AH, modulename, "could not compress data: %s\n", zp->msg); ! if (((flush == Z_FINISH) && (zp->avail_out < zlibOutSize)) ! || (zp->avail_out == 0) ! || (zp->avail_in != 0) ! ) ! { ! /* ! * Extra paranoia: avoid zero-length chunks since a zero length ! * chunk is the EOF marker. This should never happen but... ! */ ! if (zp->avail_out < zlibOutSize) ! { ! /* ! * printf("Wrote %lu byte deflated chunk\n", (unsigned long) ! * (zlibOutSize - zp->avail_out)); ! */ ! WriteInt(AH, zlibOutSize - zp->avail_out); ! if (fwrite(out, 1, zlibOutSize - zp->avail_out, AH->FH) != (zlibOutSize - zp->avail_out)) ! die_horribly(AH, modulename, "could not write to output file: %s\n", strerror(errno)); ! ctx->filePos += zlibOutSize - zp->avail_out; ! } ! zp->next_out = (void *) out; ! zp->avail_out = zlibOutSize; ! } } ! else ! #endif { ! if (zp->avail_in > 0) ! { ! WriteInt(AH, zp->avail_in); ! if (fwrite(zp->next_in, 1, zp->avail_in, AH->FH) != zp->avail_in) ! die_horribly(AH, modulename, "could not write to output file: %s\n", strerror(errno)); ! ctx->filePos += zp->avail_in; ! zp->avail_in = 0; ! } else ! { ! #ifdef HAVE_LIBZ ! if (flush == Z_FINISH) ! res = Z_STREAM_END; ! #endif ! } } ! ! #ifdef HAVE_LIBZ ! return res; ! #else ! return 1; ! #endif } /* ! * Terminate zlib context and flush it's buffers. If no zlib ! * then just return. */ static void _EndDataCompressor(ArchiveHandle *AH, TocEntry *te) { ! #ifdef HAVE_LIBZ ! lclContext *ctx = (lclContext *) AH->formatData; ! z_streamp zp = ctx->zp; ! int res; ! ! if (AH->compression != 0) ! { ! zp->next_in = NULL; ! zp->avail_in = 0; ! ! do ! { ! /* printf("Ending data output\n"); */ ! res = _DoDeflate(AH, ctx, Z_FINISH); ! } while (res != Z_STREAM_END); ! ! if (deflateEnd(zp) != Z_OK) ! die_horribly(AH, modulename, "could not close compression stream: %s\n", zp->msg); ! } ! #endif /* Send the end marker */ WriteInt(AH, 0); --- 836,918 ---- } /* ! * If a compression algorithm is available, then startit up. This is called ! * from StartData & StartBlob. The buffers are setup in the Init routine. */ static void _StartDataCompressor(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; ! CompressorState *cs = ctx->cs; ! InitCompressorState(AH, cs, COMPRESSOR_DEFLATE); ! } ! static size_t ! _CustomWriteFunc(ArchiveHandle *AH, const void *buf, size_t len) ! { ! Assert(len != 0); ! /* never write 0-byte blocks (this should not happen) */ ! if (len == 0) ! return 0; ! WriteInt(AH, len); ! return _WriteBuf(AH, buf, len); } ! static size_t ! _CustomReadFunction(ArchiveHandle *AH, void **buf, size_t sizeHint) { ! lclContext *ctx = (lclContext *) AH->formatData; ! CompressorState *cs = ctx->cs; ! size_t blkLen; ! size_t cnt; ! /* ! * We deliberately ignore the sizeHint parameter because we know ! * the exact size of the next compressed block (=blkLen). ! */ ! blkLen = ReadInt(AH); ! ! if (blkLen == 0) ! return 0; ! ! if (blkLen + 1 > cs->comprInSize) { ! free(cs->comprIn); ! cs->comprIn = NULL; ! cs->comprIn = (char *) malloc(blkLen + 1); ! if (!cs->comprIn) ! die_horribly(AH, modulename, "out of memory\n"); ! cs->comprInSize = blkLen + 1; } ! cnt = _ReadBuf(AH, cs->comprIn, blkLen); ! if (cnt != blkLen) { ! if (feof(AH->FH)) ! die_horribly(AH, modulename, ! "could not read from input file: end of file\n"); else ! die_horribly(AH, modulename, ! "could not read from input file: %s\n", strerror(errno)); } ! *buf = cs->comprIn; ! return cnt; } /* ! * Terminate zlib context and flush it's buffers. */ static void _EndDataCompressor(ArchiveHandle *AH, TocEntry *te) { + lclContext *ctx = (lclContext *) AH->formatData; + CompressorState *cs = ctx->cs; ! FlushCompressorState(AH, cs, _CustomWriteFunc); /* Send the end marker */ WriteInt(AH, 0); *************** _Clone(ArchiveHandle *AH) *** 1114,1125 **** memcpy(AH->formatData, ctx, sizeof(lclContext)); ctx = (lclContext *) AH->formatData; ! ctx->zp = (z_streamp) malloc(sizeof(z_stream)); ! ctx->zlibOut = (char *) malloc(zlibOutSize + 1); ! ctx->zlibIn = (char *) malloc(ctx->inSize); ! ! if (ctx->zp == NULL || ctx->zlibOut == NULL || ctx->zlibIn == NULL) ! die_horribly(AH, modulename, "out of memory\n"); /* * Note: we do not make a local lo_buf because we expect at most one BLOBS --- 933,939 ---- memcpy(AH->formatData, ctx, sizeof(lclContext)); ctx = (lclContext *) AH->formatData; ! ctx->cs = AllocateCompressorState(AH); /* * Note: we do not make a local lo_buf because we expect at most one BLOBS *************** static void *** 1133,1141 **** _DeClone(ArchiveHandle *AH) { lclContext *ctx = (lclContext *) AH->formatData; - free(ctx->zlibOut); - free(ctx->zlibIn); - free(ctx->zp); free(ctx); } --- 947,956 ---- _DeClone(ArchiveHandle *AH) { lclContext *ctx = (lclContext *) AH->formatData; + CompressorState *cs = ctx->cs; + + FreeCompressorState(cs); free(ctx); } + diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 55ea684..04ded33 100644 *** a/src/bin/pg_dump/pg_dump.c --- b/src/bin/pg_dump/pg_dump.c *************** *** 56,61 **** --- 56,62 ---- #include "pg_backup_archiver.h" #include "dumputils.h" + #include "compress_io.h" extern char *optarg; extern int optind, *************** main(int argc, char **argv) *** 255,261 **** int numObjs; int i; enum trivalue prompt_password = TRI_DEFAULT; ! int compressLevel = -1; int plainText = 0; int outputClean = 0; int outputCreateDB = 0; --- 256,262 ---- int numObjs; int i; enum trivalue prompt_password = TRI_DEFAULT; ! int compressLevel = COMPRESSION_UNKNOWN; int plainText = 0; int outputClean = 0; int outputCreateDB = 0; *************** main(int argc, char **argv) *** 535,540 **** --- 536,547 ---- exit(1); } + /* actually we are using a zlib constant here but formats that don't + * support compression won't care and if we are not compiled with zlib + * compression we will be forced to no compression anyway. */ + if (compressLevel == COMPRESSION_UNKNOWN) + compressLevel = Z_DEFAULT_COMPRESSION; + /* open the output file */ if (pg_strcasecmp(format, "a") == 0 || pg_strcasecmp(format, "append") == 0) { *************** dumpBlobs(Archive *AH, void *arg) *** 2174,2180 **** exit_nicely(); } ! WriteData(AH, buf, cnt); } while (cnt > 0); lo_close(g_conn, loFd); --- 2181,2189 ---- exit_nicely(); } ! /* we try to avoid writing empty chunks */ ! if (cnt > 0) ! WriteData(AH, buf, cnt); } while (cnt > 0); lo_close(g_conn, loFd);