From 083f1092aaf6e32a99649f02ad986df66a1d0d82 Mon Sep 17 00:00:00 2001 From: Justin Pryzby Date: Sun, 26 Feb 2023 11:32:34 -0600 Subject: [PATCH 3/4] pg_dump/lz4: use lz4 streaming compression.. Since 0da243fed, each row was independently compressed, but that gives poor compression (especially for tables with few columns) due to not sharing compression state. Also, respect the user's requested compression level. --- src/bin/pg_dump/compress_lz4.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c index 7dacfeae469..32a8f668907 100644 --- a/src/bin/pg_dump/compress_lz4.c +++ b/src/bin/pg_dump/compress_lz4.c @@ -40,6 +40,7 @@ typedef struct LZ4CompressorState { char *outbuf; size_t outsize; + LZ4_stream_t *stream; } LZ4CompressorState; /* Private routines that support LZ4 compressed data I/O */ @@ -90,8 +91,12 @@ WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs, LZ4cs->outsize = requiredsize; } - compressed = LZ4_compress_default(data, LZ4cs->outbuf, - dLen, LZ4cs->outsize); + if (LZ4cs->stream == NULL) + LZ4cs->stream = LZ4_createStream(); + + compressed = LZ4_compress_fast_continue(LZ4cs->stream, data, LZ4cs->outbuf, + dLen, LZ4cs->outsize, + AH->compression_spec.level); if (compressed <= 0) pg_fatal("failed to LZ4 compress data"); @@ -105,6 +110,10 @@ EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs) LZ4CompressorState *LZ4cs; LZ4cs = (LZ4CompressorState *) cs->private_data; + + if (LZ4cs->stream != NULL) + LZ4_freeStream(LZ4cs->stream); + pg_free(LZ4cs->outbuf); pg_free(LZ4cs); } -- 2.34.1