From cfbf78b2fa5f478b4699f881c9e9d6cfb627b976 Mon Sep 17 00:00:00 2001 From: Justin Pryzby Date: Mon, 24 May 2021 23:32:30 -0500 Subject: [PATCH v8 5/9] (re)add wal_compression_method: lz4 --- doc/src/sgml/config.sgml | 3 ++- doc/src/sgml/install-windows.sgml | 2 +- doc/src/sgml/installation.sgml | 5 +++-- src/backend/access/transam/xloginsert.c | 17 ++++++++++++++++- src/backend/access/transam/xlogreader.c | 14 ++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/access/xlogrecord.h | 2 +- 8 files changed, 39 insertions(+), 7 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 218a98cfc6..352b31fa81 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3126,7 +3126,8 @@ include_dir 'conf.d' server compresses full page images written to WAL when is on or during a base backup. A compressed page image will be decompressed during WAL replay. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + PostgreSQL was built) lz4. The default value is off. Only superusers can change this setting. diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml index db53ee85a8..a023584722 100644 --- a/doc/src/sgml/install-windows.sgml +++ b/doc/src/sgml/install-windows.sgml @@ -299,7 +299,7 @@ $ENV{MSBFLAGS}="/m"; LZ4 Required for supporting LZ4 compression - method for compressing the table data. Binaries and source can be + method for compressing table or WAL data. Binaries and source can be downloaded from . diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 073d5089f7..c7673a4dc8 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -270,7 +270,8 @@ su - postgres You need LZ4, if you want to support compression of data with this method; see - . + and + . @@ -980,7 +981,7 @@ build-postgresql: Build with LZ4 compression support. This allows the use of LZ4 for - compression of table data. + compression of table and WAL data. diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 4f81f19c49..a8794a941a 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -889,6 +896,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 822b2612cd..753b38e58a 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -38,6 +38,10 @@ #include #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -72,6 +76,9 @@ const struct config_enum_entry wal_compression_options[] = { {"1", WAL_COMPRESSION_PGLZ, true}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; @@ -1622,6 +1629,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 3991d35afd..dbf3155502 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -218,7 +218,7 @@ #full_page_writes = on # recover from partial page writes #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) -#wal_compression = off # enable compression of full-page writes: off, pglz, zlib +#wal_compression = off # enable compression of full-page writes: off, pglz, zlib, lz4 #wal_init_zero = on # zero-fill new WAL files #wal_recycle = on # recycle WAL files #wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index 8b740af66d..0287592cd4 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -330,6 +330,7 @@ typedef enum WalCompression WAL_COMPRESSION_NONE, WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/access/xlogrecord.h b/src/include/access/xlogrecord.h index 2a60c0fb92..abb42b364d 100644 --- a/src/include/access/xlogrecord.h +++ b/src/include/access/xlogrecord.h @@ -147,7 +147,7 @@ typedef struct XLogRecordBlockImageHeader #define BKPIMAGE_APPLY 0x02 /* page image should be restored during * replay */ #define BKPIMAGE_COMPRESS_METHOD1 0x04 /* bits to encode compression method */ -#define BKPIMAGE_COMPRESS_METHOD2 0x08 /* 0=none, 1=pglz, 2=zlib */ +#define BKPIMAGE_COMPRESS_METHOD2 0x08 /* 0=none, 1=pglz, 2=zlib, 3=lz4 */ /* How many bits to shift to extract compression */ #define BKPIMAGE_COMPRESS_OFFSET_BITS 2 -- 2.17.0