From 2fd2b76238618e62e37eb011c0fe145162602a9d Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Fri, 27 Dec 2024 12:03:34 +0900 Subject: [PATCH v2 2/3] Add RMGR and WAL-logging API for pgstats This commit adds a new facility in the backend that offers to pgstats kinds the possibility to WAL-log data that need to be persisted across restarts: - A new callback called redo_cb can be defined by a pgstats_kind. - pgstat_xlog.c includes one API able to register some payload data and its size. Stats kinds doing WAL-logging require a definition of redo_cb. - pg_waldump is able to show the data logged, as hexadedimal data. This facility has applications for in-core and custom stats kinds, with one primary case being the possibility to WAL-log some statistics related to relations so as these are not lost post-crash. This is left as future work. --- src/include/access/rmgrlist.h | 1 + src/include/utils/pgstat_internal.h | 8 +++ src/include/utils/pgstat_xlog.h | 41 +++++++++++++ src/backend/access/rmgrdesc/Makefile | 1 + src/backend/access/rmgrdesc/meson.build | 1 + src/backend/access/rmgrdesc/pgstatdesc.c | 48 +++++++++++++++ src/backend/access/transam/rmgr.c | 1 + src/backend/utils/activity/Makefile | 1 + src/backend/utils/activity/meson.build | 1 + src/backend/utils/activity/pgstat_xlog.c | 77 ++++++++++++++++++++++++ src/bin/pg_waldump/.gitignore | 1 + src/bin/pg_waldump/rmgrdesc.c | 1 + src/bin/pg_waldump/t/001_basic.pl | 3 +- src/tools/pgindent/typedefs.list | 1 + 14 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 src/include/utils/pgstat_xlog.h create mode 100644 src/backend/access/rmgrdesc/pgstatdesc.c create mode 100644 src/backend/utils/activity/pgstat_xlog.c diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h index 78e6b908c6..abf9393bbe 100644 --- a/src/include/access/rmgrlist.h +++ b/src/include/access/rmgrlist.h @@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL, NULL) PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask, NULL) PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL, logicalmsg_decode) +PG_RMGR(RM_PGSTAT_ID, "PgStat", pgstat_redo, pgstat_desc, pgstat_identify, NULL, NULL, NULL, NULL) diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 811ed9b005..47d7f8988e 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -258,6 +258,14 @@ typedef struct PgStat_KindInfo */ void (*init_backend_cb) (void); + /* + * Perform custom actions when replaying WAL related to a statistics kind. + * + * "data" is a pointer to the stats information that can be used by the + * stats kind at redo. + */ + void (*redo_cb) (void *data, size_t data_size); + /* * For variable-numbered stats: flush pending stats. Required if pending * data is used. See flush_fixed_cb for fixed-numbered stats. diff --git a/src/include/utils/pgstat_xlog.h b/src/include/utils/pgstat_xlog.h new file mode 100644 index 0000000000..25b7a78dd5 --- /dev/null +++ b/src/include/utils/pgstat_xlog.h @@ -0,0 +1,41 @@ +/* ---------- + * pgstat_xlog.h + * Exports from utils/activity/pgstat_xlog.c + * + * Copyright (c) 2001-2024, PostgreSQL Global Development Group + * + * src/include/utils/pgstat_xlog.h + * ---------- + */ +#ifndef PGSTAT_XLOG_H +#define PGSTAT_XLOG_H + +#include "access/xlog.h" +#include "access/xlogdefs.h" +#include "access/xlogreader.h" +#include "utils/pgstat_kind.h" + +/* + * Generic WAL record for pgstat data. + */ +typedef struct xl_pgstat_data +{ + PgStat_Kind stats_kind; + size_t data_size; /* size of the data */ + /* data payload, worth data_size */ + char data[FLEXIBLE_ARRAY_MEMBER]; +} xl_pgstat_data; + +#define SizeOfPgStatData (offsetof(xl_pgstat_data, data)) + +extern XLogRecPtr pgstat_xlog_data(PgStat_Kind stats_kind, + const void *data, + size_t data_size); + +/* RMGR API */ +#define XLOG_PGSTAT_DATA 0x00 +extern void pgstat_redo(XLogReaderState *record); +extern void pgstat_desc(StringInfo buf, XLogReaderState *record); +extern const char *pgstat_identify(uint8 info); + +#endif /* PGSTAT_XLOG_H */ diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile index cd95eec37f..3979d22946 100644 --- a/src/backend/access/rmgrdesc/Makefile +++ b/src/backend/access/rmgrdesc/Makefile @@ -21,6 +21,7 @@ OBJS = \ logicalmsgdesc.o \ mxactdesc.o \ nbtdesc.o \ + pgstatdesc.o \ relmapdesc.o \ replorigindesc.o \ rmgrdesc_utils.o \ diff --git a/src/backend/access/rmgrdesc/meson.build b/src/backend/access/rmgrdesc/meson.build index e8b7a65fc7..951dacbd72 100644 --- a/src/backend/access/rmgrdesc/meson.build +++ b/src/backend/access/rmgrdesc/meson.build @@ -14,6 +14,7 @@ rmgr_desc_sources = files( 'logicalmsgdesc.c', 'mxactdesc.c', 'nbtdesc.c', + 'pgstatdesc.c', 'relmapdesc.c', 'replorigindesc.c', 'rmgrdesc_utils.c', diff --git a/src/backend/access/rmgrdesc/pgstatdesc.c b/src/backend/access/rmgrdesc/pgstatdesc.c new file mode 100644 index 0000000000..0fc19d1e56 --- /dev/null +++ b/src/backend/access/rmgrdesc/pgstatdesc.c @@ -0,0 +1,48 @@ +/*------------------------------------------------------------------------- + * + * pgstatdesc.c + * rmgr descriptor routines for utils/activity/pgstat_xlog.c + * + * Portions Copyright (c) 2001-2024, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/access/rmgrdesc/pgstatdesc.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "utils/pgstat_xlog.h" + +void +pgstat_desc(StringInfo buf, XLogReaderState *record) +{ + char *rec = XLogRecGetData(record); + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + if (info == XLOG_PGSTAT_DATA) + { + xl_pgstat_data *xlrec = (xl_pgstat_data *) rec; + char *sep = ""; + + appendStringInfo(buf, "stats kind \"%u\"; with data (%zu bytes): ", + xlrec->stats_kind, xlrec->data_size); + /* Write data payload as a series of hex bytes */ + for (int cnt = 0; cnt < xlrec->data_size; cnt++) + { + appendStringInfo(buf, "%s%02X", sep, (unsigned char) xlrec->data[cnt]); + sep = " "; + } + } +} + +const char * +pgstat_identify(uint8 info) +{ + if ((info & ~XLR_INFO_MASK) == XLOG_PGSTAT_DATA) + return "PGSTAT_DATA"; + + return NULL; +} diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c index 1b7499726e..945843a78e 100644 --- a/src/backend/access/transam/rmgr.c +++ b/src/backend/access/transam/rmgr.c @@ -39,6 +39,7 @@ #include "replication/message.h" #include "replication/origin.h" #include "storage/standby.h" +#include "utils/pgstat_xlog.h" #include "utils/relmapper.h" /* IWYU pragma: end_keep */ diff --git a/src/backend/utils/activity/Makefile b/src/backend/utils/activity/Makefile index 24b64a2742..186108c60a 100644 --- a/src/backend/utils/activity/Makefile +++ b/src/backend/utils/activity/Makefile @@ -33,6 +33,7 @@ OBJS = \ pgstat_subscription.o \ pgstat_wal.o \ pgstat_xact.o \ + pgstat_xlog.o \ wait_event.o \ wait_event_funcs.o diff --git a/src/backend/utils/activity/meson.build b/src/backend/utils/activity/meson.build index 380d3dd70c..a79e2d3731 100644 --- a/src/backend/utils/activity/meson.build +++ b/src/backend/utils/activity/meson.build @@ -18,6 +18,7 @@ backend_sources += files( 'pgstat_subscription.c', 'pgstat_wal.c', 'pgstat_xact.c', + 'pgstat_xlog.c', ) # this includes a .c file with contents generated in ../../../include/activity, diff --git a/src/backend/utils/activity/pgstat_xlog.c b/src/backend/utils/activity/pgstat_xlog.c new file mode 100644 index 0000000000..397a49a33a --- /dev/null +++ b/src/backend/utils/activity/pgstat_xlog.c @@ -0,0 +1,77 @@ +/* ---------- + * pgstat_xlog.c + * WAL replay logic for cumulative statistics. + * + * + * Copyright (c) 2001-2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/utils/activity/pgstat_xlog.c + * ---------- + */ + +#include "postgres.h" + +#include + +#include "access/xlog.h" +#include "access/xloginsert.h" +#include "utils/pgstat_internal.h" +#include "utils/pgstat_xlog.h" + +/* + * Write pgstats record data into WAL. + */ +XLogRecPtr +pgstat_xlog_data(PgStat_Kind stats_kind, const void *data, + size_t data_size) +{ + xl_pgstat_data xlrec; + XLogRecPtr lsn; + + xlrec.stats_kind = stats_kind; + xlrec.data_size = data_size; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfPgStatData); + XLogRegisterData(data, data_size); + + lsn = XLogInsert(RM_PGSTAT_ID, XLOG_PGSTAT_DATA); + return lsn; +} + +/* + * Redo just passes down to the stats kind expected by this record + * data included in it. + */ +void +pgstat_redo(XLogReaderState *record) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + xl_pgstat_data *xlrec; + PgStat_Kind stats_kind; + size_t data_size; + void *data; + const PgStat_KindInfo *kind_info; + + if (info != XLOG_PGSTAT_DATA) + elog(PANIC, "pgstat_redo: unknown op code %u", info); + + xlrec = (xl_pgstat_data *) XLogRecGetData(record); + + stats_kind = xlrec->stats_kind; + data_size = xlrec->data_size; + data = xlrec->data; + + kind_info = pgstat_get_kind_info(stats_kind); + + if (kind_info == NULL) + elog(FATAL, "pgstat_redo: invalid stats data found: kind=%u", + stats_kind); + + if (kind_info->redo_cb == NULL) + elog(FATAL, "pgstat_redo: no redo callback found: kind=%s", + kind_info->name); + + kind_info->redo_cb(data, data_size); +} diff --git a/src/bin/pg_waldump/.gitignore b/src/bin/pg_waldump/.gitignore index ec51f41c76..e5089b322d 100644 --- a/src/bin/pg_waldump/.gitignore +++ b/src/bin/pg_waldump/.gitignore @@ -13,6 +13,7 @@ /logicalmsgdesc.c /mxactdesc.c /nbtdesc.c +/pgstatdesc.c /relmapdesc.c /replorigindesc.c /rmgrdesc_utils.c diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c index 6b8c17bb4c..05b4f2acd1 100644 --- a/src/bin/pg_waldump/rmgrdesc.c +++ b/src/bin/pg_waldump/rmgrdesc.c @@ -30,6 +30,7 @@ #include "replication/origin.h" #include "rmgrdesc.h" #include "storage/standbydefs.h" +#include "utils/pgstat_xlog.h" #include "utils/relmapper.h" #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask,decode) \ diff --git a/src/bin/pg_waldump/t/001_basic.pl b/src/bin/pg_waldump/t/001_basic.pl index 578e473139..367ed6ab8c 100644 --- a/src/bin/pg_waldump/t/001_basic.pl +++ b/src/bin/pg_waldump/t/001_basic.pl @@ -73,7 +73,8 @@ BRIN CommitTs ReplicationOrigin Generic -LogicalMessage$/, +LogicalMessage +PgStat$/, 'rmgr list'); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index e1c4f913f8..a5b1a2c998 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -4134,6 +4134,7 @@ xl_multixact_create xl_multixact_truncate xl_overwrite_contrecord xl_parameter_change +xl_pgstat_data xl_relmap_update xl_replorigin_drop xl_replorigin_set -- 2.45.2