From 9eacf60385624419716fe33da1a9e240fa19551d Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Fri, 2 Apr 2021 17:37:37 -0700 Subject: [PATCH v60 05/17] pgstat: split bgwriter and checkpointer messages. --- src/include/pgstat.h | 61 ++++++++++--- src/backend/access/transam/xlog.c | 4 +- src/backend/postmaster/checkpointer.c | 26 +++--- src/backend/postmaster/pgstat.c | 126 +++++++++++++++++++++----- src/backend/storage/buffer/bufmgr.c | 8 +- src/backend/utils/adt/pgstatfuncs.c | 27 +++--- 6 files changed, 185 insertions(+), 67 deletions(-) diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 7cd137506e2..bac37dfd54e 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -71,6 +71,7 @@ typedef enum StatMsgType PGSTAT_MTYPE_ANALYZE, PGSTAT_MTYPE_ARCHIVER, PGSTAT_MTYPE_BGWRITER, + PGSTAT_MTYPE_CHECKPOINTER, PGSTAT_MTYPE_WAL, PGSTAT_MTYPE_SLRU, PGSTAT_MTYPE_FUNCSTAT, @@ -450,17 +451,27 @@ typedef struct PgStat_MsgBgWriter { PgStat_MsgHdr m_hdr; + PgStat_Counter m_buf_written_clean; + PgStat_Counter m_maxwritten_clean; + PgStat_Counter m_buf_alloc; +} PgStat_MsgBgWriter; + +/* ---------- + * PgStat_MsgCheckpointer Sent by the checkpointer to update statistics. + * ---------- + */ +typedef struct PgStat_MsgCheckpointer +{ + PgStat_MsgHdr m_hdr; + PgStat_Counter m_timed_checkpoints; PgStat_Counter m_requested_checkpoints; PgStat_Counter m_buf_written_checkpoints; - PgStat_Counter m_buf_written_clean; - PgStat_Counter m_maxwritten_clean; PgStat_Counter m_buf_written_backend; PgStat_Counter m_buf_fsync_backend; - PgStat_Counter m_buf_alloc; PgStat_Counter m_checkpoint_write_time; /* times in milliseconds */ PgStat_Counter m_checkpoint_sync_time; -} PgStat_MsgBgWriter; +} PgStat_MsgCheckpointer; /* ---------- * PgStat_MsgWal Sent by backends and background processes to update WAL statistics. @@ -676,6 +687,7 @@ typedef union PgStat_Msg PgStat_MsgAnalyze msg_analyze; PgStat_MsgArchiver msg_archiver; PgStat_MsgBgWriter msg_bgwriter; + PgStat_MsgCheckpointer msg_checkpointer; PgStat_MsgWal msg_wal; PgStat_MsgSLRU msg_slru; PgStat_MsgFuncstat msg_funcstat; @@ -817,9 +829,20 @@ typedef struct PgStat_ArchiverStats } PgStat_ArchiverStats; /* - * Global statistics kept in the stats collector + * Background writer statistics kept in the stats collector */ -typedef struct PgStat_GlobalStats +typedef struct PgStat_BgWriterStats +{ + PgStat_Counter buf_written_clean; + PgStat_Counter maxwritten_clean; + PgStat_Counter buf_alloc; + TimestampTz stat_reset_timestamp; +} PgStat_BgWriterStats; + +/* + * Checkpointer statistics kept in the stats collector + */ +typedef struct PgStat_CheckpointerStats { TimestampTz stats_timestamp; /* time of stats file update */ PgStat_Counter timed_checkpoints; @@ -827,12 +850,19 @@ typedef struct PgStat_GlobalStats PgStat_Counter checkpoint_write_time; /* times in milliseconds */ PgStat_Counter checkpoint_sync_time; PgStat_Counter buf_written_checkpoints; - PgStat_Counter buf_written_clean; - PgStat_Counter maxwritten_clean; PgStat_Counter buf_written_backend; PgStat_Counter buf_fsync_backend; - PgStat_Counter buf_alloc; - TimestampTz stat_reset_timestamp; +} PgStat_CheckpointerStats; + +/* + * Global statistics kept in the stats collector + */ +typedef struct PgStat_GlobalStats +{ + TimestampTz stats_timestamp; /* time of stats file update */ + + PgStat_CheckpointerStats checkpointer; + PgStat_BgWriterStats bgwriter; } PgStat_GlobalStats; /* @@ -912,7 +942,13 @@ extern char *pgstat_stat_filename; /* * BgWriter statistics counters are updated directly by bgwriter and bufmgr */ -extern PgStat_MsgBgWriter BgWriterStats; +extern PgStat_MsgBgWriter PendingBgWriterStats; + +/* + * Checkpointer statistics counters are updated directly by checkpointer and + * bufmgr. + */ +extern PgStat_MsgCheckpointer PendingCheckpointerStats; /* * WAL statistics counter is updated by backends and background processes @@ -1065,6 +1101,7 @@ extern void pgstat_twophase_postabort(TransactionId xid, uint16 info, extern void pgstat_send_archiver(const char *xlog, bool failed); extern void pgstat_send_bgwriter(void); +extern void pgstat_send_checkpointer(void); extern void pgstat_report_wal(void); extern bool pgstat_send_wal(bool force); @@ -1077,6 +1114,8 @@ extern PgStat_StatDBEntry *pgstat_fetch_stat_dbentry(Oid dbid); extern PgStat_StatTabEntry *pgstat_fetch_stat_tabentry(Oid relid); extern PgStat_StatFuncEntry *pgstat_fetch_stat_funcentry(Oid funcid); extern PgStat_ArchiverStats *pgstat_fetch_stat_archiver(void); +extern PgStat_BgWriterStats *pgstat_fetch_stat_bgwriter(void); +extern PgStat_CheckpointerStats *pgstat_fetch_stat_checkpointer(void); extern PgStat_GlobalStats *pgstat_fetch_global(void); extern PgStat_WalStats *pgstat_fetch_stat_wal(void); extern PgStat_SLRUStats *pgstat_fetch_slru(void); diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 6f8810e1490..2edf116442c 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -8724,8 +8724,8 @@ LogCheckpointEnd(bool restartpoint) CheckpointStats.ckpt_sync_end_t); /* Accumulate checkpoint timing summary data, in milliseconds. */ - BgWriterStats.m_checkpoint_write_time += write_msecs; - BgWriterStats.m_checkpoint_sync_time += sync_msecs; + PendingCheckpointerStats.m_checkpoint_write_time += write_msecs; + PendingCheckpointerStats.m_checkpoint_sync_time += sync_msecs; /* * All of the published timing statistics are accounted for. Only diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c index e7e6a2a4594..acba513c8fc 100644 --- a/src/backend/postmaster/checkpointer.c +++ b/src/backend/postmaster/checkpointer.c @@ -360,7 +360,7 @@ CheckpointerMain(void) if (((volatile CheckpointerShmemStruct *) CheckpointerShmem)->ckpt_flags) { do_checkpoint = true; - BgWriterStats.m_requested_checkpoints++; + PendingCheckpointerStats.m_requested_checkpoints++; } /* @@ -374,7 +374,7 @@ CheckpointerMain(void) if (elapsed_secs >= CheckPointTimeout) { if (!do_checkpoint) - BgWriterStats.m_timed_checkpoints++; + PendingCheckpointerStats.m_timed_checkpoints++; do_checkpoint = true; flags |= CHECKPOINT_CAUSE_TIME; } @@ -496,13 +496,9 @@ CheckpointerMain(void) CheckArchiveTimeout(); /* - * Send off activity statistics to the stats collector. (The reason - * why we re-use bgwriter-related code for this is that the bgwriter - * and checkpointer used to be just one process. It's probably not - * worth the trouble to split the stats support into two independent - * stats message types.) + * Send off activity statistics to the stats collector. */ - pgstat_send_bgwriter(); + pgstat_send_checkpointer(); /* Send WAL statistics to the stats collector. */ pgstat_report_wal(); @@ -580,9 +576,9 @@ HandleCheckpointerInterrupts(void) * updates the statistics, increment the checkpoint request and send * the statistics to the stats collector. */ - BgWriterStats.m_requested_checkpoints++; + PendingCheckpointerStats.m_requested_checkpoints++; ShutdownXLOG(0, 0); - pgstat_send_bgwriter(); + pgstat_send_checkpointer(); pgstat_report_wal(); /* Normal exit from the checkpointer is here */ @@ -722,9 +718,9 @@ CheckpointWriteDelay(int flags, double progress) CheckArchiveTimeout(); /* - * Report interim activity statistics to the stats collector. + * Report interim activity statistics. */ - pgstat_send_bgwriter(); + pgstat_send_checkpointer(); /* * This sleep used to be connected to bgwriter_delay, typically 200ms. @@ -1268,8 +1264,10 @@ AbsorbSyncRequests(void) LWLockAcquire(CheckpointerCommLock, LW_EXCLUSIVE); /* Transfer stats counts into pending pgstats message */ - BgWriterStats.m_buf_written_backend += CheckpointerShmem->num_backend_writes; - BgWriterStats.m_buf_fsync_backend += CheckpointerShmem->num_backend_fsync; + PendingCheckpointerStats.m_buf_written_backend + += CheckpointerShmem->num_backend_writes; + PendingCheckpointerStats.m_buf_fsync_backend + += CheckpointerShmem->num_backend_fsync; CheckpointerShmem->num_backend_writes = 0; CheckpointerShmem->num_backend_fsync = 0; diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 4c4b0720681..b50ec053c05 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -125,7 +125,8 @@ char *pgstat_stat_tmpname = NULL; * Stored directly in a stats message structure so they can be sent * without needing to copy things around. We assume these init to zeroes. */ -PgStat_MsgBgWriter BgWriterStats; +PgStat_MsgBgWriter PendingBgWriterStats; +PgStat_MsgCheckpointer PendingCheckpointerStats; PgStat_MsgWal WalStats; /* @@ -345,6 +346,7 @@ static void pgstat_recv_vacuum(PgStat_MsgVacuum *msg, int len); static void pgstat_recv_analyze(PgStat_MsgAnalyze *msg, int len); static void pgstat_recv_archiver(PgStat_MsgArchiver *msg, int len); static void pgstat_recv_bgwriter(PgStat_MsgBgWriter *msg, int len); +static void pgstat_recv_checkpointer(PgStat_MsgCheckpointer *msg, int len); static void pgstat_recv_wal(PgStat_MsgWal *msg, int len); static void pgstat_recv_slru(PgStat_MsgSLRU *msg, int len); static void pgstat_recv_funcstat(PgStat_MsgFuncstat *msg, int len); @@ -2748,6 +2750,37 @@ pgstat_fetch_stat_archiver(void) return &archiverStats; } +/* + * --------- + * pgstat_fetch_stat_bgwriter() - + * + * Support function for the SQL-callable pgstat* functions. Returns + * a pointer to the bgwriter statistics struct. + * --------- + */ +PgStat_BgWriterStats * +pgstat_fetch_stat_bgwriter(void) +{ + backend_read_statsfile(); + + return &globalStats.bgwriter; +} + +/* + * --------- + * pgstat_fetch_stat_checkpointer() - + * + * Support function for the SQL-callable pgstat* functions. Returns + * a pointer to the checkpointer statistics struct. + * --------- + */ +PgStat_CheckpointerStats * +pgstat_fetch_stat_checkpointer(void) +{ + backend_read_statsfile(); + + return &globalStats.checkpointer; +} /* * --------- @@ -2945,19 +2978,51 @@ pgstat_send_bgwriter(void) * this case, avoid sending a completely empty message to the stats * collector. */ - if (memcmp(&BgWriterStats, &all_zeroes, sizeof(PgStat_MsgBgWriter)) == 0) + if (memcmp(&PendingBgWriterStats, &all_zeroes, sizeof(PgStat_MsgBgWriter)) == 0) return; /* * Prepare and send the message */ - pgstat_setheader(&BgWriterStats.m_hdr, PGSTAT_MTYPE_BGWRITER); - pgstat_send(&BgWriterStats, sizeof(BgWriterStats)); + pgstat_setheader(&PendingBgWriterStats.m_hdr, PGSTAT_MTYPE_BGWRITER); + pgstat_send(&PendingBgWriterStats, sizeof(PendingBgWriterStats)); /* * Clear out the statistics buffer, so it can be re-used. */ - MemSet(&BgWriterStats, 0, sizeof(BgWriterStats)); + MemSet(&PendingBgWriterStats, 0, sizeof(PendingBgWriterStats)); +} + +/* ---------- + * pgstat_send_checkpointer() - + * + * Send checkpointer statistics to the collector + * ---------- + */ +void +pgstat_send_checkpointer(void) +{ + /* We assume this initializes to zeroes */ + static const PgStat_MsgCheckpointer all_zeroes; + + /* + * This function can be called even if nothing at all has happened. In + * this case, avoid sending a completely empty message to the stats + * collector. + */ + if (memcmp(&PendingCheckpointerStats, &all_zeroes, sizeof(PgStat_MsgCheckpointer)) == 0) + return; + + /* + * Prepare and send the message + */ + pgstat_setheader(&PendingCheckpointerStats.m_hdr, PGSTAT_MTYPE_CHECKPOINTER); + pgstat_send(&PendingCheckpointerStats, sizeof(PendingCheckpointerStats)); + + /* + * Clear out the statistics buffer, so it can be re-used. + */ + MemSet(&PendingCheckpointerStats, 0, sizeof(PendingCheckpointerStats)); } /* ---------- @@ -3295,6 +3360,10 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_bgwriter(&msg.msg_bgwriter, len); break; + case PGSTAT_MTYPE_CHECKPOINTER: + pgstat_recv_checkpointer(&msg.msg_checkpointer, len); + break; + case PGSTAT_MTYPE_WAL: pgstat_recv_wal(&msg.msg_wal, len); break; @@ -3841,6 +3910,7 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) bool found; const char *statfile = permanent ? PGSTAT_STAT_PERMANENT_FILENAME : pgstat_stat_filename; int i; + TimestampTz ts; /* * The tables will live in pgStatLocalContext. @@ -3875,21 +3945,22 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) * Set the current timestamp (will be kept only in case we can't load an * existing statsfile). */ - globalStats.stat_reset_timestamp = GetCurrentTimestamp(); - archiverStats.stat_reset_timestamp = globalStats.stat_reset_timestamp; - walStats.stat_reset_timestamp = globalStats.stat_reset_timestamp; + ts = GetCurrentTimestamp(); + globalStats.bgwriter.stat_reset_timestamp = ts; + archiverStats.stat_reset_timestamp = ts; + walStats.stat_reset_timestamp = ts; /* * Set the same reset timestamp for all SLRU items too. */ for (i = 0; i < SLRU_NUM_ELEMENTS; i++) - slruStats[i].stat_reset_timestamp = globalStats.stat_reset_timestamp; + slruStats[i].stat_reset_timestamp = ts; /* * Set the same reset timestamp for all replication slots too. */ for (i = 0; i < max_replication_slots; i++) - replSlotStats[i].stat_reset_timestamp = globalStats.stat_reset_timestamp; + replSlotStats[i].stat_reset_timestamp = ts; /* * Try to open the stats file. If it doesn't exist, the backends simply @@ -4952,9 +5023,9 @@ pgstat_recv_resetsharedcounter(PgStat_MsgResetsharedcounter *msg, int len) { if (msg->m_resettarget == RESET_BGWRITER) { - /* Reset the global background writer statistics for the cluster. */ + /* Reset the global, bgwriter and checkpointer statistics for the cluster. */ memset(&globalStats, 0, sizeof(globalStats)); - globalStats.stat_reset_timestamp = GetCurrentTimestamp(); + globalStats.bgwriter.stat_reset_timestamp = GetCurrentTimestamp(); } else if (msg->m_resettarget == RESET_ARCHIVER) { @@ -5210,16 +5281,27 @@ pgstat_recv_archiver(PgStat_MsgArchiver *msg, int len) static void pgstat_recv_bgwriter(PgStat_MsgBgWriter *msg, int len) { - globalStats.timed_checkpoints += msg->m_timed_checkpoints; - globalStats.requested_checkpoints += msg->m_requested_checkpoints; - globalStats.checkpoint_write_time += msg->m_checkpoint_write_time; - globalStats.checkpoint_sync_time += msg->m_checkpoint_sync_time; - globalStats.buf_written_checkpoints += msg->m_buf_written_checkpoints; - globalStats.buf_written_clean += msg->m_buf_written_clean; - globalStats.maxwritten_clean += msg->m_maxwritten_clean; - globalStats.buf_written_backend += msg->m_buf_written_backend; - globalStats.buf_fsync_backend += msg->m_buf_fsync_backend; - globalStats.buf_alloc += msg->m_buf_alloc; + globalStats.bgwriter.buf_written_clean += msg->m_buf_written_clean; + globalStats.bgwriter.maxwritten_clean += msg->m_maxwritten_clean; + globalStats.bgwriter.buf_alloc += msg->m_buf_alloc; +} + +/* ---------- + * pgstat_recv_checkpointer() - + * + * Process a CHECKPOINTER message. + * ---------- + */ +static void +pgstat_recv_checkpointer(PgStat_MsgCheckpointer *msg, int len) +{ + globalStats.checkpointer.timed_checkpoints += msg->m_timed_checkpoints; + globalStats.checkpointer.requested_checkpoints += msg->m_requested_checkpoints; + globalStats.checkpointer.checkpoint_write_time += msg->m_checkpoint_write_time; + globalStats.checkpointer.checkpoint_sync_time += msg->m_checkpoint_sync_time; + globalStats.checkpointer.buf_written_checkpoints += msg->m_buf_written_checkpoints; + globalStats.checkpointer.buf_written_backend += msg->m_buf_written_backend; + globalStats.checkpointer.buf_fsync_backend += msg->m_buf_fsync_backend; } /* ---------- diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 852138f9c93..231afa39884 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -2065,7 +2065,7 @@ BufferSync(int flags) if (SyncOneBuffer(buf_id, false, &wb_context) & BUF_WRITTEN) { TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id); - BgWriterStats.m_buf_written_checkpoints++; + PendingCheckpointerStats.m_buf_written_checkpoints++; num_written++; } } @@ -2175,7 +2175,7 @@ BgBufferSync(WritebackContext *wb_context) strategy_buf_id = StrategySyncStart(&strategy_passes, &recent_alloc); /* Report buffer alloc counts to pgstat */ - BgWriterStats.m_buf_alloc += recent_alloc; + PendingBgWriterStats.m_buf_alloc += recent_alloc; /* * If we're not running the LRU scan, just stop after doing the stats @@ -2365,7 +2365,7 @@ BgBufferSync(WritebackContext *wb_context) reusable_buffers++; if (++num_written >= bgwriter_lru_maxpages) { - BgWriterStats.m_maxwritten_clean++; + PendingBgWriterStats.m_maxwritten_clean++; break; } } @@ -2373,7 +2373,7 @@ BgBufferSync(WritebackContext *wb_context) reusable_buffers++; } - BgWriterStats.m_buf_written_clean += num_written; + PendingBgWriterStats.m_buf_written_clean += num_written; #ifdef BGW_DEBUG elog(DEBUG1, "bgwriter: recent_alloc=%u smoothed=%.2f delta=%ld ahead=%d density=%.2f reusable_est=%d upcoming_est=%d scanned=%d wrote=%d reusable=%d", diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 9ffbca685cd..b38cbab2597 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -35,9 +35,6 @@ #define HAS_PGSTAT_PERMISSIONS(role) (is_member_of_role(GetUserId(), ROLE_PG_READ_ALL_STATS) || has_privs_of_role(GetUserId(), role)) -/* Global bgwriter statistics, from bgwriter.c */ -extern PgStat_MsgBgWriter bgwriterStats; - Datum pg_stat_get_numscans(PG_FUNCTION_ARGS) { @@ -1726,69 +1723,71 @@ pg_stat_get_db_sessions_killed(PG_FUNCTION_ARGS) Datum pg_stat_get_bgwriter_timed_checkpoints(PG_FUNCTION_ARGS) { - PG_RETURN_INT64(pgstat_fetch_global()->timed_checkpoints); + PG_RETURN_INT64(pgstat_fetch_stat_checkpointer()->timed_checkpoints); } Datum pg_stat_get_bgwriter_requested_checkpoints(PG_FUNCTION_ARGS) { - PG_RETURN_INT64(pgstat_fetch_global()->requested_checkpoints); + PG_RETURN_INT64(pgstat_fetch_stat_checkpointer()->requested_checkpoints); } Datum pg_stat_get_bgwriter_buf_written_checkpoints(PG_FUNCTION_ARGS) { - PG_RETURN_INT64(pgstat_fetch_global()->buf_written_checkpoints); + PG_RETURN_INT64(pgstat_fetch_stat_checkpointer()->buf_written_checkpoints); } Datum pg_stat_get_bgwriter_buf_written_clean(PG_FUNCTION_ARGS) { - PG_RETURN_INT64(pgstat_fetch_global()->buf_written_clean); + PG_RETURN_INT64(pgstat_fetch_stat_bgwriter()->buf_written_clean); } Datum pg_stat_get_bgwriter_maxwritten_clean(PG_FUNCTION_ARGS) { - PG_RETURN_INT64(pgstat_fetch_global()->maxwritten_clean); + PG_RETURN_INT64(pgstat_fetch_stat_bgwriter()->maxwritten_clean); } Datum pg_stat_get_checkpoint_write_time(PG_FUNCTION_ARGS) { /* time is already in msec, just convert to double for presentation */ - PG_RETURN_FLOAT8((double) pgstat_fetch_global()->checkpoint_write_time); + PG_RETURN_FLOAT8((double) + pgstat_fetch_stat_checkpointer()->checkpoint_write_time); } Datum pg_stat_get_checkpoint_sync_time(PG_FUNCTION_ARGS) { /* time is already in msec, just convert to double for presentation */ - PG_RETURN_FLOAT8((double) pgstat_fetch_global()->checkpoint_sync_time); + PG_RETURN_FLOAT8((double) + pgstat_fetch_stat_checkpointer()->checkpoint_sync_time); } Datum pg_stat_get_bgwriter_stat_reset_time(PG_FUNCTION_ARGS) { - PG_RETURN_TIMESTAMPTZ(pgstat_fetch_global()->stat_reset_timestamp); + PG_RETURN_TIMESTAMPTZ(pgstat_fetch_stat_bgwriter()->stat_reset_timestamp); } Datum pg_stat_get_buf_written_backend(PG_FUNCTION_ARGS) { - PG_RETURN_INT64(pgstat_fetch_global()->buf_written_backend); + PG_RETURN_INT64(pgstat_fetch_stat_checkpointer()->buf_written_backend); } Datum pg_stat_get_buf_fsync_backend(PG_FUNCTION_ARGS) { - PG_RETURN_INT64(pgstat_fetch_global()->buf_fsync_backend); + PG_RETURN_INT64(pgstat_fetch_stat_checkpointer()->buf_fsync_backend); } Datum pg_stat_get_buf_alloc(PG_FUNCTION_ARGS) { - PG_RETURN_INT64(pgstat_fetch_global()->buf_alloc); + PG_RETURN_INT64(pgstat_fetch_stat_bgwriter()->buf_alloc); } /* -- 2.31.0.121.g9198c13e34