From 06f12f6174c0b6e1c5beeb4c1a8f4b33b89cc158 Mon Sep 17 00:00:00 2001 From: Nazir Bilal Yavuz Date: Fri, 4 Apr 2025 13:39:49 +0300 Subject: [PATCH v7] Add pg_buffercache_mark_dirty[_all]() functions for testing This commit introduces two new functions for marking shared buffers as dirty: pg_buffercache_mark_dirty(): Marks a specific shared buffer as dirty. pg_buffercache_mark_dirty_all(): Marks all shared buffers as dirty in a single operation. The pg_buffercache_mark_dirty_all() function provides an efficient way to dirty the entire buffer pool (e.g., ~550ms vs. ~70ms for 16GB of shared buffers), complementing pg_buffercache_mark_dirty() for more granular control. These functions are intended for developer testing and debugging scenarios, enabling users to simulate various buffer pool states and test write-back behavior. Both functions are superuser-only. Author: Nazir Bilal Yavuz Reviewed-by: Andres Freund Reviewed-by: Aidar Imamov Reviewed-by: Joseph Koshakow Discussion: https://postgr.es/m/CAN55FZ0h_YoSqqutxV6DES1RW8ig6wcA8CR9rJk358YRMxZFmw%40mail.gmail.com --- src/include/storage/bufmgr.h | 2 + src/backend/storage/buffer/bufmgr.c | 75 +++++++++++++++++++ doc/src/sgml/pgbuffercache.sgml | 54 ++++++++++++- contrib/pg_buffercache/Makefile | 2 +- .../expected/pg_buffercache.out | 30 +++++++- contrib/pg_buffercache/meson.build | 1 + .../pg_buffercache--1.6--1.7.sql | 14 ++++ contrib/pg_buffercache/pg_buffercache.control | 2 +- contrib/pg_buffercache/pg_buffercache_pages.c | 33 ++++++++ contrib/pg_buffercache/sql/pg_buffercache.sql | 10 ++- 10 files changed, 216 insertions(+), 7 deletions(-) create mode 100644 contrib/pg_buffercache/pg_buffercache--1.6--1.7.sql diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index 33a8b8c06fb..ec7fec6368a 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -312,6 +312,8 @@ extern void EvictRelUnpinnedBuffers(Relation rel, int32 *buffers_evicted, int32 *buffers_flushed, int32 *buffers_skipped); +extern bool MarkUnpinnedBufferDirty(Buffer buf); +extern void MarkAllUnpinnedBuffersDirty(int32 *buffers_dirtied); /* in buf_init.c */ extern void BufferManagerShmemInit(void); diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index db8f2b1754e..e2bdb86525b 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -6699,6 +6699,81 @@ EvictRelUnpinnedBuffers(Relation rel, int32 *buffers_evicted, } } +/* + * Try to mark the provided shared buffer as dirty. + * + * This function is intended for testing/development use only! + * + * Same as EvictUnpinnedBuffer() but with MarkBufferDirty() call inside. + * + * Returns true if the buffer was already dirty or it has successfully been + * marked as dirty. + */ +bool +MarkUnpinnedBufferDirty(Buffer buf) +{ + BufferDesc *desc; + uint32 buf_state; + + Assert(!BufferIsLocal(buf)); + + /* Make sure we can pin the buffer. */ + ResourceOwnerEnlarge(CurrentResourceOwner); + ReservePrivateRefCountEntry(); + + desc = GetBufferDescriptor(buf - 1); + + /* Lock the header and check if it's valid. */ + buf_state = LockBufHdr(desc); + if ((buf_state & BM_VALID) == 0) + { + UnlockBufHdr(desc, buf_state); + return false; + } + + /* Check that it's not pinned already. */ + if (BUF_STATE_GET_REFCOUNT(buf_state) > 0) + { + UnlockBufHdr(desc, buf_state); + return false; + } + + PinBuffer_Locked(desc); /* releases spinlock */ + + /* If it was not already dirty, mark it as dirty. */ + if (!(buf_state & BM_DIRTY)) + { + LWLockAcquire(BufferDescriptorGetContentLock(desc), LW_EXCLUSIVE); + MarkBufferDirty(buf); + LWLockRelease(BufferDescriptorGetContentLock(desc)); + } + + UnpinBuffer(desc); + + return true; +} + +/* + * Try to mark all the shared buffers as dirty. + * + * This function is intended for testing/development use only! See + * MarkUnpinnedBufferDirty(). + * + * The buffers_dirtied parameter is mandatory and indicate the total count of + * buffers were dirtied. + */ +void +MarkAllUnpinnedBuffersDirty(int32 *buffers_dirtied) +{ + *buffers_dirtied = 0; + + for (int buf = 1; buf <= NBuffers; buf++) + { + if (MarkUnpinnedBufferDirty(buf)) + (*buffers_dirtied)++; + } +} + /* * Generic implementation of the AIO handle staging callback for readv/writev * on local/shared buffers. diff --git a/doc/src/sgml/pgbuffercache.sgml b/doc/src/sgml/pgbuffercache.sgml index 537d6014942..bfccabd9c5e 100644 --- a/doc/src/sgml/pgbuffercache.sgml +++ b/doc/src/sgml/pgbuffercache.sgml @@ -35,6 +35,14 @@ pg_buffercache_evict_all + + pg_buffercache_mark_dirty + + + + pg_buffercache_mark_dirty_all + + This module provides the pg_buffercache_pages() function (wrapped in the pg_buffercache view), @@ -42,9 +50,11 @@ pg_buffercache_numa view), the pg_buffercache_summary() function, the pg_buffercache_usage_counts() function, the - pg_buffercache_evict(), the - pg_buffercache_evict_relation() function and the - pg_buffercache_evict_all() function. + pg_buffercache_evict() function, the + pg_buffercache_evict_relation() function, the + pg_buffercache_evict_all() function, the + pg_buffercache_mark_dirty() function and the + pg_buffercache_mark_dirty_all() function. @@ -99,6 +109,18 @@ function is restricted to superusers only. + + The pg_buffercache_mark_dirty() function allows a block + to be marked as dirty from the buffer pool given a buffer identifier. Use of + this function is restricted to superusers only. + + + + The pg_buffercache_mark_dirty_all() function tries to + mark all buffers dirty in the buffer pool. Use of this function is + restricted to superusers only. + + The <structname>pg_buffercache</structname> View @@ -522,6 +544,32 @@ + + The <structname>pg_buffercache_mark_dirty</structname> Function + + The pg_buffercache_mark_dirty() function takes a + buffer identifier, as shown in the bufferid + column of the pg_buffercache view. It returns + true on success, and false if the buffer wasn't valid or if it couldn't be + marked as dirty because it was pinned. The result is immediately out of + date upon return, as the buffer might become valid again at any time due to + concurrent activity. The function is intended for developer testing only. + + + + + The <structname>pg_buffercache_mark_dirty_all</structname> Function + + The pg_buffercache_mark_dirty_all() function is very + similar to the pg_buffercache_mark_dirty() function. + The difference is, the pg_buffercache_mark_dirty_all() + function does not take an argument; instead it tries to mark all buffers + dirty in the buffer pool. The result is immediately out of date upon + return, as the buffer might become valid again at any time due to + concurrent activity. The function is intended for developer testing only. + + + Sample Output diff --git a/contrib/pg_buffercache/Makefile b/contrib/pg_buffercache/Makefile index 5f748543e2e..0e618f66aec 100644 --- a/contrib/pg_buffercache/Makefile +++ b/contrib/pg_buffercache/Makefile @@ -9,7 +9,7 @@ EXTENSION = pg_buffercache DATA = pg_buffercache--1.2.sql pg_buffercache--1.2--1.3.sql \ pg_buffercache--1.1--1.2.sql pg_buffercache--1.0--1.1.sql \ pg_buffercache--1.3--1.4.sql pg_buffercache--1.4--1.5.sql \ - pg_buffercache--1.5--1.6.sql + pg_buffercache--1.5--1.6.sql pg_buffercache--1.6--1.7.sql PGFILEDESC = "pg_buffercache - monitoring of shared buffer cache in real-time" REGRESS = pg_buffercache pg_buffercache_numa diff --git a/contrib/pg_buffercache/expected/pg_buffercache.out b/contrib/pg_buffercache/expected/pg_buffercache.out index 9a9216dc7b1..aa2c0328386 100644 --- a/contrib/pg_buffercache/expected/pg_buffercache.out +++ b/contrib/pg_buffercache/expected/pg_buffercache.out @@ -57,7 +57,7 @@ SELECT count(*) > 0 FROM pg_buffercache_usage_counts(); RESET role; ------ ----- Test pg_buffercache_evict* functions +---- Test pg_buffercache_evict* and pg_buffercache_mark_dirty* functions ------ CREATE ROLE regress_buffercache_normal; SET ROLE regress_buffercache_normal; @@ -68,6 +68,10 @@ SELECT * FROM pg_buffercache_evict_relation(1); ERROR: must be superuser to use pg_buffercache_evict_relation() SELECT * FROM pg_buffercache_evict_all(); ERROR: must be superuser to use pg_buffercache_evict_all() +SELECT * FROM pg_buffercache_mark_dirty(1); +ERROR: must be superuser to use pg_buffercache_mark_dirty() +SELECT * FROM pg_buffercache_mark_dirty_all(); +ERROR: must be superuser to use pg_buffercache_mark_dirty_all() RESET ROLE; -- These should return nothing, because these are STRICT functions SELECT * FROM pg_buffercache_evict(NULL); @@ -82,6 +86,12 @@ SELECT * FROM pg_buffercache_evict_relation(NULL); | | (1 row) +SELECT * FROM pg_buffercache_mark_dirty(NULL); + pg_buffercache_mark_dirty +--------------------------- + +(1 row) + -- These should fail because they are not called by valid range of buffers -- Number of the shared buffers are limited by max integer SELECT 2147483647 max_buffers \gset @@ -91,6 +101,12 @@ SELECT * FROM pg_buffercache_evict(0); ERROR: bad buffer ID: 0 SELECT * FROM pg_buffercache_evict(:max_buffers); ERROR: bad buffer ID: 2147483647 +SELECT * FROM pg_buffercache_mark_dirty(-1); +ERROR: bad buffer ID: -1 +SELECT * FROM pg_buffercache_mark_dirty(0); +ERROR: bad buffer ID: 0 +SELECT * FROM pg_buffercache_mark_dirty(:max_buffers); +ERROR: bad buffer ID: 2147483647 -- This should fail because pg_buffercache_evict_relation() doesn't accept -- local relations CREATE TEMP TABLE temp_pg_buffercache(); @@ -118,4 +134,16 @@ SELECT buffers_evicted IS NOT NULL FROM pg_buffercache_evict_relation('shared_pg (1 row) DROP TABLE shared_pg_buffercache; +SELECT pg_buffercache_mark_dirty(1) IS NOT NULL; + ?column? +---------- + t +(1 row) + +SELECT pg_buffercache_mark_dirty_all() IS NOT NULL; + ?column? +---------- + t +(1 row) + DROP ROLE regress_buffercache_normal; diff --git a/contrib/pg_buffercache/meson.build b/contrib/pg_buffercache/meson.build index 7cd039a1df9..7c31141881f 100644 --- a/contrib/pg_buffercache/meson.build +++ b/contrib/pg_buffercache/meson.build @@ -24,6 +24,7 @@ install_data( 'pg_buffercache--1.3--1.4.sql', 'pg_buffercache--1.4--1.5.sql', 'pg_buffercache--1.5--1.6.sql', + 'pg_buffercache--1.6--1.7.sql', 'pg_buffercache.control', kwargs: contrib_data_args, ) diff --git a/contrib/pg_buffercache/pg_buffercache--1.6--1.7.sql b/contrib/pg_buffercache/pg_buffercache--1.6--1.7.sql new file mode 100644 index 00000000000..db55c38e9b9 --- /dev/null +++ b/contrib/pg_buffercache/pg_buffercache--1.6--1.7.sql @@ -0,0 +1,14 @@ +/* contrib/pg_buffercache/pg_buffercache--1.6--1.7.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION pg_buffercache UPDATE TO '1.7'" to load this file. \quit + +CREATE FUNCTION pg_buffercache_mark_dirty(IN int) +RETURNS bool +AS 'MODULE_PATHNAME', 'pg_buffercache_mark_dirty' +LANGUAGE C PARALLEL SAFE VOLATILE STRICT; + +CREATE FUNCTION pg_buffercache_mark_dirty_all() +RETURNS INT4 +AS 'MODULE_PATHNAME', 'pg_buffercache_mark_dirty_all' +LANGUAGE C PARALLEL SAFE VOLATILE; diff --git a/contrib/pg_buffercache/pg_buffercache.control b/contrib/pg_buffercache/pg_buffercache.control index b030ba3a6fa..11499550945 100644 --- a/contrib/pg_buffercache/pg_buffercache.control +++ b/contrib/pg_buffercache/pg_buffercache.control @@ -1,5 +1,5 @@ # pg_buffercache extension comment = 'examine the shared buffer cache' -default_version = '1.6' +default_version = '1.7' module_pathname = '$libdir/pg_buffercache' relocatable = true diff --git a/contrib/pg_buffercache/pg_buffercache_pages.c b/contrib/pg_buffercache/pg_buffercache_pages.c index e1701bd56ef..6136ab3a0ae 100644 --- a/contrib/pg_buffercache/pg_buffercache_pages.c +++ b/contrib/pg_buffercache/pg_buffercache_pages.c @@ -100,6 +100,8 @@ PG_FUNCTION_INFO_V1(pg_buffercache_usage_counts); PG_FUNCTION_INFO_V1(pg_buffercache_evict); PG_FUNCTION_INFO_V1(pg_buffercache_evict_relation); PG_FUNCTION_INFO_V1(pg_buffercache_evict_all); +PG_FUNCTION_INFO_V1(pg_buffercache_mark_dirty); +PG_FUNCTION_INFO_V1(pg_buffercache_mark_dirty_all); /* Only need to touch memory once per backend process lifetime */ @@ -772,3 +774,34 @@ pg_buffercache_evict_all(PG_FUNCTION_ARGS) PG_RETURN_DATUM(result); } + +/* + * Try to mark a shared buffer as dirty. + */ +Datum +pg_buffercache_mark_dirty(PG_FUNCTION_ARGS) +{ + Buffer buf = PG_GETARG_INT32(0); + + pg_buffercache_superuser_check("pg_buffercache_mark_dirty"); + + if (buf < 1 || buf > NBuffers) + elog(ERROR, "bad buffer ID: %d", buf); + + PG_RETURN_BOOL(MarkUnpinnedBufferDirty(buf)); +} + +/* + * Try to mark all the shared buffers as dirty. + */ +Datum +pg_buffercache_mark_dirty_all(PG_FUNCTION_ARGS) +{ + int32 buffers_dirtied = 0; + + pg_buffercache_superuser_check("pg_buffercache_mark_dirty_all"); + + MarkAllUnpinnedBuffersDirty(&buffers_dirtied); + + PG_RETURN_INT32(buffers_dirtied); +} diff --git a/contrib/pg_buffercache/sql/pg_buffercache.sql b/contrib/pg_buffercache/sql/pg_buffercache.sql index 47cca1907c7..9eac5214825 100644 --- a/contrib/pg_buffercache/sql/pg_buffercache.sql +++ b/contrib/pg_buffercache/sql/pg_buffercache.sql @@ -30,7 +30,7 @@ RESET role; ------ ----- Test pg_buffercache_evict* functions +---- Test pg_buffercache_evict* and pg_buffercache_mark_dirty* functions ------ CREATE ROLE regress_buffercache_normal; @@ -40,12 +40,15 @@ SET ROLE regress_buffercache_normal; SELECT * FROM pg_buffercache_evict(1); SELECT * FROM pg_buffercache_evict_relation(1); SELECT * FROM pg_buffercache_evict_all(); +SELECT * FROM pg_buffercache_mark_dirty(1); +SELECT * FROM pg_buffercache_mark_dirty_all(); RESET ROLE; -- These should return nothing, because these are STRICT functions SELECT * FROM pg_buffercache_evict(NULL); SELECT * FROM pg_buffercache_evict_relation(NULL); +SELECT * FROM pg_buffercache_mark_dirty(NULL); -- These should fail because they are not called by valid range of buffers -- Number of the shared buffers are limited by max integer @@ -53,6 +56,9 @@ SELECT 2147483647 max_buffers \gset SELECT * FROM pg_buffercache_evict(-1); SELECT * FROM pg_buffercache_evict(0); SELECT * FROM pg_buffercache_evict(:max_buffers); +SELECT * FROM pg_buffercache_mark_dirty(-1); +SELECT * FROM pg_buffercache_mark_dirty(0); +SELECT * FROM pg_buffercache_mark_dirty(:max_buffers); -- This should fail because pg_buffercache_evict_relation() doesn't accept -- local relations @@ -66,5 +72,7 @@ SELECT buffers_evicted IS NOT NULL FROM pg_buffercache_evict_all(); CREATE TABLE shared_pg_buffercache(); SELECT buffers_evicted IS NOT NULL FROM pg_buffercache_evict_relation('shared_pg_buffercache'); DROP TABLE shared_pg_buffercache; +SELECT pg_buffercache_mark_dirty(1) IS NOT NULL; +SELECT pg_buffercache_mark_dirty_all() IS NOT NULL; DROP ROLE regress_buffercache_normal; -- 2.49.0