From fd6f751c14cc5e637a60f8f2bcae8871861321c9 Mon Sep 17 00:00:00 2001 From: Nazir Bilal Yavuz Date: Fri, 4 Apr 2025 13:35:17 +0300 Subject: [PATCH v6 2/4] Add pg_buffercache_evict_[relation | all]() functions for testing pg_buffercache_evict_relation(): Evicts all shared buffers in a relation at once. pg_buffercache_evict_all(): Evicts all shared buffers at once. Both functions provide mechanism to evict multiple shared buffers at once. They are designed to address the inefficiency of repeatedly calling pg_buffercache_evict() for each individual buffer, which can be time-consuming when dealing with large shared buffer pools. (e.g., ~790ms vs. ~270ms for 16GB of shared buffers). These functions are intended for developer testing and debugging purposes and are available to superusers only. Author: Nazir Bilal Yavuz Reviewed-by: Andres Freund Reviewed-by: Aidar Imamov Reviewed-by: Joseph Koshakow Discussion: https://postgr.es/m/CAN55FZ0h_YoSqqutxV6DES1RW8ig6wcA8CR9rJk358YRMxZFmw%40mail.gmail.com --- src/include/storage/bufmgr.h | 2 + src/backend/storage/buffer/bufmgr.c | 92 +++++++++++++++++ doc/src/sgml/pgbuffercache.sgml | 58 ++++++++++- .../pg_buffercache--1.5--1.6.sql | 13 +++ contrib/pg_buffercache/pg_buffercache_pages.c | 98 +++++++++++++++++++ 5 files changed, 261 insertions(+), 2 deletions(-) diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index fab65824b18..48f5182635c 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -305,6 +305,8 @@ extern void LimitAdditionalPins(uint32 *additional_pins); extern void LimitAdditionalLocalPins(uint32 *additional_pins); extern bool EvictUnpinnedBuffer(Buffer buf, bool *flushed); +extern void EvictRelUnpinnedBuffers(Relation rel, int32 *buffers_evicted, + int32 *buffers_flushed); /* in buf_init.c */ extern void BufferManagerShmemInit(void); diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index e585b72e5fa..e2bc155b700 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -6568,6 +6568,98 @@ EvictUnpinnedBuffer(Buffer buf, bool *flushed) return result; } +/* + * Try to evict all the shared buffers containing provided relation's pages. + * + * This function is intended for testing/development use only! + * + * We need this helper function because of the following reason. + * ReservePrivateRefCountEntry() needs to be called before acquiring the + * buffer header lock but ReservePrivateRefCountEntry() is static and it would + * be better to have it as static. Hence, it can't be called from outside of + * this file. This helper function is created to bypass that problem. + * + * Also, we should minimize the amount of code outside of storage/buffer that + * needs to know about BuferDescs etc., it is a layering violation to access + * that outside. + * + * The caller must hold at least AccessShareLock on the relation to prevent + * the relation from being dropped. + * + * buffers_evicted and buffers_flushed are optional. If they are provided, + * they are set the total number of buffers evicted and flushed respectively. + */ +void +EvictRelUnpinnedBuffers(Relation rel, int32 *buffers_evicted, int32 *buffers_flushed) +{ + if (buffers_evicted) + *buffers_evicted = 0; + if (buffers_flushed) + *buffers_flushed = 0; + + Assert(!RelationUsesLocalBuffers(rel)); + + for (int buf = 1; buf <= NBuffers; buf++) + { + uint32 buf_state = 0; + BufferDesc *bufHdr = GetBufferDescriptor(buf - 1); + + /* An unlocked precheck should be safe and saves some cycles. */ + if (!BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator)) + continue; + + /* Make sure we can pin the buffer. */ + ResourceOwnerEnlarge(CurrentResourceOwner); + ReservePrivateRefCountEntry(); + + buf_state = LockBufHdr(bufHdr); + if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator)) + { + /* + * This is basically copy of the EvictUnpinnedBuffer(). This is + * required because buffer header lock needs to be acquired before + * calling the EvictUnpinnedBuffer(). + */ + + if ((buf_state & BM_VALID) == 0) + { + UnlockBufHdr(bufHdr, buf_state); + continue; + } + + /* Check that it's not pinned already. */ + if (BUF_STATE_GET_REFCOUNT(buf_state) > 0) + { + UnlockBufHdr(bufHdr, buf_state); + continue; + } + + PinBuffer_Locked(bufHdr); /* releases spinlock */ + + /* If it was dirty, try to clean it once. */ + if (buf_state & BM_DIRTY) + { + LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED); + FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + if (buffers_flushed) + (*buffers_flushed)++; + LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); + } + + /* + * This will return false if it becomes dirty or someone else pins + * it. + */ + if (InvalidateVictimBuffer(bufHdr) && buffers_evicted) + (*buffers_evicted)++; + + UnpinBuffer(bufHdr); + } + else + UnlockBufHdr(bufHdr, buf_state); + } +} + /* * Generic implementation of the AIO handle staging callback for readv/writev * on local/shared buffers. diff --git a/doc/src/sgml/pgbuffercache.sgml b/doc/src/sgml/pgbuffercache.sgml index 187e13e8cda..c7e10f97ab6 100644 --- a/doc/src/sgml/pgbuffercache.sgml +++ b/doc/src/sgml/pgbuffercache.sgml @@ -27,12 +27,22 @@ pg_buffercache_evict + + pg_buffercache_evict_relation + + + + pg_buffercache_evict_all + + This module provides the pg_buffercache_pages() function (wrapped in the pg_buffercache view), the pg_buffercache_summary() function, the - pg_buffercache_usage_counts() function and - the pg_buffercache_evict() function. + pg_buffercache_usage_counts() function, the + pg_buffercache_evict(), the + pg_buffercache_evict_relation() function and the + pg_buffercache_evict_all() function. @@ -65,6 +75,19 @@ function is restricted to superusers only. + + The pg_buffercache_evict_relation() function allows all + unpinned shared buffers in the relation to be evicted from the buffer pool + given a relation identifier. Use of this function is restricted to + superusers only. + + + + The pg_buffercache_evict_all() function allows all + unpinned shared buffers to be evicted in the buffer pool. Use of this + function is restricted to superusers only. + + The <structname>pg_buffercache</structname> View @@ -381,6 +404,37 @@ + + The <structname>pg_buffercache_evict_relation</structname> Function + + The pg_buffercache_evict_relation() function is very + similar to the pg_buffercache_evict() function. The + difference is that the pg_buffercache_evict_relation() + takes a relation identifier instead of buffer identifier. Then, it tries + to evict all buffers for all forks in that relation. It returns the number + of evicted and flushed buffers. Flushed buffers aren't necessarily flushed + by us, they might be flushed by someone else. The result is immediately + out of date upon return, as the buffer might become valid again at any time + due to concurrent activity. The function is intended for developer testing + only. + + + + + The <structname>pg_buffercache_evict_all</structname> Function + + The pg_buffercache_evict_all() function is very + similar to the pg_buffercache_evict() function. The + difference is, the pg_buffercache_evict_all() function + does not take an argument; instead it tries to evict all buffers in the + buffer pool. It returns the number of evicted and flushed buffers. + Flushed buffers aren't necessarily flushed by us, they might be flushed by + someone else. The result is immediately out of date upon return, as the + buffer might become valid again at any time due to concurrent activity. + The function is intended for developer testing only. + + + Sample Output diff --git a/contrib/pg_buffercache/pg_buffercache--1.5--1.6.sql b/contrib/pg_buffercache/pg_buffercache--1.5--1.6.sql index 18597923af8..d54bb1fd6f8 100644 --- a/contrib/pg_buffercache/pg_buffercache--1.5--1.6.sql +++ b/contrib/pg_buffercache/pg_buffercache--1.5--1.6.sql @@ -7,3 +7,16 @@ CREATE FUNCTION pg_buffercache_evict( OUT flushed boolean) AS 'MODULE_PATHNAME', 'pg_buffercache_evict' LANGUAGE C PARALLEL SAFE VOLATILE STRICT; + +CREATE FUNCTION pg_buffercache_evict_relation( + IN regclass, + OUT buffers_evicted int4, + OUT buffers_flushed int4) +AS 'MODULE_PATHNAME', 'pg_buffercache_evict_relation' +LANGUAGE C PARALLEL SAFE VOLATILE STRICT; + +CREATE FUNCTION pg_buffercache_evict_all( + OUT buffers_evicted int4, + OUT buffers_flushed int4) +AS 'MODULE_PATHNAME', 'pg_buffercache_evict_all' +LANGUAGE C PARALLEL SAFE VOLATILE; diff --git a/contrib/pg_buffercache/pg_buffercache_pages.c b/contrib/pg_buffercache/pg_buffercache_pages.c index 9493a40e804..b6eba76fd27 100644 --- a/contrib/pg_buffercache/pg_buffercache_pages.c +++ b/contrib/pg_buffercache/pg_buffercache_pages.c @@ -9,10 +9,12 @@ #include "postgres.h" #include "access/htup_details.h" +#include "access/relation.h" #include "catalog/pg_type.h" #include "funcapi.h" #include "storage/buf_internals.h" #include "storage/bufmgr.h" +#include "utils/rel.h" #define NUM_BUFFERCACHE_PAGES_MIN_ELEM 8 @@ -20,6 +22,8 @@ #define NUM_BUFFERCACHE_SUMMARY_ELEM 5 #define NUM_BUFFERCACHE_USAGE_COUNTS_ELEM 4 #define NUM_BUFFERCACHE_EVICT_ELEM 2 +#define NUM_BUFFERCACHE_EVICT_RELATION_ELEM 2 +#define NUM_BUFFERCACHE_EVICT_ALL_ELEM 2 PG_MODULE_MAGIC_EXT( .name = "pg_buffercache", @@ -68,6 +72,8 @@ PG_FUNCTION_INFO_V1(pg_buffercache_pages); PG_FUNCTION_INFO_V1(pg_buffercache_summary); PG_FUNCTION_INFO_V1(pg_buffercache_usage_counts); PG_FUNCTION_INFO_V1(pg_buffercache_evict); +PG_FUNCTION_INFO_V1(pg_buffercache_evict_relation); +PG_FUNCTION_INFO_V1(pg_buffercache_evict_all); Datum pg_buffercache_pages(PG_FUNCTION_ARGS) @@ -388,3 +394,95 @@ pg_buffercache_evict(PG_FUNCTION_ARGS) PG_RETURN_DATUM(result); } + +/* + * Try to evict specified relation. + */ +Datum +pg_buffercache_evict_relation(PG_FUNCTION_ARGS) +{ + Datum result; + TupleDesc tupledesc; + HeapTuple tuple; + Datum values[NUM_BUFFERCACHE_EVICT_RELATION_ELEM]; + bool nulls[NUM_BUFFERCACHE_EVICT_RELATION_ELEM] = {0}; + + Oid relOid; + Relation rel; + int32 buffers_evicted = 0; + int32 buffers_flushed = 0; + + if (get_call_result_type(fcinfo, NULL, &tupledesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to use %s()", + "pg_buffercache_evict_relation"))); + + relOid = PG_GETARG_OID(0); + + rel = relation_open(relOid, AccessShareLock); + + if (RelationUsesLocalBuffers(rel)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("relation uses local buffers, %s() is intended to be used for shared buffers only", + "pg_buffercache_evict_relation"))); + + EvictRelUnpinnedBuffers(rel, &buffers_evicted, &buffers_flushed); + + relation_close(rel, AccessShareLock); + + values[0] = Int32GetDatum(buffers_evicted); + values[1] = Int32GetDatum(buffers_flushed); + + tuple = heap_form_tuple(tupledesc, values, nulls); + result = HeapTupleGetDatum(tuple); + + PG_RETURN_DATUM(result); +} + + +/* + * Try to evict all shared buffers. + */ +Datum +pg_buffercache_evict_all(PG_FUNCTION_ARGS) +{ + Datum result; + TupleDesc tupledesc; + HeapTuple tuple; + Datum values[NUM_BUFFERCACHE_EVICT_ALL_ELEM]; + bool nulls[NUM_BUFFERCACHE_EVICT_ALL_ELEM] = {0}; + + int32 buffers_evicted = 0; + int32 buffers_flushed = 0; + bool flushed; + + if (get_call_result_type(fcinfo, NULL, &tupledesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to use %s()", + "pg_buffercache_evict_all"))); + + for (int buf = 1; buf <= NBuffers; buf++) + { + if (EvictUnpinnedBuffer(buf, &flushed)) + buffers_evicted++; + if (flushed) + buffers_flushed++; + } + + values[0] = Int32GetDatum(buffers_evicted); + values[1] = Int32GetDatum(buffers_flushed); + + tuple = heap_form_tuple(tupledesc, values, nulls); + result = HeapTupleGetDatum(tuple); + + PG_RETURN_DATUM(result); +} -- 2.47.2