From 7a793b13803d5defd6d5154a075d1d4cb6826103 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi Date: Thu, 14 Nov 2019 19:24:36 +0900 Subject: [PATCH] catcache benchmark extension Provides the function catcachebench(bench_no int), which runs CPU intensive benchmark on catcache search. The test table is created by a script separately provided. catcachebench(0): prewarm catcache with provided test tables. catcachebench(1): fetches all attribute stats of all tables. This benchmark loads a vast number of unique entries. Expriration doesn't work since it runs in a transaction. catcachebench(2): fetches all attribute stats of a tables many times. This benchmark repeatedly accesses already loaded entries. Expriration doesn't work since it runs in a transaction. catcachebench(3): fetches all attribute stats of all tables four times. Different from other modes, this runs expiration by forcibly updates reference clock variable every 1000 entries. At this point, variables needed for the expiration feature is not added so SetCatCacheClock is a dummy macro that just replaces it with its parameter. --- contrib/catcachebench/Makefile | 17 + contrib/catcachebench/catcachebench--0.0.sql | 14 + contrib/catcachebench/catcachebench.c | 330 +++++++++++++++++++ contrib/catcachebench/catcachebench.control | 6 + src/backend/utils/cache/catcache.c | 35 ++ src/backend/utils/cache/syscache.c | 2 +- src/include/utils/catcache.h | 3 + 7 files changed, 406 insertions(+), 1 deletion(-) create mode 100644 contrib/catcachebench/Makefile create mode 100644 contrib/catcachebench/catcachebench--0.0.sql create mode 100644 contrib/catcachebench/catcachebench.c create mode 100644 contrib/catcachebench/catcachebench.control diff --git a/contrib/catcachebench/Makefile b/contrib/catcachebench/Makefile new file mode 100644 index 0000000000..0478818b25 --- /dev/null +++ b/contrib/catcachebench/Makefile @@ -0,0 +1,17 @@ +MODULE_big = catcachebench +OBJS = catcachebench.o + +EXTENSION = catcachebench +DATA = catcachebench--0.0.sql +PGFILEDESC = "catcachebench - benchmark for catcache pruning feature" + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/catcachebench +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/catcachebench/catcachebench--0.0.sql b/contrib/catcachebench/catcachebench--0.0.sql new file mode 100644 index 0000000000..ea9cd62abb --- /dev/null +++ b/contrib/catcachebench/catcachebench--0.0.sql @@ -0,0 +1,14 @@ +/* contrib/catcachebench/catcachebench--0.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION catcachebench" to load this file. \quit + +CREATE FUNCTION catcachebench(IN type int) +RETURNS double precision +AS 'MODULE_PATHNAME', 'catcachebench' +LANGUAGE C STRICT VOLATILE; + +CREATE FUNCTION catcachereadstats(OUT catid int, OUT reloid oid, OUT searches bigint, OUT hits bigint, OUT neg_hits bigint) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'catcachereadstats' +LANGUAGE C STRICT VOLATILE; diff --git a/contrib/catcachebench/catcachebench.c b/contrib/catcachebench/catcachebench.c new file mode 100644 index 0000000000..b6c2b8f577 --- /dev/null +++ b/contrib/catcachebench/catcachebench.c @@ -0,0 +1,330 @@ +/* + * catcachebench: test code for cache pruning feature + */ +/* #define CATCACHE_STATS */ +#include "postgres.h" +#include "catalog/pg_type.h" +#include "catalog/pg_statistic.h" +#include "executor/spi.h" +#include "funcapi.h" +#include "libpq/pqsignal.h" +#include "utils/catcache.h" +#include "utils/syscache.h" +#include "utils/timestamp.h" + +Oid tableoids[10000]; +int ntables = 0; +int16 attnums[1000]; +int natts = 0; + +PG_MODULE_MAGIC; + +double catcachebench1(void); +double catcachebench2(void); +double catcachebench3(void); +void collectinfo(void); +void catcachewarmup(void); + +PG_FUNCTION_INFO_V1(catcachebench); +PG_FUNCTION_INFO_V1(catcachereadstats); + +extern void CatalogCacheFlushCatalog2(Oid catId); +extern int64 catcache_called; +extern CatCache *SysCache[]; + +typedef struct catcachestatsstate +{ + TupleDesc tupd; + int catId; +} catcachestatsstate; + +Datum +catcachereadstats(PG_FUNCTION_ARGS) +{ + catcachestatsstate *state_data = NULL; + FuncCallContext *fctx; + + if (SRF_IS_FIRSTCALL()) + { + TupleDesc tupdesc; + MemoryContext mctx; + + fctx = SRF_FIRSTCALL_INIT(); + mctx = MemoryContextSwitchTo(fctx->multi_call_memory_ctx); + + state_data = palloc(sizeof(catcachestatsstate)); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + state_data->tupd = tupdesc; + state_data->catId = 0; + + fctx->user_fctx = state_data; + + MemoryContextSwitchTo(mctx); + } + + fctx = SRF_PERCALL_SETUP(); + state_data = fctx->user_fctx; + + if (state_data->catId < SysCacheSize) + { + Datum values[5]; + bool nulls[5]; + HeapTuple resulttup; + Datum result; + int catId = state_data->catId++; + + memset(nulls, 0, sizeof(nulls)); + memset(values, 0, sizeof(values)); + values[0] = Int16GetDatum(catId); + values[1] = ObjectIdGetDatum(SysCache[catId]->cc_reloid); +#ifdef CATCACHE_STATS + values[2] = Int64GetDatum(SysCache[catId]->cc_searches); + values[3] = Int64GetDatum(SysCache[catId]->cc_hits); + values[4] = Int64GetDatum(SysCache[catId]->cc_neg_hits); +#endif + resulttup = heap_form_tuple(state_data->tupd, values, nulls); + result = HeapTupleGetDatum(resulttup); + + SRF_RETURN_NEXT(fctx, result); + } + + SRF_RETURN_DONE(fctx); +} + +Datum +catcachebench(PG_FUNCTION_ARGS) +{ + int testtype = PG_GETARG_INT32(0); + double ms; + + collectinfo(); + + /* flush the catalog -- safe? don't mind. */ + CatalogCacheFlushCatalog2(StatisticRelationId); + + switch (testtype) + { + case 0: + catcachewarmup(); /* prewarm of syscatalog */ + PG_RETURN_NULL(); + case 1: + ms = catcachebench1(); break; + case 2: + ms = catcachebench2(); break; + case 3: + ms = catcachebench3(); break; + default: + elog(ERROR, "Invalid test type: %d", testtype); + } + + PG_RETURN_DATUM(Float8GetDatum(ms)); +} + +/* + * fetch all attribute entires of all tables. + */ +double +catcachebench1(void) +{ + int t, a; + instr_time start, + duration; + + PG_SETMASK(&BlockSig); + INSTR_TIME_SET_CURRENT(start); + for (t = 0 ; t < ntables ; t++) + { + for (a = 0 ; a < natts ; a++) + { + HeapTuple tup; + + tup = SearchSysCache3(STATRELATTINH, + ObjectIdGetDatum(tableoids[t]), + Int16GetDatum(attnums[a]), + BoolGetDatum(false)); + /* should be null, but.. */ + if (HeapTupleIsValid(tup)) + ReleaseSysCache(tup); + } + } + INSTR_TIME_SET_CURRENT(duration); + INSTR_TIME_SUBTRACT(duration, start); + PG_SETMASK(&UnBlockSig); + + return INSTR_TIME_GET_MILLISEC(duration); +}; + +/* + * fetch all attribute entires of a table many times. + */ +double +catcachebench2(void) +{ + int t, a; + instr_time start, + duration; + + PG_SETMASK(&BlockSig); + INSTR_TIME_SET_CURRENT(start); + for (t = 0 ; t < 240000 ; t++) + { + for (a = 0 ; a < natts ; a++) + { + HeapTuple tup; + + tup = SearchSysCache3(STATRELATTINH, + ObjectIdGetDatum(tableoids[0]), + Int16GetDatum(attnums[a]), + BoolGetDatum(false)); + /* should be null, but.. */ + if (HeapTupleIsValid(tup)) + ReleaseSysCache(tup); + } + } + INSTR_TIME_SET_CURRENT(duration); + INSTR_TIME_SUBTRACT(duration, start); + PG_SETMASK(&UnBlockSig); + + return INSTR_TIME_GET_MILLISEC(duration); +}; + +/* + * fetch all attribute entires of all tables several times with having + * expiration happen. + */ +double +catcachebench3(void) +{ + const int clock_step = 1000; + int i, t, a; + instr_time start, + duration; + + PG_SETMASK(&BlockSig); + INSTR_TIME_SET_CURRENT(start); + for (i = 0 ; i < 4 ; i++) + { + int ct = clock_step; + + for (t = 0 ; t < ntables ; t++) + { + /* + * catcacheclock is updated by transaction timestamp, so needs to + * be updated by other means for this test to work. Here I choosed + * to update the clock every 1000 tables scan. + */ + if (--ct < 0) + { + SetCatCacheClock(GetCurrentTimestamp()); + ct = clock_step; + } + for (a = 0 ; a < natts ; a++) + { + HeapTuple tup; + + tup = SearchSysCache3(STATRELATTINH, + ObjectIdGetDatum(tableoids[t]), + Int16GetDatum(attnums[a]), + BoolGetDatum(false)); + /* should be null, but.. */ + if (HeapTupleIsValid(tup)) + ReleaseSysCache(tup); + } + } + } + INSTR_TIME_SET_CURRENT(duration); + INSTR_TIME_SUBTRACT(duration, start); + PG_SETMASK(&UnBlockSig); + + return INSTR_TIME_GET_MILLISEC(duration); +}; + +void +catcachewarmup(void) +{ + int t, a; + + /* load up catalog tables */ + for (t = 0 ; t < ntables ; t++) + { + for (a = 0 ; a < natts ; a++) + { + HeapTuple tup; + + tup = SearchSysCache3(STATRELATTINH, + ObjectIdGetDatum(tableoids[t]), + Int16GetDatum(attnums[a]), + BoolGetDatum(false)); + /* should be null, but.. */ + if (HeapTupleIsValid(tup)) + ReleaseSysCache(tup); + } + } +} + +void +collectinfo(void) +{ + int ret; + Datum values[10000]; + bool nulls[10000]; + Oid types0[] = {OIDOID}; + int i; + + ntables = 0; + natts = 0; + + SPI_connect(); + /* collect target tables */ + ret = SPI_execute("select oid from pg_class where relnamespace = (select oid from pg_namespace where nspname = \'test\')", + true, 0); + if (ret != SPI_OK_SELECT) + elog(ERROR, "Failed 1"); + if (SPI_processed == 0) + elog(ERROR, "no relation found in schema \"test\""); + if (SPI_processed > 10000) + elog(ERROR, "too many relation found in schema \"test\""); + + for (i = 0 ; i < SPI_processed ; i++) + { + heap_deform_tuple(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, + values, nulls); + if (nulls[0]) + elog(ERROR, "Failed 2"); + + tableoids[ntables++] = DatumGetObjectId(values[0]); + } + SPI_finish(); + elog(DEBUG1, "%d tables found", ntables); + + values[0] = ObjectIdGetDatum(tableoids[0]); + nulls[0] = false; + SPI_connect(); + ret = SPI_execute_with_args("select attnum from pg_attribute where attrelid = (select oid from pg_class where oid = $1)", + 1, types0, values, NULL, true, 0); + if (SPI_processed == 0) + elog(ERROR, "no attribute found in table %d", tableoids[0]); + if (SPI_processed > 10000) + elog(ERROR, "too many relation found in table %d", tableoids[0]); + + /* collect target attributes. assuming all tables have the same attnums */ + for (i = 0 ; i < SPI_processed ; i++) + { + int16 attnum; + + heap_deform_tuple(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, + values, nulls); + if (nulls[0]) + elog(ERROR, "Failed 3"); + attnum = DatumGetInt16(values[0]); + + if (attnum > 0) + attnums[natts++] = attnum; + } + SPI_finish(); + elog(DEBUG1, "%d attributes found", natts); +} diff --git a/contrib/catcachebench/catcachebench.control b/contrib/catcachebench/catcachebench.control new file mode 100644 index 0000000000..3fc9d2e420 --- /dev/null +++ b/contrib/catcachebench/catcachebench.control @@ -0,0 +1,6 @@ +# catcachebench + +comment = 'benchmark for catcache pruning' +default_version = '0.0' +module_pathname = '$libdir/catcachebench' +relocatable = true diff --git a/src/backend/utils/cache/catcache.c b/src/backend/utils/cache/catcache.c index 0c68c04caa..35e1a07e57 100644 --- a/src/backend/utils/cache/catcache.c +++ b/src/backend/utils/cache/catcache.c @@ -814,6 +814,41 @@ CatalogCacheFlushCatalog(Oid catId) CACHE_elog(DEBUG2, "end of CatalogCacheFlushCatalog call"); } + +/* FUNCTION FOR BENCHMARKING */ +void +CatalogCacheFlushCatalog2(Oid catId) +{ + slist_iter iter; + + CACHE_elog(DEBUG2, "CatalogCacheFlushCatalog called for %u", catId); + + slist_foreach(iter, &CacheHdr->ch_caches) + { + CatCache *cache = slist_container(CatCache, cc_next, iter.cur); + + /* Does this cache store tuples of the target catalog? */ + if (cache->cc_reloid == catId) + { + /* Yes, so flush all its contents */ + ResetCatalogCache(cache); + + /* Tell inval.c to call syscache callbacks for this cache */ + CallSyscacheCallbacks(cache->id, 0); + + cache->cc_nbuckets = 128; + pfree(cache->cc_bucket); + cache->cc_bucket = palloc0(128 * sizeof(dlist_head)); + ereport(DEBUG1, + (errmsg("Catcache reset"), + errhidestmt(true))); + } + } + + CACHE_elog(DEBUG2, "end of CatalogCacheFlushCatalog call"); +} +/* END: FUNCTION FOR BENCHMARKING */ + /* * InitCatCache * diff --git a/src/backend/utils/cache/syscache.c b/src/backend/utils/cache/syscache.c index 53d9ddf159..1c79a85a8c 100644 --- a/src/backend/utils/cache/syscache.c +++ b/src/backend/utils/cache/syscache.c @@ -983,7 +983,7 @@ static const struct cachedesc cacheinfo[] = { } }; -static CatCache *SysCache[SysCacheSize]; +CatCache *SysCache[SysCacheSize]; static bool CacheInitialized = false; diff --git a/src/include/utils/catcache.h b/src/include/utils/catcache.h index f9e9889339..dc0ad1a268 100644 --- a/src/include/utils/catcache.h +++ b/src/include/utils/catcache.h @@ -278,4 +278,7 @@ extern void PrepareToInvalidateCacheTuple(Relation relation, extern void PrintCatCacheLeakWarning(HeapTuple tuple); extern void PrintCatCacheListLeakWarning(CatCList *list); +/* tentative change to allow benchmark on master branch */ +#define SetCatCacheClock(ts) (ts) + #endif /* CATCACHE_H */ -- 2.23.0