diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index a5cd4e44c7..caf958310a 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -2079,6 +2079,32 @@ include_dir 'conf.d' + + max_total_backend_memory (integer) + + max_total_backend_memory configuration parameter + + + + + Specifies a limit to the amount of memory (MB) that may be allocated to + backends in total (i.e. this is not a per user or per backend limit). + If unset, or set to 0 it is disabled. A backend request that would push + the total over the limit will be denied with an out of memory error + causing that backends current query/transaction to fail. Due to the dynamic + nature of memory allocations, this limit is not exact. If within 1.5MB of + the limit and two backends request 1MB each at the same time both may be + allocated exceeding the limit. Further requests will not be allocated until + dropping below the limit. Keep this in mind when setting this value. This + limit does not affect auxiliary backend processes + . Backend memory allocations + (backend_mem_allocated) are displayed in the + pg_stat_activity + view. + + + + diff --git a/src/backend/storage/ipc/dsm_impl.c b/src/backend/storage/ipc/dsm_impl.c index 269ad2fe53..808ffe75f2 100644 --- a/src/backend/storage/ipc/dsm_impl.c +++ b/src/backend/storage/ipc/dsm_impl.c @@ -253,6 +253,10 @@ dsm_impl_posix(dsm_op op, dsm_handle handle, Size request_size, return true; } + /* Do not exceed maximum allowed memory allocation */ + if (op == DSM_OP_CREATE && exceeds_max_total_bkend_mem(request_size)) + return false; + /* * Create new segment or open an existing one for attach. * @@ -524,6 +528,10 @@ dsm_impl_sysv(dsm_op op, dsm_handle handle, Size request_size, int flags = IPCProtection; size_t segsize; + /* Do not exceed maximum allowed memory allocation */ + if (op == DSM_OP_CREATE && exceeds_max_total_bkend_mem(request_size)) + return false; + /* * Allocate the memory BEFORE acquiring the resource, so that we don't * leak the resource if memory allocation fails. @@ -718,6 +726,10 @@ dsm_impl_windows(dsm_op op, dsm_handle handle, Size request_size, return true; } + /* Do not exceed maximum allowed memory allocation */ + if (op == DSM_OP_CREATE && exceeds_max_total_bkend_mem(request_size)) + return false; + /* Create new segment or open an existing one for attach. */ if (op == DSM_OP_CREATE) { diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c index 17a00587f8..9137a000ae 100644 --- a/src/backend/utils/activity/backend_status.c +++ b/src/backend/utils/activity/backend_status.c @@ -44,6 +44,8 @@ */ bool pgstat_track_activities = false; int pgstat_track_activity_query_size = 1024; +/* Max backend memory allocation allowed (MB). 0 = disabled */ +int max_total_bkend_mem = 0; /* exposed so that backend_progress.c can access it */ @@ -1253,3 +1255,107 @@ pgstat_report_backend_mem_allocated_decrease(uint64 deallocation) beentry->backend_mem_allocated -= deallocation; PGSTAT_END_WRITE_ACTIVITY(beentry); } + +/* ---------- + * pgstat_get_all_backend_memory_allocated() - + * + * Return a uint64 representing the current shared memory allocated to all + * backends. This looks directly at the BackendStatusArray, and so will + * provide current information regardless of the age of our transaction's + * snapshot of the status array. + * In the future we will likely utilize additional values - perhaps limit + * backend allocation by user/role, etc. + * ---------- + */ +uint64 +pgstat_get_all_backend_memory_allocated(void) +{ + PgBackendStatus *beentry; + int i; + uint64 all_backend_memory_allocated = 0; + + beentry = BackendStatusArray; + + /* + * We probably shouldn't get here before shared memory has been set up, + * but be safe. + */ + if (beentry == NULL || BackendActivityBuffer == NULL) + return 0; + + /* + * We include AUX procs in all backend memory calculation + */ + for (i = 1; i <= NumBackendStatSlots; i++) + { + /* + * We use a volatile pointer here to ensure the compiler doesn't try to + * get cute. + */ + volatile PgBackendStatus *vbeentry = beentry; + bool found; + uint64 backend_mem_allocated = 0; + + for (;;) + { + int before_changecount; + int after_changecount; + + pgstat_begin_read_activity(vbeentry, before_changecount); + + /* Ignore invalid entries, which may contain invalid data. + * See pgstat_beshutdown_hook() + */ + if (vbeentry->st_procpid > 0) + backend_mem_allocated = vbeentry->backend_mem_allocated; + + pgstat_end_read_activity(vbeentry, after_changecount); + + if ((found = pgstat_read_activity_complete(before_changecount, + after_changecount))) + break; + + /* Make sure we can break out of loop if stuck... */ + CHECK_FOR_INTERRUPTS(); + } + + if (found) + all_backend_memory_allocated += backend_mem_allocated; + + beentry++; + } + + return all_backend_memory_allocated; +} + +/* + * Determine if allocation request will exceed max backend memory allowed. + * Do not apply to auxiliary processes. + */ +bool +exceeds_max_total_bkend_mem(uint64 allocation_request) +{ + bool result = false; + + if (MyAuxProcType != NotAnAuxProcess) + return result; + + /* Convert max_total_bkend_mem to bytes for comparison */ + if (max_total_bkend_mem && + pgstat_get_all_backend_memory_allocated() + + allocation_request > (uint64)max_total_bkend_mem * 1024 * 1024) + { + /* + * Explicitely identify the OOM being a result of this + * configuration parameter vs a system failure to allocate OOM. + */ + elog(WARNING, + "request will exceed postgresql.conf defined max_total_backend_memory limit (%lu > %lu)", + pgstat_get_all_backend_memory_allocated() + + allocation_request, (uint64)max_total_bkend_mem * 1024 * 1024); + + result = true; + } + + return result; +} diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 9fbbfb1be5..ab8d83c235 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -3664,6 +3664,17 @@ static struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"max_total_backend_memory", PGC_SIGHUP, RESOURCES_MEM, + gettext_noop("Restrict total backend memory allocations to this max."), + gettext_noop("0 turns this feature off."), + GUC_UNIT_MB + }, + &max_total_bkend_mem, + 0, 0, INT_MAX, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, 0, 0, 0, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 90bec0502c..8e944f6511 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -155,6 +155,9 @@ # mmap # (change requires restart) #min_dynamic_shared_memory = 0MB # (change requires restart) +#max_total_backend_memory = 0MB # Restrict total backend memory allocations + # to this max (in MB). 0 turns this feature + # off. # - Disk - diff --git a/src/backend/utils/mmgr/aset.c b/src/backend/utils/mmgr/aset.c index c91f8efa4d..ac9a1ced3f 100644 --- a/src/backend/utils/mmgr/aset.c +++ b/src/backend/utils/mmgr/aset.c @@ -428,6 +428,10 @@ AllocSetContextCreateInternal(MemoryContext parent, else firstBlockSize = Max(firstBlockSize, initBlockSize); + /* Do not exceed maximum allowed memory allocation */ + if (exceeds_max_total_bkend_mem(firstBlockSize)) + return NULL; + /* * Allocate the initial block. Unlike other aset.c blocks, it starts with * the context header and its block header follows that. @@ -720,6 +724,11 @@ AllocSetAlloc(MemoryContext context, Size size) { chunk_size = MAXALIGN(size); blksize = chunk_size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ; + + /* Do not exceed maximum allowed memory allocation */ + if (exceeds_max_total_bkend_mem(blksize)) + return NULL; + block = (AllocBlock) malloc(blksize); if (block == NULL) return NULL; @@ -911,6 +920,10 @@ AllocSetAlloc(MemoryContext context, Size size) while (blksize < required_size) blksize <<= 1; + /* Do not exceed maximum allowed memory allocation */ + if (exceeds_max_total_bkend_mem(blksize)) + return NULL; + /* Try to allocate it */ block = (AllocBlock) malloc(blksize); @@ -1133,6 +1146,10 @@ AllocSetRealloc(void *pointer, Size size) blksize = chksize + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ; oldblksize = block->endptr - ((char *) block); + /* Do not exceed maximum allowed memory allocation */ + if (blksize > oldblksize && exceeds_max_total_bkend_mem(blksize - oldblksize)) + return NULL; + block = (AllocBlock) realloc(block, blksize); if (block == NULL) { diff --git a/src/backend/utils/mmgr/generation.c b/src/backend/utils/mmgr/generation.c index 36e5b3f94d..1d5720836c 100644 --- a/src/backend/utils/mmgr/generation.c +++ b/src/backend/utils/mmgr/generation.c @@ -192,6 +192,9 @@ GenerationContextCreate(MemoryContext parent, else allocSize = Max(allocSize, initBlockSize); + if (exceeds_max_total_bkend_mem(allocSize)) + return NULL; + /* * Allocate the initial block. Unlike other generation.c blocks, it * starts with the context header and its block header follows that. @@ -361,6 +364,9 @@ GenerationAlloc(MemoryContext context, Size size) { Size blksize = required_size + Generation_BLOCKHDRSZ; + if (exceeds_max_total_bkend_mem(blksize)) + return NULL; + block = (GenerationBlock *) malloc(blksize); if (block == NULL) return NULL; @@ -464,6 +470,9 @@ GenerationAlloc(MemoryContext context, Size size) if (blksize < required_size) blksize = pg_nextpower2_size_t(required_size); + if (exceeds_max_total_bkend_mem(blksize)) + return NULL; + block = (GenerationBlock *) malloc(blksize); if (block == NULL) diff --git a/src/backend/utils/mmgr/slab.c b/src/backend/utils/mmgr/slab.c index efc8bcfaa7..5cf0cfff86 100644 --- a/src/backend/utils/mmgr/slab.c +++ b/src/backend/utils/mmgr/slab.c @@ -177,6 +177,10 @@ SlabContextCreate(MemoryContext parent, headerSize += chunksPerBlock * sizeof(bool); #endif + /* Do not exceed maximum allowed memory allocation */ + if (exceeds_max_total_bkend_mem(headerSize)) + return NULL; + slab = (SlabContext *) malloc(headerSize); if (slab == NULL) { @@ -331,6 +335,10 @@ SlabAlloc(MemoryContext context, Size size) */ if (slab->minFreeChunks == 0) { + /* Do not exceed maximum allowed memory allocation */ + if (exceeds_max_total_bkend_mem(slab->blockSize)) + return NULL; + block = (SlabBlock *) malloc(slab->blockSize); if (block == NULL) diff --git a/src/include/utils/backend_status.h b/src/include/utils/backend_status.h index 9bdc4197bd..3b940ff98e 100644 --- a/src/include/utils/backend_status.h +++ b/src/include/utils/backend_status.h @@ -270,6 +270,7 @@ typedef struct LocalPgBackendStatus */ extern PGDLLIMPORT bool pgstat_track_activities; extern PGDLLIMPORT int pgstat_track_activity_query_size; +extern PGDLLIMPORT int max_total_bkend_mem; /* ---------- @@ -321,6 +322,7 @@ extern int pgstat_fetch_stat_numbackends(void); extern PgBackendStatus *pgstat_fetch_stat_beentry(int beid); extern LocalPgBackendStatus *pgstat_fetch_stat_local_beentry(int beid); extern char *pgstat_clip_activity(const char *raw_activity); +extern bool exceeds_max_total_bkend_mem(uint64 allocation_request); #endif /* BACKEND_STATUS_H */