diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c index 78c7d41ac48d95d6648baa3951563ba9e4ad3496..7de93647759ce8dc58d0a9f3c8aad9ed7f36f5e6 100644 *** a/src/backend/storage/file/fd.c --- b/src/backend/storage/file/fd.c *************** *** 43,50 **** * wrappers around fopen(3), opendir(3), popen(3) and open(2), respectively. * They behave like the corresponding native functions, except that the handle * is registered with the current subtransaction, and will be automatically ! * closed at abort. These are intended for short operations like reading a ! * configuration file, and there is a fixed limit on the number of files that * can be opened using these functions at any one time. * * Finally, BasicOpenFile is just a thin wrapper around open() that can --- 43,50 ---- * wrappers around fopen(3), opendir(3), popen(3) and open(2), respectively. * They behave like the corresponding native functions, except that the handle * is registered with the current subtransaction, and will be automatically ! * closed at abort. These are intended mainly for short operations like ! * reading a configuration file; there is a limit on the number of files that * can be opened using these functions at any one time. * * Finally, BasicOpenFile is just a thin wrapper around open() that can *************** static uint64 temporary_files_size = 0; *** 198,210 **** /* * List of OS handles opened with AllocateFile, AllocateDir and * OpenTransientFile. - * - * Since we don't want to encourage heavy use of those functions, - * it seems OK to put a pretty small maximum limit on the number of - * simultaneously allocated descs. */ - #define MAX_ALLOCATED_DESCS 32 - typedef enum { AllocateDescFile, --- 198,204 ---- *************** typedef enum *** 216,232 **** typedef struct { AllocateDescKind kind; union { FILE *file; DIR *dir; int fd; } desc; - SubTransactionId create_subid; } AllocateDesc; static int numAllocatedDescs = 0; ! static AllocateDesc allocatedDescs[MAX_ALLOCATED_DESCS]; /* * Number of temporary files opened during the current session; --- 210,227 ---- typedef struct { AllocateDescKind kind; + SubTransactionId create_subid; union { FILE *file; DIR *dir; int fd; } desc; } AllocateDesc; static int numAllocatedDescs = 0; ! static int maxAllocatedDescs = 0; ! static AllocateDesc *allocatedDescs = NULL; /* * Number of temporary files opened during the current session; *************** static void FreeVfd(File file); *** 284,289 **** --- 279,286 ---- static int FileAccess(File file); static File OpenTemporaryFileInTablespace(Oid tblspcOid, bool rejectError); + static bool reserveAllocatedDesc(void); + static int FreeDesc(AllocateDesc *desc); static void AtProcExit_Files(int code, Datum arg); static void CleanupTempFiles(bool isProcExit); static void RemovePgTempFilesInDir(const char *tmpdirname); *************** FilePathName(File file) *** 1491,1496 **** --- 1488,1553 ---- /* + * Create/enlarge the allocatedDescs[] array if needed and possible. + * Returns true if an array element is available. + */ + static bool + reserveAllocatedDesc(void) + { + AllocateDesc *newDescs; + int newMax; + + /* Quick out if array already has a free slot. */ + if (numAllocatedDescs < maxAllocatedDescs) + return true; + + /* + * If the array hasn't yet been created in the current process, initialize + * it with FD_MINFREE / 2 elements. In many scenarios this is as many as + * we will ever need, anyway. We don't want to look at max_safe_fds + * immediately because set_max_safe_fds() may not have run yet. + */ + if (allocatedDescs == NULL) + { + newMax = FD_MINFREE / 2; + newDescs = (AllocateDesc *) malloc(newMax * sizeof(AllocateDesc)); + /* Out of memory already? Treat as fatal error. */ + if (newDescs == NULL) + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"))); + allocatedDescs = newDescs; + maxAllocatedDescs = newMax; + return true; + } + + /* + * Consider enlarging the array beyond the initial allocation used above. + * By the time this happens, max_safe_fds should be known accurately. + * + * We mustn't let allocated descriptors hog all the available FDs, and in + * practice we'd better leave a reasonable number of FDs for VFD use. So + * set the maximum to max_safe_fds / 2. (This should certainly be at + * least as large as the initial size, FD_MINFREE / 2.) + */ + newMax = max_safe_fds / 2; + if (newMax > maxAllocatedDescs) + { + newDescs = (AllocateDesc *) realloc(allocatedDescs, + newMax * sizeof(AllocateDesc)); + /* Treat out-of-memory as a non-fatal error. */ + if (newDescs == NULL) + return false; + allocatedDescs = newDescs; + maxAllocatedDescs = newMax; + return true; + } + + /* Can't enlarge allocatedDescs[] any more. */ + return false; + } + + /* * Routines that want to use stdio (ie, FILE*) should use AllocateFile * rather than plain fopen(). This lets fd.c deal with freeing FDs if * necessary to open the file. When done, call FreeFile rather than fclose. *************** AllocateFile(const char *name, const cha *** 1516,1530 **** numAllocatedDescs, name)); /* ! * The test against MAX_ALLOCATED_DESCS prevents us from overflowing ! * allocatedFiles[]; the test against max_safe_fds prevents AllocateFile ! * from hogging every one of the available FDs, which'd lead to infinite ! * looping. */ ! if (numAllocatedDescs >= MAX_ALLOCATED_DESCS || ! numAllocatedDescs >= max_safe_fds - 1) ! elog(ERROR, "exceeded MAX_ALLOCATED_DESCS while trying to open file \"%s\"", ! name); TryAgain: if ((file = fopen(name, mode)) != NULL) --- 1573,1587 ---- numAllocatedDescs, name)); /* ! * This test protects us both against overrunning the allocatedDescs[] ! * array and against trying to eat more than max_safe_fds file descriptors ! * for allocatedDescs (which would result in an infinite loop below). */ ! if (!reserveAllocatedDesc()) ! ereport(ERROR, ! (errcode(ERRCODE_INSUFFICIENT_RESOURCES), ! errmsg("exceeded maxAllocatedDescs (%d) while trying to open file \"%s\"", ! maxAllocatedDescs, name))); TryAgain: if ((file = fopen(name, mode)) != NULL) *************** OpenTransientFile(FileName fileName, int *** 1567,1581 **** numAllocatedDescs, fileName)); /* ! * The test against MAX_ALLOCATED_DESCS prevents us from overflowing ! * allocatedFiles[]; the test against max_safe_fds prevents BasicOpenFile ! * from hogging every one of the available FDs, which'd lead to infinite ! * looping. */ ! if (numAllocatedDescs >= MAX_ALLOCATED_DESCS || ! numAllocatedDescs >= max_safe_fds - 1) ! elog(ERROR, "exceeded MAX_ALLOCATED_DESCS while trying to open file \"%s\"", ! fileName); fd = BasicOpenFile(fileName, fileFlags, fileMode); --- 1624,1638 ---- numAllocatedDescs, fileName)); /* ! * This test protects us both against overrunning the allocatedDescs[] ! * array and against trying to eat more than max_safe_fds file descriptors ! * for allocatedDescs (which would result in an infinite loop below). */ ! if (!reserveAllocatedDesc()) ! ereport(ERROR, ! (errcode(ERRCODE_INSUFFICIENT_RESOURCES), ! errmsg("exceeded maxAllocatedDescs (%d) while trying to open file \"%s\"", ! maxAllocatedDescs, fileName))); fd = BasicOpenFile(fileName, fileFlags, fileMode); *************** OpenPipeStream(const char *command, cons *** 1608,1622 **** numAllocatedDescs, command)); /* ! * The test against MAX_ALLOCATED_DESCS prevents us from overflowing ! * allocatedFiles[]; the test against max_safe_fds prevents AllocateFile ! * from hogging every one of the available FDs, which'd lead to infinite ! * looping. */ ! if (numAllocatedDescs >= MAX_ALLOCATED_DESCS || ! numAllocatedDescs >= max_safe_fds - 1) ! elog(ERROR, "exceeded MAX_ALLOCATED_DESCS while trying to execute command \"%s\"", ! command); TryAgain: fflush(stdout); --- 1665,1679 ---- numAllocatedDescs, command)); /* ! * This test protects us both against overrunning the allocatedDescs[] ! * array and against trying to eat more than max_safe_fds file descriptors ! * for allocatedDescs (which would result in an infinite loop below). */ ! if (!reserveAllocatedDesc()) ! ereport(ERROR, ! (errcode(ERRCODE_INSUFFICIENT_RESOURCES), ! errmsg("exceeded maxAllocatedDescs (%d) while trying to execute command \"%s\"", ! maxAllocatedDescs, command))); TryAgain: fflush(stdout); *************** AllocateDir(const char *dirname) *** 1760,1774 **** numAllocatedDescs, dirname)); /* ! * The test against MAX_ALLOCATED_DESCS prevents us from overflowing ! * allocatedDescs[]; the test against max_safe_fds prevents AllocateDir ! * from hogging every one of the available FDs, which'd lead to infinite ! * looping. */ ! if (numAllocatedDescs >= MAX_ALLOCATED_DESCS || ! numAllocatedDescs >= max_safe_fds - 1) ! elog(ERROR, "exceeded MAX_ALLOCATED_DESCS while trying to open directory \"%s\"", ! dirname); TryAgain: if ((dir = opendir(dirname)) != NULL) --- 1817,1831 ---- numAllocatedDescs, dirname)); /* ! * This test protects us both against overrunning the allocatedDescs[] ! * array and against trying to eat more than max_safe_fds file descriptors ! * for allocatedDescs (which would result in an infinite loop below). */ ! if (!reserveAllocatedDesc()) ! ereport(ERROR, ! (errcode(ERRCODE_INSUFFICIENT_RESOURCES), ! errmsg("exceeded maxAllocatedDescs (%d) while trying to open directory \"%s\"", ! maxAllocatedDescs, dirname))); TryAgain: if ((dir = opendir(dirname)) != NULL)