[PATCH] Speedup truncates of relation forks

From c058bba2bfc47490f231d6f067bb470cdefccbc0 Mon Sep 17 00:00:00 2001
From: Kirk Jamison <k.jamison@jp.fujitsu.com>
Date: Tue, 11 Jun 2019 01:41:43 +0000
Subject: [PATCH] Speedup truncates of relation forks

Whenever we truncate relations, it involves several scans of the
shared buffers for every call of smgrtruncate() for each fork which
is time-consuming. This patch reduces the scan for all forks into one 
instead of three, and improves the relation truncates by initially
marking the pages-to-be-truncated of relation forks, then
simultaneously truncating them, resulting to an improved performance
in VACUUM, autovacuum operations and their recovery performance.
---
 contrib/pg_visibility/pg_visibility.c     |  17 +++-
 src/backend/access/heap/visibilitymap.c   |  31 +++-----
 src/backend/catalog/storage.c             | 126 ++++++++++++++++++++++++++----
 src/backend/storage/buffer/bufmgr.c       |  45 ++++++++---
 src/backend/storage/freespace/freespace.c |  38 +++------
 src/backend/storage/smgr/smgr.c           |  24 +++---
 src/include/access/visibilitymap.h        |   2 +-
 src/include/storage/bufmgr.h              |   4 +-
 src/include/storage/freespace.h           |   2 +-
 src/include/storage/smgr.h                |   7 +-
 10 files changed, 207 insertions(+), 89 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index 1372bb6..2499415 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -383,6 +383,10 @@ pg_truncate_visibility_map(PG_FUNCTION_ARGS)
 {
 	Oid			relid = PG_GETARG_OID(0);
 	Relation	rel;
+	ForkNumber	forks[MAX_FORKNUM];
+	BlockNumber	blocks[MAX_FORKNUM];
+	BlockNumber	newnblocks = InvalidBlockNumber;
+	int		nforks = 0;
 
 	rel = relation_open(relid, AccessExclusiveLock);
 
@@ -392,7 +396,18 @@ pg_truncate_visibility_map(PG_FUNCTION_ARGS)
 	RelationOpenSmgr(rel);
 	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
 
-	visibilitymap_truncate(rel, 0);
+	blocks[nforks] = visibilitymap_mark_truncate(rel, 0);
+	if (BlockNumberIsValid(blocks[nforks]))
+	{
+		forks[nforks] = VISIBILITYMAP_FORKNUM;
+		newnblocks = blocks[nforks];
+		nforks++;
+	}
+	smgrtruncate(rel->rd_smgr, forks, blocks, nforks);
+
+	/* Update the local smgr_vm_nblocks setting */
+	if (rel->rd_smgr)
+		rel->rd_smgr->smgr_vm_nblocks = newnblocks;
 
 	if (RelationNeedsWAL(rel))
 	{
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index 64dfe06..2f1379c 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -17,7 +17,7 @@
  *		visibilitymap_set	 - set a bit in a previously pinned page
  *		visibilitymap_get_status - get status of bits
  *		visibilitymap_count  - count number of bits set in visibility map
- *		visibilitymap_truncate	- truncate the visibility map
+ *		visibilitymap_mark_truncate - mark the about-to-be-truncated VM
  *
  * NOTES
  *
@@ -430,7 +430,10 @@ visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_fro
 }
 
 /*
- *	visibilitymap_truncate - truncate the visibility map
+ *	visibilitymap_mark_truncate - mark the about-to-be-truncated VM
+ *
+ * Formerly, this function truncates VM relation forks. Instead, this just
+ * marks the dirty buffers.
  *
  * The caller must hold AccessExclusiveLock on the relation, to ensure that
  * other backends receive the smgr invalidation event that this function sends
@@ -438,8 +441,8 @@ visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_fro
  *
  * nheapblocks is the new size of the heap.
  */
-void
-visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
+BlockNumber
+visibilitymap_mark_truncate(Relation rel, BlockNumber nheapblocks)
 {
 	BlockNumber newnblocks;
 
@@ -459,7 +462,7 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 	 * nothing to truncate.
 	 */
 	if (!smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM))
-		return;
+		return InvalidBlockNumber;
 
 	/*
 	 * Unless the new size is exactly at a visibility map page boundary, the
@@ -480,7 +483,7 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 		if (!BufferIsValid(mapBuffer))
 		{
 			/* nothing to do, the file was already smaller */
-			return;
+			return InvalidBlockNumber;
 		}
 
 		page = BufferGetPage(mapBuffer);
@@ -528,20 +531,10 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 	if (smgrnblocks(rel->rd_smgr, VISIBILITYMAP_FORKNUM) <= newnblocks)
 	{
 		/* nothing to do, the file was already smaller than requested size */
-		return;
+		return InvalidBlockNumber;
 	}
-
-	/* Truncate the unused VM pages, and send smgr inval message */
-	smgrtruncate(rel->rd_smgr, VISIBILITYMAP_FORKNUM, newnblocks);
-
-	/*
-	 * We might as well update the local smgr_vm_nblocks setting. smgrtruncate
-	 * sent an smgr cache inval message, which will cause other backends to
-	 * invalidate their copy of smgr_vm_nblocks, and this one too at the next
-	 * command boundary.  But this ensures it isn't outright wrong until then.
-	 */
-	if (rel->rd_smgr)
-		rel->rd_smgr->smgr_vm_nblocks = newnblocks;
+	else
+		return newnblocks;
 }
 
 /*
diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c
index 3cc886f..3151632 100644
--- a/src/backend/catalog/storage.c
+++ b/src/backend/catalog/storage.c
@@ -231,6 +231,11 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 {
 	bool		fsm;
 	bool		vm;
+	ForkNumber	forks[MAX_FORKNUM];
+	BlockNumber	blocks[MAX_FORKNUM];
+	BlockNumber	new_nfsmblocks = InvalidBlockNumber;	/* FSM blocks */
+	BlockNumber	newnblocks = InvalidBlockNumber;	/* VM blocks */
+	int		nforks = 0;
 
 	/* Open it at the smgr level if not already done */
 	RelationOpenSmgr(rel);
@@ -242,15 +247,34 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 	rel->rd_smgr->smgr_fsm_nblocks = InvalidBlockNumber;
 	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
 
-	/* Truncate the FSM first if it exists */
+	/*
+	 * We used to truncate FSM and VM forks here. Now we only mark the
+	 * dirty buffers of all forks about-to-be-truncated if they exist.
+	 */
+
 	fsm = smgrexists(rel->rd_smgr, FSM_FORKNUM);
 	if (fsm)
-		FreeSpaceMapTruncateRel(rel, nblocks);
+	{
+		blocks[nforks] = MarkFreeSpaceMapTruncateRel(rel, nblocks);
+		if (BlockNumberIsValid(blocks[nforks]))
+		{
+			forks[nforks] = FSM_FORKNUM;
+			new_nfsmblocks= blocks[nforks];	/* FSM blocks */
+			nforks++;
+		}
+	}
 
-	/* Truncate the visibility map too if it exists. */
 	vm = smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM);
 	if (vm)
-		visibilitymap_truncate(rel, nblocks);
+	{
+		blocks[nforks] = visibilitymap_mark_truncate(rel, nblocks);
+		if (BlockNumberIsValid(blocks[nforks]))
+		{
+			forks[nforks] = VISIBILITYMAP_FORKNUM;
+			newnblocks = blocks[nforks]; 	/* VM blocks */
+			nforks++;
+		}
+	}
 
 	/*
 	 * We WAL-log the truncation before actually truncating, which means
@@ -263,9 +287,7 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 	 */
 	if (RelationNeedsWAL(rel))
 	{
-		/*
-		 * Make an XLOG entry reporting the file truncation.
-		 */
+		/* Make an XLOG entry reporting the file truncation */
 		XLogRecPtr	lsn;
 		xl_smgr_truncate xlrec;
 
@@ -290,8 +312,33 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 			XLogFlush(lsn);
 	}
 
-	/* Do the real work */
-	smgrtruncate(rel->rd_smgr, MAIN_FORKNUM, nblocks);
+	/* Mark the MAIN fork */
+	forks[nforks] = MAIN_FORKNUM;
+	blocks[nforks] = nblocks;
+	nforks++;
+
+	/* Truncate relation forks simultaneously */
+	smgrtruncate(rel->rd_smgr, forks, blocks, nforks);
+
+	/*
+	 * We might as well update the local smgr_fsm_nblocks and smgr_vm_nblocks
+	 * setting. smgrtruncate sent an smgr cache inval message, which will cause
+	 * other backends to invalidate their copy of smgr_fsm_nblocks and
+	 * smgr_vm_nblocks, and this one too at the next command boundary. But this
+	 * ensures it isn't outright wrong until then.
+	 */
+	if (rel->rd_smgr)
+	{
+		rel->rd_smgr->smgr_fsm_nblocks = new_nfsmblocks;
+		rel->rd_smgr->smgr_vm_nblocks = newnblocks;
+	}
+
+	/*
+	 * Update upper-level FSM pages to account for the truncation.  This is
+	 * important because the just-truncated pages were likely marked as
+	 * all-free, and would be preferentially selected.
+	 */
+	FreeSpaceMapVacuumRange(rel, new_nfsmblocks, InvalidBlockNumber);
 }
 
 /*
@@ -588,6 +635,14 @@ smgr_redo(XLogReaderState *record)
 		xl_smgr_truncate *xlrec = (xl_smgr_truncate *) XLogRecGetData(record);
 		SMgrRelation reln;
 		Relation	rel;
+		ForkNumber	forks[MAX_FORKNUM];
+		BlockNumber	blocks[MAX_FORKNUM];
+		BlockNumber	new_nfsmblocks = InvalidBlockNumber;
+		BlockNumber	newnblocks = InvalidBlockNumber;
+		int		nforks = 0;
+		bool		fsm_fork = false;
+		bool		main_fork = false;
+		bool		vm_fork = false;
 
 		reln = smgropen(xlrec->rnode, InvalidBackendId);
 
@@ -616,23 +671,62 @@ smgr_redo(XLogReaderState *record)
 		 */
 		XLogFlush(lsn);
 
+		/*
+		 * To speedup recovery, we mark the about-to-be-truncated blocks of
+		 * relation forks first, then truncate those simultaneously later.
+		 */
 		if ((xlrec->flags & SMGR_TRUNCATE_HEAP) != 0)
 		{
-			smgrtruncate(reln, MAIN_FORKNUM, xlrec->blkno);
-
-			/* Also tell xlogutils.c about it */
-			XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
+			forks[nforks] = MAIN_FORKNUM;
+			blocks[nforks] = xlrec->blkno;
+			nforks++;
+			main_fork = true;
 		}
 
-		/* Truncate FSM and VM too */
 		rel = CreateFakeRelcacheEntry(xlrec->rnode);
 
 		if ((xlrec->flags & SMGR_TRUNCATE_FSM) != 0 &&
 			smgrexists(reln, FSM_FORKNUM))
-			FreeSpaceMapTruncateRel(rel, xlrec->blkno);
+		{
+			blocks[nforks] = MarkFreeSpaceMapTruncateRel(rel, xlrec->blkno);
+			if (BlockNumberIsValid(blocks[nforks]))
+			{
+				forks[nforks] = FSM_FORKNUM;
+				new_nfsmblocks= blocks[nforks];
+				nforks++;
+				fsm_fork = true;
+			}
+		}
 		if ((xlrec->flags & SMGR_TRUNCATE_VM) != 0 &&
 			smgrexists(reln, VISIBILITYMAP_FORKNUM))
-			visibilitymap_truncate(rel, xlrec->blkno);
+		{
+			blocks[nforks] = visibilitymap_mark_truncate(rel, xlrec->blkno);
+			if (BlockNumberIsValid(blocks[nforks]))
+			{
+				forks[nforks] = VISIBILITYMAP_FORKNUM;
+				newnblocks = blocks[nforks];
+				nforks++;
+				vm_fork = true;
+			}
+		}
+
+		/* Truncate relation forks simultaneously */
+		if (main_fork || fsm_fork || vm_fork)
+			smgrtruncate(reln, forks, blocks, nforks);
+
+		/* Also tell xlogutils.c about it */
+		if (main_fork)
+			XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
+
+		/* Update the local smgr_fsm_nblocks and smgr_vm_nblocks setting */
+		if (rel->rd_smgr)
+		{
+			rel->rd_smgr->smgr_fsm_nblocks = new_nfsmblocks;
+			rel->rd_smgr->smgr_vm_nblocks = newnblocks;
+		}
+
+		/* Update upper-level FSM pages to account for the truncation */
+		FreeSpaceMapVacuumRange(rel, new_nfsmblocks, InvalidBlockNumber);
 
 		FreeFakeRelcacheEntry(rel);
 	}
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index 7332e6b..dad8158 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -2899,8 +2899,8 @@ BufferGetLSNAtomic(Buffer buffer)
 /* ---------------------------------------------------------------------
  *		DropRelFileNodeBuffers
  *
- *		This function removes from the buffer pool all the pages of the
- *		specified relation fork that have block numbers >= firstDelBlock.
+ *		This function simultaneously removes from the buffer pool all the
+ *		pages of the relation forks that have block numbers >= firstDelBlock.
  *		(In particular, with firstDelBlock = 0, all pages are removed.)
  *		Dirty pages are simply dropped, without bothering to write them
  *		out first.  Therefore, this is NOT rollback-able, and so should be
@@ -2923,23 +2923,37 @@ BufferGetLSNAtomic(Buffer buffer)
  * --------------------------------------------------------------------
  */
 void
-DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
-					   BlockNumber firstDelBlock)
+DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
+					   BlockNumber *firstDelBlock, int nforks)
 {
 	int			i;
+	BlockNumber minBlock = InvalidBlockNumber;
 
 	/* If it's a local relation, it's localbuf.c's problem. */
 	if (RelFileNodeBackendIsTemp(rnode))
 	{
 		if (rnode.backend == MyBackendId)
-			DropRelFileNodeLocalBuffers(rnode.node, forkNum, firstDelBlock);
+		{
+			for (int j = 0; j < nforks; j++)
+				DropRelFileNodeLocalBuffers(rnode.node, forkNum[j],
+											firstDelBlock[j]);
+		}
 		return;
 	}
 
+	/* Get the lower bound of target block number we're interested in */
+	for (i = 0; i < nforks; i++)
+	{
+		if (!BlockNumberIsValid(minBlock) ||
+			minBlock > firstDelBlock[i])
+			minBlock = firstDelBlock[i];
+	}
+
 	for (i = 0; i < NBuffers; i++)
 	{
 		BufferDesc *bufHdr = GetBufferDescriptor(i);
 		uint32		buf_state;
+		int		k = 0;
 
 		/*
 		 * We can make this a tad faster by prechecking the buffer tag before
@@ -2960,12 +2974,23 @@ DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
 		if (!RelFileNodeEquals(bufHdr->tag.rnode, rnode.node))
 			continue;
 
+		/* Check with the lower bound block number and skip the loop */
+		if (bufHdr->tag.blockNum < minBlock)
+			continue; /* skip checking the buffer pool scan */
+
 		buf_state = LockBufHdr(bufHdr);
-		if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
-			bufHdr->tag.forkNum == forkNum &&
-			bufHdr->tag.blockNum >= firstDelBlock)
-			InvalidateBuffer(bufHdr);	/* releases spinlock */
-		else
+
+		for (k = 0; k < nforks; k++)
+		{
+			if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
+				bufHdr->tag.forkNum == forkNum[k] &&
+				bufHdr->tag.blockNum >= firstDelBlock[k])
+			{
+				InvalidateBuffer(bufHdr); /* releases spinlock */
+				break;
+			}
+		}
+		if (k >= nforks)
 			UnlockBufHdr(bufHdr, buf_state);
 	}
 }
diff --git a/src/backend/storage/freespace/freespace.c b/src/backend/storage/freespace/freespace.c
index c17b3f4..708c7cb 100644
--- a/src/backend/storage/freespace/freespace.c
+++ b/src/backend/storage/freespace/freespace.c
@@ -247,7 +247,10 @@ GetRecordedFreeSpace(Relation rel, BlockNumber heapBlk)
 }
 
 /*
- * FreeSpaceMapTruncateRel - adjust for truncation of a relation.
+ * MarkFreeSpaceMapTruncateRel - adjust for truncation of a relation.
+ *
+ * Formerly, this function truncates FSM relation forks. Instead, this just
+ * marks the dirty buffers and returns a block number.
  *
  * The caller must hold AccessExclusiveLock on the relation, to ensure that
  * other backends receive the smgr invalidation event that this function sends
@@ -255,8 +258,8 @@ GetRecordedFreeSpace(Relation rel, BlockNumber heapBlk)
  *
  * nblocks is the new size of the heap.
  */
-void
-FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
+BlockNumber
+MarkFreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 {
 	BlockNumber new_nfsmblocks;
 	FSMAddress	first_removed_address;
@@ -270,7 +273,7 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 	 * truncate.
 	 */
 	if (!smgrexists(rel->rd_smgr, FSM_FORKNUM))
-		return;
+		return InvalidBlockNumber;
 
 	/* Get the location in the FSM of the first removed heap block */
 	first_removed_address = fsm_get_location(nblocks, &first_removed_slot);
@@ -285,7 +288,7 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 	{
 		buf = fsm_readbuf(rel, first_removed_address, false);
 		if (!BufferIsValid(buf))
-			return;				/* nothing to do; the FSM was already smaller */
+			return InvalidBlockNumber;	/* nothing to do; the FSM was already smaller */
 		LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
 
 		/* NO EREPORT(ERROR) from here till changes are logged */
@@ -310,33 +313,16 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 		UnlockReleaseBuffer(buf);
 
 		new_nfsmblocks = fsm_logical_to_physical(first_removed_address) + 1;
+		return new_nfsmblocks;
 	}
 	else
 	{
 		new_nfsmblocks = fsm_logical_to_physical(first_removed_address);
 		if (smgrnblocks(rel->rd_smgr, FSM_FORKNUM) <= new_nfsmblocks)
-			return;				/* nothing to do; the FSM was already smaller */
+			return InvalidBlockNumber;	/* nothing to do; the FSM was already smaller */
+		else
+			return new_nfsmblocks;
 	}
-
-	/* Truncate the unused FSM pages, and send smgr inval message */
-	smgrtruncate(rel->rd_smgr, FSM_FORKNUM, new_nfsmblocks);
-
-	/*
-	 * We might as well update the local smgr_fsm_nblocks setting.
-	 * smgrtruncate sent an smgr cache inval message, which will cause other
-	 * backends to invalidate their copy of smgr_fsm_nblocks, and this one too
-	 * at the next command boundary.  But this ensures it isn't outright wrong
-	 * until then.
-	 */
-	if (rel->rd_smgr)
-		rel->rd_smgr->smgr_fsm_nblocks = new_nfsmblocks;
-
-	/*
-	 * Update upper-level FSM pages to account for the truncation.  This is
-	 * important because the just-truncated pages were likely marked as
-	 * all-free, and would be preferentially selected.
-	 */
-	FreeSpaceMapVacuumRange(rel, nblocks, InvalidBlockNumber);
 }
 
 /*
diff --git a/src/backend/storage/smgr/smgr.c b/src/backend/storage/smgr/smgr.c
index dba8c39..b37560e 100644
--- a/src/backend/storage/smgr/smgr.c
+++ b/src/backend/storage/smgr/smgr.c
@@ -508,19 +508,21 @@ smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo)
  *		already.
  */
 void
-smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo)
+smgrdounlinkfork(SMgrRelation reln, ForkNumber *forknum, bool isRedo, int nforks)
 {
 	RelFileNodeBackend rnode = reln->smgr_rnode;
 	int			which = reln->smgr_which;
+	int			i;
 
 	/* Close the fork at smgr level */
-	smgrsw[which].smgr_close(reln, forknum);
+	for (i = 0; i < nforks; i++)
+		smgrsw[which].smgr_close(reln, forknum[i]);
 
 	/*
 	 * Get rid of any remaining buffers for the fork.  bufmgr will just drop
 	 * them without bothering to write the contents.
 	 */
-	DropRelFileNodeBuffers(rnode, forknum, 0);
+	DropRelFileNodeBuffers(rnode, forknum, 0, nforks);
 
 	/*
 	 * It'd be nice to tell the stats collector to forget it immediately, too.
@@ -546,7 +548,8 @@ smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo)
 	 * ERROR, because we've already decided to commit or abort the current
 	 * xact.
 	 */
-	smgrsw[which].smgr_unlink(rnode, forknum, isRedo);
+	for (i = 0; i < nforks; i++)
+		smgrsw[which].smgr_unlink(rnode, forknum[i], isRedo);
 }
 
 /*
@@ -643,13 +646,15 @@ smgrnblocks(SMgrRelation reln, ForkNumber forknum)
  * The truncation is done immediately, so this can't be rolled back.
  */
 void
-smgrtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks)
+smgrtruncate(SMgrRelation reln, ForkNumber *forknum, BlockNumber *nblocks, int nforks)
 {
+	int		i;
+
 	/*
 	 * Get rid of any buffers for the about-to-be-deleted blocks. bufmgr will
 	 * just drop them without bothering to write the contents.
 	 */
-	DropRelFileNodeBuffers(reln->smgr_rnode, forknum, nblocks);
+	DropRelFileNodeBuffers(reln->smgr_rnode, forknum, nblocks, nforks);
 
 	/*
 	 * Send a shared-inval message to force other backends to close any smgr
@@ -663,10 +668,9 @@ smgrtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks)
 	 */
 	CacheInvalidateSmgr(reln->smgr_rnode);
 
-	/*
-	 * Do the truncation.
-	 */
-	smgrsw[reln->smgr_which].smgr_truncate(reln, forknum, nblocks);
+	/* Do the truncation */
+	for (i = 0; i < nforks; i++)
+		smgrsw[reln->smgr_which].smgr_truncate(reln, forknum[i], nblocks[i]);
 }
 
 /*
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index 2d88043..4735d5f 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -44,6 +44,6 @@ extern void visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf,
 							  uint8 flags);
 extern uint8 visibilitymap_get_status(Relation rel, BlockNumber heapBlk, Buffer *vmbuf);
 extern void visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_frozen);
-extern void visibilitymap_truncate(Relation rel, BlockNumber nheapblocks);
+extern BlockNumber visibilitymap_mark_truncate(Relation rel, BlockNumber nheapblocks);
 
 #endif							/* VISIBILITYMAP_H */
diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h
index 509f4b7..5be6c0d 100644
--- a/src/include/storage/bufmgr.h
+++ b/src/include/storage/bufmgr.h
@@ -190,8 +190,8 @@ extern BlockNumber RelationGetNumberOfBlocksInFork(Relation relation,
 extern void FlushOneBuffer(Buffer buffer);
 extern void FlushRelationBuffers(Relation rel);
 extern void FlushDatabaseBuffers(Oid dbid);
-extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode,
-								   ForkNumber forkNum, BlockNumber firstDelBlock);
+extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
+								   BlockNumber *firstDelBlock, int nforks);
 extern void DropRelFileNodesAllBuffers(RelFileNodeBackend *rnodes, int nnodes);
 extern void DropDatabaseBuffers(Oid dbid);
 
diff --git a/src/include/storage/freespace.h b/src/include/storage/freespace.h
index 8d8c465..bf19a67 100644
--- a/src/include/storage/freespace.h
+++ b/src/include/storage/freespace.h
@@ -30,7 +30,7 @@ extern void RecordPageWithFreeSpace(Relation rel, BlockNumber heapBlk,
 extern void XLogRecordPageWithFreeSpace(RelFileNode rnode, BlockNumber heapBlk,
 										Size spaceAvail);
 
-extern void FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks);
+extern BlockNumber MarkFreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks);
 extern void FreeSpaceMapVacuum(Relation rel);
 extern void FreeSpaceMapVacuumRange(Relation rel, BlockNumber start,
 									BlockNumber end);
diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h
index d286c8c..ff70b09 100644
--- a/src/include/storage/smgr.h
+++ b/src/include/storage/smgr.h
@@ -90,7 +90,8 @@ extern void smgrclosenode(RelFileNodeBackend rnode);
 extern void smgrcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo);
 extern void smgrdounlink(SMgrRelation reln, bool isRedo);
 extern void smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo);
-extern void smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo);
+extern void smgrdounlinkfork(SMgrRelation reln, ForkNumber *forknum,
+							 bool isRedo, int nforks);
 extern void smgrextend(SMgrRelation reln, ForkNumber forknum,
 					   BlockNumber blocknum, char *buffer, bool skipFsync);
 extern void smgrprefetch(SMgrRelation reln, ForkNumber forknum,
@@ -102,8 +103,8 @@ extern void smgrwrite(SMgrRelation reln, ForkNumber forknum,
 extern void smgrwriteback(SMgrRelation reln, ForkNumber forknum,
 						  BlockNumber blocknum, BlockNumber nblocks);
 extern BlockNumber smgrnblocks(SMgrRelation reln, ForkNumber forknum);
-extern void smgrtruncate(SMgrRelation reln, ForkNumber forknum,
-						 BlockNumber nblocks);
+extern void smgrtruncate(SMgrRelation reln, ForkNumber *forknum,
+						 BlockNumber *nblocks, int nforks);
 extern void smgrimmedsync(SMgrRelation reln, ForkNumber forknum);
 extern void AtEOXact_SMgr(void);
 
-- 
1.8.3.1

From 93b4cefe99087e6738827d3cda518883e723a4a9 Mon Sep 17 00:00:00 2001
From: Kirk Jamison <k.jamison@jp.fujitsu.com>
Date: Thu, 4 Jul 2019 10:59:05 +0000
Subject: [PATCH] Speedup truncates of relation forks

Whenever we truncate relations, it involves several scans of the
shared buffers for every call of smgrtruncate() for each fork which
is time-consuming. This patch reduces the scan for all forks into one
instead of three, and improves the relation truncates by initially
marking the pages-to-be-truncated of relation forks, then
simultaneously truncating them, resulting to an improved performance
in VACUUM, autovacuum operations and their recovery performance.
---
 contrib/pg_visibility/pg_visibility.c     | 11 +++-
 src/backend/access/heap/visibilitymap.c   | 36 +++++------
 src/backend/catalog/storage.c             | 99 +++++++++++++++++++++++++++----
 src/backend/storage/buffer/bufmgr.c       | 48 +++++++++++----
 src/backend/storage/freespace/freespace.c | 41 ++++---------
 src/backend/storage/smgr/smgr.c           | 49 ++++++++++-----
 src/include/access/visibilitymap.h        |  2 +-
 src/include/storage/bufmgr.h              |  4 +-
 src/include/storage/freespace.h           |  2 +-
 src/include/storage/smgr.h                |  7 ++-
 10 files changed, 199 insertions(+), 100 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index 1372bb6..1aabde2 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -383,6 +383,9 @@ pg_truncate_visibility_map(PG_FUNCTION_ARGS)
 {
 	Oid			relid = PG_GETARG_OID(0);
 	Relation	rel;
+	ForkNumber	fork;
+	BlockNumber	block;
+	BlockNumber	newnblocks = InvalidBlockNumber;
 
 	rel = relation_open(relid, AccessExclusiveLock);
 
@@ -392,7 +395,13 @@ pg_truncate_visibility_map(PG_FUNCTION_ARGS)
 	RelationOpenSmgr(rel);
 	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
 
-	visibilitymap_truncate(rel, 0);
+	block = visibilitymap_truncate_prepare(rel, 0);
+	if (BlockNumberIsValid(block)
+	{
+		fork = VISIBILITYMAP_FORKNUM;
+		newnblocks = block;
+	}
+	smgrtruncate(rel->rd_smgr, &fork, 1, &block);
 
 	if (RelationNeedsWAL(rel))
 	{
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index 64dfe06..4cc7977 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -17,7 +17,7 @@
  *		visibilitymap_set	 - set a bit in a previously pinned page
  *		visibilitymap_get_status - get status of bits
  *		visibilitymap_count  - count number of bits set in visibility map
- *		visibilitymap_truncate	- truncate the visibility map
+ *		visibilitymap_truncate_prepare - truncate only tail bits of map pages
  *
  * NOTES
  *
@@ -430,16 +430,18 @@ visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_fro
 }
 
 /*
- *	visibilitymap_truncate - truncate the visibility map
+ *	visibilitymap_truncate_prepare - truncate only tail bits of map page
+ *									 and return the block number for actual
+ *									 truncation later
  *
- * The caller must hold AccessExclusiveLock on the relation, to ensure that
- * other backends receive the smgr invalidation event that this function sends
- * before they access the VM again.
+ * Note that this does not truncate the actual visibility map pages.
+ * When this function is called, the caller must eventually follow it with
+ * smgrtruncate() call to actually truncate visibility map pages.
  *
  * nheapblocks is the new size of the heap.
  */
-void
-visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
+BlockNumber
+visibilitymap_truncate_prepare(Relation rel, BlockNumber nheapblocks)
 {
 	BlockNumber newnblocks;
 
@@ -459,7 +461,7 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 	 * nothing to truncate.
 	 */
 	if (!smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM))
-		return;
+		return InvalidBlockNumber;
 
 	/*
 	 * Unless the new size is exactly at a visibility map page boundary, the
@@ -480,7 +482,7 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 		if (!BufferIsValid(mapBuffer))
 		{
 			/* nothing to do, the file was already smaller */
-			return;
+			return InvalidBlockNumber;
 		}
 
 		page = BufferGetPage(mapBuffer);
@@ -528,20 +530,10 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 	if (smgrnblocks(rel->rd_smgr, VISIBILITYMAP_FORKNUM) <= newnblocks)
 	{
 		/* nothing to do, the file was already smaller than requested size */
-		return;
+		return InvalidBlockNumber;
 	}
-
-	/* Truncate the unused VM pages, and send smgr inval message */
-	smgrtruncate(rel->rd_smgr, VISIBILITYMAP_FORKNUM, newnblocks);
-
-	/*
-	 * We might as well update the local smgr_vm_nblocks setting. smgrtruncate
-	 * sent an smgr cache inval message, which will cause other backends to
-	 * invalidate their copy of smgr_vm_nblocks, and this one too at the next
-	 * command boundary.  But this ensures it isn't outright wrong until then.
-	 */
-	if (rel->rd_smgr)
-		rel->rd_smgr->smgr_vm_nblocks = newnblocks;
+	else
+		return newnblocks;
 }
 
 /*
diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c
index 3cc886f..646d26c 100644
--- a/src/backend/catalog/storage.c
+++ b/src/backend/catalog/storage.c
@@ -231,6 +231,11 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 {
 	bool		fsm;
 	bool		vm;
+	ForkNumber	forks[MAX_FORKNUM];
+	BlockNumber	blocks[MAX_FORKNUM];
+	BlockNumber	new_nfsmblocks = InvalidBlockNumber;
+	BlockNumber	newnblocks = InvalidBlockNumber;
+	int		nforks = 0;
 
 	/* Open it at the smgr level if not already done */
 	RelationOpenSmgr(rel);
@@ -242,15 +247,34 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 	rel->rd_smgr->smgr_fsm_nblocks = InvalidBlockNumber;
 	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
 
-	/* Truncate the FSM first if it exists */
+	/* Mark the dirty FSM page and return a block number. */
 	fsm = smgrexists(rel->rd_smgr, FSM_FORKNUM);
 	if (fsm)
-		FreeSpaceMapTruncateRel(rel, nblocks);
+	{
+		blocks[nforks] = MarkFreeSpaceMapTruncateRel(rel, nblocks);
+		if (BlockNumberIsValid(blocks[nforks]))
+		{
+			forks[nforks] = FSM_FORKNUM;
+			new_nfsmblocks= blocks[nforks];
+			nforks++;
+		}
+	}
 
-	/* Truncate the visibility map too if it exists. */
+	/*
+	 * Truncate only the tail bits of VM and return the block number
+	 * for actual truncation later in smgrtruncate.
+	 */
 	vm = smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM);
 	if (vm)
-		visibilitymap_truncate(rel, nblocks);
+	{
+		blocks[nforks] = visibilitymap_truncate_prepare(rel, nblocks);
+		if (BlockNumberIsValid(blocks[nforks]))
+		{
+			forks[nforks] = VISIBILITYMAP_FORKNUM;
+			newnblocks = blocks[nforks];
+			nforks++;
+		}
+	}
 
 	/*
 	 * We WAL-log the truncation before actually truncating, which means
@@ -290,8 +314,20 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 			XLogFlush(lsn);
 	}
 
-	/* Do the real work */
-	smgrtruncate(rel->rd_smgr, MAIN_FORKNUM, nblocks);
+	/* Mark the MAIN fork */
+	forks[nforks] = MAIN_FORKNUM;
+	blocks[nforks] = nblocks;
+	nforks++;
+
+	/* Truncate relation forks simultaneously */
+	smgrtruncate(rel->rd_smgr, forks, nforks, blocks);
+
+	/*
+	 * Update upper-level FSM pages to account for the truncation.
+	 * This is important because the just-truncated pages were likely
+	 * marked as all-free, and would be preferentially selected.
+	 */
+	FreeSpaceMapVacuumRange(rel->rd_smgr, new_nfsmblocks, InvalidBlockNumber);
 }
 
 /*
@@ -588,6 +624,14 @@ smgr_redo(XLogReaderState *record)
 		xl_smgr_truncate *xlrec = (xl_smgr_truncate *) XLogRecGetData(record);
 		SMgrRelation reln;
 		Relation	rel;
+		ForkNumber	forks[MAX_FORKNUM];
+		BlockNumber	blocks[MAX_FORKNUM];
+		BlockNumber	new_nfsmblocks = InvalidBlockNumber;
+		BlockNumber	newnblocks = InvalidBlockNumber;
+		int		nforks = 0;
+		bool		fsm_fork = false;
+		bool		main_fork = false;
+		bool		vm_fork = false;
 
 		reln = smgropen(xlrec->rnode, InvalidBackendId);
 
@@ -616,23 +660,52 @@ smgr_redo(XLogReaderState *record)
 		 */
 		XLogFlush(lsn);
 
+		/*
+		 * To speedup recovery, we mark the about-to-be-truncated blocks of
+		 * relation forks first, then truncate those simultaneously later.
+		 */
 		if ((xlrec->flags & SMGR_TRUNCATE_HEAP) != 0)
 		{
-			smgrtruncate(reln, MAIN_FORKNUM, xlrec->blkno);
-
-			/* Also tell xlogutils.c about it */
-			XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
+			forks[nforks] = MAIN_FORKNUM;
+			blocks[nforks] = xlrec->blkno;
+			nforks++;
+			main_fork = true;
 		}
 
-		/* Truncate FSM and VM too */
 		rel = CreateFakeRelcacheEntry(xlrec->rnode);
 
 		if ((xlrec->flags & SMGR_TRUNCATE_FSM) != 0 &&
 			smgrexists(reln, FSM_FORKNUM))
-			FreeSpaceMapTruncateRel(rel, xlrec->blkno);
+		{
+			blocks[nforks] = MarkFreeSpaceMapTruncateRel(rel, xlrec->blkno);
+			if (BlockNumberIsValid(blocks[nforks]))
+			{
+				forks[nforks] = FSM_FORKNUM;
+				new_nfsmblocks= blocks[nforks];
+				nforks++;
+				fsm_fork = true;
+			}
+		}
 		if ((xlrec->flags & SMGR_TRUNCATE_VM) != 0 &&
 			smgrexists(reln, VISIBILITYMAP_FORKNUM))
-			visibilitymap_truncate(rel, xlrec->blkno);
+		{
+			blocks[nforks] = visibilitymap_truncate_prepare(rel, xlrec->blkno);
+			if (BlockNumberIsValid(blocks[nforks]))
+			{
+				forks[nforks] = VISIBILITYMAP_FORKNUM;
+				newnblocks = blocks[nforks];
+				nforks++;
+				vm_fork = true;
+			}
+		}
+
+		/* Truncate relation forks simultaneously */
+		if (main_fork || fsm_fork || vm_fork)
+			smgrtruncate(reln, forks, nforks, blocks);
+
+		/* Also tell xlogutils.c about it */
+		if (main_fork)
+			XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
 
 		FreeFakeRelcacheEntry(rel);
 	}
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index 7332e6b..512c8a1 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -2899,8 +2899,8 @@ BufferGetLSNAtomic(Buffer buffer)
 /* ---------------------------------------------------------------------
  *		DropRelFileNodeBuffers
  *
- *		This function removes from the buffer pool all the pages of the
- *		specified relation fork that have block numbers >= firstDelBlock.
+ *		This function simultaneously removes from the buffer pool all the
+ *		pages of the relation forks that have block numbers >= firstDelBlock.
  *		(In particular, with firstDelBlock = 0, all pages are removed.)
  *		Dirty pages are simply dropped, without bothering to write them
  *		out first.  Therefore, this is NOT rollback-able, and so should be
@@ -2923,23 +2923,36 @@ BufferGetLSNAtomic(Buffer buffer)
  * --------------------------------------------------------------------
  */
 void
-DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
-					   BlockNumber firstDelBlock)
+DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
+					   int nforks, BlockNumber *firstDelBlock)
 {
-	int			i;
+	BlockNumber minBlock = InvalidBlockNumber;
 
 	/* If it's a local relation, it's localbuf.c's problem. */
 	if (RelFileNodeBackendIsTemp(rnode))
 	{
 		if (rnode.backend == MyBackendId)
-			DropRelFileNodeLocalBuffers(rnode.node, forkNum, firstDelBlock);
+		{
+			for (int i = 0; i < nforks; i++)
+				DropRelFileNodeLocalBuffers(rnode.node, forkNum[i],
+											firstDelBlock[i]);
+		}
 		return;
 	}
 
-	for (i = 0; i < NBuffers; i++)
+	/* Get the lower bound of target block number we're interested in */
+	for (int i = 0; i < nforks; i++)
+	{
+		if (!BlockNumberIsValid(minBlock) ||
+			minBlock > firstDelBlock[i])
+			minBlock = firstDelBlock[i];
+	}
+
+	for (int i = 0; i < NBuffers; i++)
 	{
 		BufferDesc *bufHdr = GetBufferDescriptor(i);
 		uint32		buf_state;
+		int		j = 0;
 
 		/*
 		 * We can make this a tad faster by prechecking the buffer tag before
@@ -2960,12 +2973,23 @@ DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
 		if (!RelFileNodeEquals(bufHdr->tag.rnode, rnode.node))
 			continue;
 
+		/* Check with the lower bound block number and skip the loop */
+		if (bufHdr->tag.blockNum < minBlock)
+			continue; /* skip checking the buffer pool scan */
+
 		buf_state = LockBufHdr(bufHdr);
-		if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
-			bufHdr->tag.forkNum == forkNum &&
-			bufHdr->tag.blockNum >= firstDelBlock)
-			InvalidateBuffer(bufHdr);	/* releases spinlock */
-		else
+
+		for (j = 0; j < nforks; j++)
+		{
+			if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
+				bufHdr->tag.forkNum == forkNum[j] &&
+				bufHdr->tag.blockNum >= firstDelBlock[j])
+			{
+				InvalidateBuffer(bufHdr); /* releases spinlock */
+				break;
+			}
+		}
+		if (j >= nforks)
 			UnlockBufHdr(bufHdr, buf_state);
 	}
 }
diff --git a/src/backend/storage/freespace/freespace.c b/src/backend/storage/freespace/freespace.c
index c17b3f4..9c29604 100644
--- a/src/backend/storage/freespace/freespace.c
+++ b/src/backend/storage/freespace/freespace.c
@@ -247,16 +247,16 @@ GetRecordedFreeSpace(Relation rel, BlockNumber heapBlk)
 }
 
 /*
- * FreeSpaceMapTruncateRel - adjust for truncation of a relation.
+ * MarkFreeSpaceMapTruncateRel - adjust for truncation of a relation.
  *
- * The caller must hold AccessExclusiveLock on the relation, to ensure that
- * other backends receive the smgr invalidation event that this function sends
- * before they access the FSM again.
+ * This function marks the dirty page and returns a block number.
+ * The caller of this function must eventually call smgrtruncate() to actually
+ * truncate FSM pages.
  *
  * nblocks is the new size of the heap.
  */
-void
-FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
+BlockNumber
+MarkFreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 {
 	BlockNumber new_nfsmblocks;
 	FSMAddress	first_removed_address;
@@ -270,7 +270,7 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 	 * truncate.
 	 */
 	if (!smgrexists(rel->rd_smgr, FSM_FORKNUM))
-		return;
+		return InvalidBlockNumber;
 
 	/* Get the location in the FSM of the first removed heap block */
 	first_removed_address = fsm_get_location(nblocks, &first_removed_slot);
@@ -285,7 +285,7 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 	{
 		buf = fsm_readbuf(rel, first_removed_address, false);
 		if (!BufferIsValid(buf))
-			return;				/* nothing to do; the FSM was already smaller */
+			return InvalidBlockNumber;	/* nothing to do; the FSM was already smaller */
 		LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
 
 		/* NO EREPORT(ERROR) from here till changes are logged */
@@ -310,33 +310,16 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 		UnlockReleaseBuffer(buf);
 
 		new_nfsmblocks = fsm_logical_to_physical(first_removed_address) + 1;
+		return new_nfsmblocks;
 	}
 	else
 	{
 		new_nfsmblocks = fsm_logical_to_physical(first_removed_address);
 		if (smgrnblocks(rel->rd_smgr, FSM_FORKNUM) <= new_nfsmblocks)
-			return;				/* nothing to do; the FSM was already smaller */
+			return InvalidBlockNumber;	/* nothing to do; the FSM was already smaller */
+		else
+			return new_nfsmblocks;
 	}
-
-	/* Truncate the unused FSM pages, and send smgr inval message */
-	smgrtruncate(rel->rd_smgr, FSM_FORKNUM, new_nfsmblocks);
-
-	/*
-	 * We might as well update the local smgr_fsm_nblocks setting.
-	 * smgrtruncate sent an smgr cache inval message, which will cause other
-	 * backends to invalidate their copy of smgr_fsm_nblocks, and this one too
-	 * at the next command boundary.  But this ensures it isn't outright wrong
-	 * until then.
-	 */
-	if (rel->rd_smgr)
-		rel->rd_smgr->smgr_fsm_nblocks = new_nfsmblocks;
-
-	/*
-	 * Update upper-level FSM pages to account for the truncation.  This is
-	 * important because the just-truncated pages were likely marked as
-	 * all-free, and would be preferentially selected.
-	 */
-	FreeSpaceMapVacuumRange(rel, nblocks, InvalidBlockNumber);
 }
 
 /*
diff --git a/src/backend/storage/smgr/smgr.c b/src/backend/storage/smgr/smgr.c
index dba8c39..ea57de7 100644
--- a/src/backend/storage/smgr/smgr.c
+++ b/src/backend/storage/smgr/smgr.c
@@ -498,29 +498,30 @@ smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo)
 }
 
 /*
- *	smgrdounlinkfork() -- Immediately unlink one fork of a relation.
+ *	smgrdounlinkfork() -- Immediately unlink each fork of a relation.
  *
- *		The specified fork of the relation is removed from the store.  This
- *		should not be used during transactional operations, since it can't be
- *		undone.
+ *		Each fork of the relation is removed from the store.  This should
+ *		not be used during transactional operations, since it can't be undone.
  *
  *		If isRedo is true, it is okay for the underlying file to be gone
  *		already.
  */
 void
-smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo)
+smgrdounlinkfork(SMgrRelation reln, ForkNumber *forknum, int nforks, bool isRedo)
 {
 	RelFileNodeBackend rnode = reln->smgr_rnode;
 	int			which = reln->smgr_which;
+	int			i;
 
-	/* Close the fork at smgr level */
-	smgrsw[which].smgr_close(reln, forknum);
+	/* Close each fork at smgr level */
+	for (i = 0; i < nforks; i++)
+		smgrsw[which].smgr_close(reln, forknum[i]);
 
 	/*
-	 * Get rid of any remaining buffers for the fork.  bufmgr will just drop
+	 * Get rid of any remaining buffers for each fork. bufmgr will just drop
 	 * them without bothering to write the contents.
 	 */
-	DropRelFileNodeBuffers(rnode, forknum, 0);
+	DropRelFileNodeBuffers(rnode, forknum, nforks, 0);
 
 	/*
 	 * It'd be nice to tell the stats collector to forget it immediately, too.
@@ -546,7 +547,8 @@ smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo)
 	 * ERROR, because we've already decided to commit or abort the current
 	 * xact.
 	 */
-	smgrsw[which].smgr_unlink(rnode, forknum, isRedo);
+	for (i = 0; i < nforks; i++)
+		smgrsw[which].smgr_unlink(rnode, forknum[i], isRedo);
 }
 
 /*
@@ -643,13 +645,15 @@ smgrnblocks(SMgrRelation reln, ForkNumber forknum)
  * The truncation is done immediately, so this can't be rolled back.
  */
 void
-smgrtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks)
+smgrtruncate(SMgrRelation reln, ForkNumber *forknum, int nforks, BlockNumber *nblocks)
 {
+	int		i;
+
 	/*
 	 * Get rid of any buffers for the about-to-be-deleted blocks. bufmgr will
 	 * just drop them without bothering to write the contents.
 	 */
-	DropRelFileNodeBuffers(reln->smgr_rnode, forknum, nblocks);
+	DropRelFileNodeBuffers(reln->smgr_rnode, forknum, nforks, nblocks);
 
 	/*
 	 * Send a shared-inval message to force other backends to close any smgr
@@ -663,10 +667,23 @@ smgrtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks)
 	 */
 	CacheInvalidateSmgr(reln->smgr_rnode);
 
-	/*
-	 * Do the truncation.
-	 */
-	smgrsw[reln->smgr_which].smgr_truncate(reln, forknum, nblocks);
+	/* Do the truncation */
+	for (i = 0; i < nforks; i++)
+	{
+		smgrsw[reln->smgr_which].smgr_truncate(reln, forknum[i], nblocks[i]);
+
+		/*
+		 * We might as well update the local smgr_fsm_nblocks and smgr_vm_nblocks
+		 * setting. smgrtruncate sent an smgr cache inval message, which will
+		 * cause other backends to invalidate their copy of smgr_fsm_nblocks and
+		 * smgr_vm_nblocks, and these ones too at the next command boundary. But
+		 * this ensures these aren't outright wrong until then.
+		 */
+		if (forknum[i] == FSM_FORKNUM)
+			reln->smgr_fsm_nblocks = nblocks[i];
+		if (forknum[i] == VISIBILITYMAP_FORKNUM)
+			reln->smgr_vm_nblocks = nblocks[i];
+	}
 }
 
 /*
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index 2d88043..1ab6a81 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -44,6 +44,6 @@ extern void visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf,
 							  uint8 flags);
 extern uint8 visibilitymap_get_status(Relation rel, BlockNumber heapBlk, Buffer *vmbuf);
 extern void visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_frozen);
-extern void visibilitymap_truncate(Relation rel, BlockNumber nheapblocks);
+extern BlockNumber visibilitymap_truncate_prepare(Relation rel, BlockNumber nheapblocks);
 
 #endif							/* VISIBILITYMAP_H */
diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h
index 509f4b7..17b97f7 100644
--- a/src/include/storage/bufmgr.h
+++ b/src/include/storage/bufmgr.h
@@ -190,8 +190,8 @@ extern BlockNumber RelationGetNumberOfBlocksInFork(Relation relation,
 extern void FlushOneBuffer(Buffer buffer);
 extern void FlushRelationBuffers(Relation rel);
 extern void FlushDatabaseBuffers(Oid dbid);
-extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode,
-								   ForkNumber forkNum, BlockNumber firstDelBlock);
+extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
+								   int nforks, BlockNumber *firstDelBlock);
 extern void DropRelFileNodesAllBuffers(RelFileNodeBackend *rnodes, int nnodes);
 extern void DropDatabaseBuffers(Oid dbid);
 
diff --git a/src/include/storage/freespace.h b/src/include/storage/freespace.h
index 8d8c465..bf19a67 100644
--- a/src/include/storage/freespace.h
+++ b/src/include/storage/freespace.h
@@ -30,7 +30,7 @@ extern void RecordPageWithFreeSpace(Relation rel, BlockNumber heapBlk,
 extern void XLogRecordPageWithFreeSpace(RelFileNode rnode, BlockNumber heapBlk,
 										Size spaceAvail);
 
-extern void FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks);
+extern BlockNumber MarkFreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks);
 extern void FreeSpaceMapVacuum(Relation rel);
 extern void FreeSpaceMapVacuumRange(Relation rel, BlockNumber start,
 									BlockNumber end);
diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h
index d286c8c..a24532c 100644
--- a/src/include/storage/smgr.h
+++ b/src/include/storage/smgr.h
@@ -90,7 +90,8 @@ extern void smgrclosenode(RelFileNodeBackend rnode);
 extern void smgrcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo);
 extern void smgrdounlink(SMgrRelation reln, bool isRedo);
 extern void smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo);
-extern void smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo);
+extern void smgrdounlinkfork(SMgrRelation reln, ForkNumber *forknum,
+							 int nforks, bool isRedo);
 extern void smgrextend(SMgrRelation reln, ForkNumber forknum,
 					   BlockNumber blocknum, char *buffer, bool skipFsync);
 extern void smgrprefetch(SMgrRelation reln, ForkNumber forknum,
@@ -102,8 +103,8 @@ extern void smgrwrite(SMgrRelation reln, ForkNumber forknum,
 extern void smgrwriteback(SMgrRelation reln, ForkNumber forknum,
 						  BlockNumber blocknum, BlockNumber nblocks);
 extern BlockNumber smgrnblocks(SMgrRelation reln, ForkNumber forknum);
-extern void smgrtruncate(SMgrRelation reln, ForkNumber forknum,
-						 BlockNumber nblocks);
+extern void smgrtruncate(SMgrRelation reln, ForkNumber *forknum,
+						 int nforks, BlockNumber *nblocks);
 extern void smgrimmedsync(SMgrRelation reln, ForkNumber forknum);
 extern void AtEOXact_SMgr(void);
 
-- 
1.8.3.1

From 969e07a05b0b067cba1ca02f549c522fba5873d0 Mon Sep 17 00:00:00 2001
From: Kirk Jamison <k.jamison@jp.fujitsu.com>
Date: Thu, 4 Jul 2019 10:59:05 +0000
Subject: [PATCH] Speedup truncates of relation forks

Whenever we truncate relations, it involves several scans of the
shared buffers for every call of smgrtruncate() for each fork which
is time-consuming. This patch reduces the scan for all forks into one
instead of three, and improves the relation truncates by initially
marking the pages-to-be-truncated of relation forks, then
simultaneously truncating them, resulting to an improved performance
in VACUUM, autovacuum operations and their recovery performance.
---
 contrib/pg_visibility/pg_visibility.c     |  11 ++-
 src/backend/access/heap/visibilitymap.c   |  36 ++++------
 src/backend/catalog/storage.c             | 111 ++++++++++++++++++++++++++----
 src/backend/storage/buffer/bufmgr.c       |  48 +++++++++----
 src/backend/storage/freespace/freespace.c |  41 ++++-------
 src/backend/storage/smgr/smgr.c           |  49 ++++++++-----
 src/include/access/visibilitymap.h        |   2 +-
 src/include/storage/bufmgr.h              |   4 +-
 src/include/storage/freespace.h           |   2 +-
 src/include/storage/smgr.h                |   7 +-
 10 files changed, 211 insertions(+), 100 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index 1372bb6..1aabde2 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -383,6 +383,9 @@ pg_truncate_visibility_map(PG_FUNCTION_ARGS)
 {
 	Oid			relid = PG_GETARG_OID(0);
 	Relation	rel;
+	ForkNumber	fork;
+	BlockNumber	block;
+	BlockNumber	newnblocks = InvalidBlockNumber;
 
 	rel = relation_open(relid, AccessExclusiveLock);
 
@@ -392,7 +395,13 @@ pg_truncate_visibility_map(PG_FUNCTION_ARGS)
 	RelationOpenSmgr(rel);
 	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
 
-	visibilitymap_truncate(rel, 0);
+	block = visibilitymap_truncate_prepare(rel, 0);
+	if (BlockNumberIsValid(block)
+	{
+		fork = VISIBILITYMAP_FORKNUM;
+		newnblocks = block;
+	}
+	smgrtruncate(rel->rd_smgr, &fork, 1, &block);
 
 	if (RelationNeedsWAL(rel))
 	{
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index 64dfe06..4cc7977 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -17,7 +17,7 @@
  *		visibilitymap_set	 - set a bit in a previously pinned page
  *		visibilitymap_get_status - get status of bits
  *		visibilitymap_count  - count number of bits set in visibility map
- *		visibilitymap_truncate	- truncate the visibility map
+ *		visibilitymap_truncate_prepare - truncate only tail bits of map pages
  *
  * NOTES
  *
@@ -430,16 +430,18 @@ visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_fro
 }
 
 /*
- *	visibilitymap_truncate - truncate the visibility map
+ *	visibilitymap_truncate_prepare - truncate only tail bits of map page
+ *									 and return the block number for actual
+ *									 truncation later
  *
- * The caller must hold AccessExclusiveLock on the relation, to ensure that
- * other backends receive the smgr invalidation event that this function sends
- * before they access the VM again.
+ * Note that this does not truncate the actual visibility map pages.
+ * When this function is called, the caller must eventually follow it with
+ * smgrtruncate() call to actually truncate visibility map pages.
  *
  * nheapblocks is the new size of the heap.
  */
-void
-visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
+BlockNumber
+visibilitymap_truncate_prepare(Relation rel, BlockNumber nheapblocks)
 {
 	BlockNumber newnblocks;
 
@@ -459,7 +461,7 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 	 * nothing to truncate.
 	 */
 	if (!smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM))
-		return;
+		return InvalidBlockNumber;
 
 	/*
 	 * Unless the new size is exactly at a visibility map page boundary, the
@@ -480,7 +482,7 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 		if (!BufferIsValid(mapBuffer))
 		{
 			/* nothing to do, the file was already smaller */
-			return;
+			return InvalidBlockNumber;
 		}
 
 		page = BufferGetPage(mapBuffer);
@@ -528,20 +530,10 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 	if (smgrnblocks(rel->rd_smgr, VISIBILITYMAP_FORKNUM) <= newnblocks)
 	{
 		/* nothing to do, the file was already smaller than requested size */
-		return;
+		return InvalidBlockNumber;
 	}
-
-	/* Truncate the unused VM pages, and send smgr inval message */
-	smgrtruncate(rel->rd_smgr, VISIBILITYMAP_FORKNUM, newnblocks);
-
-	/*
-	 * We might as well update the local smgr_vm_nblocks setting. smgrtruncate
-	 * sent an smgr cache inval message, which will cause other backends to
-	 * invalidate their copy of smgr_vm_nblocks, and this one too at the next
-	 * command boundary.  But this ensures it isn't outright wrong until then.
-	 */
-	if (rel->rd_smgr)
-		rel->rd_smgr->smgr_vm_nblocks = newnblocks;
+	else
+		return newnblocks;
 }
 
 /*
diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c
index 3cc886f..c9fd637 100644
--- a/src/backend/catalog/storage.c
+++ b/src/backend/catalog/storage.c
@@ -231,6 +231,12 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 {
 	bool		fsm;
 	bool		vm;
+	ForkNumber	forks[MAX_FORKNUM];
+	BlockNumber	blocks[MAX_FORKNUM];
+	BlockNumber	first_removed_nblocks = InvalidBlockNumber;
+	BlockNumber	new_nfsmblocks = InvalidBlockNumber;
+	BlockNumber	newnblocks = InvalidBlockNumber;
+	int		nforks = 0;
 
 	/* Open it at the smgr level if not already done */
 	RelationOpenSmgr(rel);
@@ -242,15 +248,35 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 	rel->rd_smgr->smgr_fsm_nblocks = InvalidBlockNumber;
 	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
 
-	/* Truncate the FSM first if it exists */
+	/* Mark the dirty FSM page and return a block number. */
 	fsm = smgrexists(rel->rd_smgr, FSM_FORKNUM);
 	if (fsm)
-		FreeSpaceMapTruncateRel(rel, nblocks);
+	{
+		blocks[nforks] = MarkFreeSpaceMapTruncateRel(rel, nblocks);
+		if (BlockNumberIsValid(blocks[nforks]))
+		{
+			first_removed_nblocks = nblocks;
+			forks[nforks] = FSM_FORKNUM;
+			new_nfsmblocks= blocks[nforks];
+			nforks++;
+		}
+	}
 
-	/* Truncate the visibility map too if it exists. */
+	/*
+	 * Truncate only the tail bits of VM and return the block number
+	 * for actual truncation later in smgrtruncate.
+	 */
 	vm = smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM);
 	if (vm)
-		visibilitymap_truncate(rel, nblocks);
+	{
+		blocks[nforks] = visibilitymap_truncate_prepare(rel, nblocks);
+		if (BlockNumberIsValid(blocks[nforks]))
+		{
+			forks[nforks] = VISIBILITYMAP_FORKNUM;
+			newnblocks = blocks[nforks];
+			nforks++;
+		}
+	}
 
 	/*
 	 * We WAL-log the truncation before actually truncating, which means
@@ -290,8 +316,21 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 			XLogFlush(lsn);
 	}
 
-	/* Do the real work */
-	smgrtruncate(rel->rd_smgr, MAIN_FORKNUM, nblocks);
+	/* Mark the MAIN fork */
+	forks[nforks] = MAIN_FORKNUM;
+	blocks[nforks] = nblocks;
+	nforks++;
+
+	/* Truncate relation forks simultaneously */
+	smgrtruncate(rel->rd_smgr, forks, nforks, blocks);
+
+	/*
+	 * Update upper-level FSM pages to account for the truncation.
+	 * This is important because the just-truncated pages were likely
+	 * marked as all-free, and would be preferentially selected.
+	 */
+	//FreeSpaceMapVacuumRange(rel, new_nfsmblocks, InvalidBlockNumber);
+	FreeSpaceMapVacuumRange(rel, first_removed_nblocks, InvalidBlockNumber);
 }
 
 /*
@@ -588,6 +627,15 @@ smgr_redo(XLogReaderState *record)
 		xl_smgr_truncate *xlrec = (xl_smgr_truncate *) XLogRecGetData(record);
 		SMgrRelation reln;
 		Relation	rel;
+		ForkNumber	forks[MAX_FORKNUM];
+		BlockNumber	blocks[MAX_FORKNUM];
+		BlockNumber	first_removed_nblocks = InvalidBlockNumber;
+		BlockNumber	new_nfsmblocks = InvalidBlockNumber;
+		BlockNumber	newnblocks = InvalidBlockNumber;
+		int		nforks = 0;
+		bool		fsm_fork = false;
+		bool		main_fork = false;
+		bool		vm_fork = false;
 
 		reln = smgropen(xlrec->rnode, InvalidBackendId);
 
@@ -616,23 +664,60 @@ smgr_redo(XLogReaderState *record)
 		 */
 		XLogFlush(lsn);
 
+		/*
+		 * To speedup recovery, we mark the about-to-be-truncated blocks of
+		 * relation forks first, then truncate those simultaneously later.
+		 */
 		if ((xlrec->flags & SMGR_TRUNCATE_HEAP) != 0)
 		{
-			smgrtruncate(reln, MAIN_FORKNUM, xlrec->blkno);
-
-			/* Also tell xlogutils.c about it */
-			XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
+			forks[nforks] = MAIN_FORKNUM;
+			blocks[nforks] = xlrec->blkno;
+			nforks++;
+			main_fork = true;
 		}
 
-		/* Truncate FSM and VM too */
 		rel = CreateFakeRelcacheEntry(xlrec->rnode);
 
 		if ((xlrec->flags & SMGR_TRUNCATE_FSM) != 0 &&
 			smgrexists(reln, FSM_FORKNUM))
-			FreeSpaceMapTruncateRel(rel, xlrec->blkno);
+		{
+			blocks[nforks] = MarkFreeSpaceMapTruncateRel(rel, xlrec->blkno);
+			if (BlockNumberIsValid(blocks[nforks]))
+			{
+				first_removed_nblocks = xlrec->blkno;
+				forks[nforks] = FSM_FORKNUM;
+				new_nfsmblocks= blocks[nforks];
+				nforks++;
+				fsm_fork = true;
+			}
+		}
 		if ((xlrec->flags & SMGR_TRUNCATE_VM) != 0 &&
 			smgrexists(reln, VISIBILITYMAP_FORKNUM))
-			visibilitymap_truncate(rel, xlrec->blkno);
+		{
+			blocks[nforks] = visibilitymap_truncate_prepare(rel, xlrec->blkno);
+			if (BlockNumberIsValid(blocks[nforks]))
+			{
+				forks[nforks] = VISIBILITYMAP_FORKNUM;
+				newnblocks = blocks[nforks];
+				nforks++;
+				vm_fork = true;
+			}
+		}
+
+		/* Truncate relation forks simultaneously */
+		if (main_fork || fsm_fork || vm_fork)
+			smgrtruncate(reln, forks, nforks, blocks);
+
+		/* Also tell xlogutils.c about it */
+		if (main_fork)
+			XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
+
+		/*
+		 * Update upper-level FSM pages to account for the truncation.
+		 * This is important because the just-truncated pages were likely
+		 * marked as all-free, and would be preferentially selected.
+		 */
+		FreeSpaceMapVacuumRange(rel, first_removed_nblocks, InvalidBlockNumber);
 
 		FreeFakeRelcacheEntry(rel);
 	}
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index 7332e6b..512c8a1 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -2899,8 +2899,8 @@ BufferGetLSNAtomic(Buffer buffer)
 /* ---------------------------------------------------------------------
  *		DropRelFileNodeBuffers
  *
- *		This function removes from the buffer pool all the pages of the
- *		specified relation fork that have block numbers >= firstDelBlock.
+ *		This function simultaneously removes from the buffer pool all the
+ *		pages of the relation forks that have block numbers >= firstDelBlock.
  *		(In particular, with firstDelBlock = 0, all pages are removed.)
  *		Dirty pages are simply dropped, without bothering to write them
  *		out first.  Therefore, this is NOT rollback-able, and so should be
@@ -2923,23 +2923,36 @@ BufferGetLSNAtomic(Buffer buffer)
  * --------------------------------------------------------------------
  */
 void
-DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
-					   BlockNumber firstDelBlock)
+DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
+					   int nforks, BlockNumber *firstDelBlock)
 {
-	int			i;
+	BlockNumber minBlock = InvalidBlockNumber;
 
 	/* If it's a local relation, it's localbuf.c's problem. */
 	if (RelFileNodeBackendIsTemp(rnode))
 	{
 		if (rnode.backend == MyBackendId)
-			DropRelFileNodeLocalBuffers(rnode.node, forkNum, firstDelBlock);
+		{
+			for (int i = 0; i < nforks; i++)
+				DropRelFileNodeLocalBuffers(rnode.node, forkNum[i],
+											firstDelBlock[i]);
+		}
 		return;
 	}
 
-	for (i = 0; i < NBuffers; i++)
+	/* Get the lower bound of target block number we're interested in */
+	for (int i = 0; i < nforks; i++)
+	{
+		if (!BlockNumberIsValid(minBlock) ||
+			minBlock > firstDelBlock[i])
+			minBlock = firstDelBlock[i];
+	}
+
+	for (int i = 0; i < NBuffers; i++)
 	{
 		BufferDesc *bufHdr = GetBufferDescriptor(i);
 		uint32		buf_state;
+		int		j = 0;
 
 		/*
 		 * We can make this a tad faster by prechecking the buffer tag before
@@ -2960,12 +2973,23 @@ DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
 		if (!RelFileNodeEquals(bufHdr->tag.rnode, rnode.node))
 			continue;
 
+		/* Check with the lower bound block number and skip the loop */
+		if (bufHdr->tag.blockNum < minBlock)
+			continue; /* skip checking the buffer pool scan */
+
 		buf_state = LockBufHdr(bufHdr);
-		if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
-			bufHdr->tag.forkNum == forkNum &&
-			bufHdr->tag.blockNum >= firstDelBlock)
-			InvalidateBuffer(bufHdr);	/* releases spinlock */
-		else
+
+		for (j = 0; j < nforks; j++)
+		{
+			if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
+				bufHdr->tag.forkNum == forkNum[j] &&
+				bufHdr->tag.blockNum >= firstDelBlock[j])
+			{
+				InvalidateBuffer(bufHdr); /* releases spinlock */
+				break;
+			}
+		}
+		if (j >= nforks)
 			UnlockBufHdr(bufHdr, buf_state);
 	}
 }
diff --git a/src/backend/storage/freespace/freespace.c b/src/backend/storage/freespace/freespace.c
index c17b3f4..9c29604 100644
--- a/src/backend/storage/freespace/freespace.c
+++ b/src/backend/storage/freespace/freespace.c
@@ -247,16 +247,16 @@ GetRecordedFreeSpace(Relation rel, BlockNumber heapBlk)
 }
 
 /*
- * FreeSpaceMapTruncateRel - adjust for truncation of a relation.
+ * MarkFreeSpaceMapTruncateRel - adjust for truncation of a relation.
  *
- * The caller must hold AccessExclusiveLock on the relation, to ensure that
- * other backends receive the smgr invalidation event that this function sends
- * before they access the FSM again.
+ * This function marks the dirty page and returns a block number.
+ * The caller of this function must eventually call smgrtruncate() to actually
+ * truncate FSM pages.
  *
  * nblocks is the new size of the heap.
  */
-void
-FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
+BlockNumber
+MarkFreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 {
 	BlockNumber new_nfsmblocks;
 	FSMAddress	first_removed_address;
@@ -270,7 +270,7 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 	 * truncate.
 	 */
 	if (!smgrexists(rel->rd_smgr, FSM_FORKNUM))
-		return;
+		return InvalidBlockNumber;
 
 	/* Get the location in the FSM of the first removed heap block */
 	first_removed_address = fsm_get_location(nblocks, &first_removed_slot);
@@ -285,7 +285,7 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 	{
 		buf = fsm_readbuf(rel, first_removed_address, false);
 		if (!BufferIsValid(buf))
-			return;				/* nothing to do; the FSM was already smaller */
+			return InvalidBlockNumber;	/* nothing to do; the FSM was already smaller */
 		LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
 
 		/* NO EREPORT(ERROR) from here till changes are logged */
@@ -310,33 +310,16 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 		UnlockReleaseBuffer(buf);
 
 		new_nfsmblocks = fsm_logical_to_physical(first_removed_address) + 1;
+		return new_nfsmblocks;
 	}
 	else
 	{
 		new_nfsmblocks = fsm_logical_to_physical(first_removed_address);
 		if (smgrnblocks(rel->rd_smgr, FSM_FORKNUM) <= new_nfsmblocks)
-			return;				/* nothing to do; the FSM was already smaller */
+			return InvalidBlockNumber;	/* nothing to do; the FSM was already smaller */
+		else
+			return new_nfsmblocks;
 	}
-
-	/* Truncate the unused FSM pages, and send smgr inval message */
-	smgrtruncate(rel->rd_smgr, FSM_FORKNUM, new_nfsmblocks);
-
-	/*
-	 * We might as well update the local smgr_fsm_nblocks setting.
-	 * smgrtruncate sent an smgr cache inval message, which will cause other
-	 * backends to invalidate their copy of smgr_fsm_nblocks, and this one too
-	 * at the next command boundary.  But this ensures it isn't outright wrong
-	 * until then.
-	 */
-	if (rel->rd_smgr)
-		rel->rd_smgr->smgr_fsm_nblocks = new_nfsmblocks;
-
-	/*
-	 * Update upper-level FSM pages to account for the truncation.  This is
-	 * important because the just-truncated pages were likely marked as
-	 * all-free, and would be preferentially selected.
-	 */
-	FreeSpaceMapVacuumRange(rel, nblocks, InvalidBlockNumber);
 }
 
 /*
diff --git a/src/backend/storage/smgr/smgr.c b/src/backend/storage/smgr/smgr.c
index dba8c39..ea57de7 100644
--- a/src/backend/storage/smgr/smgr.c
+++ b/src/backend/storage/smgr/smgr.c
@@ -498,29 +498,30 @@ smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo)
 }
 
 /*
- *	smgrdounlinkfork() -- Immediately unlink one fork of a relation.
+ *	smgrdounlinkfork() -- Immediately unlink each fork of a relation.
  *
- *		The specified fork of the relation is removed from the store.  This
- *		should not be used during transactional operations, since it can't be
- *		undone.
+ *		Each fork of the relation is removed from the store.  This should
+ *		not be used during transactional operations, since it can't be undone.
  *
  *		If isRedo is true, it is okay for the underlying file to be gone
  *		already.
  */
 void
-smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo)
+smgrdounlinkfork(SMgrRelation reln, ForkNumber *forknum, int nforks, bool isRedo)
 {
 	RelFileNodeBackend rnode = reln->smgr_rnode;
 	int			which = reln->smgr_which;
+	int			i;
 
-	/* Close the fork at smgr level */
-	smgrsw[which].smgr_close(reln, forknum);
+	/* Close each fork at smgr level */
+	for (i = 0; i < nforks; i++)
+		smgrsw[which].smgr_close(reln, forknum[i]);
 
 	/*
-	 * Get rid of any remaining buffers for the fork.  bufmgr will just drop
+	 * Get rid of any remaining buffers for each fork. bufmgr will just drop
 	 * them without bothering to write the contents.
 	 */
-	DropRelFileNodeBuffers(rnode, forknum, 0);
+	DropRelFileNodeBuffers(rnode, forknum, nforks, 0);
 
 	/*
 	 * It'd be nice to tell the stats collector to forget it immediately, too.
@@ -546,7 +547,8 @@ smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo)
 	 * ERROR, because we've already decided to commit or abort the current
 	 * xact.
 	 */
-	smgrsw[which].smgr_unlink(rnode, forknum, isRedo);
+	for (i = 0; i < nforks; i++)
+		smgrsw[which].smgr_unlink(rnode, forknum[i], isRedo);
 }
 
 /*
@@ -643,13 +645,15 @@ smgrnblocks(SMgrRelation reln, ForkNumber forknum)
  * The truncation is done immediately, so this can't be rolled back.
  */
 void
-smgrtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks)
+smgrtruncate(SMgrRelation reln, ForkNumber *forknum, int nforks, BlockNumber *nblocks)
 {
+	int		i;
+
 	/*
 	 * Get rid of any buffers for the about-to-be-deleted blocks. bufmgr will
 	 * just drop them without bothering to write the contents.
 	 */
-	DropRelFileNodeBuffers(reln->smgr_rnode, forknum, nblocks);
+	DropRelFileNodeBuffers(reln->smgr_rnode, forknum, nforks, nblocks);
 
 	/*
 	 * Send a shared-inval message to force other backends to close any smgr
@@ -663,10 +667,23 @@ smgrtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks)
 	 */
 	CacheInvalidateSmgr(reln->smgr_rnode);
 
-	/*
-	 * Do the truncation.
-	 */
-	smgrsw[reln->smgr_which].smgr_truncate(reln, forknum, nblocks);
+	/* Do the truncation */
+	for (i = 0; i < nforks; i++)
+	{
+		smgrsw[reln->smgr_which].smgr_truncate(reln, forknum[i], nblocks[i]);
+
+		/*
+		 * We might as well update the local smgr_fsm_nblocks and smgr_vm_nblocks
+		 * setting. smgrtruncate sent an smgr cache inval message, which will
+		 * cause other backends to invalidate their copy of smgr_fsm_nblocks and
+		 * smgr_vm_nblocks, and these ones too at the next command boundary. But
+		 * this ensures these aren't outright wrong until then.
+		 */
+		if (forknum[i] == FSM_FORKNUM)
+			reln->smgr_fsm_nblocks = nblocks[i];
+		if (forknum[i] == VISIBILITYMAP_FORKNUM)
+			reln->smgr_vm_nblocks = nblocks[i];
+	}
 }
 
 /*
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index 2d88043..1ab6a81 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -44,6 +44,6 @@ extern void visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf,
 							  uint8 flags);
 extern uint8 visibilitymap_get_status(Relation rel, BlockNumber heapBlk, Buffer *vmbuf);
 extern void visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_frozen);
-extern void visibilitymap_truncate(Relation rel, BlockNumber nheapblocks);
+extern BlockNumber visibilitymap_truncate_prepare(Relation rel, BlockNumber nheapblocks);
 
 #endif							/* VISIBILITYMAP_H */
diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h
index 509f4b7..17b97f7 100644
--- a/src/include/storage/bufmgr.h
+++ b/src/include/storage/bufmgr.h
@@ -190,8 +190,8 @@ extern BlockNumber RelationGetNumberOfBlocksInFork(Relation relation,
 extern void FlushOneBuffer(Buffer buffer);
 extern void FlushRelationBuffers(Relation rel);
 extern void FlushDatabaseBuffers(Oid dbid);
-extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode,
-								   ForkNumber forkNum, BlockNumber firstDelBlock);
+extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
+								   int nforks, BlockNumber *firstDelBlock);
 extern void DropRelFileNodesAllBuffers(RelFileNodeBackend *rnodes, int nnodes);
 extern void DropDatabaseBuffers(Oid dbid);
 
diff --git a/src/include/storage/freespace.h b/src/include/storage/freespace.h
index 8d8c465..bf19a67 100644
--- a/src/include/storage/freespace.h
+++ b/src/include/storage/freespace.h
@@ -30,7 +30,7 @@ extern void RecordPageWithFreeSpace(Relation rel, BlockNumber heapBlk,
 extern void XLogRecordPageWithFreeSpace(RelFileNode rnode, BlockNumber heapBlk,
 										Size spaceAvail);
 
-extern void FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks);
+extern BlockNumber MarkFreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks);
 extern void FreeSpaceMapVacuum(Relation rel);
 extern void FreeSpaceMapVacuumRange(Relation rel, BlockNumber start,
 									BlockNumber end);
diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h
index d286c8c..a24532c 100644
--- a/src/include/storage/smgr.h
+++ b/src/include/storage/smgr.h
@@ -90,7 +90,8 @@ extern void smgrclosenode(RelFileNodeBackend rnode);
 extern void smgrcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo);
 extern void smgrdounlink(SMgrRelation reln, bool isRedo);
 extern void smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo);
-extern void smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo);
+extern void smgrdounlinkfork(SMgrRelation reln, ForkNumber *forknum,
+							 int nforks, bool isRedo);
 extern void smgrextend(SMgrRelation reln, ForkNumber forknum,
 					   BlockNumber blocknum, char *buffer, bool skipFsync);
 extern void smgrprefetch(SMgrRelation reln, ForkNumber forknum,
@@ -102,8 +103,8 @@ extern void smgrwrite(SMgrRelation reln, ForkNumber forknum,
 extern void smgrwriteback(SMgrRelation reln, ForkNumber forknum,
 						  BlockNumber blocknum, BlockNumber nblocks);
 extern BlockNumber smgrnblocks(SMgrRelation reln, ForkNumber forknum);
-extern void smgrtruncate(SMgrRelation reln, ForkNumber forknum,
-						 BlockNumber nblocks);
+extern void smgrtruncate(SMgrRelation reln, ForkNumber *forknum,
+						 int nforks, BlockNumber *nblocks);
 extern void smgrimmedsync(SMgrRelation reln, ForkNumber forknum);
 extern void AtEOXact_SMgr(void);
 
-- 
1.8.3.1

From e661b6ec1c0cee764830850d68799bf9a08bb99f Mon Sep 17 00:00:00 2001
From: Kirk Jamison <k.jamison@jp.fujitsu.com>
Date: Thu, 4 Jul 2019 10:59:05 +0000
Subject: [PATCH] Speedup truncates of relation forks

Whenever we truncate relations, it involves several scans of the
shared buffers for every call of smgrtruncate() for each fork which
is time-consuming. This patch reduces the scan for all forks into one
instead of three, and improves the relation truncates by initially
marking and preparing the pages-to-be-truncated of relation forks,
then simultaneously truncating them, resulting to an improved
performance in VACUUM, autovacuum operations, and their recovery.
---
 contrib/pg_visibility/pg_visibility.c     |   8 ++-
 src/backend/access/heap/visibilitymap.c   |  36 ++++-------
 src/backend/catalog/storage.c             | 102 ++++++++++++++++++++++++++----
 src/backend/storage/buffer/bufmgr.c       |  48 ++++++++++----
 src/backend/storage/freespace/freespace.c |  41 ++++--------
 src/backend/storage/smgr/smgr.c           |  49 +++++++++-----
 src/include/access/visibilitymap.h        |   2 +-
 src/include/storage/bufmgr.h              |   4 +-
 src/include/storage/freespace.h           |   2 +-
 src/include/storage/smgr.h                |   7 +-
 10 files changed, 199 insertions(+), 100 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index 1372bb6..60eff7f 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -383,6 +383,8 @@ pg_truncate_visibility_map(PG_FUNCTION_ARGS)
 {
 	Oid			relid = PG_GETARG_OID(0);
 	Relation	rel;
+	ForkNumber	fork;
+	BlockNumber	block;
 
 	rel = relation_open(relid, AccessExclusiveLock);
 
@@ -392,7 +394,11 @@ pg_truncate_visibility_map(PG_FUNCTION_ARGS)
 	RelationOpenSmgr(rel);
 	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
 
-	visibilitymap_truncate(rel, 0);
+	block = visibilitymap_truncate_prepare(rel, 0);
+	if (BlockNumberIsValid(block))
+		fork = VISIBILITYMAP_FORKNUM;
+
+	smgrtruncate(rel->rd_smgr, &fork, 1, &block);
 
 	if (RelationNeedsWAL(rel))
 	{
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index 64dfe06..4cc7977 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -17,7 +17,7 @@
  *		visibilitymap_set	 - set a bit in a previously pinned page
  *		visibilitymap_get_status - get status of bits
  *		visibilitymap_count  - count number of bits set in visibility map
- *		visibilitymap_truncate	- truncate the visibility map
+ *		visibilitymap_truncate_prepare - truncate only tail bits of map pages
  *
  * NOTES
  *
@@ -430,16 +430,18 @@ visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_fro
 }
 
 /*
- *	visibilitymap_truncate - truncate the visibility map
+ *	visibilitymap_truncate_prepare - truncate only tail bits of map page
+ *									 and return the block number for actual
+ *									 truncation later
  *
- * The caller must hold AccessExclusiveLock on the relation, to ensure that
- * other backends receive the smgr invalidation event that this function sends
- * before they access the VM again.
+ * Note that this does not truncate the actual visibility map pages.
+ * When this function is called, the caller must eventually follow it with
+ * smgrtruncate() call to actually truncate visibility map pages.
  *
  * nheapblocks is the new size of the heap.
  */
-void
-visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
+BlockNumber
+visibilitymap_truncate_prepare(Relation rel, BlockNumber nheapblocks)
 {
 	BlockNumber newnblocks;
 
@@ -459,7 +461,7 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 	 * nothing to truncate.
 	 */
 	if (!smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM))
-		return;
+		return InvalidBlockNumber;
 
 	/*
 	 * Unless the new size is exactly at a visibility map page boundary, the
@@ -480,7 +482,7 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 		if (!BufferIsValid(mapBuffer))
 		{
 			/* nothing to do, the file was already smaller */
-			return;
+			return InvalidBlockNumber;
 		}
 
 		page = BufferGetPage(mapBuffer);
@@ -528,20 +530,10 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 	if (smgrnblocks(rel->rd_smgr, VISIBILITYMAP_FORKNUM) <= newnblocks)
 	{
 		/* nothing to do, the file was already smaller than requested size */
-		return;
+		return InvalidBlockNumber;
 	}
-
-	/* Truncate the unused VM pages, and send smgr inval message */
-	smgrtruncate(rel->rd_smgr, VISIBILITYMAP_FORKNUM, newnblocks);
-
-	/*
-	 * We might as well update the local smgr_vm_nblocks setting. smgrtruncate
-	 * sent an smgr cache inval message, which will cause other backends to
-	 * invalidate their copy of smgr_vm_nblocks, and this one too at the next
-	 * command boundary.  But this ensures it isn't outright wrong until then.
-	 */
-	if (rel->rd_smgr)
-		rel->rd_smgr->smgr_vm_nblocks = newnblocks;
+	else
+		return newnblocks;
 }
 
 /*
diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c
index 3cc886f..395ef0f 100644
--- a/src/backend/catalog/storage.c
+++ b/src/backend/catalog/storage.c
@@ -231,6 +231,10 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 {
 	bool		fsm;
 	bool		vm;
+	ForkNumber	forks[MAX_FORKNUM];
+	BlockNumber	blocks[MAX_FORKNUM];
+	BlockNumber	first_removed_nblocks = InvalidBlockNumber;
+	int		nforks = 0;
 
 	/* Open it at the smgr level if not already done */
 	RelationOpenSmgr(rel);
@@ -242,15 +246,33 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 	rel->rd_smgr->smgr_fsm_nblocks = InvalidBlockNumber;
 	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
 
-	/* Truncate the FSM first if it exists */
+	/* Mark the dirty FSM page and return a block number. */
 	fsm = smgrexists(rel->rd_smgr, FSM_FORKNUM);
 	if (fsm)
-		FreeSpaceMapTruncateRel(rel, nblocks);
+	{
+		blocks[nforks] = MarkFreeSpaceMapTruncateRel(rel, nblocks);
+		if (BlockNumberIsValid(blocks[nforks]))
+		{
+			first_removed_nblocks = nblocks;
+			forks[nforks] = FSM_FORKNUM;
+			nforks++;
+		}
+	}
 
-	/* Truncate the visibility map too if it exists. */
+	/*
+	 * Truncate only the tail bits of VM and return the block number
+	 * for actual truncation later in smgrtruncate.
+	 */
 	vm = smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM);
 	if (vm)
-		visibilitymap_truncate(rel, nblocks);
+	{
+		blocks[nforks] = visibilitymap_truncate_prepare(rel, nblocks);
+		if (BlockNumberIsValid(blocks[nforks]))
+		{
+			forks[nforks] = VISIBILITYMAP_FORKNUM;
+			nforks++;
+		}
+	}
 
 	/*
 	 * We WAL-log the truncation before actually truncating, which means
@@ -290,8 +312,20 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 			XLogFlush(lsn);
 	}
 
-	/* Do the real work */
-	smgrtruncate(rel->rd_smgr, MAIN_FORKNUM, nblocks);
+	/* Mark the MAIN fork */
+	forks[nforks] = MAIN_FORKNUM;
+	blocks[nforks] = nblocks;
+	nforks++;
+
+	/* Truncate relation forks simultaneously */
+	smgrtruncate(rel->rd_smgr, forks, nforks, blocks);
+
+	/*
+	 * Update upper-level FSM pages to account for the truncation.
+	 * This is important because the just-truncated pages were likely
+	 * marked as all-free, and would be preferentially selected.
+	 */
+	FreeSpaceMapVacuumRange(rel, first_removed_nblocks, InvalidBlockNumber);
 }
 
 /*
@@ -588,6 +622,13 @@ smgr_redo(XLogReaderState *record)
 		xl_smgr_truncate *xlrec = (xl_smgr_truncate *) XLogRecGetData(record);
 		SMgrRelation reln;
 		Relation	rel;
+		ForkNumber	forks[MAX_FORKNUM];
+		BlockNumber	blocks[MAX_FORKNUM];
+		BlockNumber	first_removed_nblocks = InvalidBlockNumber;
+		int		nforks = 0;
+		bool		fsm_fork = false;
+		bool		main_fork = false;
+		bool		vm_fork = false;
 
 		reln = smgropen(xlrec->rnode, InvalidBackendId);
 
@@ -616,23 +657,58 @@ smgr_redo(XLogReaderState *record)
 		 */
 		XLogFlush(lsn);
 
+		/*
+		 * To speedup recovery, we mark the about-to-be-truncated blocks of
+		 * relation forks first, then truncate those simultaneously later.
+		 */
 		if ((xlrec->flags & SMGR_TRUNCATE_HEAP) != 0)
 		{
-			smgrtruncate(reln, MAIN_FORKNUM, xlrec->blkno);
-
-			/* Also tell xlogutils.c about it */
-			XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
+			forks[nforks] = MAIN_FORKNUM;
+			blocks[nforks] = xlrec->blkno;
+			nforks++;
+			main_fork = true;
 		}
 
-		/* Truncate FSM and VM too */
 		rel = CreateFakeRelcacheEntry(xlrec->rnode);
 
 		if ((xlrec->flags & SMGR_TRUNCATE_FSM) != 0 &&
 			smgrexists(reln, FSM_FORKNUM))
-			FreeSpaceMapTruncateRel(rel, xlrec->blkno);
+		{
+			blocks[nforks] = MarkFreeSpaceMapTruncateRel(rel, xlrec->blkno);
+			if (BlockNumberIsValid(blocks[nforks]))
+			{
+				first_removed_nblocks = xlrec->blkno;
+				forks[nforks] = FSM_FORKNUM;
+				nforks++;
+				fsm_fork = true;
+			}
+		}
 		if ((xlrec->flags & SMGR_TRUNCATE_VM) != 0 &&
 			smgrexists(reln, VISIBILITYMAP_FORKNUM))
-			visibilitymap_truncate(rel, xlrec->blkno);
+		{
+			blocks[nforks] = visibilitymap_truncate_prepare(rel, xlrec->blkno);
+			if (BlockNumberIsValid(blocks[nforks]))
+			{
+				forks[nforks] = VISIBILITYMAP_FORKNUM;
+				nforks++;
+				vm_fork = true;
+			}
+		}
+
+		/* Truncate relation forks simultaneously */
+		if (main_fork || fsm_fork || vm_fork)
+			smgrtruncate(reln, forks, nforks, blocks);
+
+		/* Also tell xlogutils.c about it */
+		if (main_fork)
+			XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
+
+		/*
+		 * Update upper-level FSM pages to account for the truncation.
+		 * This is important because the just-truncated pages were likely
+		 * marked as all-free, and would be preferentially selected.
+		 */
+		FreeSpaceMapVacuumRange(rel, first_removed_nblocks, InvalidBlockNumber);
 
 		FreeFakeRelcacheEntry(rel);
 	}
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index 7332e6b..512c8a1 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -2899,8 +2899,8 @@ BufferGetLSNAtomic(Buffer buffer)
 /* ---------------------------------------------------------------------
  *		DropRelFileNodeBuffers
  *
- *		This function removes from the buffer pool all the pages of the
- *		specified relation fork that have block numbers >= firstDelBlock.
+ *		This function simultaneously removes from the buffer pool all the
+ *		pages of the relation forks that have block numbers >= firstDelBlock.
  *		(In particular, with firstDelBlock = 0, all pages are removed.)
  *		Dirty pages are simply dropped, without bothering to write them
  *		out first.  Therefore, this is NOT rollback-able, and so should be
@@ -2923,23 +2923,36 @@ BufferGetLSNAtomic(Buffer buffer)
  * --------------------------------------------------------------------
  */
 void
-DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
-					   BlockNumber firstDelBlock)
+DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
+					   int nforks, BlockNumber *firstDelBlock)
 {
-	int			i;
+	BlockNumber minBlock = InvalidBlockNumber;
 
 	/* If it's a local relation, it's localbuf.c's problem. */
 	if (RelFileNodeBackendIsTemp(rnode))
 	{
 		if (rnode.backend == MyBackendId)
-			DropRelFileNodeLocalBuffers(rnode.node, forkNum, firstDelBlock);
+		{
+			for (int i = 0; i < nforks; i++)
+				DropRelFileNodeLocalBuffers(rnode.node, forkNum[i],
+											firstDelBlock[i]);
+		}
 		return;
 	}
 
-	for (i = 0; i < NBuffers; i++)
+	/* Get the lower bound of target block number we're interested in */
+	for (int i = 0; i < nforks; i++)
+	{
+		if (!BlockNumberIsValid(minBlock) ||
+			minBlock > firstDelBlock[i])
+			minBlock = firstDelBlock[i];
+	}
+
+	for (int i = 0; i < NBuffers; i++)
 	{
 		BufferDesc *bufHdr = GetBufferDescriptor(i);
 		uint32		buf_state;
+		int		j = 0;
 
 		/*
 		 * We can make this a tad faster by prechecking the buffer tag before
@@ -2960,12 +2973,23 @@ DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
 		if (!RelFileNodeEquals(bufHdr->tag.rnode, rnode.node))
 			continue;
 
+		/* Check with the lower bound block number and skip the loop */
+		if (bufHdr->tag.blockNum < minBlock)
+			continue; /* skip checking the buffer pool scan */
+
 		buf_state = LockBufHdr(bufHdr);
-		if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
-			bufHdr->tag.forkNum == forkNum &&
-			bufHdr->tag.blockNum >= firstDelBlock)
-			InvalidateBuffer(bufHdr);	/* releases spinlock */
-		else
+
+		for (j = 0; j < nforks; j++)
+		{
+			if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
+				bufHdr->tag.forkNum == forkNum[j] &&
+				bufHdr->tag.blockNum >= firstDelBlock[j])
+			{
+				InvalidateBuffer(bufHdr); /* releases spinlock */
+				break;
+			}
+		}
+		if (j >= nforks)
 			UnlockBufHdr(bufHdr, buf_state);
 	}
 }
diff --git a/src/backend/storage/freespace/freespace.c b/src/backend/storage/freespace/freespace.c
index c17b3f4..9c29604 100644
--- a/src/backend/storage/freespace/freespace.c
+++ b/src/backend/storage/freespace/freespace.c
@@ -247,16 +247,16 @@ GetRecordedFreeSpace(Relation rel, BlockNumber heapBlk)
 }
 
 /*
- * FreeSpaceMapTruncateRel - adjust for truncation of a relation.
+ * MarkFreeSpaceMapTruncateRel - adjust for truncation of a relation.
  *
- * The caller must hold AccessExclusiveLock on the relation, to ensure that
- * other backends receive the smgr invalidation event that this function sends
- * before they access the FSM again.
+ * This function marks the dirty page and returns a block number.
+ * The caller of this function must eventually call smgrtruncate() to actually
+ * truncate FSM pages.
  *
  * nblocks is the new size of the heap.
  */
-void
-FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
+BlockNumber
+MarkFreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 {
 	BlockNumber new_nfsmblocks;
 	FSMAddress	first_removed_address;
@@ -270,7 +270,7 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 	 * truncate.
 	 */
 	if (!smgrexists(rel->rd_smgr, FSM_FORKNUM))
-		return;
+		return InvalidBlockNumber;
 
 	/* Get the location in the FSM of the first removed heap block */
 	first_removed_address = fsm_get_location(nblocks, &first_removed_slot);
@@ -285,7 +285,7 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 	{
 		buf = fsm_readbuf(rel, first_removed_address, false);
 		if (!BufferIsValid(buf))
-			return;				/* nothing to do; the FSM was already smaller */
+			return InvalidBlockNumber;	/* nothing to do; the FSM was already smaller */
 		LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
 
 		/* NO EREPORT(ERROR) from here till changes are logged */
@@ -310,33 +310,16 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 		UnlockReleaseBuffer(buf);
 
 		new_nfsmblocks = fsm_logical_to_physical(first_removed_address) + 1;
+		return new_nfsmblocks;
 	}
 	else
 	{
 		new_nfsmblocks = fsm_logical_to_physical(first_removed_address);
 		if (smgrnblocks(rel->rd_smgr, FSM_FORKNUM) <= new_nfsmblocks)
-			return;				/* nothing to do; the FSM was already smaller */
+			return InvalidBlockNumber;	/* nothing to do; the FSM was already smaller */
+		else
+			return new_nfsmblocks;
 	}
-
-	/* Truncate the unused FSM pages, and send smgr inval message */
-	smgrtruncate(rel->rd_smgr, FSM_FORKNUM, new_nfsmblocks);
-
-	/*
-	 * We might as well update the local smgr_fsm_nblocks setting.
-	 * smgrtruncate sent an smgr cache inval message, which will cause other
-	 * backends to invalidate their copy of smgr_fsm_nblocks, and this one too
-	 * at the next command boundary.  But this ensures it isn't outright wrong
-	 * until then.
-	 */
-	if (rel->rd_smgr)
-		rel->rd_smgr->smgr_fsm_nblocks = new_nfsmblocks;
-
-	/*
-	 * Update upper-level FSM pages to account for the truncation.  This is
-	 * important because the just-truncated pages were likely marked as
-	 * all-free, and would be preferentially selected.
-	 */
-	FreeSpaceMapVacuumRange(rel, nblocks, InvalidBlockNumber);
 }
 
 /*
diff --git a/src/backend/storage/smgr/smgr.c b/src/backend/storage/smgr/smgr.c
index dba8c39..ea57de7 100644
--- a/src/backend/storage/smgr/smgr.c
+++ b/src/backend/storage/smgr/smgr.c
@@ -498,29 +498,30 @@ smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo)
 }
 
 /*
- *	smgrdounlinkfork() -- Immediately unlink one fork of a relation.
+ *	smgrdounlinkfork() -- Immediately unlink each fork of a relation.
  *
- *		The specified fork of the relation is removed from the store.  This
- *		should not be used during transactional operations, since it can't be
- *		undone.
+ *		Each fork of the relation is removed from the store.  This should
+ *		not be used during transactional operations, since it can't be undone.
  *
  *		If isRedo is true, it is okay for the underlying file to be gone
  *		already.
  */
 void
-smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo)
+smgrdounlinkfork(SMgrRelation reln, ForkNumber *forknum, int nforks, bool isRedo)
 {
 	RelFileNodeBackend rnode = reln->smgr_rnode;
 	int			which = reln->smgr_which;
+	int			i;
 
-	/* Close the fork at smgr level */
-	smgrsw[which].smgr_close(reln, forknum);
+	/* Close each fork at smgr level */
+	for (i = 0; i < nforks; i++)
+		smgrsw[which].smgr_close(reln, forknum[i]);
 
 	/*
-	 * Get rid of any remaining buffers for the fork.  bufmgr will just drop
+	 * Get rid of any remaining buffers for each fork. bufmgr will just drop
 	 * them without bothering to write the contents.
 	 */
-	DropRelFileNodeBuffers(rnode, forknum, 0);
+	DropRelFileNodeBuffers(rnode, forknum, nforks, 0);
 
 	/*
 	 * It'd be nice to tell the stats collector to forget it immediately, too.
@@ -546,7 +547,8 @@ smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo)
 	 * ERROR, because we've already decided to commit or abort the current
 	 * xact.
 	 */
-	smgrsw[which].smgr_unlink(rnode, forknum, isRedo);
+	for (i = 0; i < nforks; i++)
+		smgrsw[which].smgr_unlink(rnode, forknum[i], isRedo);
 }
 
 /*
@@ -643,13 +645,15 @@ smgrnblocks(SMgrRelation reln, ForkNumber forknum)
  * The truncation is done immediately, so this can't be rolled back.
  */
 void
-smgrtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks)
+smgrtruncate(SMgrRelation reln, ForkNumber *forknum, int nforks, BlockNumber *nblocks)
 {
+	int		i;
+
 	/*
 	 * Get rid of any buffers for the about-to-be-deleted blocks. bufmgr will
 	 * just drop them without bothering to write the contents.
 	 */
-	DropRelFileNodeBuffers(reln->smgr_rnode, forknum, nblocks);
+	DropRelFileNodeBuffers(reln->smgr_rnode, forknum, nforks, nblocks);
 
 	/*
 	 * Send a shared-inval message to force other backends to close any smgr
@@ -663,10 +667,23 @@ smgrtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks)
 	 */
 	CacheInvalidateSmgr(reln->smgr_rnode);
 
-	/*
-	 * Do the truncation.
-	 */
-	smgrsw[reln->smgr_which].smgr_truncate(reln, forknum, nblocks);
+	/* Do the truncation */
+	for (i = 0; i < nforks; i++)
+	{
+		smgrsw[reln->smgr_which].smgr_truncate(reln, forknum[i], nblocks[i]);
+
+		/*
+		 * We might as well update the local smgr_fsm_nblocks and smgr_vm_nblocks
+		 * setting. smgrtruncate sent an smgr cache inval message, which will
+		 * cause other backends to invalidate their copy of smgr_fsm_nblocks and
+		 * smgr_vm_nblocks, and these ones too at the next command boundary. But
+		 * this ensures these aren't outright wrong until then.
+		 */
+		if (forknum[i] == FSM_FORKNUM)
+			reln->smgr_fsm_nblocks = nblocks[i];
+		if (forknum[i] == VISIBILITYMAP_FORKNUM)
+			reln->smgr_vm_nblocks = nblocks[i];
+	}
 }
 
 /*
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index 2d88043..1ab6a81 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -44,6 +44,6 @@ extern void visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf,
 							  uint8 flags);
 extern uint8 visibilitymap_get_status(Relation rel, BlockNumber heapBlk, Buffer *vmbuf);
 extern void visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_frozen);
-extern void visibilitymap_truncate(Relation rel, BlockNumber nheapblocks);
+extern BlockNumber visibilitymap_truncate_prepare(Relation rel, BlockNumber nheapblocks);
 
 #endif							/* VISIBILITYMAP_H */
diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h
index 509f4b7..17b97f7 100644
--- a/src/include/storage/bufmgr.h
+++ b/src/include/storage/bufmgr.h
@@ -190,8 +190,8 @@ extern BlockNumber RelationGetNumberOfBlocksInFork(Relation relation,
 extern void FlushOneBuffer(Buffer buffer);
 extern void FlushRelationBuffers(Relation rel);
 extern void FlushDatabaseBuffers(Oid dbid);
-extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode,
-								   ForkNumber forkNum, BlockNumber firstDelBlock);
+extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
+								   int nforks, BlockNumber *firstDelBlock);
 extern void DropRelFileNodesAllBuffers(RelFileNodeBackend *rnodes, int nnodes);
 extern void DropDatabaseBuffers(Oid dbid);
 
diff --git a/src/include/storage/freespace.h b/src/include/storage/freespace.h
index 8d8c465..bf19a67 100644
--- a/src/include/storage/freespace.h
+++ b/src/include/storage/freespace.h
@@ -30,7 +30,7 @@ extern void RecordPageWithFreeSpace(Relation rel, BlockNumber heapBlk,
 extern void XLogRecordPageWithFreeSpace(RelFileNode rnode, BlockNumber heapBlk,
 										Size spaceAvail);
 
-extern void FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks);
+extern BlockNumber MarkFreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks);
 extern void FreeSpaceMapVacuum(Relation rel);
 extern void FreeSpaceMapVacuumRange(Relation rel, BlockNumber start,
 									BlockNumber end);
diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h
index d286c8c..a24532c 100644
--- a/src/include/storage/smgr.h
+++ b/src/include/storage/smgr.h
@@ -90,7 +90,8 @@ extern void smgrclosenode(RelFileNodeBackend rnode);
 extern void smgrcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo);
 extern void smgrdounlink(SMgrRelation reln, bool isRedo);
 extern void smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo);
-extern void smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo);
+extern void smgrdounlinkfork(SMgrRelation reln, ForkNumber *forknum,
+							 int nforks, bool isRedo);
 extern void smgrextend(SMgrRelation reln, ForkNumber forknum,
 					   BlockNumber blocknum, char *buffer, bool skipFsync);
 extern void smgrprefetch(SMgrRelation reln, ForkNumber forknum,
@@ -102,8 +103,8 @@ extern void smgrwrite(SMgrRelation reln, ForkNumber forknum,
 extern void smgrwriteback(SMgrRelation reln, ForkNumber forknum,
 						  BlockNumber blocknum, BlockNumber nblocks);
 extern BlockNumber smgrnblocks(SMgrRelation reln, ForkNumber forknum);
-extern void smgrtruncate(SMgrRelation reln, ForkNumber forknum,
-						 BlockNumber nblocks);
+extern void smgrtruncate(SMgrRelation reln, ForkNumber *forknum,
+						 int nforks, BlockNumber *nblocks);
 extern void smgrimmedsync(SMgrRelation reln, ForkNumber forknum);
 extern void AtEOXact_SMgr(void);
 
-- 
1.8.3.1

From e661b6ec1c0cee764830850d68799bf9a08bb99f Mon Sep 17 00:00:00 2001
From: Kirk Jamison <k.jamison@jp.fujitsu.com>
Date: Thu, 4 Jul 2019 10:59:05 +0000
Subject: [PATCH] Speedup truncates of relation forks

Whenever we truncate relations, it involves several scans of the
shared buffers for every call of smgrtruncate() for each fork which
is time-consuming. This patch reduces the scan for all forks into one
instead of three, and improves the relation truncates by initially
marking and preparing the pages-to-be-truncated of relation forks,
then simultaneously truncating them, resulting to an improved
performance in VACUUM, autovacuum operations, and their recovery.
---
 contrib/pg_visibility/pg_visibility.c     |   8 ++-
 src/backend/access/heap/visibilitymap.c   |  36 ++++-------
 src/backend/catalog/storage.c             | 102 ++++++++++++++++++++++++++----
 src/backend/storage/buffer/bufmgr.c       |  48 ++++++++++----
 src/backend/storage/freespace/freespace.c |  41 ++++--------
 src/backend/storage/smgr/smgr.c           |  49 +++++++++-----
 src/include/access/visibilitymap.h        |   2 +-
 src/include/storage/bufmgr.h              |   4 +-
 src/include/storage/freespace.h           |   2 +-
 src/include/storage/smgr.h                |   7 +-
 10 files changed, 199 insertions(+), 100 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index 1372bb6..60eff7f 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -383,6 +383,8 @@ pg_truncate_visibility_map(PG_FUNCTION_ARGS)
 {
 	Oid			relid = PG_GETARG_OID(0);
 	Relation	rel;
+	ForkNumber	fork;
+	BlockNumber	block;
 
 	rel = relation_open(relid, AccessExclusiveLock);
 
@@ -392,7 +394,11 @@ pg_truncate_visibility_map(PG_FUNCTION_ARGS)
 	RelationOpenSmgr(rel);
 	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
 
-	visibilitymap_truncate(rel, 0);
+	block = visibilitymap_truncate_prepare(rel, 0);
+	if (BlockNumberIsValid(block))
+		fork = VISIBILITYMAP_FORKNUM;
+
+	smgrtruncate(rel->rd_smgr, &fork, 1, &block);
 
 	if (RelationNeedsWAL(rel))
 	{
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index 64dfe06..4cc7977 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -17,7 +17,7 @@
  *		visibilitymap_set	 - set a bit in a previously pinned page
  *		visibilitymap_get_status - get status of bits
  *		visibilitymap_count  - count number of bits set in visibility map
- *		visibilitymap_truncate	- truncate the visibility map
+ *		visibilitymap_truncate_prepare - truncate only tail bits of map pages
  *
  * NOTES
  *
@@ -430,16 +430,18 @@ visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_fro
 }
 
 /*
- *	visibilitymap_truncate - truncate the visibility map
+ *	visibilitymap_truncate_prepare - truncate only tail bits of map page
+ *									 and return the block number for actual
+ *									 truncation later
  *
- * The caller must hold AccessExclusiveLock on the relation, to ensure that
- * other backends receive the smgr invalidation event that this function sends
- * before they access the VM again.
+ * Note that this does not truncate the actual visibility map pages.
+ * When this function is called, the caller must eventually follow it with
+ * smgrtruncate() call to actually truncate visibility map pages.
  *
  * nheapblocks is the new size of the heap.
  */
-void
-visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
+BlockNumber
+visibilitymap_truncate_prepare(Relation rel, BlockNumber nheapblocks)
 {
 	BlockNumber newnblocks;
 
@@ -459,7 +461,7 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 	 * nothing to truncate.
 	 */
 	if (!smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM))
-		return;
+		return InvalidBlockNumber;
 
 	/*
 	 * Unless the new size is exactly at a visibility map page boundary, the
@@ -480,7 +482,7 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 		if (!BufferIsValid(mapBuffer))
 		{
 			/* nothing to do, the file was already smaller */
-			return;
+			return InvalidBlockNumber;
 		}
 
 		page = BufferGetPage(mapBuffer);
@@ -528,20 +530,10 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 	if (smgrnblocks(rel->rd_smgr, VISIBILITYMAP_FORKNUM) <= newnblocks)
 	{
 		/* nothing to do, the file was already smaller than requested size */
-		return;
+		return InvalidBlockNumber;
 	}
-
-	/* Truncate the unused VM pages, and send smgr inval message */
-	smgrtruncate(rel->rd_smgr, VISIBILITYMAP_FORKNUM, newnblocks);
-
-	/*
-	 * We might as well update the local smgr_vm_nblocks setting. smgrtruncate
-	 * sent an smgr cache inval message, which will cause other backends to
-	 * invalidate their copy of smgr_vm_nblocks, and this one too at the next
-	 * command boundary.  But this ensures it isn't outright wrong until then.
-	 */
-	if (rel->rd_smgr)
-		rel->rd_smgr->smgr_vm_nblocks = newnblocks;
+	else
+		return newnblocks;
 }
 
 /*
diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c
index 3cc886f..395ef0f 100644
--- a/src/backend/catalog/storage.c
+++ b/src/backend/catalog/storage.c
@@ -231,6 +231,10 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 {
 	bool		fsm;
 	bool		vm;
+	ForkNumber	forks[MAX_FORKNUM];
+	BlockNumber	blocks[MAX_FORKNUM];
+	BlockNumber	first_removed_nblocks = InvalidBlockNumber;
+	int		nforks = 0;
 
 	/* Open it at the smgr level if not already done */
 	RelationOpenSmgr(rel);
@@ -242,15 +246,33 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 	rel->rd_smgr->smgr_fsm_nblocks = InvalidBlockNumber;
 	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
 
-	/* Truncate the FSM first if it exists */
+	/* Mark the dirty FSM page and return a block number. */
 	fsm = smgrexists(rel->rd_smgr, FSM_FORKNUM);
 	if (fsm)
-		FreeSpaceMapTruncateRel(rel, nblocks);
+	{
+		blocks[nforks] = MarkFreeSpaceMapTruncateRel(rel, nblocks);
+		if (BlockNumberIsValid(blocks[nforks]))
+		{
+			first_removed_nblocks = nblocks;
+			forks[nforks] = FSM_FORKNUM;
+			nforks++;
+		}
+	}
 
-	/* Truncate the visibility map too if it exists. */
+	/*
+	 * Truncate only the tail bits of VM and return the block number
+	 * for actual truncation later in smgrtruncate.
+	 */
 	vm = smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM);
 	if (vm)
-		visibilitymap_truncate(rel, nblocks);
+	{
+		blocks[nforks] = visibilitymap_truncate_prepare(rel, nblocks);
+		if (BlockNumberIsValid(blocks[nforks]))
+		{
+			forks[nforks] = VISIBILITYMAP_FORKNUM;
+			nforks++;
+		}
+	}
 
 	/*
 	 * We WAL-log the truncation before actually truncating, which means
@@ -290,8 +312,20 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 			XLogFlush(lsn);
 	}
 
-	/* Do the real work */
-	smgrtruncate(rel->rd_smgr, MAIN_FORKNUM, nblocks);
+	/* Mark the MAIN fork */
+	forks[nforks] = MAIN_FORKNUM;
+	blocks[nforks] = nblocks;
+	nforks++;
+
+	/* Truncate relation forks simultaneously */
+	smgrtruncate(rel->rd_smgr, forks, nforks, blocks);
+
+	/*
+	 * Update upper-level FSM pages to account for the truncation.
+	 * This is important because the just-truncated pages were likely
+	 * marked as all-free, and would be preferentially selected.
+	 */
+	FreeSpaceMapVacuumRange(rel, first_removed_nblocks, InvalidBlockNumber);
 }
 
 /*
@@ -588,6 +622,13 @@ smgr_redo(XLogReaderState *record)
 		xl_smgr_truncate *xlrec = (xl_smgr_truncate *) XLogRecGetData(record);
 		SMgrRelation reln;
 		Relation	rel;
+		ForkNumber	forks[MAX_FORKNUM];
+		BlockNumber	blocks[MAX_FORKNUM];
+		BlockNumber	first_removed_nblocks = InvalidBlockNumber;
+		int		nforks = 0;
+		bool		fsm_fork = false;
+		bool		main_fork = false;
+		bool		vm_fork = false;
 
 		reln = smgropen(xlrec->rnode, InvalidBackendId);
 
@@ -616,23 +657,58 @@ smgr_redo(XLogReaderState *record)
 		 */
 		XLogFlush(lsn);
 
+		/*
+		 * To speedup recovery, we mark the about-to-be-truncated blocks of
+		 * relation forks first, then truncate those simultaneously later.
+		 */
 		if ((xlrec->flags & SMGR_TRUNCATE_HEAP) != 0)
 		{
-			smgrtruncate(reln, MAIN_FORKNUM, xlrec->blkno);
-
-			/* Also tell xlogutils.c about it */
-			XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
+			forks[nforks] = MAIN_FORKNUM;
+			blocks[nforks] = xlrec->blkno;
+			nforks++;
+			main_fork = true;
 		}
 
-		/* Truncate FSM and VM too */
 		rel = CreateFakeRelcacheEntry(xlrec->rnode);
 
 		if ((xlrec->flags & SMGR_TRUNCATE_FSM) != 0 &&
 			smgrexists(reln, FSM_FORKNUM))
-			FreeSpaceMapTruncateRel(rel, xlrec->blkno);
+		{
+			blocks[nforks] = MarkFreeSpaceMapTruncateRel(rel, xlrec->blkno);
+			if (BlockNumberIsValid(blocks[nforks]))
+			{
+				first_removed_nblocks = xlrec->blkno;
+				forks[nforks] = FSM_FORKNUM;
+				nforks++;
+				fsm_fork = true;
+			}
+		}
 		if ((xlrec->flags & SMGR_TRUNCATE_VM) != 0 &&
 			smgrexists(reln, VISIBILITYMAP_FORKNUM))
-			visibilitymap_truncate(rel, xlrec->blkno);
+		{
+			blocks[nforks] = visibilitymap_truncate_prepare(rel, xlrec->blkno);
+			if (BlockNumberIsValid(blocks[nforks]))
+			{
+				forks[nforks] = VISIBILITYMAP_FORKNUM;
+				nforks++;
+				vm_fork = true;
+			}
+		}
+
+		/* Truncate relation forks simultaneously */
+		if (main_fork || fsm_fork || vm_fork)
+			smgrtruncate(reln, forks, nforks, blocks);
+
+		/* Also tell xlogutils.c about it */
+		if (main_fork)
+			XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
+
+		/*
+		 * Update upper-level FSM pages to account for the truncation.
+		 * This is important because the just-truncated pages were likely
+		 * marked as all-free, and would be preferentially selected.
+		 */
+		FreeSpaceMapVacuumRange(rel, first_removed_nblocks, InvalidBlockNumber);
 
 		FreeFakeRelcacheEntry(rel);
 	}
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index 7332e6b..512c8a1 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -2899,8 +2899,8 @@ BufferGetLSNAtomic(Buffer buffer)
 /* ---------------------------------------------------------------------
  *		DropRelFileNodeBuffers
  *
- *		This function removes from the buffer pool all the pages of the
- *		specified relation fork that have block numbers >= firstDelBlock.
+ *		This function simultaneously removes from the buffer pool all the
+ *		pages of the relation forks that have block numbers >= firstDelBlock.
  *		(In particular, with firstDelBlock = 0, all pages are removed.)
  *		Dirty pages are simply dropped, without bothering to write them
  *		out first.  Therefore, this is NOT rollback-able, and so should be
@@ -2923,23 +2923,36 @@ BufferGetLSNAtomic(Buffer buffer)
  * --------------------------------------------------------------------
  */
 void
-DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
-					   BlockNumber firstDelBlock)
+DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
+					   int nforks, BlockNumber *firstDelBlock)
 {
-	int			i;
+	BlockNumber minBlock = InvalidBlockNumber;
 
 	/* If it's a local relation, it's localbuf.c's problem. */
 	if (RelFileNodeBackendIsTemp(rnode))
 	{
 		if (rnode.backend == MyBackendId)
-			DropRelFileNodeLocalBuffers(rnode.node, forkNum, firstDelBlock);
+		{
+			for (int i = 0; i < nforks; i++)
+				DropRelFileNodeLocalBuffers(rnode.node, forkNum[i],
+											firstDelBlock[i]);
+		}
 		return;
 	}
 
-	for (i = 0; i < NBuffers; i++)
+	/* Get the lower bound of target block number we're interested in */
+	for (int i = 0; i < nforks; i++)
+	{
+		if (!BlockNumberIsValid(minBlock) ||
+			minBlock > firstDelBlock[i])
+			minBlock = firstDelBlock[i];
+	}
+
+	for (int i = 0; i < NBuffers; i++)
 	{
 		BufferDesc *bufHdr = GetBufferDescriptor(i);
 		uint32		buf_state;
+		int		j = 0;
 
 		/*
 		 * We can make this a tad faster by prechecking the buffer tag before
@@ -2960,12 +2973,23 @@ DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
 		if (!RelFileNodeEquals(bufHdr->tag.rnode, rnode.node))
 			continue;
 
+		/* Check with the lower bound block number and skip the loop */
+		if (bufHdr->tag.blockNum < minBlock)
+			continue; /* skip checking the buffer pool scan */
+
 		buf_state = LockBufHdr(bufHdr);
-		if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
-			bufHdr->tag.forkNum == forkNum &&
-			bufHdr->tag.blockNum >= firstDelBlock)
-			InvalidateBuffer(bufHdr);	/* releases spinlock */
-		else
+
+		for (j = 0; j < nforks; j++)
+		{
+			if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
+				bufHdr->tag.forkNum == forkNum[j] &&
+				bufHdr->tag.blockNum >= firstDelBlock[j])
+			{
+				InvalidateBuffer(bufHdr); /* releases spinlock */
+				break;
+			}
+		}
+		if (j >= nforks)
 			UnlockBufHdr(bufHdr, buf_state);
 	}
 }
diff --git a/src/backend/storage/freespace/freespace.c b/src/backend/storage/freespace/freespace.c
index c17b3f4..9c29604 100644
--- a/src/backend/storage/freespace/freespace.c
+++ b/src/backend/storage/freespace/freespace.c
@@ -247,16 +247,16 @@ GetRecordedFreeSpace(Relation rel, BlockNumber heapBlk)
 }
 
 /*
- * FreeSpaceMapTruncateRel - adjust for truncation of a relation.
+ * MarkFreeSpaceMapTruncateRel - adjust for truncation of a relation.
  *
- * The caller must hold AccessExclusiveLock on the relation, to ensure that
- * other backends receive the smgr invalidation event that this function sends
- * before they access the FSM again.
+ * This function marks the dirty page and returns a block number.
+ * The caller of this function must eventually call smgrtruncate() to actually
+ * truncate FSM pages.
  *
  * nblocks is the new size of the heap.
  */
-void
-FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
+BlockNumber
+MarkFreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 {
 	BlockNumber new_nfsmblocks;
 	FSMAddress	first_removed_address;
@@ -270,7 +270,7 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 	 * truncate.
 	 */
 	if (!smgrexists(rel->rd_smgr, FSM_FORKNUM))
-		return;
+		return InvalidBlockNumber;
 
 	/* Get the location in the FSM of the first removed heap block */
 	first_removed_address = fsm_get_location(nblocks, &first_removed_slot);
@@ -285,7 +285,7 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 	{
 		buf = fsm_readbuf(rel, first_removed_address, false);
 		if (!BufferIsValid(buf))
-			return;				/* nothing to do; the FSM was already smaller */
+			return InvalidBlockNumber;	/* nothing to do; the FSM was already smaller */
 		LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
 
 		/* NO EREPORT(ERROR) from here till changes are logged */
@@ -310,33 +310,16 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 		UnlockReleaseBuffer(buf);
 
 		new_nfsmblocks = fsm_logical_to_physical(first_removed_address) + 1;
+		return new_nfsmblocks;
 	}
 	else
 	{
 		new_nfsmblocks = fsm_logical_to_physical(first_removed_address);
 		if (smgrnblocks(rel->rd_smgr, FSM_FORKNUM) <= new_nfsmblocks)
-			return;				/* nothing to do; the FSM was already smaller */
+			return InvalidBlockNumber;	/* nothing to do; the FSM was already smaller */
+		else
+			return new_nfsmblocks;
 	}
-
-	/* Truncate the unused FSM pages, and send smgr inval message */
-	smgrtruncate(rel->rd_smgr, FSM_FORKNUM, new_nfsmblocks);
-
-	/*
-	 * We might as well update the local smgr_fsm_nblocks setting.
-	 * smgrtruncate sent an smgr cache inval message, which will cause other
-	 * backends to invalidate their copy of smgr_fsm_nblocks, and this one too
-	 * at the next command boundary.  But this ensures it isn't outright wrong
-	 * until then.
-	 */
-	if (rel->rd_smgr)
-		rel->rd_smgr->smgr_fsm_nblocks = new_nfsmblocks;
-
-	/*
-	 * Update upper-level FSM pages to account for the truncation.  This is
-	 * important because the just-truncated pages were likely marked as
-	 * all-free, and would be preferentially selected.
-	 */
-	FreeSpaceMapVacuumRange(rel, nblocks, InvalidBlockNumber);
 }
 
 /*
diff --git a/src/backend/storage/smgr/smgr.c b/src/backend/storage/smgr/smgr.c
index dba8c39..ea57de7 100644
--- a/src/backend/storage/smgr/smgr.c
+++ b/src/backend/storage/smgr/smgr.c
@@ -498,29 +498,30 @@ smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo)
 }
 
 /*
- *	smgrdounlinkfork() -- Immediately unlink one fork of a relation.
+ *	smgrdounlinkfork() -- Immediately unlink each fork of a relation.
  *
- *		The specified fork of the relation is removed from the store.  This
- *		should not be used during transactional operations, since it can't be
- *		undone.
+ *		Each fork of the relation is removed from the store.  This should
+ *		not be used during transactional operations, since it can't be undone.
  *
  *		If isRedo is true, it is okay for the underlying file to be gone
  *		already.
  */
 void
-smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo)
+smgrdounlinkfork(SMgrRelation reln, ForkNumber *forknum, int nforks, bool isRedo)
 {
 	RelFileNodeBackend rnode = reln->smgr_rnode;
 	int			which = reln->smgr_which;
+	int			i;
 
-	/* Close the fork at smgr level */
-	smgrsw[which].smgr_close(reln, forknum);
+	/* Close each fork at smgr level */
+	for (i = 0; i < nforks; i++)
+		smgrsw[which].smgr_close(reln, forknum[i]);
 
 	/*
-	 * Get rid of any remaining buffers for the fork.  bufmgr will just drop
+	 * Get rid of any remaining buffers for each fork. bufmgr will just drop
 	 * them without bothering to write the contents.
 	 */
-	DropRelFileNodeBuffers(rnode, forknum, 0);
+	DropRelFileNodeBuffers(rnode, forknum, nforks, 0);
 
 	/*
 	 * It'd be nice to tell the stats collector to forget it immediately, too.
@@ -546,7 +547,8 @@ smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo)
 	 * ERROR, because we've already decided to commit or abort the current
 	 * xact.
 	 */
-	smgrsw[which].smgr_unlink(rnode, forknum, isRedo);
+	for (i = 0; i < nforks; i++)
+		smgrsw[which].smgr_unlink(rnode, forknum[i], isRedo);
 }
 
 /*
@@ -643,13 +645,15 @@ smgrnblocks(SMgrRelation reln, ForkNumber forknum)
  * The truncation is done immediately, so this can't be rolled back.
  */
 void
-smgrtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks)
+smgrtruncate(SMgrRelation reln, ForkNumber *forknum, int nforks, BlockNumber *nblocks)
 {
+	int		i;
+
 	/*
 	 * Get rid of any buffers for the about-to-be-deleted blocks. bufmgr will
 	 * just drop them without bothering to write the contents.
 	 */
-	DropRelFileNodeBuffers(reln->smgr_rnode, forknum, nblocks);
+	DropRelFileNodeBuffers(reln->smgr_rnode, forknum, nforks, nblocks);
 
 	/*
 	 * Send a shared-inval message to force other backends to close any smgr
@@ -663,10 +667,23 @@ smgrtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks)
 	 */
 	CacheInvalidateSmgr(reln->smgr_rnode);
 
-	/*
-	 * Do the truncation.
-	 */
-	smgrsw[reln->smgr_which].smgr_truncate(reln, forknum, nblocks);
+	/* Do the truncation */
+	for (i = 0; i < nforks; i++)
+	{
+		smgrsw[reln->smgr_which].smgr_truncate(reln, forknum[i], nblocks[i]);
+
+		/*
+		 * We might as well update the local smgr_fsm_nblocks and smgr_vm_nblocks
+		 * setting. smgrtruncate sent an smgr cache inval message, which will
+		 * cause other backends to invalidate their copy of smgr_fsm_nblocks and
+		 * smgr_vm_nblocks, and these ones too at the next command boundary. But
+		 * this ensures these aren't outright wrong until then.
+		 */
+		if (forknum[i] == FSM_FORKNUM)
+			reln->smgr_fsm_nblocks = nblocks[i];
+		if (forknum[i] == VISIBILITYMAP_FORKNUM)
+			reln->smgr_vm_nblocks = nblocks[i];
+	}
 }
 
 /*
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index 2d88043..1ab6a81 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -44,6 +44,6 @@ extern void visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf,
 							  uint8 flags);
 extern uint8 visibilitymap_get_status(Relation rel, BlockNumber heapBlk, Buffer *vmbuf);
 extern void visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_frozen);
-extern void visibilitymap_truncate(Relation rel, BlockNumber nheapblocks);
+extern BlockNumber visibilitymap_truncate_prepare(Relation rel, BlockNumber nheapblocks);
 
 #endif							/* VISIBILITYMAP_H */
diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h
index 509f4b7..17b97f7 100644
--- a/src/include/storage/bufmgr.h
+++ b/src/include/storage/bufmgr.h
@@ -190,8 +190,8 @@ extern BlockNumber RelationGetNumberOfBlocksInFork(Relation relation,
 extern void FlushOneBuffer(Buffer buffer);
 extern void FlushRelationBuffers(Relation rel);
 extern void FlushDatabaseBuffers(Oid dbid);
-extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode,
-								   ForkNumber forkNum, BlockNumber firstDelBlock);
+extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
+								   int nforks, BlockNumber *firstDelBlock);
 extern void DropRelFileNodesAllBuffers(RelFileNodeBackend *rnodes, int nnodes);
 extern void DropDatabaseBuffers(Oid dbid);
 
diff --git a/src/include/storage/freespace.h b/src/include/storage/freespace.h
index 8d8c465..bf19a67 100644
--- a/src/include/storage/freespace.h
+++ b/src/include/storage/freespace.h
@@ -30,7 +30,7 @@ extern void RecordPageWithFreeSpace(Relation rel, BlockNumber heapBlk,
 extern void XLogRecordPageWithFreeSpace(RelFileNode rnode, BlockNumber heapBlk,
 										Size spaceAvail);
 
-extern void FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks);
+extern BlockNumber MarkFreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks);
 extern void FreeSpaceMapVacuum(Relation rel);
 extern void FreeSpaceMapVacuumRange(Relation rel, BlockNumber start,
 									BlockNumber end);
diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h
index d286c8c..a24532c 100644
--- a/src/include/storage/smgr.h
+++ b/src/include/storage/smgr.h
@@ -90,7 +90,8 @@ extern void smgrclosenode(RelFileNodeBackend rnode);
 extern void smgrcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo);
 extern void smgrdounlink(SMgrRelation reln, bool isRedo);
 extern void smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo);
-extern void smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo);
+extern void smgrdounlinkfork(SMgrRelation reln, ForkNumber *forknum,
+							 int nforks, bool isRedo);
 extern void smgrextend(SMgrRelation reln, ForkNumber forknum,
 					   BlockNumber blocknum, char *buffer, bool skipFsync);
 extern void smgrprefetch(SMgrRelation reln, ForkNumber forknum,
@@ -102,8 +103,8 @@ extern void smgrwrite(SMgrRelation reln, ForkNumber forknum,
 extern void smgrwriteback(SMgrRelation reln, ForkNumber forknum,
 						  BlockNumber blocknum, BlockNumber nblocks);
 extern BlockNumber smgrnblocks(SMgrRelation reln, ForkNumber forknum);
-extern void smgrtruncate(SMgrRelation reln, ForkNumber forknum,
-						 BlockNumber nblocks);
+extern void smgrtruncate(SMgrRelation reln, ForkNumber *forknum,
+						 int nforks, BlockNumber *nblocks);
 extern void smgrimmedsync(SMgrRelation reln, ForkNumber forknum);
 extern void AtEOXact_SMgr(void);
 
-- 
1.8.3.1

From ea70f5734ca6be1ea4d6ec701c419debf9cc5743 Mon Sep 17 00:00:00 2001
From: Kirk Jamison <k.jamison@jp.fujitsu.com>
Date: Thu, 4 Jul 2019 10:59:05 +0000
Subject: [PATCH] Speedup truncates of relation forks

Whenever we truncate relations, it involves several scans of the
shared buffers for every call of smgrtruncate() for each fork which
is time-consuming. This patch reduces the scan for all forks into one
instead of three, and improves the relation truncates by initially
marking and preparing the pages-to-be-truncated of relation forks,
then simultaneously truncating them, resulting to an improved
performance in VACUUM, autovacuum operations, and their recovery.
---
 contrib/pg_visibility/pg_visibility.c     |   8 ++-
 src/backend/access/heap/visibilitymap.c   |  36 ++++-------
 src/backend/catalog/storage.c             | 102 ++++++++++++++++++++++++++----
 src/backend/storage/buffer/bufmgr.c       |  33 +++++++---
 src/backend/storage/freespace/freespace.c |  41 ++++--------
 src/backend/storage/smgr/smgr.c           |  49 +++++++++-----
 src/include/access/visibilitymap.h        |   2 +-
 src/include/storage/bufmgr.h              |   4 +-
 src/include/storage/freespace.h           |   2 +-
 src/include/storage/smgr.h                |   7 +-
 10 files changed, 186 insertions(+), 98 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index 1372bb6..60eff7f 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -383,6 +383,8 @@ pg_truncate_visibility_map(PG_FUNCTION_ARGS)
 {
 	Oid			relid = PG_GETARG_OID(0);
 	Relation	rel;
+	ForkNumber	fork;
+	BlockNumber	block;
 
 	rel = relation_open(relid, AccessExclusiveLock);
 
@@ -392,7 +394,11 @@ pg_truncate_visibility_map(PG_FUNCTION_ARGS)
 	RelationOpenSmgr(rel);
 	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
 
-	visibilitymap_truncate(rel, 0);
+	block = visibilitymap_truncate_prepare(rel, 0);
+	if (BlockNumberIsValid(block))
+		fork = VISIBILITYMAP_FORKNUM;
+
+	smgrtruncate(rel->rd_smgr, &fork, 1, &block);
 
 	if (RelationNeedsWAL(rel))
 	{
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index a08922b..351fc31 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -17,7 +17,7 @@
  *		visibilitymap_set	 - set a bit in a previously pinned page
  *		visibilitymap_get_status - get status of bits
  *		visibilitymap_count  - count number of bits set in visibility map
- *		visibilitymap_truncate	- truncate the visibility map
+ *		visibilitymap_truncate_prepare - truncate only tail bits of map pages
  *
  * NOTES
  *
@@ -430,16 +430,18 @@ visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_fro
 }
 
 /*
- *	visibilitymap_truncate - truncate the visibility map
+ *	visibilitymap_truncate_prepare - truncate only tail bits of map page
+ *									 and return the block number for actual
+ *									 truncation later
  *
- * The caller must hold AccessExclusiveLock on the relation, to ensure that
- * other backends receive the smgr invalidation event that this function sends
- * before they access the VM again.
+ * Note that this does not truncate the actual visibility map pages.
+ * When this function is called, the caller must eventually follow it with
+ * smgrtruncate() call to actually truncate visibility map pages.
  *
  * nheapblocks is the new size of the heap.
  */
-void
-visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
+BlockNumber
+visibilitymap_truncate_prepare(Relation rel, BlockNumber nheapblocks)
 {
 	BlockNumber newnblocks;
 
@@ -459,7 +461,7 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 	 * nothing to truncate.
 	 */
 	if (!smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM))
-		return;
+		return InvalidBlockNumber;
 
 	/*
 	 * Unless the new size is exactly at a visibility map page boundary, the
@@ -480,7 +482,7 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 		if (!BufferIsValid(mapBuffer))
 		{
 			/* nothing to do, the file was already smaller */
-			return;
+			return InvalidBlockNumber;
 		}
 
 		page = BufferGetPage(mapBuffer);
@@ -528,20 +530,10 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 	if (smgrnblocks(rel->rd_smgr, VISIBILITYMAP_FORKNUM) <= newnblocks)
 	{
 		/* nothing to do, the file was already smaller than requested size */
-		return;
+		return InvalidBlockNumber;
 	}
-
-	/* Truncate the unused VM pages, and send smgr inval message */
-	smgrtruncate(rel->rd_smgr, VISIBILITYMAP_FORKNUM, newnblocks);
-
-	/*
-	 * We might as well update the local smgr_vm_nblocks setting. smgrtruncate
-	 * sent an smgr cache inval message, which will cause other backends to
-	 * invalidate their copy of smgr_vm_nblocks, and this one too at the next
-	 * command boundary.  But this ensures it isn't outright wrong until then.
-	 */
-	if (rel->rd_smgr)
-		rel->rd_smgr->smgr_vm_nblocks = newnblocks;
+	else
+		return newnblocks;
 }
 
 /*
diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c
index 3cc886f..395ef0f 100644
--- a/src/backend/catalog/storage.c
+++ b/src/backend/catalog/storage.c
@@ -231,6 +231,10 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 {
 	bool		fsm;
 	bool		vm;
+	ForkNumber	forks[MAX_FORKNUM];
+	BlockNumber	blocks[MAX_FORKNUM];
+	BlockNumber	first_removed_nblocks = InvalidBlockNumber;
+	int		nforks = 0;
 
 	/* Open it at the smgr level if not already done */
 	RelationOpenSmgr(rel);
@@ -242,15 +246,33 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 	rel->rd_smgr->smgr_fsm_nblocks = InvalidBlockNumber;
 	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
 
-	/* Truncate the FSM first if it exists */
+	/* Mark the dirty FSM page and return a block number. */
 	fsm = smgrexists(rel->rd_smgr, FSM_FORKNUM);
 	if (fsm)
-		FreeSpaceMapTruncateRel(rel, nblocks);
+	{
+		blocks[nforks] = MarkFreeSpaceMapTruncateRel(rel, nblocks);
+		if (BlockNumberIsValid(blocks[nforks]))
+		{
+			first_removed_nblocks = nblocks;
+			forks[nforks] = FSM_FORKNUM;
+			nforks++;
+		}
+	}
 
-	/* Truncate the visibility map too if it exists. */
+	/*
+	 * Truncate only the tail bits of VM and return the block number
+	 * for actual truncation later in smgrtruncate.
+	 */
 	vm = smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM);
 	if (vm)
-		visibilitymap_truncate(rel, nblocks);
+	{
+		blocks[nforks] = visibilitymap_truncate_prepare(rel, nblocks);
+		if (BlockNumberIsValid(blocks[nforks]))
+		{
+			forks[nforks] = VISIBILITYMAP_FORKNUM;
+			nforks++;
+		}
+	}
 
 	/*
 	 * We WAL-log the truncation before actually truncating, which means
@@ -290,8 +312,20 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 			XLogFlush(lsn);
 	}
 
-	/* Do the real work */
-	smgrtruncate(rel->rd_smgr, MAIN_FORKNUM, nblocks);
+	/* Mark the MAIN fork */
+	forks[nforks] = MAIN_FORKNUM;
+	blocks[nforks] = nblocks;
+	nforks++;
+
+	/* Truncate relation forks simultaneously */
+	smgrtruncate(rel->rd_smgr, forks, nforks, blocks);
+
+	/*
+	 * Update upper-level FSM pages to account for the truncation.
+	 * This is important because the just-truncated pages were likely
+	 * marked as all-free, and would be preferentially selected.
+	 */
+	FreeSpaceMapVacuumRange(rel, first_removed_nblocks, InvalidBlockNumber);
 }
 
 /*
@@ -588,6 +622,13 @@ smgr_redo(XLogReaderState *record)
 		xl_smgr_truncate *xlrec = (xl_smgr_truncate *) XLogRecGetData(record);
 		SMgrRelation reln;
 		Relation	rel;
+		ForkNumber	forks[MAX_FORKNUM];
+		BlockNumber	blocks[MAX_FORKNUM];
+		BlockNumber	first_removed_nblocks = InvalidBlockNumber;
+		int		nforks = 0;
+		bool		fsm_fork = false;
+		bool		main_fork = false;
+		bool		vm_fork = false;
 
 		reln = smgropen(xlrec->rnode, InvalidBackendId);
 
@@ -616,23 +657,58 @@ smgr_redo(XLogReaderState *record)
 		 */
 		XLogFlush(lsn);
 
+		/*
+		 * To speedup recovery, we mark the about-to-be-truncated blocks of
+		 * relation forks first, then truncate those simultaneously later.
+		 */
 		if ((xlrec->flags & SMGR_TRUNCATE_HEAP) != 0)
 		{
-			smgrtruncate(reln, MAIN_FORKNUM, xlrec->blkno);
-
-			/* Also tell xlogutils.c about it */
-			XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
+			forks[nforks] = MAIN_FORKNUM;
+			blocks[nforks] = xlrec->blkno;
+			nforks++;
+			main_fork = true;
 		}
 
-		/* Truncate FSM and VM too */
 		rel = CreateFakeRelcacheEntry(xlrec->rnode);
 
 		if ((xlrec->flags & SMGR_TRUNCATE_FSM) != 0 &&
 			smgrexists(reln, FSM_FORKNUM))
-			FreeSpaceMapTruncateRel(rel, xlrec->blkno);
+		{
+			blocks[nforks] = MarkFreeSpaceMapTruncateRel(rel, xlrec->blkno);
+			if (BlockNumberIsValid(blocks[nforks]))
+			{
+				first_removed_nblocks = xlrec->blkno;
+				forks[nforks] = FSM_FORKNUM;
+				nforks++;
+				fsm_fork = true;
+			}
+		}
 		if ((xlrec->flags & SMGR_TRUNCATE_VM) != 0 &&
 			smgrexists(reln, VISIBILITYMAP_FORKNUM))
-			visibilitymap_truncate(rel, xlrec->blkno);
+		{
+			blocks[nforks] = visibilitymap_truncate_prepare(rel, xlrec->blkno);
+			if (BlockNumberIsValid(blocks[nforks]))
+			{
+				forks[nforks] = VISIBILITYMAP_FORKNUM;
+				nforks++;
+				vm_fork = true;
+			}
+		}
+
+		/* Truncate relation forks simultaneously */
+		if (main_fork || fsm_fork || vm_fork)
+			smgrtruncate(reln, forks, nforks, blocks);
+
+		/* Also tell xlogutils.c about it */
+		if (main_fork)
+			XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
+
+		/*
+		 * Update upper-level FSM pages to account for the truncation.
+		 * This is important because the just-truncated pages were likely
+		 * marked as all-free, and would be preferentially selected.
+		 */
+		FreeSpaceMapVacuumRange(rel, first_removed_nblocks, InvalidBlockNumber);
 
 		FreeFakeRelcacheEntry(rel);
 	}
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index 6f3a402..1f2b600 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -2900,8 +2900,8 @@ BufferGetLSNAtomic(Buffer buffer)
 /* ---------------------------------------------------------------------
  *		DropRelFileNodeBuffers
  *
- *		This function removes from the buffer pool all the pages of the
- *		specified relation fork that have block numbers >= firstDelBlock.
+ *		This function simultaneously removes from the buffer pool all the
+ *		pages of the relation forks that have block numbers >= firstDelBlock.
  *		(In particular, with firstDelBlock = 0, all pages are removed.)
  *		Dirty pages are simply dropped, without bothering to write them
  *		out first.  Therefore, this is NOT rollback-able, and so should be
@@ -2924,8 +2924,8 @@ BufferGetLSNAtomic(Buffer buffer)
  * --------------------------------------------------------------------
  */
 void
-DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
-					   BlockNumber firstDelBlock)
+DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
+					   int nforks, BlockNumber *firstDelBlock)
 {
 	int			i;
 
@@ -2933,7 +2933,12 @@ DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
 	if (RelFileNodeBackendIsTemp(rnode))
 	{
 		if (rnode.backend == MyBackendId)
-			DropRelFileNodeLocalBuffers(rnode.node, forkNum, firstDelBlock);
+		{
+			int		j;
+			for (j = 0; j < nforks; j++)
+				DropRelFileNodeLocalBuffers(rnode.node, forkNum[j],
+											firstDelBlock[j]);
+		}
 		return;
 	}
 
@@ -2941,6 +2946,7 @@ DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
 	{
 		BufferDesc *bufHdr = GetBufferDescriptor(i);
 		uint32		buf_state;
+		int		j = 0;
 
 		/*
 		 * We can make this a tad faster by prechecking the buffer tag before
@@ -2962,11 +2968,18 @@ DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
 			continue;
 
 		buf_state = LockBufHdr(bufHdr);
-		if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
-			bufHdr->tag.forkNum == forkNum &&
-			bufHdr->tag.blockNum >= firstDelBlock)
-			InvalidateBuffer(bufHdr);	/* releases spinlock */
-		else
+
+		for (j = 0; j < nforks; j++)
+		{
+			if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
+				bufHdr->tag.forkNum == forkNum[j] &&
+				bufHdr->tag.blockNum >= firstDelBlock[j])
+			{
+				InvalidateBuffer(bufHdr); /* releases spinlock */
+				break;
+			}
+		}
+		if (j >= nforks)
 			UnlockBufHdr(bufHdr, buf_state);
 	}
 }
diff --git a/src/backend/storage/freespace/freespace.c b/src/backend/storage/freespace/freespace.c
index 2383094..437181f 100644
--- a/src/backend/storage/freespace/freespace.c
+++ b/src/backend/storage/freespace/freespace.c
@@ -247,16 +247,16 @@ GetRecordedFreeSpace(Relation rel, BlockNumber heapBlk)
 }
 
 /*
- * FreeSpaceMapTruncateRel - adjust for truncation of a relation.
+ * MarkFreeSpaceMapTruncateRel - adjust for truncation of a relation.
  *
- * The caller must hold AccessExclusiveLock on the relation, to ensure that
- * other backends receive the smgr invalidation event that this function sends
- * before they access the FSM again.
+ * This function marks the dirty page and returns a block number.
+ * The caller of this function must eventually call smgrtruncate() to actually
+ * truncate FSM pages.
  *
  * nblocks is the new size of the heap.
  */
-void
-FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
+BlockNumber
+MarkFreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 {
 	BlockNumber new_nfsmblocks;
 	FSMAddress	first_removed_address;
@@ -270,7 +270,7 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 	 * truncate.
 	 */
 	if (!smgrexists(rel->rd_smgr, FSM_FORKNUM))
-		return;
+		return InvalidBlockNumber;
 
 	/* Get the location in the FSM of the first removed heap block */
 	first_removed_address = fsm_get_location(nblocks, &first_removed_slot);
@@ -285,7 +285,7 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 	{
 		buf = fsm_readbuf(rel, first_removed_address, false);
 		if (!BufferIsValid(buf))
-			return;				/* nothing to do; the FSM was already smaller */
+			return InvalidBlockNumber;	/* nothing to do; the FSM was already smaller */
 		LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
 
 		/* NO EREPORT(ERROR) from here till changes are logged */
@@ -310,33 +310,16 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 		UnlockReleaseBuffer(buf);
 
 		new_nfsmblocks = fsm_logical_to_physical(first_removed_address) + 1;
+		return new_nfsmblocks;
 	}
 	else
 	{
 		new_nfsmblocks = fsm_logical_to_physical(first_removed_address);
 		if (smgrnblocks(rel->rd_smgr, FSM_FORKNUM) <= new_nfsmblocks)
-			return;				/* nothing to do; the FSM was already smaller */
+			return InvalidBlockNumber;	/* nothing to do; the FSM was already smaller */
+		else
+			return new_nfsmblocks;
 	}
-
-	/* Truncate the unused FSM pages, and send smgr inval message */
-	smgrtruncate(rel->rd_smgr, FSM_FORKNUM, new_nfsmblocks);
-
-	/*
-	 * We might as well update the local smgr_fsm_nblocks setting.
-	 * smgrtruncate sent an smgr cache inval message, which will cause other
-	 * backends to invalidate their copy of smgr_fsm_nblocks, and this one too
-	 * at the next command boundary.  But this ensures it isn't outright wrong
-	 * until then.
-	 */
-	if (rel->rd_smgr)
-		rel->rd_smgr->smgr_fsm_nblocks = new_nfsmblocks;
-
-	/*
-	 * Update upper-level FSM pages to account for the truncation.  This is
-	 * important because the just-truncated pages were likely marked as
-	 * all-free, and would be preferentially selected.
-	 */
-	FreeSpaceMapVacuumRange(rel, nblocks, InvalidBlockNumber);
 }
 
 /*
diff --git a/src/backend/storage/smgr/smgr.c b/src/backend/storage/smgr/smgr.c
index b0d9f21..82a2977 100644
--- a/src/backend/storage/smgr/smgr.c
+++ b/src/backend/storage/smgr/smgr.c
@@ -473,29 +473,30 @@ smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo)
 }
 
 /*
- *	smgrdounlinkfork() -- Immediately unlink one fork of a relation.
+ *	smgrdounlinkfork() -- Immediately unlink each fork of a relation.
  *
- *		The specified fork of the relation is removed from the store.  This
- *		should not be used during transactional operations, since it can't be
- *		undone.
+ *		Each fork of the relation is removed from the store.  This should
+ *		not be used during transactional operations, since it can't be undone.
  *
  *		If isRedo is true, it is okay for the underlying file to be gone
  *		already.
  */
 void
-smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo)
+smgrdounlinkfork(SMgrRelation reln, ForkNumber *forknum, int nforks, bool isRedo)
 {
 	RelFileNodeBackend rnode = reln->smgr_rnode;
 	int			which = reln->smgr_which;
+	int			i;
 
-	/* Close the fork at smgr level */
-	smgrsw[which].smgr_close(reln, forknum);
+	/* Close each fork at smgr level */
+	for (i = 0; i < nforks; i++)
+		smgrsw[which].smgr_close(reln, forknum[i]);
 
 	/*
-	 * Get rid of any remaining buffers for the fork.  bufmgr will just drop
+	 * Get rid of any remaining buffers for each fork. bufmgr will just drop
 	 * them without bothering to write the contents.
 	 */
-	DropRelFileNodeBuffers(rnode, forknum, 0);
+	DropRelFileNodeBuffers(rnode, forknum, nforks, 0);
 
 	/*
 	 * It'd be nice to tell the stats collector to forget it immediately, too.
@@ -521,7 +522,8 @@ smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo)
 	 * ERROR, because we've already decided to commit or abort the current
 	 * xact.
 	 */
-	smgrsw[which].smgr_unlink(rnode, forknum, isRedo);
+	for (i = 0; i < nforks; i++)
+		smgrsw[which].smgr_unlink(rnode, forknum[i], isRedo);
 }
 
 /*
@@ -618,13 +620,15 @@ smgrnblocks(SMgrRelation reln, ForkNumber forknum)
  * The truncation is done immediately, so this can't be rolled back.
  */
 void
-smgrtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks)
+smgrtruncate(SMgrRelation reln, ForkNumber *forknum, int nforks, BlockNumber *nblocks)
 {
+	int		i;
+
 	/*
 	 * Get rid of any buffers for the about-to-be-deleted blocks. bufmgr will
 	 * just drop them without bothering to write the contents.
 	 */
-	DropRelFileNodeBuffers(reln->smgr_rnode, forknum, nblocks);
+	DropRelFileNodeBuffers(reln->smgr_rnode, forknum, nforks, nblocks);
 
 	/*
 	 * Send a shared-inval message to force other backends to close any smgr
@@ -638,10 +642,23 @@ smgrtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks)
 	 */
 	CacheInvalidateSmgr(reln->smgr_rnode);
 
-	/*
-	 * Do the truncation.
-	 */
-	smgrsw[reln->smgr_which].smgr_truncate(reln, forknum, nblocks);
+	/* Do the truncation */
+	for (i = 0; i < nforks; i++)
+	{
+		smgrsw[reln->smgr_which].smgr_truncate(reln, forknum[i], nblocks[i]);
+
+		/*
+		 * We might as well update the local smgr_fsm_nblocks and smgr_vm_nblocks
+		 * setting. smgrtruncate sent an smgr cache inval message, which will
+		 * cause other backends to invalidate their copy of smgr_fsm_nblocks and
+		 * smgr_vm_nblocks, and these ones too at the next command boundary. But
+		 * this ensures these aren't outright wrong until then.
+		 */
+		if (forknum[i] == FSM_FORKNUM)
+			reln->smgr_fsm_nblocks = nblocks[i];
+		if (forknum[i] == VISIBILITYMAP_FORKNUM)
+			reln->smgr_vm_nblocks = nblocks[i];
+	}
 }
 
 /*
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index 2d88043..1ab6a81 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -44,6 +44,6 @@ extern void visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf,
 							  uint8 flags);
 extern uint8 visibilitymap_get_status(Relation rel, BlockNumber heapBlk, Buffer *vmbuf);
 extern void visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_frozen);
-extern void visibilitymap_truncate(Relation rel, BlockNumber nheapblocks);
+extern BlockNumber visibilitymap_truncate_prepare(Relation rel, BlockNumber nheapblocks);
 
 #endif							/* VISIBILITYMAP_H */
diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h
index 509f4b7..17b97f7 100644
--- a/src/include/storage/bufmgr.h
+++ b/src/include/storage/bufmgr.h
@@ -190,8 +190,8 @@ extern BlockNumber RelationGetNumberOfBlocksInFork(Relation relation,
 extern void FlushOneBuffer(Buffer buffer);
 extern void FlushRelationBuffers(Relation rel);
 extern void FlushDatabaseBuffers(Oid dbid);
-extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode,
-								   ForkNumber forkNum, BlockNumber firstDelBlock);
+extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
+								   int nforks, BlockNumber *firstDelBlock);
 extern void DropRelFileNodesAllBuffers(RelFileNodeBackend *rnodes, int nnodes);
 extern void DropDatabaseBuffers(Oid dbid);
 
diff --git a/src/include/storage/freespace.h b/src/include/storage/freespace.h
index 8d8c465..bf19a67 100644
--- a/src/include/storage/freespace.h
+++ b/src/include/storage/freespace.h
@@ -30,7 +30,7 @@ extern void RecordPageWithFreeSpace(Relation rel, BlockNumber heapBlk,
 extern void XLogRecordPageWithFreeSpace(RelFileNode rnode, BlockNumber heapBlk,
 										Size spaceAvail);
 
-extern void FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks);
+extern BlockNumber MarkFreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks);
 extern void FreeSpaceMapVacuum(Relation rel);
 extern void FreeSpaceMapVacuumRange(Relation rel, BlockNumber start,
 									BlockNumber end);
diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h
index d286c8c..a24532c 100644
--- a/src/include/storage/smgr.h
+++ b/src/include/storage/smgr.h
@@ -90,7 +90,8 @@ extern void smgrclosenode(RelFileNodeBackend rnode);
 extern void smgrcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo);
 extern void smgrdounlink(SMgrRelation reln, bool isRedo);
 extern void smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo);
-extern void smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo);
+extern void smgrdounlinkfork(SMgrRelation reln, ForkNumber *forknum,
+							 int nforks, bool isRedo);
 extern void smgrextend(SMgrRelation reln, ForkNumber forknum,
 					   BlockNumber blocknum, char *buffer, bool skipFsync);
 extern void smgrprefetch(SMgrRelation reln, ForkNumber forknum,
@@ -102,8 +103,8 @@ extern void smgrwrite(SMgrRelation reln, ForkNumber forknum,
 extern void smgrwriteback(SMgrRelation reln, ForkNumber forknum,
 						  BlockNumber blocknum, BlockNumber nblocks);
 extern BlockNumber smgrnblocks(SMgrRelation reln, ForkNumber forknum);
-extern void smgrtruncate(SMgrRelation reln, ForkNumber forknum,
-						 BlockNumber nblocks);
+extern void smgrtruncate(SMgrRelation reln, ForkNumber *forknum,
+						 int nforks, BlockNumber *nblocks);
 extern void smgrimmedsync(SMgrRelation reln, ForkNumber forknum);
 extern void AtEOXact_SMgr(void);
 
-- 
1.8.3.1

From 3aeda104f3c2cc9b0841e536cb39fcd16cb8d881 Mon Sep 17 00:00:00 2001
From: Kirk Jamison <k.jamison@jp.fujitsu.com>
Date: Mon, 9 Sep 2019 06:09:04 +0000
Subject: [PATCH] Remove deadcode smgrdounlinkfork()

---
 src/backend/storage/smgr/smgr.c | 55 -----------------------------------------
 src/include/storage/smgr.h      |  1 -
 2 files changed, 56 deletions(-)

diff --git a/src/backend/storage/smgr/smgr.c b/src/backend/storage/smgr/smgr.c
index b0d9f21..5b5a80e 100644
--- a/src/backend/storage/smgr/smgr.c
+++ b/src/backend/storage/smgr/smgr.c
@@ -343,9 +343,6 @@ smgrcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo)
  *
  *		If isRedo is true, it is okay for the underlying file(s) to be gone
  *		already.
- *
- *		This is equivalent to calling smgrdounlinkfork for each fork, but
- *		it's significantly quicker so should be preferred when possible.
  */
 void
 smgrdounlink(SMgrRelation reln, bool isRedo)
@@ -473,58 +470,6 @@ smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo)
 }
 
 /*
- *	smgrdounlinkfork() -- Immediately unlink one fork of a relation.
- *
- *		The specified fork of the relation is removed from the store.  This
- *		should not be used during transactional operations, since it can't be
- *		undone.
- *
- *		If isRedo is true, it is okay for the underlying file to be gone
- *		already.
- */
-void
-smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo)
-{
-	RelFileNodeBackend rnode = reln->smgr_rnode;
-	int			which = reln->smgr_which;
-
-	/* Close the fork at smgr level */
-	smgrsw[which].smgr_close(reln, forknum);
-
-	/*
-	 * Get rid of any remaining buffers for the fork.  bufmgr will just drop
-	 * them without bothering to write the contents.
-	 */
-	DropRelFileNodeBuffers(rnode, forknum, 0);
-
-	/*
-	 * It'd be nice to tell the stats collector to forget it immediately, too.
-	 * But we can't because we don't know the OID (and in cases involving
-	 * relfilenode swaps, it's not always clear which table OID to forget,
-	 * anyway).
-	 */
-
-	/*
-	 * Send a shared-inval message to force other backends to close any
-	 * dangling smgr references they may have for this rel.  We should do this
-	 * before starting the actual unlinking, in case we fail partway through
-	 * that step.  Note that the sinval message will eventually come back to
-	 * this backend, too, and thereby provide a backstop that we closed our
-	 * own smgr rel.
-	 */
-	CacheInvalidateSmgr(rnode);
-
-	/*
-	 * Delete the physical file(s).
-	 *
-	 * Note: smgr_unlink must treat deletion failure as a WARNING, not an
-	 * ERROR, because we've already decided to commit or abort the current
-	 * xact.
-	 */
-	smgrsw[which].smgr_unlink(rnode, forknum, isRedo);
-}
-
-/*
  *	smgrextend() -- Add a new block to a file.
  *
  *		The semantics are nearly the same as smgrwrite(): write at the
diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h
index d286c8c..7393727 100644
--- a/src/include/storage/smgr.h
+++ b/src/include/storage/smgr.h
@@ -90,7 +90,6 @@ extern void smgrclosenode(RelFileNodeBackend rnode);
 extern void smgrcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo);
 extern void smgrdounlink(SMgrRelation reln, bool isRedo);
 extern void smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo);
-extern void smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo);
 extern void smgrextend(SMgrRelation reln, ForkNumber forknum,
 					   BlockNumber blocknum, char *buffer, bool skipFsync);
 extern void smgrprefetch(SMgrRelation reln, ForkNumber forknum,
-- 
1.8.3.1

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index 1372bb6..60eff7f 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -383,6 +383,8 @@ pg_truncate_visibility_map(PG_FUNCTION_ARGS)
 {
 	Oid			relid = PG_GETARG_OID(0);
 	Relation	rel;
+	ForkNumber	fork;
+	BlockNumber	block;
 
 	rel = relation_open(relid, AccessExclusiveLock);
 
@@ -392,7 +394,11 @@ pg_truncate_visibility_map(PG_FUNCTION_ARGS)
 	RelationOpenSmgr(rel);
 	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
 
-	visibilitymap_truncate(rel, 0);
+	block = visibilitymap_truncate_prepare(rel, 0);
+	if (BlockNumberIsValid(block))
+		fork = VISIBILITYMAP_FORKNUM;
+
+	smgrtruncate(rel->rd_smgr, &fork, 1, &block);
 
 	if (RelationNeedsWAL(rel))
 	{
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index a08922b..351fc31 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -17,7 +17,7 @@
  *		visibilitymap_set	 - set a bit in a previously pinned page
  *		visibilitymap_get_status - get status of bits
  *		visibilitymap_count  - count number of bits set in visibility map
- *		visibilitymap_truncate	- truncate the visibility map
+ *		visibilitymap_truncate_prepare - truncate only tail bits of map pages
  *
  * NOTES
  *
@@ -430,16 +430,18 @@ visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_fro
 }
 
 /*
- *	visibilitymap_truncate - truncate the visibility map
+ *	visibilitymap_truncate_prepare - truncate only tail bits of map page
+ *									 and return the block number for actual
+ *									 truncation later
  *
- * The caller must hold AccessExclusiveLock on the relation, to ensure that
- * other backends receive the smgr invalidation event that this function sends
- * before they access the VM again.
+ * Note that this does not truncate the actual visibility map pages.
+ * When this function is called, the caller must eventually follow it with
+ * smgrtruncate() call to actually truncate visibility map pages.
  *
  * nheapblocks is the new size of the heap.
  */
-void
-visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
+BlockNumber
+visibilitymap_truncate_prepare(Relation rel, BlockNumber nheapblocks)
 {
 	BlockNumber newnblocks;
 
@@ -459,7 +461,7 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 	 * nothing to truncate.
 	 */
 	if (!smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM))
-		return;
+		return InvalidBlockNumber;
 
 	/*
 	 * Unless the new size is exactly at a visibility map page boundary, the
@@ -480,7 +482,7 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 		if (!BufferIsValid(mapBuffer))
 		{
 			/* nothing to do, the file was already smaller */
-			return;
+			return InvalidBlockNumber;
 		}
 
 		page = BufferGetPage(mapBuffer);
@@ -528,20 +530,10 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 	if (smgrnblocks(rel->rd_smgr, VISIBILITYMAP_FORKNUM) <= newnblocks)
 	{
 		/* nothing to do, the file was already smaller than requested size */
-		return;
+		return InvalidBlockNumber;
 	}
-
-	/* Truncate the unused VM pages, and send smgr inval message */
-	smgrtruncate(rel->rd_smgr, VISIBILITYMAP_FORKNUM, newnblocks);
-
-	/*
-	 * We might as well update the local smgr_vm_nblocks setting. smgrtruncate
-	 * sent an smgr cache inval message, which will cause other backends to
-	 * invalidate their copy of smgr_vm_nblocks, and this one too at the next
-	 * command boundary.  But this ensures it isn't outright wrong until then.
-	 */
-	if (rel->rd_smgr)
-		rel->rd_smgr->smgr_vm_nblocks = newnblocks;
+	else
+		return newnblocks;
 }
 
 /*
diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c
index 3cc886f..a11e54a 100644
--- a/src/backend/catalog/storage.c
+++ b/src/backend/catalog/storage.c
@@ -231,6 +231,10 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 {
 	bool		fsm;
 	bool		vm;
+	ForkNumber	forks[MAX_FORKNUM];
+	BlockNumber	blocks[MAX_FORKNUM];
+	BlockNumber	first_removed_nblocks = InvalidBlockNumber;
+	int		nforks = 0;
 
 	/* Open it at the smgr level if not already done */
 	RelationOpenSmgr(rel);
@@ -242,15 +246,33 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 	rel->rd_smgr->smgr_fsm_nblocks = InvalidBlockNumber;
 	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
 
-	/* Truncate the FSM first if it exists */
+	/* Find the dirty FSM page and return a block number. */
 	fsm = smgrexists(rel->rd_smgr, FSM_FORKNUM);
 	if (fsm)
-		FreeSpaceMapTruncateRel(rel, nblocks);
+	{
+		blocks[nforks] = FreeSpaceMapLocateBlock(rel, nblocks);
+		if (BlockNumberIsValid(blocks[nforks]))
+		{
+			first_removed_nblocks = nblocks;
+			forks[nforks] = FSM_FORKNUM;
+			nforks++;
+		}
+	}
 
-	/* Truncate the visibility map too if it exists. */
+	/*
+	 * Truncate only the tail bits of VM and return the block number
+	 * for actual truncation later in smgrtruncate.
+	 */
 	vm = smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM);
 	if (vm)
-		visibilitymap_truncate(rel, nblocks);
+	{
+		blocks[nforks] = visibilitymap_truncate_prepare(rel, nblocks);
+		if (BlockNumberIsValid(blocks[nforks]))
+		{
+			forks[nforks] = VISIBILITYMAP_FORKNUM;
+			nforks++;
+		}
+	}
 
 	/*
 	 * We WAL-log the truncation before actually truncating, which means
@@ -290,8 +312,20 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 			XLogFlush(lsn);
 	}
 
-	/* Do the real work */
-	smgrtruncate(rel->rd_smgr, MAIN_FORKNUM, nblocks);
+	/* Pinpoint the MAIN fork and its blocks */
+	forks[nforks] = MAIN_FORKNUM;
+	blocks[nforks] = nblocks;
+	nforks++;
+
+	/* Truncate relation forks simultaneously */
+	smgrtruncate(rel->rd_smgr, forks, nforks, blocks);
+
+	/*
+	 * Update upper-level FSM pages to account for the truncation.
+	 * This is important because the just-truncated pages were likely
+	 * marked as all-free, and would be preferentially selected.
+	 */
+	FreeSpaceMapVacuumRange(rel, first_removed_nblocks, InvalidBlockNumber);
 }
 
 /*
@@ -588,6 +622,13 @@ smgr_redo(XLogReaderState *record)
 		xl_smgr_truncate *xlrec = (xl_smgr_truncate *) XLogRecGetData(record);
 		SMgrRelation reln;
 		Relation	rel;
+		ForkNumber	forks[MAX_FORKNUM];
+		BlockNumber	blocks[MAX_FORKNUM];
+		BlockNumber	first_removed_nblocks = InvalidBlockNumber;
+		int		nforks = 0;
+		bool		fsm_fork = false;
+		bool		main_fork = false;
+		bool		vm_fork = false;
 
 		reln = smgropen(xlrec->rnode, InvalidBackendId);
 
@@ -616,23 +657,58 @@ smgr_redo(XLogReaderState *record)
 		 */
 		XLogFlush(lsn);
 
+		/*
+		 * To speedup recovery, we identify the about-to-be-truncated blocks
+		 * of relation forks first, then truncate those simultaneously later.
+		 */
 		if ((xlrec->flags & SMGR_TRUNCATE_HEAP) != 0)
 		{
-			smgrtruncate(reln, MAIN_FORKNUM, xlrec->blkno);
-
-			/* Also tell xlogutils.c about it */
-			XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
+			forks[nforks] = MAIN_FORKNUM;
+			blocks[nforks] = xlrec->blkno;
+			nforks++;
+			main_fork = true;
 		}
 
-		/* Truncate FSM and VM too */
 		rel = CreateFakeRelcacheEntry(xlrec->rnode);
 
 		if ((xlrec->flags & SMGR_TRUNCATE_FSM) != 0 &&
 			smgrexists(reln, FSM_FORKNUM))
-			FreeSpaceMapTruncateRel(rel, xlrec->blkno);
+		{
+			blocks[nforks] = FreeSpaceMapLocateBlock(rel, xlrec->blkno);
+			if (BlockNumberIsValid(blocks[nforks]))
+			{
+				first_removed_nblocks = xlrec->blkno;
+				forks[nforks] = FSM_FORKNUM;
+				nforks++;
+				fsm_fork = true;
+			}
+		}
 		if ((xlrec->flags & SMGR_TRUNCATE_VM) != 0 &&
 			smgrexists(reln, VISIBILITYMAP_FORKNUM))
-			visibilitymap_truncate(rel, xlrec->blkno);
+		{
+			blocks[nforks] = visibilitymap_truncate_prepare(rel, xlrec->blkno);
+			if (BlockNumberIsValid(blocks[nforks]))
+			{
+				forks[nforks] = VISIBILITYMAP_FORKNUM;
+				nforks++;
+				vm_fork = true;
+			}
+		}
+
+		/* Truncate relation forks simultaneously */
+		if (main_fork || fsm_fork || vm_fork)
+			smgrtruncate(reln, forks, nforks, blocks);
+
+		/* Also tell xlogutils.c about it */
+		if (main_fork)
+			XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
+
+		/*
+		 * Update upper-level FSM pages to account for the truncation.
+		 * This is important because the just-truncated pages were likely
+		 * marked as all-free, and would be preferentially selected.
+		 */
+		FreeSpaceMapVacuumRange(rel, first_removed_nblocks, InvalidBlockNumber);
 
 		FreeFakeRelcacheEntry(rel);
 	}
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index 6f3a402..1f2b600 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -2900,8 +2900,8 @@ BufferGetLSNAtomic(Buffer buffer)
 /* ---------------------------------------------------------------------
  *		DropRelFileNodeBuffers
  *
- *		This function removes from the buffer pool all the pages of the
- *		specified relation fork that have block numbers >= firstDelBlock.
+ *		This function simultaneously removes from the buffer pool all the
+ *		pages of the relation forks that have block numbers >= firstDelBlock.
  *		(In particular, with firstDelBlock = 0, all pages are removed.)
  *		Dirty pages are simply dropped, without bothering to write them
  *		out first.  Therefore, this is NOT rollback-able, and so should be
@@ -2924,8 +2924,8 @@ BufferGetLSNAtomic(Buffer buffer)
  * --------------------------------------------------------------------
  */
 void
-DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
-					   BlockNumber firstDelBlock)
+DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
+					   int nforks, BlockNumber *firstDelBlock)
 {
 	int			i;
 
@@ -2933,7 +2933,12 @@ DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
 	if (RelFileNodeBackendIsTemp(rnode))
 	{
 		if (rnode.backend == MyBackendId)
-			DropRelFileNodeLocalBuffers(rnode.node, forkNum, firstDelBlock);
+		{
+			int		j;
+			for (j = 0; j < nforks; j++)
+				DropRelFileNodeLocalBuffers(rnode.node, forkNum[j],
+											firstDelBlock[j]);
+		}
 		return;
 	}
 
@@ -2941,6 +2946,7 @@ DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
 	{
 		BufferDesc *bufHdr = GetBufferDescriptor(i);
 		uint32		buf_state;
+		int		j = 0;
 
 		/*
 		 * We can make this a tad faster by prechecking the buffer tag before
@@ -2962,11 +2968,18 @@ DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
 			continue;
 
 		buf_state = LockBufHdr(bufHdr);
-		if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
-			bufHdr->tag.forkNum == forkNum &&
-			bufHdr->tag.blockNum >= firstDelBlock)
-			InvalidateBuffer(bufHdr);	/* releases spinlock */
-		else
+
+		for (j = 0; j < nforks; j++)
+		{
+			if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
+				bufHdr->tag.forkNum == forkNum[j] &&
+				bufHdr->tag.blockNum >= firstDelBlock[j])
+			{
+				InvalidateBuffer(bufHdr); /* releases spinlock */
+				break;
+			}
+		}
+		if (j >= nforks)
 			UnlockBufHdr(bufHdr, buf_state);
 	}
 }
diff --git a/src/backend/storage/freespace/freespace.c b/src/backend/storage/freespace/freespace.c
index 2383094..473c2e5 100644
--- a/src/backend/storage/freespace/freespace.c
+++ b/src/backend/storage/freespace/freespace.c
@@ -247,16 +247,17 @@ GetRecordedFreeSpace(Relation rel, BlockNumber heapBlk)
 }
 
 /*
- * FreeSpaceMapTruncateRel - adjust for truncation of a relation.
+ * FreeSpaceMapLocateBlock - adjust for truncation of a relation.
  *
- * The caller must hold AccessExclusiveLock on the relation, to ensure that
- * other backends receive the smgr invalidation event that this function sends
- * before they access the FSM again.
+ * This function finds the dirty page and returns a block number.
+ *
+ * The caller of this function must eventually call smgrtruncate() to actually
+ * truncate FSM pages.
  *
  * nblocks is the new size of the heap.
  */
-void
-FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
+BlockNumber
+FreeSpaceMapLocateBlock(Relation rel, BlockNumber nblocks)
 {
 	BlockNumber new_nfsmblocks;
 	FSMAddress	first_removed_address;
@@ -270,7 +271,7 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 	 * truncate.
 	 */
 	if (!smgrexists(rel->rd_smgr, FSM_FORKNUM))
-		return;
+		return InvalidBlockNumber;
 
 	/* Get the location in the FSM of the first removed heap block */
 	first_removed_address = fsm_get_location(nblocks, &first_removed_slot);
@@ -285,7 +286,7 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 	{
 		buf = fsm_readbuf(rel, first_removed_address, false);
 		if (!BufferIsValid(buf))
-			return;				/* nothing to do; the FSM was already smaller */
+			return InvalidBlockNumber;	/* nothing to do; the FSM was already smaller */
 		LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
 
 		/* NO EREPORT(ERROR) from here till changes are logged */
@@ -310,33 +311,16 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 		UnlockReleaseBuffer(buf);
 
 		new_nfsmblocks = fsm_logical_to_physical(first_removed_address) + 1;
+		return new_nfsmblocks;
 	}
 	else
 	{
 		new_nfsmblocks = fsm_logical_to_physical(first_removed_address);
 		if (smgrnblocks(rel->rd_smgr, FSM_FORKNUM) <= new_nfsmblocks)
-			return;				/* nothing to do; the FSM was already smaller */
+			return InvalidBlockNumber;	/* nothing to do; the FSM was already smaller */
+		else
+			return new_nfsmblocks;
 	}
-
-	/* Truncate the unused FSM pages, and send smgr inval message */
-	smgrtruncate(rel->rd_smgr, FSM_FORKNUM, new_nfsmblocks);
-
-	/*
-	 * We might as well update the local smgr_fsm_nblocks setting.
-	 * smgrtruncate sent an smgr cache inval message, which will cause other
-	 * backends to invalidate their copy of smgr_fsm_nblocks, and this one too
-	 * at the next command boundary.  But this ensures it isn't outright wrong
-	 * until then.
-	 */
-	if (rel->rd_smgr)
-		rel->rd_smgr->smgr_fsm_nblocks = new_nfsmblocks;
-
-	/*
-	 * Update upper-level FSM pages to account for the truncation.  This is
-	 * important because the just-truncated pages were likely marked as
-	 * all-free, and would be preferentially selected.
-	 */
-	FreeSpaceMapVacuumRange(rel, nblocks, InvalidBlockNumber);
 }
 
 /*
diff --git a/src/backend/storage/smgr/smgr.c b/src/backend/storage/smgr/smgr.c
index 5b5a80e..0fc1f76 100644
--- a/src/backend/storage/smgr/smgr.c
+++ b/src/backend/storage/smgr/smgr.c
@@ -469,6 +469,7 @@ smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo)
 	pfree(rnodes);
 }
 
+
 /*
  *	smgrextend() -- Add a new block to a file.
  *
@@ -561,15 +562,21 @@ smgrnblocks(SMgrRelation reln, ForkNumber forknum)
  *					  of blocks
  *
  * The truncation is done immediately, so this can't be rolled back.
+ *
+ * The caller must hold AccessExclusiveLock on the relation, to ensure that
+ * other backends receive the smgr invalidation event that this function sends
+ * before they access the relation again.
  */
 void
-smgrtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks)
+smgrtruncate(SMgrRelation reln, ForkNumber *forknum, int nforks, BlockNumber *nblocks)
 {
+	int		i;
+
 	/*
 	 * Get rid of any buffers for the about-to-be-deleted blocks. bufmgr will
 	 * just drop them without bothering to write the contents.
 	 */
-	DropRelFileNodeBuffers(reln->smgr_rnode, forknum, nblocks);
+	DropRelFileNodeBuffers(reln->smgr_rnode, forknum, nforks, nblocks);
 
 	/*
 	 * Send a shared-inval message to force other backends to close any smgr
@@ -583,10 +590,23 @@ smgrtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks)
 	 */
 	CacheInvalidateSmgr(reln->smgr_rnode);
 
-	/*
-	 * Do the truncation.
-	 */
-	smgrsw[reln->smgr_which].smgr_truncate(reln, forknum, nblocks);
+	/* Do the truncation */
+	for (i = 0; i < nforks; i++)
+	{
+		smgrsw[reln->smgr_which].smgr_truncate(reln, forknum[i], nblocks[i]);
+
+		/*
+		 * We might as well update the local smgr_fsm_nblocks and smgr_vm_nblocks
+		 * setting. smgrtruncate sent an smgr cache inval message, which will
+		 * cause other backends to invalidate their copy of smgr_fsm_nblocks and
+		 * smgr_vm_nblocks, and these ones too at the next command boundary. But
+		 * this ensures these aren't outright wrong until then.
+		 */
+		if (forknum[i] == FSM_FORKNUM)
+			reln->smgr_fsm_nblocks = nblocks[i];
+		if (forknum[i] == VISIBILITYMAP_FORKNUM)
+			reln->smgr_vm_nblocks = nblocks[i];
+	}
 }
 
 /*
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index 2d88043..1ab6a81 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -44,6 +44,6 @@ extern void visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf,
 							  uint8 flags);
 extern uint8 visibilitymap_get_status(Relation rel, BlockNumber heapBlk, Buffer *vmbuf);
 extern void visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_frozen);
-extern void visibilitymap_truncate(Relation rel, BlockNumber nheapblocks);
+extern BlockNumber visibilitymap_truncate_prepare(Relation rel, BlockNumber nheapblocks);
 
 #endif							/* VISIBILITYMAP_H */
diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h
index 509f4b7..17b97f7 100644
--- a/src/include/storage/bufmgr.h
+++ b/src/include/storage/bufmgr.h
@@ -190,8 +190,8 @@ extern BlockNumber RelationGetNumberOfBlocksInFork(Relation relation,
 extern void FlushOneBuffer(Buffer buffer);
 extern void FlushRelationBuffers(Relation rel);
 extern void FlushDatabaseBuffers(Oid dbid);
-extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode,
-								   ForkNumber forkNum, BlockNumber firstDelBlock);
+extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
+								   int nforks, BlockNumber *firstDelBlock);
 extern void DropRelFileNodesAllBuffers(RelFileNodeBackend *rnodes, int nnodes);
 extern void DropDatabaseBuffers(Oid dbid);
 
diff --git a/src/include/storage/freespace.h b/src/include/storage/freespace.h
index 8d8c465..0b834cb 100644
--- a/src/include/storage/freespace.h
+++ b/src/include/storage/freespace.h
@@ -30,7 +30,7 @@ extern void RecordPageWithFreeSpace(Relation rel, BlockNumber heapBlk,
 extern void XLogRecordPageWithFreeSpace(RelFileNode rnode, BlockNumber heapBlk,
 										Size spaceAvail);
 
-extern void FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks);
+extern BlockNumber FreeSpaceMapLocateBlock(Relation rel, BlockNumber nblocks);
 extern void FreeSpaceMapVacuum(Relation rel);
 extern void FreeSpaceMapVacuumRange(Relation rel, BlockNumber start,
 									BlockNumber end);
diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h
index 7393727..1543d8d 100644
--- a/src/include/storage/smgr.h
+++ b/src/include/storage/smgr.h
@@ -101,8 +101,8 @@ extern void smgrwrite(SMgrRelation reln, ForkNumber forknum,
 extern void smgrwriteback(SMgrRelation reln, ForkNumber forknum,
 						  BlockNumber blocknum, BlockNumber nblocks);
 extern BlockNumber smgrnblocks(SMgrRelation reln, ForkNumber forknum);
-extern void smgrtruncate(SMgrRelation reln, ForkNumber forknum,
-						 BlockNumber nblocks);
+extern void smgrtruncate(SMgrRelation reln, ForkNumber *forknum,
+						 int nforks, BlockNumber *nblocks);
 extern void smgrimmedsync(SMgrRelation reln, ForkNumber forknum);
 extern void AtEOXact_SMgr(void);
 

From 3aeda104f3c2cc9b0841e536cb39fcd16cb8d881 Mon Sep 17 00:00:00 2001
From: Kirk Jamison <k.jamison@jp.fujitsu.com>
Date: Mon, 9 Sep 2019 06:09:04 +0000
Subject: [PATCH] Remove deadcode smgrdounlinkfork()

---
 src/backend/storage/smgr/smgr.c | 55 -----------------------------------------
 src/include/storage/smgr.h      |  1 -
 2 files changed, 56 deletions(-)

diff --git a/src/backend/storage/smgr/smgr.c b/src/backend/storage/smgr/smgr.c
index b0d9f21..5b5a80e 100644
--- a/src/backend/storage/smgr/smgr.c
+++ b/src/backend/storage/smgr/smgr.c
@@ -343,9 +343,6 @@ smgrcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo)
  *
  *		If isRedo is true, it is okay for the underlying file(s) to be gone
  *		already.
- *
- *		This is equivalent to calling smgrdounlinkfork for each fork, but
- *		it's significantly quicker so should be preferred when possible.
  */
 void
 smgrdounlink(SMgrRelation reln, bool isRedo)
@@ -473,58 +470,6 @@ smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo)
 }
 
 /*
- *	smgrdounlinkfork() -- Immediately unlink one fork of a relation.
- *
- *		The specified fork of the relation is removed from the store.  This
- *		should not be used during transactional operations, since it can't be
- *		undone.
- *
- *		If isRedo is true, it is okay for the underlying file to be gone
- *		already.
- */
-void
-smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo)
-{
-	RelFileNodeBackend rnode = reln->smgr_rnode;
-	int			which = reln->smgr_which;
-
-	/* Close the fork at smgr level */
-	smgrsw[which].smgr_close(reln, forknum);
-
-	/*
-	 * Get rid of any remaining buffers for the fork.  bufmgr will just drop
-	 * them without bothering to write the contents.
-	 */
-	DropRelFileNodeBuffers(rnode, forknum, 0);
-
-	/*
-	 * It'd be nice to tell the stats collector to forget it immediately, too.
-	 * But we can't because we don't know the OID (and in cases involving
-	 * relfilenode swaps, it's not always clear which table OID to forget,
-	 * anyway).
-	 */
-
-	/*
-	 * Send a shared-inval message to force other backends to close any
-	 * dangling smgr references they may have for this rel.  We should do this
-	 * before starting the actual unlinking, in case we fail partway through
-	 * that step.  Note that the sinval message will eventually come back to
-	 * this backend, too, and thereby provide a backstop that we closed our
-	 * own smgr rel.
-	 */
-	CacheInvalidateSmgr(rnode);
-
-	/*
-	 * Delete the physical file(s).
-	 *
-	 * Note: smgr_unlink must treat deletion failure as a WARNING, not an
-	 * ERROR, because we've already decided to commit or abort the current
-	 * xact.
-	 */
-	smgrsw[which].smgr_unlink(rnode, forknum, isRedo);
-}
-
-/*
  *	smgrextend() -- Add a new block to a file.
  *
  *		The semantics are nearly the same as smgrwrite(): write at the
diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h
index d286c8c..7393727 100644
--- a/src/include/storage/smgr.h
+++ b/src/include/storage/smgr.h
@@ -90,7 +90,6 @@ extern void smgrclosenode(RelFileNodeBackend rnode);
 extern void smgrcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo);
 extern void smgrdounlink(SMgrRelation reln, bool isRedo);
 extern void smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo);
-extern void smgrdounlinkfork(SMgrRelation reln, ForkNumber forknum, bool isRedo);
 extern void smgrextend(SMgrRelation reln, ForkNumber forknum,
 					   BlockNumber blocknum, char *buffer, bool skipFsync);
 extern void smgrprefetch(SMgrRelation reln, ForkNumber forknum,
-- 
1.8.3.1

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index 1372bb6..4d342ea 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -383,6 +383,8 @@ pg_truncate_visibility_map(PG_FUNCTION_ARGS)
 {
 	Oid			relid = PG_GETARG_OID(0);
 	Relation	rel;
+	ForkNumber	fork;
+	BlockNumber	block;
 
 	rel = relation_open(relid, AccessExclusiveLock);
 
@@ -392,20 +394,26 @@ pg_truncate_visibility_map(PG_FUNCTION_ARGS)
 	RelationOpenSmgr(rel);
 	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
 
-	visibilitymap_truncate(rel, 0);
-
-	if (RelationNeedsWAL(rel))
+	block = visibilitymap_truncate_prepare(rel, 0);
+	if (BlockNumberIsValid(block))
 	{
-		xl_smgr_truncate xlrec;
+		fork = VISIBILITYMAP_FORKNUM;
+		smgrtruncate(rel->rd_smgr, &fork, 1, &block);
+
+		if (RelationNeedsWAL(rel))
+		{
+			xl_smgr_truncate xlrec;
 
-		xlrec.blkno = 0;
-		xlrec.rnode = rel->rd_node;
-		xlrec.flags = SMGR_TRUNCATE_VM;
+			xlrec.blkno = 0;
+			xlrec.rnode = rel->rd_node;
+			xlrec.flags = SMGR_TRUNCATE_VM;
 
-		XLogBeginInsert();
-		XLogRegisterData((char *) &xlrec, sizeof(xlrec));
+			XLogBeginInsert();
+			XLogRegisterData((char *) &xlrec, sizeof(xlrec));
 
-		XLogInsert(RM_SMGR_ID, XLOG_SMGR_TRUNCATE | XLR_SPECIAL_REL_UPDATE);
+			XLogInsert(RM_SMGR_ID, XLOG_SMGR_TRUNCATE |
+					   XLR_SPECIAL_REL_UPDATE);
+		}
 	}
 
 	/*
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index a08922b..351fc31 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -17,7 +17,7 @@
  *		visibilitymap_set	 - set a bit in a previously pinned page
  *		visibilitymap_get_status - get status of bits
  *		visibilitymap_count  - count number of bits set in visibility map
- *		visibilitymap_truncate	- truncate the visibility map
+ *		visibilitymap_truncate_prepare - truncate only tail bits of map pages
  *
  * NOTES
  *
@@ -430,16 +430,18 @@ visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_fro
 }
 
 /*
- *	visibilitymap_truncate - truncate the visibility map
+ *	visibilitymap_truncate_prepare - truncate only tail bits of map page
+ *									 and return the block number for actual
+ *									 truncation later
  *
- * The caller must hold AccessExclusiveLock on the relation, to ensure that
- * other backends receive the smgr invalidation event that this function sends
- * before they access the VM again.
+ * Note that this does not truncate the actual visibility map pages.
+ * When this function is called, the caller must eventually follow it with
+ * smgrtruncate() call to actually truncate visibility map pages.
  *
  * nheapblocks is the new size of the heap.
  */
-void
-visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
+BlockNumber
+visibilitymap_truncate_prepare(Relation rel, BlockNumber nheapblocks)
 {
 	BlockNumber newnblocks;
 
@@ -459,7 +461,7 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 	 * nothing to truncate.
 	 */
 	if (!smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM))
-		return;
+		return InvalidBlockNumber;
 
 	/*
 	 * Unless the new size is exactly at a visibility map page boundary, the
@@ -480,7 +482,7 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 		if (!BufferIsValid(mapBuffer))
 		{
 			/* nothing to do, the file was already smaller */
-			return;
+			return InvalidBlockNumber;
 		}
 
 		page = BufferGetPage(mapBuffer);
@@ -528,20 +530,10 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
 	if (smgrnblocks(rel->rd_smgr, VISIBILITYMAP_FORKNUM) <= newnblocks)
 	{
 		/* nothing to do, the file was already smaller than requested size */
-		return;
+		return InvalidBlockNumber;
 	}
-
-	/* Truncate the unused VM pages, and send smgr inval message */
-	smgrtruncate(rel->rd_smgr, VISIBILITYMAP_FORKNUM, newnblocks);
-
-	/*
-	 * We might as well update the local smgr_vm_nblocks setting. smgrtruncate
-	 * sent an smgr cache inval message, which will cause other backends to
-	 * invalidate their copy of smgr_vm_nblocks, and this one too at the next
-	 * command boundary.  But this ensures it isn't outright wrong until then.
-	 */
-	if (rel->rd_smgr)
-		rel->rd_smgr->smgr_vm_nblocks = newnblocks;
+	else
+		return newnblocks;
 }
 
 /*
diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c
index 3cc886f..623cf9f 100644
--- a/src/backend/catalog/storage.c
+++ b/src/backend/catalog/storage.c
@@ -231,6 +231,10 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 {
 	bool		fsm;
 	bool		vm;
+	ForkNumber	forks[MAX_FORKNUM];
+	BlockNumber	blocks[MAX_FORKNUM];
+	BlockNumber	first_removed_nblocks = InvalidBlockNumber;
+	int		nforks = 0;
 
 	/* Open it at the smgr level if not already done */
 	RelationOpenSmgr(rel);
@@ -242,15 +246,33 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 	rel->rd_smgr->smgr_fsm_nblocks = InvalidBlockNumber;
 	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
 
-	/* Truncate the FSM first if it exists */
+	/* Find the dirty FSM page and return a block number. */
 	fsm = smgrexists(rel->rd_smgr, FSM_FORKNUM);
 	if (fsm)
-		FreeSpaceMapTruncateRel(rel, nblocks);
+	{
+		blocks[nforks] = FreeSpaceMapLocateBlock(rel, nblocks);
+		if (BlockNumberIsValid(blocks[nforks]))
+		{
+			first_removed_nblocks = nblocks;
+			forks[nforks] = FSM_FORKNUM;
+			nforks++;
+		}
+	}
 
-	/* Truncate the visibility map too if it exists. */
+	/*
+	 * Truncate only the tail bits of VM and return the block number
+	 * for actual truncation later in smgrtruncate.
+	 */
 	vm = smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM);
 	if (vm)
-		visibilitymap_truncate(rel, nblocks);
+	{
+		blocks[nforks] = visibilitymap_truncate_prepare(rel, nblocks);
+		if (BlockNumberIsValid(blocks[nforks]))
+		{
+			forks[nforks] = VISIBILITYMAP_FORKNUM;
+			nforks++;
+		}
+	}
 
 	/*
 	 * We WAL-log the truncation before actually truncating, which means
@@ -290,8 +312,22 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
 			XLogFlush(lsn);
 	}
 
-	/* Do the real work */
-	smgrtruncate(rel->rd_smgr, MAIN_FORKNUM, nblocks);
+	/* Pinpoint the MAIN fork and its blocks */
+	forks[nforks] = MAIN_FORKNUM;
+	blocks[nforks] = nblocks;
+	nforks++;
+
+	/* Truncate relation forks simultaneously */
+	smgrtruncate(rel->rd_smgr, forks, nforks, blocks);
+
+	/*
+	 * Update upper-level FSM pages to account for the truncation.
+	 * This is important because the just-truncated pages were likely
+	 * marked as all-free, and would be preferentially selected.
+	 */
+	if (fsm)
+		FreeSpaceMapVacuumRange(rel, first_removed_nblocks,
+								InvalidBlockNumber);
 }
 
 /*
@@ -588,6 +624,13 @@ smgr_redo(XLogReaderState *record)
 		xl_smgr_truncate *xlrec = (xl_smgr_truncate *) XLogRecGetData(record);
 		SMgrRelation reln;
 		Relation	rel;
+		ForkNumber	forks[MAX_FORKNUM];
+		BlockNumber	blocks[MAX_FORKNUM];
+		BlockNumber	first_removed_nblocks = InvalidBlockNumber;
+		int		nforks = 0;
+		bool		fsm = false;
+		bool		main = false;
+		bool		vm = false;
 
 		reln = smgropen(xlrec->rnode, InvalidBackendId);
 
@@ -616,23 +659,60 @@ smgr_redo(XLogReaderState *record)
 		 */
 		XLogFlush(lsn);
 
+		/*
+		 * To speedup recovery, we identify the about-to-be-truncated blocks
+		 * of relation forks first, then truncate those simultaneously later.
+		 */
 		if ((xlrec->flags & SMGR_TRUNCATE_HEAP) != 0)
 		{
-			smgrtruncate(reln, MAIN_FORKNUM, xlrec->blkno);
-
-			/* Also tell xlogutils.c about it */
-			XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
+			forks[nforks] = MAIN_FORKNUM;
+			blocks[nforks] = xlrec->blkno;
+			nforks++;
+			main = true;
 		}
 
-		/* Truncate FSM and VM too */
 		rel = CreateFakeRelcacheEntry(xlrec->rnode);
 
 		if ((xlrec->flags & SMGR_TRUNCATE_FSM) != 0 &&
 			smgrexists(reln, FSM_FORKNUM))
-			FreeSpaceMapTruncateRel(rel, xlrec->blkno);
+		{
+			blocks[nforks] = FreeSpaceMapLocateBlock(rel, xlrec->blkno);
+			if (BlockNumberIsValid(blocks[nforks]))
+			{
+				first_removed_nblocks = xlrec->blkno;
+				forks[nforks] = FSM_FORKNUM;
+				nforks++;
+				fsm = true;
+			}
+		}
 		if ((xlrec->flags & SMGR_TRUNCATE_VM) != 0 &&
 			smgrexists(reln, VISIBILITYMAP_FORKNUM))
-			visibilitymap_truncate(rel, xlrec->blkno);
+		{
+			blocks[nforks] = visibilitymap_truncate_prepare(rel, xlrec->blkno);
+			if (BlockNumberIsValid(blocks[nforks]))
+			{
+				forks[nforks] = VISIBILITYMAP_FORKNUM;
+				nforks++;
+				vm = true;
+			}
+		}
+
+		/* Truncate relation forks simultaneously */
+		if (main || fsm || vm)
+			smgrtruncate(reln, forks, nforks, blocks);
+
+		/* Also tell xlogutils.c about it */
+		if (main)
+			XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
+
+		/*
+		 * Update upper-level FSM pages to account for the truncation.
+		 * This is important because the just-truncated pages were likely
+		 * marked as all-free, and would be preferentially selected.
+		 */
+		if (fsm)
+			FreeSpaceMapVacuumRange(rel, first_removed_nblocks,
+									InvalidBlockNumber);
 
 		FreeFakeRelcacheEntry(rel);
 	}
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index 6f3a402..1f2b600 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -2900,8 +2900,8 @@ BufferGetLSNAtomic(Buffer buffer)
 /* ---------------------------------------------------------------------
  *		DropRelFileNodeBuffers
  *
- *		This function removes from the buffer pool all the pages of the
- *		specified relation fork that have block numbers >= firstDelBlock.
+ *		This function simultaneously removes from the buffer pool all the
+ *		pages of the relation forks that have block numbers >= firstDelBlock.
  *		(In particular, with firstDelBlock = 0, all pages are removed.)
  *		Dirty pages are simply dropped, without bothering to write them
  *		out first.  Therefore, this is NOT rollback-able, and so should be
@@ -2924,8 +2924,8 @@ BufferGetLSNAtomic(Buffer buffer)
  * --------------------------------------------------------------------
  */
 void
-DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
-					   BlockNumber firstDelBlock)
+DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
+					   int nforks, BlockNumber *firstDelBlock)
 {
 	int			i;
 
@@ -2933,7 +2933,12 @@ DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
 	if (RelFileNodeBackendIsTemp(rnode))
 	{
 		if (rnode.backend == MyBackendId)
-			DropRelFileNodeLocalBuffers(rnode.node, forkNum, firstDelBlock);
+		{
+			int		j;
+			for (j = 0; j < nforks; j++)
+				DropRelFileNodeLocalBuffers(rnode.node, forkNum[j],
+											firstDelBlock[j]);
+		}
 		return;
 	}
 
@@ -2941,6 +2946,7 @@ DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
 	{
 		BufferDesc *bufHdr = GetBufferDescriptor(i);
 		uint32		buf_state;
+		int		j = 0;
 
 		/*
 		 * We can make this a tad faster by prechecking the buffer tag before
@@ -2962,11 +2968,18 @@ DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
 			continue;
 
 		buf_state = LockBufHdr(bufHdr);
-		if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
-			bufHdr->tag.forkNum == forkNum &&
-			bufHdr->tag.blockNum >= firstDelBlock)
-			InvalidateBuffer(bufHdr);	/* releases spinlock */
-		else
+
+		for (j = 0; j < nforks; j++)
+		{
+			if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
+				bufHdr->tag.forkNum == forkNum[j] &&
+				bufHdr->tag.blockNum >= firstDelBlock[j])
+			{
+				InvalidateBuffer(bufHdr); /* releases spinlock */
+				break;
+			}
+		}
+		if (j >= nforks)
 			UnlockBufHdr(bufHdr, buf_state);
 	}
 }
diff --git a/src/backend/storage/freespace/freespace.c b/src/backend/storage/freespace/freespace.c
index 2383094..473c2e5 100644
--- a/src/backend/storage/freespace/freespace.c
+++ b/src/backend/storage/freespace/freespace.c
@@ -247,16 +247,17 @@ GetRecordedFreeSpace(Relation rel, BlockNumber heapBlk)
 }
 
 /*
- * FreeSpaceMapTruncateRel - adjust for truncation of a relation.
+ * FreeSpaceMapLocateBlock - adjust for truncation of a relation.
  *
- * The caller must hold AccessExclusiveLock on the relation, to ensure that
- * other backends receive the smgr invalidation event that this function sends
- * before they access the FSM again.
+ * This function finds the dirty page and returns a block number.
+ *
+ * The caller of this function must eventually call smgrtruncate() to actually
+ * truncate FSM pages.
  *
  * nblocks is the new size of the heap.
  */
-void
-FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
+BlockNumber
+FreeSpaceMapLocateBlock(Relation rel, BlockNumber nblocks)
 {
 	BlockNumber new_nfsmblocks;
 	FSMAddress	first_removed_address;
@@ -270,7 +271,7 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 	 * truncate.
 	 */
 	if (!smgrexists(rel->rd_smgr, FSM_FORKNUM))
-		return;
+		return InvalidBlockNumber;
 
 	/* Get the location in the FSM of the first removed heap block */
 	first_removed_address = fsm_get_location(nblocks, &first_removed_slot);
@@ -285,7 +286,7 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 	{
 		buf = fsm_readbuf(rel, first_removed_address, false);
 		if (!BufferIsValid(buf))
-			return;				/* nothing to do; the FSM was already smaller */
+			return InvalidBlockNumber;	/* nothing to do; the FSM was already smaller */
 		LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
 
 		/* NO EREPORT(ERROR) from here till changes are logged */
@@ -310,33 +311,16 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
 		UnlockReleaseBuffer(buf);
 
 		new_nfsmblocks = fsm_logical_to_physical(first_removed_address) + 1;
+		return new_nfsmblocks;
 	}
 	else
 	{
 		new_nfsmblocks = fsm_logical_to_physical(first_removed_address);
 		if (smgrnblocks(rel->rd_smgr, FSM_FORKNUM) <= new_nfsmblocks)
-			return;				/* nothing to do; the FSM was already smaller */
+			return InvalidBlockNumber;	/* nothing to do; the FSM was already smaller */
+		else
+			return new_nfsmblocks;
 	}
-
-	/* Truncate the unused FSM pages, and send smgr inval message */
-	smgrtruncate(rel->rd_smgr, FSM_FORKNUM, new_nfsmblocks);
-
-	/*
-	 * We might as well update the local smgr_fsm_nblocks setting.
-	 * smgrtruncate sent an smgr cache inval message, which will cause other
-	 * backends to invalidate their copy of smgr_fsm_nblocks, and this one too
-	 * at the next command boundary.  But this ensures it isn't outright wrong
-	 * until then.
-	 */
-	if (rel->rd_smgr)
-		rel->rd_smgr->smgr_fsm_nblocks = new_nfsmblocks;
-
-	/*
-	 * Update upper-level FSM pages to account for the truncation.  This is
-	 * important because the just-truncated pages were likely marked as
-	 * all-free, and would be preferentially selected.
-	 */
-	FreeSpaceMapVacuumRange(rel, nblocks, InvalidBlockNumber);
 }
 
 /*
diff --git a/src/backend/storage/smgr/smgr.c b/src/backend/storage/smgr/smgr.c
index 5b5a80e..b6d4d23 100644
--- a/src/backend/storage/smgr/smgr.c
+++ b/src/backend/storage/smgr/smgr.c
@@ -469,6 +469,7 @@ smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo)
 	pfree(rnodes);
 }
 
+
 /*
  *	smgrextend() -- Add a new block to a file.
  *
@@ -561,15 +562,21 @@ smgrnblocks(SMgrRelation reln, ForkNumber forknum)
  *					  of blocks
  *
  * The truncation is done immediately, so this can't be rolled back.
+ *
+ * The caller must hold AccessExclusiveLock on the relation, to ensure that
+ * other backends receive the smgr invalidation event that this function sends
+ * before they access the relation again.
  */
 void
-smgrtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks)
+smgrtruncate(SMgrRelation reln, ForkNumber *forknum, int nforks, BlockNumber *nblocks)
 {
+	int		i;
+
 	/*
 	 * Get rid of any buffers for the about-to-be-deleted blocks. bufmgr will
 	 * just drop them without bothering to write the contents.
 	 */
-	DropRelFileNodeBuffers(reln->smgr_rnode, forknum, nblocks);
+	DropRelFileNodeBuffers(reln->smgr_rnode, forknum, nforks, nblocks);
 
 	/*
 	 * Send a shared-inval message to force other backends to close any smgr
@@ -583,10 +590,23 @@ smgrtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks)
 	 */
 	CacheInvalidateSmgr(reln->smgr_rnode);
 
-	/*
-	 * Do the truncation.
-	 */
-	smgrsw[reln->smgr_which].smgr_truncate(reln, forknum, nblocks);
+	/* Do the truncation */
+	for (i = 0; i < nforks; i++)
+	{
+		smgrsw[reln->smgr_which].smgr_truncate(reln, forknum[i], nblocks[i]);
+
+		/*
+		 * We might as well update the local smgr_fsm_nblocks and smgr_vm_nblocks
+		 * setting. The smgr cache inval message we sent will cause other backends
+		 * to invalidate their copy of smgr_fsm_nblocks and smgr_vm_nblocks, and
+		 * these ones too at the next command boundary. But these ensure these
+		 * aren't outright wrong until then.
+		 */
+		if (forknum[i] == FSM_FORKNUM)
+			reln->smgr_fsm_nblocks = nblocks[i];
+		if (forknum[i] == VISIBILITYMAP_FORKNUM)
+			reln->smgr_vm_nblocks = nblocks[i];
+	}
 }
 
 /*
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index 2d88043..1ab6a81 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -44,6 +44,6 @@ extern void visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf,
 							  uint8 flags);
 extern uint8 visibilitymap_get_status(Relation rel, BlockNumber heapBlk, Buffer *vmbuf);
 extern void visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_frozen);
-extern void visibilitymap_truncate(Relation rel, BlockNumber nheapblocks);
+extern BlockNumber visibilitymap_truncate_prepare(Relation rel, BlockNumber nheapblocks);
 
 #endif							/* VISIBILITYMAP_H */
diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h
index 509f4b7..17b97f7 100644
--- a/src/include/storage/bufmgr.h
+++ b/src/include/storage/bufmgr.h
@@ -190,8 +190,8 @@ extern BlockNumber RelationGetNumberOfBlocksInFork(Relation relation,
 extern void FlushOneBuffer(Buffer buffer);
 extern void FlushRelationBuffers(Relation rel);
 extern void FlushDatabaseBuffers(Oid dbid);
-extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode,
-								   ForkNumber forkNum, BlockNumber firstDelBlock);
+extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
+								   int nforks, BlockNumber *firstDelBlock);
 extern void DropRelFileNodesAllBuffers(RelFileNodeBackend *rnodes, int nnodes);
 extern void DropDatabaseBuffers(Oid dbid);
 
diff --git a/src/include/storage/freespace.h b/src/include/storage/freespace.h
index 8d8c465..0b834cb 100644
--- a/src/include/storage/freespace.h
+++ b/src/include/storage/freespace.h
@@ -30,7 +30,7 @@ extern void RecordPageWithFreeSpace(Relation rel, BlockNumber heapBlk,
 extern void XLogRecordPageWithFreeSpace(RelFileNode rnode, BlockNumber heapBlk,
 										Size spaceAvail);
 
-extern void FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks);
+extern BlockNumber FreeSpaceMapLocateBlock(Relation rel, BlockNumber nblocks);
 extern void FreeSpaceMapVacuum(Relation rel);
 extern void FreeSpaceMapVacuumRange(Relation rel, BlockNumber start,
 									BlockNumber end);
diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h
index 7393727..1543d8d 100644
--- a/src/include/storage/smgr.h
+++ b/src/include/storage/smgr.h
@@ -101,8 +101,8 @@ extern void smgrwrite(SMgrRelation reln, ForkNumber forknum,
 extern void smgrwriteback(SMgrRelation reln, ForkNumber forknum,
 						  BlockNumber blocknum, BlockNumber nblocks);
 extern BlockNumber smgrnblocks(SMgrRelation reln, ForkNumber forknum);
-extern void smgrtruncate(SMgrRelation reln, ForkNumber forknum,
-						 BlockNumber nblocks);
+extern void smgrtruncate(SMgrRelation reln, ForkNumber *forknum,
+						 int nforks, BlockNumber *nblocks);
 extern void smgrimmedsync(SMgrRelation reln, ForkNumber forknum);
 extern void AtEOXact_SMgr(void);
 

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index 1372bb638f..75b6d96440 100644
*** a/contrib/pg_visibility/pg_visibility.c
--- b/contrib/pg_visibility/pg_visibility.c
***************
*** 383,388 **** pg_truncate_visibility_map(PG_FUNCTION_ARGS)
--- 383,390 ----
  {
  	Oid			relid = PG_GETARG_OID(0);
  	Relation	rel;
+ 	ForkNumber	fork;
+ 	BlockNumber	block;
  
  	rel = relation_open(relid, AccessExclusiveLock);
  
***************
*** 392,398 **** pg_truncate_visibility_map(PG_FUNCTION_ARGS)
  	RelationOpenSmgr(rel);
  	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
  
! 	visibilitymap_truncate(rel, 0);
  
  	if (RelationNeedsWAL(rel))
  	{
--- 394,405 ----
  	RelationOpenSmgr(rel);
  	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
  
! 	block = visibilitymap_prepare_truncate(rel, 0);
! 	if (BlockNumberIsValid(block))
! 	{
! 		fork = VISIBILITYMAP_FORKNUM;
! 		smgrtruncate(rel->rd_smgr, &fork, 1, &block);
! 	}
  
  	if (RelationNeedsWAL(rel))
  	{
***************
*** 418,424 **** pg_truncate_visibility_map(PG_FUNCTION_ARGS)
  	 * here and when we sent the messages at our eventual commit.  However,
  	 * we're currently only sending a non-transactional smgr invalidation,
  	 * which will have been posted to shared memory immediately from within
! 	 * visibilitymap_truncate.  Therefore, there should be no race here.
  	 *
  	 * The reason why it's desirable to release the lock early here is because
  	 * of the possibility that someone will need to use this to blow away many
--- 425,431 ----
  	 * here and when we sent the messages at our eventual commit.  However,
  	 * we're currently only sending a non-transactional smgr invalidation,
  	 * which will have been posted to shared memory immediately from within
! 	 * smgr_truncate.  Therefore, there should be no race here.
  	 *
  	 * The reason why it's desirable to release the lock early here is because
  	 * of the possibility that someone will need to use this to blow away many
diff --git a/src/backend/access/heap/visibilindex a08922b079..262876772f 100644
*** a/src/backend/access/heap/visibilitymap.c
--- b/src/backend/access/heap/visibilitymap.c
***************
*** 17,23 ****
   *		visibilitymap_set	 - set a bit in a previously pinned page
   *		visibilitymap_get_status - get status of bits
   *		visibilitymap_count  - count number of bits set in visibility map
!  *		visibilitymap_truncate	- truncate the visibility map
   *
   * NOTES
   *
--- 17,24 ----
   *		visibilitymap_set	 - set a bit in a previously pinned page
   *		visibilitymap_get_status - get status of bits
   *		visibilitymap_count  - count number of bits set in visibility map
!  *		visibilitymap_prepare_truncate -
!  *			prepare for truncation of the visibility map
   *
   * NOTES
   *
***************
*** 430,445 **** visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_fro
  }
  
  /*
!  *	visibilitymap_truncate - truncate the visibility map
!  *
!  * The caller must hold AccessExclusiveLock on the relation, to ensure that
!  * other backends receive the smgr invalidation event that this function sends
!  * before they access the VM again.
   *
   * nheapblocks is the new size of the heap.
   */
! void
! visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
  {
  	BlockNumber newnblocks;
  
--- 431,448 ----
  }
  
  /*
!  *	visibilitymap_prepare_truncate -
!  *			prepare for truncation of the visibility map
   *
   * nheapblocks is the new size of the heap.
+  *
+  * Return the number of blocks of new visibility map after it's truncated.
+  * If it's InvalidBlockNumber, there is nothing to truncate;
+  * otherwise the caller is responsible for calling smgrtruncate()
+  * to truncate the visibility map pages.
   */
! BlockNumber
! visibilitymap_prepare_truncate(Relation rel, BlockNumber nheapblocks)
  {
  	BlockNumber newnblocks;
  
***************
*** 459,465 **** visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
  	 * nothing to truncate.
  	 */
  	if (!smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM))
! 		return;
  
  	/*
  	 * Unless the new size is exactly at a visibility map page boundary, the
--- 462,468 ----
  	 * nothing to truncate.
  	 */
  	if (!smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM))
! 		return InvalidBlockNumber;
  
  	/*
  	 * Unless the new size is exactly at a visibility map page boundary, the
***************
*** 480,486 **** visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
  		if (!BufferIsValid(mapBuffer))
  		{
  			/* nothing to do, the file was already smaller */
! 			return;
  		}
  
  		page = BufferGetPage(mapBuffer);
--- 483,489 ----
  		if (!BufferIsValid(mapBuffer))
  		{
  			/* nothing to do, the file was already smaller */
! 			return InvalidBlockNumber;
  		}
  
  		page = BufferGetPage(mapBuffer);
***************
*** 528,547 **** visibilitymap_truncate(Relation rel, BlockNumber nheapblocks)
  	if (smgrnblocks(rel->rd_smgr, VISIBILITYMAP_FORKNUM) <= newnblocks)
  	{
  		/* nothing to do, the file was already smaller than requested size */
! 		return;
  	}
  
! 	/* Truncate the unused VM pages, and send smgr inval message */
! 	smgrtruncate(rel->rd_smgr, VISIBILITYMAP_FORKNUM, newnblocks);
! 
! 	/*
! 	 * We might as well update the local smgr_vm_nblocks setting. smgrtruncate
! 	 * sent an smgr cache inval message, which will cause other backends to
! 	 * invalidate their copy of smgr_vm_nblocks, and this one too at the next
! 	 * command boundary.  But this ensures it isn't outright wrong until then.
! 	 */
! 	if (rel->rd_smgr)
! 		rel->rd_smgr->smgr_vm_nblocks = newnblocks;
  }
  
  /*
--- 531,540 ----
  	if (smgrnblocks(rel->rd_smgr, VISIBILITYMAP_FORKNUM) <= newnblocks)
  	{
  		/* nothing to do, the file was already smaller than requested size */
! 		return InvalidBlockNumber;
  	}
  
! 	return newnblocks;
  }
  
  /*
diff --git a/src/backend/catalog/storage.c b/sindex 3cc886f7fe..b8c9b6f9c6 100644
*** a/src/backend/catalog/storage.c
--- b/src/backend/catalog/storage.c
***************
*** 231,236 **** RelationTruncate(Relation rel, BlockNumber nblocks)
--- 231,240 ----
  {
  	bool		fsm;
  	bool		vm;
+ 	bool		need_fsm_vacuum = false;
+ 	ForkNumber	forks[MAX_FORKNUM];
+ 	BlockNumber	blocks[MAX_FORKNUM];
+ 	int		nforks = 0;
  
  	/* Open it at the smgr level if not already done */
  	RelationOpenSmgr(rel);
***************
*** 242,256 **** RelationTruncate(Relation rel, BlockNumber nblocks)
  	rel->rd_smgr->smgr_fsm_nblocks = InvalidBlockNumber;
  	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
  
! 	/* Truncate the FSM first if it exists */
  	fsm = smgrexists(rel->rd_smgr, FSM_FORKNUM);
  	if (fsm)
! 		FreeSpaceMapTruncateRel(rel, nblocks);
  
! 	/* Truncate the visibility map too if it exists. */
  	vm = smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM);
  	if (vm)
! 		visibilitymap_truncate(rel, nblocks);
  
  	/*
  	 * We WAL-log the truncation before actually truncating, which means
--- 246,280 ----
  	rel->rd_smgr->smgr_fsm_nblocks = InvalidBlockNumber;
  	rel->rd_smgr->smgr_vm_nblocks = InvalidBlockNumber;
  
! 	/* Prepare for truncation of MAIN fork of the relation */
! 	forks[nforks] = MAIN_FORKNUM;
! 	blocks[nforks] = nblocks;
! 	nforks++;
! 
! 	/*  Prepare for truncation of the FSM if it exists */
  	fsm = smgrexists(rel->rd_smgr, FSM_FORKNUM);
  	if (fsm)
! 	{
! 		blocks[nforks] = FreeSpaceMapPrepareTruncateRel(rel, nblocks);
! 		if (BlockNumberIsValid(blocks[nforks]))
! 		{
! 			forks[nforks] = FSM_FORKNUM;
! 			nforks++;
! 			need_fsm_vacuum = true;
! 		}
! 	}
  
! 	/* Prepare for truncation of the visibility map too if it exists */
  	vm = smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM);
  	if (vm)
! 	{
! 		blocks[nforks] = visibilitymap_prepare_truncate(rel, nblocks);
! 		if (BlockNumberIsValid(blocks[nforks]))
! 		{
! 			forks[nforks] = VISIBILITYMAP_FORKNUM;
! 			nforks++;
! 		}
! 	}
  
  	/*
  	 * We WAL-log the truncation before actually truncating, which means
***************
*** 290,297 **** RelationTruncate(Relation rel, BlockNumber nblocks)
  			XLogFlush(lsn);
  	}
  
! 	/* Do the real work */
! 	smgrtruncate(rel->rd_smgr, MAIN_FORKNUM, nblocks);
  }
  
  /*
--- 314,329 ----
  			XLogFlush(lsn);
  	}
  
! 	/* Do the real work to truncate relation forks */
! 	smgrtruncate(rel->rd_smgr, forks, nforks, blocks);
! 
! 	/*
! 	 * Update upper-level FSM pages to account for the truncation.
! 	 * This is important because the just-truncated pages were likely
! 	 * marked as all-free, and would be preferentially selected.
! 	 */
! 	if (need_fsm_vacuum)
! 		FreeSpaceMapVacuumRange(rel, nblocks, InvalidBlockNumber);
  }
  
  /*
***************
*** 588,593 **** smgr_redo(XLogReaderState *record)
--- 620,629 ----
  		xl_smgr_truncate *xlrec = (xl_smgr_truncate *) XLogRecGetData(record);
  		SMgrRelation reln;
  		Relation	rel;
+ 		ForkNumber	forks[MAX_FORKNUM];
+ 		BlockNumber	blocks[MAX_FORKNUM];
+ 		int		nforks = 0;
+ 		bool		need_fsm_vacuum = false;
  
  		reln = smgropen(xlrec->rnode, InvalidBackendId);
  
***************
*** 616,638 **** smgr_redo(XLogReaderState *record)
  		 */
  		XLogFlush(lsn);
  
  		if ((xlrec->flags & SMGR_TRUNCATE_HEAP) != 0)
  		{
! 			smgrtruncate(reln, MAIN_FORKNUM, xlrec->blkno);
  
  			/* Also tell xlogutils.c about it */
  			XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
  		}
  
! 		/* Truncate FSM and VM too */
  		rel = CreateFakeRelcacheEntry(xlrec->rnode);
  
  		if ((xlrec->flags & SMGR_TRUNCATE_FSM) != 0 &&
  			smgrexists(reln, FSM_FORKNUM))
! 			FreeSpaceMapTruncateRel(rel, xlrec->blkno);
  		if ((xlrec->flags & SMGR_TRUNCATE_VM) != 0 &&
  			smgrexists(reln, VISIBILITYMAP_FORKNUM))
! 			visibilitymap_truncate(rel, xlrec->blkno);
  
  		FreeFakeRelcacheEntry(rel);
  	}
--- 652,705 ----
  		 */
  		XLogFlush(lsn);
  
+ 		/* Prepare for truncation of MAIN fork */
  		if ((xlrec->flags & SMGR_TRUNCATE_HEAP) != 0)
  		{
! 			forks[nforks] = MAIN_FORKNUM;
! 			blocks[nforks] = xlrec->blkno;
! 			nforks++;
  
  			/* Also tell xlogutils.c about it */
  			XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
  		}
  
! 		/* Prepare for truncation of FSM and VM too */
  		rel = CreateFakeRelcacheEntry(xlrec->rnode);
  
  		if ((xlrec->flags & SMGR_TRUNCATE_FSM) != 0 &&
  			smgrexists(reln, FSM_FORKNUM))
! 		{
! 			blocks[nforks] = FreeSpaceMapPrepareTruncateRel(rel, xlrec->blkno);
! 			if (BlockNumberIsValid(blocks[nforks]))
! 			{
! 				forks[nforks] = FSM_FORKNUM;
! 				nforks++;
! 				need_fsm_vacuum = true;
! 			}
! 		}
  		if ((xlrec->flags & SMGR_TRUNCATE_VM) != 0 &&
  			smgrexists(reln, VISIBILITYMAP_FORKNUM))
! 		{
! 			blocks[nforks] = visibilitymap_prepare_truncate(rel, xlrec->blkno);
! 			if (BlockNumberIsValid(blocks[nforks]))
! 			{
! 				forks[nforks] = VISIBILITYMAP_FORKNUM;
! 				nforks++;
! 			}
! 		}
! 
! 		/* Do the real work to truncate relation forks */
! 		if (nforks > 0)
! 			smgrtruncate(reln, forks, nforks, blocks);
! 
! 		/*
! 		 * Update upper-level FSM pages to account for the truncation.
! 		 * This is important because the just-truncated pages were likely
! 		 * marked as all-free, and would be preferentially selected.
! 		 */
! 		if (need_fsm_vacuum)
! 			FreeSpaceMapVacuumRange(rel, xlrec->blkno,
! 									InvalidBlockNumber);
  
  		FreeFakeRelcacheEntry(rel);
  	}
diff --git a/src/backend/storage/bufindex 6f3a402854..483f705305 100644
*** a/src/backend/storage/buffer/bufmgr.c
--- b/src/backend/storage/buffer/bufmgr.c
***************
*** 2901,2907 **** BufferGetLSNAtomic(Buffer buffer)
   *		DropRelFileNodeBuffers
   *
   *		This function removes from the buffer pool all the pages of the
!  *		specified relation fork that have block numbers >= firstDelBlock.
   *		(In particular, with firstDelBlock = 0, all pages are removed.)
   *		Dirty pages are simply dropped, without bothering to write them
   *		out first.  Therefore, this is NOT rollback-able, and so should be
--- 2901,2907 ----
   *		DropRelFileNodeBuffers
   *
   *		This function removes from the buffer pool all the pages of the
!  *		specified relation forks that have block numbers >= firstDelBlock.
   *		(In particular, with firstDelBlock = 0, all pages are removed.)
   *		Dirty pages are simply dropped, without bothering to write them
   *		out first.  Therefore, this is NOT rollback-able, and so should be
***************
*** 2924,2939 **** BufferGetLSNAtomic(Buffer buffer)
   * --------------------------------------------------------------------
   */
  void
! DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
! 					   BlockNumber firstDelBlock)
  {
  	int			i;
  
  	/* If it's a local relation, it's localbuf.c's problem. */
  	if (RelFileNodeBackendIsTemp(rnode))
  	{
  		if (rnode.backend == MyBackendId)
! 			DropRelFileNodeLocalBuffers(rnode.node, forkNum, firstDelBlock);
  		return;
  	}
  
--- 2924,2944 ----
   * --------------------------------------------------------------------
   */
  void
! DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
! 					   int nforks, BlockNumber *firstDelBlock)
  {
  	int			i;
+ 	int			j;
  
  	/* If it's a local relation, it's localbuf.c's problem. */
  	if (RelFileNodeBackendIsTemp(rnode))
  	{
  		if (rnode.backend == MyBackendId)
! 		{
! 			for (j = 0; j < nforks; j++)
! 				DropRelFileNodeLocalBuffers(rnode.node, forkNum[j],
! 											firstDelBlock[j]);
! 		}
  		return;
  	}
  
***************
*** 2962,2972 **** DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber forkNum,
  			continue;
  
  		buf_state = LockBufHdr(bufHdr);
! 		if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
! 			bufHdr->tag.forkNum == forkNum &&
! 			bufHdr->tag.blockNum >= firstDelBlock)
! 			InvalidateBuffer(bufHdr);	/* releases spinlock */
! 		else
  			UnlockBufHdr(bufHdr, buf_state);
  	}
  }
--- 2967,2984 ----
  			continue;
  
  		buf_state = LockBufHdr(bufHdr);
! 
! 		for (j = 0; j < nforks; j++)
! 		{
! 			if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
! 				bufHdr->tag.forkNum == forkNum[j] &&
! 				bufHdr->tag.blockNum >= firstDelBlock[j])
! 			{
! 				InvalidateBuffer(bufHdr); /* releases spinlock */
! 				break;
! 			}
! 		}
! 		if (j >= nforks)
  			UnlockBufHdr(bufHdr, buf_state);
  	}
  }
diff --git a/src/backend/storage/freespaceindex 2383094cfd..b7a048c96c 100644
*** a/src/backend/storage/freespace/freespace.c
--- b/src/backend/storage/freespace/freespace.c
***************
*** 247,262 **** GetRecordedFreeSpace(Relation rel, BlockNumber heapBlk)
  }
  
  /*
!  * FreeSpaceMapTruncateRel - adjust for truncation of a relation.
!  *
!  * The caller must hold AccessExclusiveLock on the relation, to ensure that
!  * other backends receive the smgr invalidation event that this function sends
!  * before they access the FSM again.
   *
   * nblocks is the new size of the heap.
   */
! void
! FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
  {
  	BlockNumber new_nfsmblocks;
  	FSMAddress	first_removed_address;
--- 247,264 ----
  }
  
  /*
!  * FreeSpaceMapPrepareTruncateRel - prepare for truncation of a relation.
   *
   * nblocks is the new size of the heap.
+  *
+  * Return the number of blocks of new FSM after it's truncated.
+  * If it's InvalidBlockNumber, there is nothing to truncate;
+  * otherwise the caller is responsible for calling smgrtruncate()
+  * to truncate the FSM pages, and FreeSpaceMapVacuumRange()
+  * to update upper-level pages in the FSM.
   */
! BlockNumber
! FreeSpaceMapPrepareTruncateRel(Relation rel, BlockNumber nblocks)
  {
  	BlockNumber new_nfsmblocks;
  	FSMAddress	first_removed_address;
***************
*** 270,276 **** FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
  	 * truncate.
  	 */
  	if (!smgrexists(rel->rd_smgr, FSM_FORKNUM))
! 		return;
  
  	/* Get the location in the FSM of the first removed heap block */
  	first_removed_address = fsm_get_location(nblocks, &first_removed_slot);
--- 272,278 ----
  	 * truncate.
  	 */
  	if (!smgrexists(rel->rd_smgr, FSM_FORKNUM))
! 		return InvalidBlockNumber;
  
  	/* Get the location in the FSM of the first removed heap block */
  	first_removed_address = fsm_get_location(nblocks, &first_removed_slot);
***************
*** 285,291 **** FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
  	{
  		buf = fsm_readbuf(rel, first_removed_address, false);
  		if (!BufferIsValid(buf))
! 			return;				/* nothing to do; the FSM was already smaller */
  		LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
  
  		/* NO EREPORT(ERROR) from here till changes are logged */
--- 287,293 ----
  	{
  		buf = fsm_readbuf(rel, first_removed_address, false);
  		if (!BufferIsValid(buf))
! 			return InvalidBlockNumber;	/* nothing to do; the FSM was already smaller */
  		LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
  
  		/* NO EREPORT(ERROR) from here till changes are logged */
***************
*** 315,342 **** FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks)
  	{
  		new_nfsmblocks = fsm_logical_to_physical(first_removed_address);
  		if (smgrnblocks(rel->rd_smgr, FSM_FORKNUM) <= new_nfsmblocks)
! 			return;				/* nothing to do; the FSM was already smaller */
  	}
  
! 	/* Truncate the unused FSM pages, and send smgr inval message */
! 	smgrtruncate(rel->rd_smgr, FSM_FORKNUM, new_nfsmblocks);
! 
! 	/*
! 	 * We might as well update the local smgr_fsm_nblocks setting.
! 	 * smgrtruncate sent an smgr cache inval message, which will cause other
! 	 * backends to invalidate their copy of smgr_fsm_nblocks, and this one too
! 	 * at the next command boundary.  But this ensures it isn't outright wrong
! 	 * until then.
! 	 */
! 	if (rel->rd_smgr)
! 		rel->rd_smgr->smgr_fsm_nblocks = new_nfsmblocks;
! 
! 	/*
! 	 * Update upper-level FSM pages to account for the truncation.  This is
! 	 * important because the just-truncated pages were likely marked as
! 	 * all-free, and would be preferentially selected.
! 	 */
! 	FreeSpaceMapVacuumRange(rel, nblocks, InvalidBlockNumber);
  }
  
  /*
--- 317,326 ----
  	{
  		new_nfsmblocks = fsm_logical_to_physical(first_removed_address);
  		if (smgrnblocks(rel->rd_smgr, FSM_FORKNUM) <= new_nfsmblocks)
! 			return InvalidBlockNumber;	/* nothing to do; the FSM was already smaller */
  	}
  
! 	return new_nfsmblocks;
  }
  
  /*
diff --git a/src/backend/storage/smgr/smgr.c b/sindex 5b5a80e890..b50c69b438 100644
*** a/src/backend/storage/smgr/smgr.c
--- b/src/backend/storage/smgr/smgr.c
***************
*** 469,474 **** smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo)
--- 469,475 ----
  	pfree(rnodes);
  }
  
+ 
  /*
   *	smgrextend() -- Add a new block to a file.
   *
***************
*** 557,575 **** smgrnblocks(SMgrRelation reln, ForkNumber forknum)
  }
  
  /*
!  *	smgrtruncate() -- Truncate supplied relation to the specified number
!  *					  of blocks
   *
   * The truncation is done immediately, so this can't be rolled back.
   */
  void
! smgrtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks)
  {
  	/*
  	 * Get rid of any buffers for the about-to-be-deleted blocks. bufmgr will
  	 * just drop them without bothering to write the contents.
  	 */
! 	DropRelFileNodeBuffers(reln->smgr_rnode, forknum, nblocks);
  
  	/*
  	 * Send a shared-inval message to force other backends to close any smgr
--- 558,582 ----
  }
  
  /*
!  *	smgrtruncate() -- Truncate the given forks of supplied relation to
!  *					  each specified numbers of blocks
   *
   * The truncation is done immediately, so this can't be rolled back.
+  *
+  * The caller must hold AccessExclusiveLock on the relation, to ensure that
+  * other backends receive the smgr invalidation event that this function sends
+  * before they access any forks of the relation again.
   */
  void
! smgrtruncate(SMgrRelation reln, ForkNumber *forknum, int nforks, BlockNumber *nblocks)
  {
+ 	int		i;
+ 
  	/*
  	 * Get rid of any buffers for the about-to-be-deleted blocks. bufmgr will
  	 * just drop them without bothering to write the contents.
  	 */
! 	DropRelFileNodeBuffers(reln->smgr_rnode, forknum, nforks, nblocks);
  
  	/*
  	 * Send a shared-inval message to force other backends to close any smgr
***************
*** 583,592 **** smgrtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks)
  	 */
  	CacheInvalidateSmgr(reln->smgr_rnode);
  
! 	/*
! 	 * Do the truncation.
! 	 */
! 	smgrsw[reln->smgr_which].smgr_truncate(reln, forknum, nblocks);
  }
  
  /*
--- 590,613 ----
  	 */
  	CacheInvalidateSmgr(reln->smgr_rnode);
  
! 	/* Do the truncation */
! 	for (i = 0; i < nforks; i++)
! 	{
! 		smgrsw[reln->smgr_which].smgr_truncate(reln, forknum[i], nblocks[i]);
! 
! 		/*
! 		 * We might as well update the local smgr_fsm_nblocks and
! 		 * smgr_vm_nblocks settings. The smgr cache inval message that
! 		 * this function sent will cause other backends to invalidate
! 		 * their copies of smgr_fsm_nblocks and smgr_vm_nblocks,
! 		 * and these ones too at the next command boundary.
! 		 * But these ensure they aren't outright wrong until then.
! 		 */
! 		if (forknum[i] == FSM_FORKNUM)
! 			reln->smgr_fsm_nblocks = nblocks[i];
! 		if (forknum[i] == VISIBILITYMAP_FORKNUM)
! 			reln->smgr_vm_nblocks = nblocks[i];
! 	}
  }
  
  /*
diff --git a/src/include/access/visibiindex 2d8804351a..0532b04e34 100644
*** a/src/include/access/visibilitymap.h
--- b/src/include/access/visibilitymap.h
***************
*** 44,49 **** extern void visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf,
  							  uint8 flags);
  extern uint8 visibilitymap_get_status(Relation rel, BlockNumber heapBlk, Buffer *vmbuf);
  extern void visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_frozen);
! extern void visibilitymap_truncate(Relation rel, BlockNumber nheapblocks);
  
  #endif							/* VISIBILITYMAP_H */
--- 44,50 ----
  							  uint8 flags);
  extern uint8 visibilitymap_get_status(Relation rel, BlockNumber heapBlk, Buffer *vmbuf);
  extern void visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_frozen);
! extern BlockNumber visibilitymap_prepare_truncate(Relation rel,
! 							  BlockNumber nheapblocks);
  
  #endif							/* VISIBILITYMAP_H */
diff --git a/src/include/storage/bufmgr.hindex 509f4b7ef1..17b97f7e38 100644
*** a/src/include/storage/bufmgr.h
--- b/src/include/storage/bufmgr.h
***************
*** 190,197 **** extern BlockNumber RelationGetNumberOfBlocksInFork(Relation relation,
  extern void FlushOneBuffer(Buffer buffer);
  extern void FlushRelationBuffers(Relation rel);
  extern void FlushDatabaseBuffers(Oid dbid);
! extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode,
! 								   ForkNumber forkNum, BlockNumber firstDelBlock);
  extern void DropRelFileNodesAllBuffers(RelFileNodeBackend *rnodes, int nnodes);
  extern void DropDatabaseBuffers(Oid dbid);
  
--- 190,197 ----
  extern void FlushOneBuffer(Buffer buffer);
  extern void FlushRelationBuffers(Relation rel);
  extern void FlushDatabaseBuffers(Oid dbid);
! extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
! 								   int nforks, BlockNumber *firstDelBlock);
  extern void DropRelFileNodesAllBuffers(RelFileNodeBackend *rnodes, int nnodes);
  extern void DropDatabaseBuffers(Oid dbid);
  
diff --git a/src/include/storage/frindex 8d8c465d7b..b75f6fe946 100644
*** a/src/include/storage/freespace.h
--- b/src/include/storage/freespace.h
***************
*** 30,36 **** extern void RecordPageWithFreeSpace(Relation rel, BlockNumber heapBlk,
  extern void XLogRecordPageWithFreeSpace(RelFileNode rnode, BlockNumber heapBlk,
  										Size spaceAvail);
  
! extern void FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks);
  extern void FreeSpaceMapVacuum(Relation rel);
  extern void FreeSpaceMapVacuumRange(Relation rel, BlockNumber start,
  									BlockNumber end);
--- 30,37 ----
  extern void XLogRecordPageWithFreeSpace(RelFileNode rnode, BlockNumber heapBlk,
  										Size spaceAvail);
  
! extern BlockNumber FreeSpaceMapPrepareTruncateRel(Relation rel,
! 												  BlockNumber nblocks);
  extern void FreeSpaceMapVacuum(Relation rel);
  extern void FreeSpaceMapVacuumRange(Relation rel, BlockNumber start,
  									BlockNumber end);
diff --git a/src/include/storage/smgr.index 7393727a4b..1543d8d870 100644
*** a/src/include/storage/smgr.h
--- b/src/include/storage/smgr.h
***************
*** 101,108 **** extern void smgrwrite(SMgrRelation reln, ForkNumber forknum,
  extern void smgrwriteback(SMgrRelation reln, ForkNumber forknum,
  						  BlockNumber blocknum, BlockNumber nblocks);
  extern BlockNumber smgrnblocks(SMgrRelation reln, ForkNumber forknum);
! extern void smgrtruncate(SMgrRelation reln, ForkNumber forknum,
! 						 BlockNumber nblocks);
  extern void smgrimmedsync(SMgrRelation reln, ForkNumber forknum);
  extern void AtEOXact_SMgr(void);
  
--- 101,108 ----
  extern void smgrwriteback(SMgrRelation reln, ForkNumber forknum,
  						  BlockNumber blocknum, BlockNumber nblocks);
  extern BlockNumber smgrnblocks(SMgrRelation reln, ForkNumber forknum);
! extern void smgrtruncate(SMgrRelation reln, ForkNumber *forknum,
! 						 int nforks, BlockNumber *nblocks);
  extern void smgrimmedsync(SMgrRelation reln, ForkNumber forknum);
  extern void AtEOXact_SMgr(void);