diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c
index c2ba480c70..b0fd1dc5ac 100644
--- a/src/backend/access/transam/timeline.c
+++ b/src/backend/access/transam/timeline.c
@@ -281,9 +281,9 @@ findNewestTimeLine(TimeLineID startTLI)
  *	switchpoint: WAL location where the system switched to the new timeline
  *	reason: human-readable explanation of why the timeline was switched
  *
- * Currently this is only used at the end recovery, and so there are no locking
+ * Currently this is only used after the end recovery, and so there are no locking
  * considerations.  But we should be just as tense as XLogFileInit to avoid
- * emplacing a bogus file.
+ * emplacing a bogus file and need to get rid of recovered timeline-history file.
  */
 void
 writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI,
@@ -296,6 +296,7 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI,
 	int			srcfd;
 	int			fd;
 	int			nbytes;
+	bool		remove_tmphist = false;
 
 	Assert(newTLI > parentTLI); /* else bad selection of newTLI */
 
@@ -319,7 +320,8 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI,
 	if (ArchiveRecoveryRequested)
 	{
 		TLHistoryFileName(histfname, parentTLI);
-		RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0, false);
+		remove_tmphist = RestoreArchivedFile(path, histfname, "RECOVERYHISTORY",
+											 0, false);
 	}
 	else
 		TLHistoryFilePath(path, parentTLI);
@@ -375,6 +377,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI,
 			ereport(ERROR,
 					(errcode_for_file_access(),
 					 errmsg("could not close file \"%s\": %m", path)));
+
+		/*
+		 * Get rid of remaining recovered timeline-history file since we are
+		 * already exited from archive recovery mode and therefore there is no
+		 * cleanup later.
+		 */
+		if (remove_tmphist)
+		{
+			char		histpath[MAXPGPATH];
+
+			snprintf(histpath, MAXPGPATH, XLOGDIR "/RECOVERYHISTORY");
+			unlink(histpath);		/* ignore any error */
+		}
 	}
 
 	/*