diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index 33e9acab4a..f2ae497266 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -46,6 +46,7 @@
 #include "pgstat.h"
 #include "replication/slot.h"
 #include "storage/fd.h"
+#include "storage/ipc.h"
 #include "storage/proc.h"
 #include "storage/procarray.h"
 #include "utils/builtins.h"
@@ -101,6 +102,8 @@ int			max_replication_slots = 0;	/* the maximum number of replication
 
 static void ReplicationSlotDropAcquired(void);
 static void ReplicationSlotDropPtr(ReplicationSlot *slot);
+static void RegisterReplicationSlotCallback(void);
+static void ReplicationSlotBeforeShmemExit(int code, Datum arg);
 
 /* internal persistency functions */
 static void RestoreSlotFromDisk(const char *name);
@@ -301,6 +304,12 @@ ReplicationSlotCreate(const char *name, bool db_specific,
 	 */
 	CreateSlotOnDisk(slot);
 
+	/*
+	 * Register callback to make sure cleanup and releasing the replication
+	 * slot on exit.
+	 */
+	RegisterReplicationSlotCallback();
+
 	/*
 	 * We need to briefly prevent any other backend from iterating over the
 	 * slots while we flip the in_use flag. We also need to set the active
@@ -455,6 +464,9 @@ retry:
 	/* Let everybody know we've modified this slot */
 	ConditionVariableBroadcast(&s->active_cv);
 
+	/* Register callback to make sure releasing the replication slot on exit */
+	RegisterReplicationSlotCallback();
+
 	/* We made this slot active, so it's ours now. */
 	MyReplicationSlot = s;
 }
@@ -702,6 +714,45 @@ ReplicationSlotDropPtr(ReplicationSlot *slot)
 	LWLockRelease(ReplicationSlotAllocationLock);
 }
 
+/*
+ * Register the callback for replication slot cleanup and releasing if not
+ * registered yet.
+ */
+static void
+RegisterReplicationSlotCallback(void)
+{
+	static bool callback_registered = false;
+
+	if (callback_registered)
+		return;
+
+	/* The callback must be registered before acquiring a replication slot */
+	Assert(MyReplicationSlot == NULL);
+
+	/*
+	 * Register before-shmem-exit hook to ensure releasing and cleanup
+	 * replication slots while we can still report stats. On dropping a
+	 * replication slot, we report the message to drop the replication slot
+	 * stats.
+	 */
+	before_shmem_exit(ReplicationSlotBeforeShmemExit, 0);
+	callback_registered = true;
+}
+
+/*
+ * Release and cleanup replication slots.
+ */
+static void
+ReplicationSlotBeforeShmemExit(int code, Datum arg)
+{
+	/* Make sure active replication slots are released */
+	if (MyReplicationSlot != NULL)
+		ReplicationSlotRelease();
+
+	/* Also cleanup all the temporary slots. */
+	ReplicationSlotCleanup();
+}
+
 /*
  * Serialize the currently acquired slot's state from memory to disk, thereby
  * guaranteeing the current state will survive a crash.
@@ -972,6 +1023,13 @@ ReplicationSlotsDropDBSlots(Oid dboid)
 	if (max_replication_slots <= 0)
 		return;
 
+	/*
+	 * While dropping replication slots, we acquire a slot to reuse
+	 * ReplicationSlotDropAcquired().  So we register callback to make sure
+	 * releasing the replication slot on exit.
+	 */
+	RegisterReplicationSlotCallback();
+
 restart:
 	LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
 	for (i = 0; i < max_replication_slots; i++)
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index b7d9da0aa9..031a73104e 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -847,13 +847,6 @@ ProcKill(int code, Datum arg)
 	/* Cancel any pending condition variable sleep, too */
 	ConditionVariableCancelSleep();
 
-	/* Make sure active replication slots are released */
-	if (MyReplicationSlot != NULL)
-		ReplicationSlotRelease();
-
-	/* Also cleanup all the temporary slots. */
-	ReplicationSlotCleanup();
-
 	/*
 	 * Detach from any lock group of which we are a member.  If the leader
 	 * exist before all other group members, its PGPROC will remain allocated
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 58b5960e27..36e39c50c0 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -4249,8 +4249,9 @@ PostgresMain(int argc, char *argv[],
 		 * We can't release replication slots inside AbortTransaction() as we
 		 * need to be able to start and abort transactions while having a slot
 		 * acquired. But we never need to hold them across top level errors,
-		 * so releasing here is fine. There's another cleanup in ProcKill()
-		 * ensuring we'll correctly cleanup on FATAL errors as well.
+		 * so releasing here is fine. There's another cleanup in
+		 * ReplicationSlotBeforeShmemExit() callback ensuring we'll correctly
+		 * cleanup on FATAL errors as well.
 		 */
 		if (MyReplicationSlot != NULL)
 			ReplicationSlotRelease();