Switching XLog source from archive to streaming when primary available

From 756d28ab2284e6eb8c057dae45280a38212cec48 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Tue, 24 May 2022 16:05:10 +0000
Subject: [PATCH v1] Switch to stream mode from archive occasionally

This patch enables standby to switch to stream mode i.e. get
WAL from primary even before it fails to receive from archive
location. Currently, if receive from archive location fails, it
switches back to stream mode and by then WAL receiver could have
come up. Since fetching WAL from archvie isn't cheap always,
switch to stream mode occasionally.

Right now, this switching is based on wal_retrieve_retry_interval
i.e. if the standby has received the WAL from archive for
wal_retrieve_retry_interval after the last failed attempt from
the stream, it switches back to stream mode to see if it can
receivefrom primary, if yes it's a good bet otherwise falls back
to archive.
---
 src/backend/access/transam/xlogrecovery.c | 53 +++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 6eba626420..3746e24a18 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -241,6 +241,7 @@ static XLogSource readSource = XLOG_FROM_ANY;
  * walreceiver restart.  This is only valid in XLOG_FROM_STREAM state.
  */
 static XLogSource currentSource = XLOG_FROM_ANY;
+static XLogSource lastSource = XLOG_FROM_ANY;
 static bool lastSourceFailed = false;
 static bool pendingWalRcvRestart = false;
 
@@ -3065,6 +3066,7 @@ ReadRecord(XLogPrefetcher *xlogprefetcher, int emode,
 				 * so that we will check the archive next.
 				 */
 				lastSourceFailed = false;
+				lastSource = currentSource;
 				currentSource = XLOG_FROM_ANY;
 
 				continue;
@@ -3375,11 +3377,15 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 *-------
 	 */
 	if (!InArchiveRecovery)
+	{
+		lastSource = currentSource;
 		currentSource = XLOG_FROM_PG_WAL;
+	}
 	else if (currentSource == XLOG_FROM_ANY ||
 			 (!StandbyMode && currentSource == XLOG_FROM_STREAM))
 	{
 		lastSourceFailed = false;
+		lastSource = currentSource;
 		currentSource = XLOG_FROM_ARCHIVE;
 	}
 
@@ -3432,7 +3438,15 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					 * Move to XLOG_FROM_STREAM state, and set to start a
 					 * walreceiver if necessary.
 					 */
+					lastSource = currentSource;
 					currentSource = XLOG_FROM_STREAM;
+					/*
+					 * XXX: we might have to see if the WAL receiver is already
+					 * running before even we just go ahead and start it
+					 * relying only on startWalReceiver flag. The WAL receiver
+					 * could've come up by then, if yes, there can be multiple
+					 * WAL receivers???
+					 */
 					startWalReceiver = true;
 					break;
 
@@ -3475,6 +3489,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					{
 						if (rescanLatestTimeLine(replayTLI, replayLSN))
 						{
+							lastSource = currentSource;
 							currentSource = XLOG_FROM_ARCHIVE;
 							break;
 						}
@@ -3512,6 +3527,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 						HandleStartupProcInterrupts();
 					}
 					last_fail_time = now;
+					lastSource = currentSource;
 					currentSource = XLOG_FROM_ARCHIVE;
 					break;
 
@@ -3527,7 +3543,10 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 			 * from the archive first.
 			 */
 			if (InArchiveRecovery)
+			{
+				lastSource = currentSource;
 				currentSource = XLOG_FROM_ARCHIVE;
+			}
 		}
 
 		if (currentSource != oldSource)
@@ -3562,6 +3581,40 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/*
+				 * Try to stream WAL from primary after a specified amount of
+				 * time fetching from archive. This is because fetching WAL
+				 * from archive isn't always cheaper and the primary could have
+				 * come up meanwhile.
+				 */
+				if (StandbyMode && lastSource == XLOG_FROM_STREAM)
+				{
+					TimestampTz now;
+
+					now = GetCurrentTimestamp();
+
+					if (TimestampDifferenceExceeds(last_fail_time, now,
+												   wal_retrieve_retry_interval))
+					{
+						elog(DEBUG2,
+							 "switching WAL source from %s to %s after \"wal_retrieve_retry_interval\" %d milliseconds",
+							 xlogSourceNames[currentSource],
+							 xlogSourceNames[lastSource],
+							 wal_retrieve_retry_interval);
+
+						currentSource =	lastSource;
+						last_fail_time = 0;
+
+						/*
+						 * Treat this as a failure to read from current source,
+						 * even though it is actually not, so that the state
+						 * machine moves to read it from XLOG_FROM_STREAM.
+						 */
+						lastSourceFailed = true;
+						break;
+					}
+				}
+
 				/*
 				 * Try to restore the file from archive, or read an existing
 				 * file from pg_wal.
-- 
2.25.1

From 2147194165ebc66232b6e7ff9e3a8b3d07d50a72 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Thu, 11 Aug 2022 15:35:20 +0000
Subject: [PATCH v2] Switch WAL source to stream from archive

This patch enables standby to switch to stream mode i.e. get
WAL from primary even before it fails to receive from archive
location. Currently, the standby switches to stream mode, only
when receive from archive location fails.

The standby makes an attempt to read WAL from primary after
wal_retrieve_retry_interval milliseconds reading from archive.
Reading WAL from archive may not always be efficient and cheaper
because network latencies, disk IO cost might differ on the archive
as compared to the primary and often the archive may sit far from
the standbys - all adding to the recovery performance on the
standbys.

Hence reading WAL from primary as opposed to archive enables
standbys to catch up with the primary sooner thus reducing
replication lag and avoiding WAL files accumulation on the primary.

This can benefit in any of the following situations:
1) standby in initial recovery after start/restart.
2) standby stopped streaming from primary because of connectivity
issues with the primary (either due to network issues or crash in
the primary or something else) or walreceiver got killed or
crashed for whatever reasons.
---
 doc/src/sgml/config.sgml                      |  31 +++++
 src/backend/access/transam/xlogrecovery.c     |  96 ++++++++++++-
 src/backend/utils/misc/guc.c                  |  11 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/t/034_wal_source_switch.pl  | 126 ++++++++++++++++++
 6 files changed, 265 insertions(+), 4 deletions(-)
 create mode 100644 src/test/recovery/t/034_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 2522f4c8c5..d9a6a2ec78 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4837,6 +4837,37 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+    <varlistentry id="guc-wal-source-switch-interval" xreflabel="wal_source_switch_interval">
+      <term><varname>wal_source_switch_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>wal_source_switch_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies how long the standby server should wait before switching WAL
+        source from WAL archive to primary (streaming replication). This can
+        happen either during the standby initial recovery or after a previous
+        failed attempt to stream WAL from the primary.
+        If this value is specified without units, it is taken as milliseconds.
+        The default value is 5 seconds. A setting of <literal>0</literal>
+        disables the feature.
+        This parameter can only be set in
+        the <filename>postgresql.conf</filename> file or on the server
+        command line.
+       </para>
+       <para>
+        Reading WAL from archive may not always be efficient and cheaper
+        because network latencies, disk IO cost might differ on the archive as
+        compared to primary and often the archive may sit far from standbys
+        impacting the recovery performance on the standbys. Hence reading WAL
+        from the primary, by setting this parameter, as opposed to the archive
+        enables the standbys to catch up with the primary sooner thus reducing
+        replication lag and avoiding WAL files accumulation on the primary.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index a59a0e826b..f22af22ba7 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -88,6 +88,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			wal_source_switch_interval = 5000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -3397,6 +3398,9 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	static bool first_time = true;
+	static TimestampTz last_switch_time = 0;
+	bool	intentionalSourceSwitch = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
 
@@ -3418,6 +3422,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after every wal_source_switch_interval
+	 * milliseconds, when state machine is in XLOG_FROM_ARCHIVE state. If
+	 * successful, the state machine moves to XLOG_FROM_STREAM state, otherwise
+	 * it falls back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3452,8 +3461,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 			 * Don't allow any retry loops to occur during nonblocking
 			 * readahead.  Let the caller process everything that has been
 			 * decoded already first.
+			 *
+			 * Continue retrying for requested WAL when there was an
+			 * intentional source switch from archive to stream.
 			 */
-			if (nonblocking)
+			if (nonblocking && !intentionalSourceSwitch)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3583,15 +3595,30 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			if (intentionalSourceSwitch)
+			{
+				elog(DEBUG2,
+					 "switched WAL source to %s after fetching WAL from %s for %d milliseconds",
+					 xlogSourceNames[oldSource],
+					 xlogSourceNames[currentSource],
+					 wal_source_switch_interval);
+			}
+			else
+			{
+				elog(DEBUG2, "switched WAL source from %s to %s after %s",
+					 xlogSourceNames[oldSource],
+					 xlogSourceNames[currentSource],
+					 lastSourceFailed ? "failure" : "success");
+			}
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
 		 * source.
 		 */
 		lastSourceFailed = false;
+		intentionalSourceSwitch = false;
 
 		switch (currentSource)
 		{
@@ -3614,6 +3641,67 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/*
+				 * Make an attempt to read WAL from primary after
+				 * wal_retrieve_retry_interval milliseconds reading from
+				 * archive.
+				 *
+				 * Reading WAL from archive may not always be efficient and
+				 * cheaper because network latencies, disk IO cost might differ
+				 * on the archive as compared to the primary and often the
+				 * archive may sit far from the standbys - all adding to the
+				 * recovery performance on the standbys.
+				 *
+				 * Hence reading WAL from primary as opposed to archive enables
+				 * standbys to catch up with the primary sooner thus reducing
+				 * replication lag and avoiding WAL files accumulation on the
+				 * primary.
+				 *
+				 * We are here for any of the following reasons:
+				 * 1) standby in initial recovery after start/restart.
+				 * 2) standby stopped streaming from primary because of
+				 * connectivity issues with the primary (either due to network
+				 * issues or crash in the primary or something else) or
+				 * walreceiver got killed or crashed for whatever reasons.
+				 */
+				if (StandbyMode && currentSource == XLOG_FROM_ARCHIVE)
+				{
+					TimestampTz curr_time;
+
+					curr_time = GetCurrentTimestamp();
+
+					/* Assume last_switch_time as curr_time for the first time */
+					if (first_time)
+						last_switch_time = curr_time;
+
+					if (!first_time &&
+						TimestampDifferenceExceeds(last_switch_time, curr_time,
+												   wal_source_switch_interval))
+					{
+						elog(DEBUG2,
+							 "trying to switch WAL source to %s after fetching WAL from %s for %d milliseconds",
+							 xlogSourceNames[XLOG_FROM_STREAM],
+							 xlogSourceNames[currentSource],
+							 wal_source_switch_interval);
+
+						last_switch_time = curr_time;
+
+						/*
+						 * Treat this as a failure to read from archive, even
+						 * though it is actually not, so that the state machine
+						 * will move on to stream the WAL from primary.
+						 */
+						lastSourceFailed = true;
+						intentionalSourceSwitch = true;
+
+						break;
+					}
+
+					/* We're not here for the first time any more */
+					if (first_time)
+						first_time = false;
+				}
+
 				/*
 				 * Try to restore the file from archive, or read an existing
 				 * file from pg_wal.
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 5db5df6285..ce50e6596d 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -3324,6 +3324,17 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"wal_source_switch_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time to wait before switching WAL source from archive to primary"),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&wal_source_switch_interval,
+		5000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 90bec0502c..ec70a76a11 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -351,6 +351,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#wal_source_switch_interval = 5s	# time to wait before switching WAL
+					# source from archive to primary
+					# 0 disables the feature, > 0 indicates the
+					# interval in milliseconds.
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index 0aa85d90e8..a4f8e9c804 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int wal_source_switch_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/t/034_wal_source_switch.pl b/src/test/recovery/t/034_wal_source_switch.pl
new file mode 100644
index 0000000000..b5579e745d
--- /dev/null
+++ b/src/test/recovery/t/034_wal_source_switch.pl
@@ -0,0 +1,126 @@
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Test for WAL source switch feature
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+$ENV{PGDATABASE} = 'postgres';
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(
+    allows_streaming => 1,
+    has_archiving => 1,
+    extra => ['--wal-segsize=1']);
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf(
+	'postgresql.conf', qq(
+    min_wal_size = 2MB
+    max_wal_size = 1GB
+    checkpoint_timeout=1h
+    wal_recycle = off
+));
+$node_primary->start;
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('rep1')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create a standby linking to it using the replication slot
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup($node_primary, $backup_name,
+	has_streaming => 1,
+    has_restoring => 1);
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+primary_slot_name = 'rep1'
+min_wal_size = 2MB
+max_wal_size = 1GB
+checkpoint_timeout=1h
+wal_source_switch_interval = 100ms
+wal_recycle = off
+log_min_messages = 'debug2'
+));
+
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+# Stop standby
+$node_standby->stop;
+
+# Advance WAL by 100 segments (= 100MB) on primary
+advance_wal($node_primary, 100);
+
+# Wait for primary to generate requested WAL files
+$node_primary->poll_query_until('postgres',
+	q|SELECT COUNT(*) >= 100 FROM pg_ls_waldir()|, 't');
+
+# The standby now connects to primary during inital recovery after
+# fetching WAL from archive for about wal_source_switch_interval milliseconds.
+$node_standby->start;
+
+$node_primary->wait_for_catchup($node_standby);
+
+ok(find_in_log(
+		$node_standby,
+        qr/restored log file ".*" from archive/),
+	    'check that some of WAL segments were fetched from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/trying to switch WAL source to .* after fetching WAL from .* for .* milliseconds/),
+	    'check that standby tried to switch WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/switched WAL source to .* after fetching WAL from .* for .* milliseconds/),
+	    'check that standby actually switched WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/started streaming WAL from primary at .* on timeline .*/),
+	    'check that standby strated streaming from primary');
+
+# Stop standby
+$node_standby->stop;
+
+# Stop primary
+$node_primary->stop;
+#####################################
+# Advance WAL of $node by $n segments
+sub advance_wal
+{
+	my ($node, $n) = @_;
+
+	# Advance by $n segments (= (wal_segment_size * $n) bytes) on primary.
+	for (my $i = 0; $i < $n; $i++)
+	{
+		$node->safe_psql('postgres',
+			"CREATE TABLE t (); DROP TABLE t; SELECT pg_switch_wal();");
+	}
+	return;
+}
+
+# find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
+
+done_testing();
-- 
2.34.1

From c9c7601707b97ebcf029cfdc2db69593b7f5f4a0 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Thu, 8 Sep 2022 10:07:34 +0000
Subject: [PATCH v3] Allow standby to switch WAL source from archive to
 streaming replication

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be efficient and cheaper because network
latencies, disk IO cost might differ on the archive as compared
to primary and often the archive may sit far from standby
impacting recovery performance on the standby. Hence reading WAL
from the primary, by setting this parameter, as opposed to the
archive enables the standby to catch up with the primary sooner
thus reducing replication lag and avoiding WAL files accumulation
on the primary.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication. If the standby fails to switch to stream
mode, it falls back to archive mode.

Reported-By: SATYANARAYANA NARLAPURAM
Author: Bharath Rupireddy
Reviewed-By: Cary Huang, Nathan Bossart
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  36 +++++
 src/backend/access/transam/xlogrecovery.c     |  95 ++++++++++++-
 src/backend/utils/misc/guc.c                  |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/t/034_wal_source_switch.pl  | 126 ++++++++++++++++++
 6 files changed, 270 insertions(+), 4 deletions(-)
 create mode 100644 src/test/recovery/t/034_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index a5cd4e44c7..278ecd54c3 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4840,6 +4840,42 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+    <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from WAL archive to streaming replication (get WAL from
+        primary). If the standby fails to switch to stream mode, it falls back
+        to archive mode.
+        If this value is specified without units, it is taken as milliseconds.
+        The default is five minutes (<literal>5min</literal>).
+        With a lower setting of this parameter, the standby makes frequent
+        WAL source switch attempts when the primary is lost for quite longer.
+        To avoid this, set a reasonable value.
+        A setting of <literal>0</literal> disables the feature. When disabled,
+        the standby typically switches to stream mode, only when receive from
+        WAL archive finishes (no more WAL left there) or fails for any reason.
+        This parameter can only be set in
+        the <filename>postgresql.conf</filename> file or on the server
+        command line.
+       </para>
+       <para>
+        Reading WAL from archive may not always be efficient and cheaper
+        because network latencies, disk IO cost might differ on the archive as
+        compared to primary and often the archive may sit far from standby
+        impacting recovery performance on the standby. Hence reading WAL
+        from the primary, by setting this parameter, as opposed to the archive
+        enables the standby to catch up with the primary sooner thus reducing
+        replication lag and avoiding WAL files accumulation on the primary.
+      </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index ae2af5ae3d..142fd19e7f 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -88,6 +88,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 300000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -3411,6 +3412,9 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	static bool first_time = true;
+	static TimestampTz last_switch_time = 0;
+	bool	intentionalSourceSwitch = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
 
@@ -3432,6 +3436,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after every
+	 * streaming_replication_retry_interval milliseconds, when state machine is
+	 * in XLOG_FROM_ARCHIVE state. If successful, the state machine moves to
+	 * XLOG_FROM_STREAM state, otherwise it falls back to XLOG_FROM_ARCHIVE
+	 * state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3466,8 +3476,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 			 * Don't allow any retry loops to occur during nonblocking
 			 * readahead.  Let the caller process everything that has been
 			 * decoded already first.
+			 *
+			 * Continue retrying for requested WAL when there was an
+			 * intentional source switch from archive to stream.
 			 */
-			if (nonblocking)
+			if (nonblocking && !intentionalSourceSwitch)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3597,15 +3610,30 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			if (intentionalSourceSwitch)
+			{
+				elog(DEBUG2,
+					 "switched WAL source to %s after fetching WAL from %s for %d milliseconds",
+					 xlogSourceNames[oldSource],
+					 xlogSourceNames[currentSource],
+					 streaming_replication_retry_interval);
+			}
+			else
+			{
+				elog(DEBUG2, "switched WAL source from %s to %s after %s",
+					 xlogSourceNames[oldSource],
+					 xlogSourceNames[currentSource],
+					 lastSourceFailed ? "failure" : "success");
+			}
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
 		 * source.
 		 */
 		lastSourceFailed = false;
+		intentionalSourceSwitch = false;
 
 		switch (currentSource)
 		{
@@ -3628,6 +3656,65 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/*
+				 * Make an attempt to read WAL from primary after reading from
+				 * archive for streaming_replication_retry_interval
+				 * milliseconds. Reading WAL from the archive may not always be
+				 * efficient and cheaper because network latencies, disk IO
+				 * cost might differ on the archive as compared to the primary
+				 * and often the archive may sit far from the standby - all
+				 * adding to recovery performance on the standby. Hence reading
+				 * WAL from the primary as opposed to the archive enables the
+				 * standby to catch up with the primary sooner thus reducing
+				 * replication lag and avoiding WAL files accumulation on the
+				 * primary.
+				 *
+				 * We are here for any of the following reasons:
+				 * 1) standby in initial recovery after start/restart.
+				 * 2) standby stopped streaming from primary because of
+				 * connectivity issues with the primary (either due to network
+				 * issues or crash in the primary or something else) or
+				 * walreceiver got killed or crashed for whatever reasons.
+				 */
+				if (streaming_replication_retry_interval > 0 &&
+					StandbyMode &&
+					currentSource == XLOG_FROM_ARCHIVE)
+				{
+					TimestampTz curr_time;
+
+					curr_time = GetCurrentTimestamp();
+
+					/* Assume last_switch_time as curr_time for the first time */
+					if (first_time)
+						last_switch_time = curr_time;
+
+					if (!first_time &&
+						TimestampDifferenceExceeds(last_switch_time, curr_time,
+												   streaming_replication_retry_interval))
+					{
+						elog(DEBUG2,
+							 "trying to switch WAL source to %s after fetching WAL from %s for %d milliseconds",
+							 xlogSourceNames[XLOG_FROM_STREAM],
+							 xlogSourceNames[currentSource],
+							 streaming_replication_retry_interval);
+
+						last_switch_time = curr_time;
+
+						/*
+						 * Treat this as a failure to read from archive, even
+						 * though it is actually not, so that the state machine
+						 * will move on to stream the WAL from primary.
+						 */
+						lastSourceFailed = true;
+						intentionalSourceSwitch = true;
+						break;
+					}
+
+					/* We're not here for the first time any more */
+					if (first_time)
+						first_time = false;
+				}
+
 				/*
 				 * Try to restore the file from archive, or read an existing
 				 * file from pg_wal.
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 55bf998511..b5f0575fdc 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -3321,6 +3321,18 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication"),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&streaming_replication_retry_interval,
+		300000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 90bec0502c..6e083c72da 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -351,6 +351,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 5min	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication
+					# in milliseconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index 0aa85d90e8..2d5c815246 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/t/034_wal_source_switch.pl b/src/test/recovery/t/034_wal_source_switch.pl
new file mode 100644
index 0000000000..03c92af753
--- /dev/null
+++ b/src/test/recovery/t/034_wal_source_switch.pl
@@ -0,0 +1,126 @@
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Test for WAL source switch feature
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+$ENV{PGDATABASE} = 'postgres';
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(
+    allows_streaming => 1,
+    has_archiving => 1,
+    extra => ['--wal-segsize=1']);
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf(
+	'postgresql.conf', qq(
+    min_wal_size = 2MB
+    max_wal_size = 1GB
+    checkpoint_timeout = 1h
+    wal_recycle = off
+));
+$node_primary->start;
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('rep1')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create a standby linking to it using the replication slot
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup($node_primary, $backup_name,
+	has_streaming => 1,
+    has_restoring => 1);
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+primary_slot_name = 'rep1'
+min_wal_size = 2MB
+max_wal_size = 1GB
+checkpoint_timeout = 1h
+streaming_replication_retry_interval = 100ms
+wal_recycle = off
+log_min_messages = 'debug2'
+));
+
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+# Stop standby
+$node_standby->stop;
+
+# Advance WAL by 100 segments (= 100MB) on primary
+advance_wal($node_primary, 100);
+
+# Wait for primary to generate requested WAL files
+$node_primary->poll_query_until('postgres',
+	q|SELECT COUNT(*) >= 100 FROM pg_ls_waldir()|, 't');
+
+# Standby now connects to primary during inital recovery after fetching WAL
+# from archive for about streaming_replication_retry_interval milliseconds.
+$node_standby->start;
+
+$node_primary->wait_for_catchup($node_standby);
+
+ok(find_in_log(
+		$node_standby,
+        qr/restored log file ".*" from archive/),
+	    'check that some of WAL segments were fetched from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/trying to switch WAL source to .* after fetching WAL from .* for .* milliseconds/),
+	    'check that standby tried to switch WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/switched WAL source to .* after fetching WAL from .* for .* milliseconds/),
+	    'check that standby actually switched WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/started streaming WAL from primary at .* on timeline .*/),
+	    'check that standby strated streaming from primary');
+
+# Stop standby
+$node_standby->stop;
+
+# Stop primary
+$node_primary->stop;
+#####################################
+# Advance WAL of $node by $n segments
+sub advance_wal
+{
+	my ($node, $n) = @_;
+
+	# Advance by $n segments (= (wal_segment_size * $n) bytes) on primary.
+	for (my $i = 0; $i < $n; $i++)
+	{
+		$node->safe_psql('postgres',
+			"CREATE TABLE t (); DROP TABLE t; SELECT pg_switch_wal();");
+	}
+	return;
+}
+
+# find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
+
+done_testing();
-- 
2.34.1

From 02766cd6bfa533070b8488c6172f62f93ecbb855 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Mon, 12 Sep 2022 03:28:23 +0000
Subject: [PATCH v4] Allow standby to switch WAL source from archive to
 streaming replication

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be efficient and cheaper because network
latencies, disk IO cost might differ on the archive as compared
to primary and often the archive may sit far from standby
impacting recovery performance on the standby. Hence reading WAL
from the primary, by setting this parameter, as opposed to the
archive enables the standby to catch up with the primary sooner
thus reducing replication lag and avoiding WAL files accumulation
on the primary.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication. If the standby fails to switch to stream
mode, it falls back to archive mode.

Reported-by: SATYANARAYANA NARLAPURAM
Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  45 +++++++
 src/backend/access/transam/xlogrecovery.c     | 124 +++++++++++++++--
 src/backend/utils/misc/guc.c                  |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/t/034_wal_source_switch.pl  | 126 ++++++++++++++++++
 6 files changed, 301 insertions(+), 11 deletions(-)
 create mode 100644 src/test/recovery/t/034_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index a5cd4e44c7..694da93b8c 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4840,6 +4840,51 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+    <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from WAL archive to streaming replication (get WAL from
+        primary). If the standby fails to switch to stream mode, it falls back
+        to archive mode.
+        If this value is specified without units, it is taken as milliseconds.
+        The default is five minutes (<literal>5min</literal>).
+        With a lower setting of this parameter, the standby makes frequent
+        WAL source switch attempts when the primary is lost for quite longer.
+        To avoid this, set a reasonable value.
+        A setting of <literal>0</literal> disables the feature. When disabled,
+        the standby typically switches to stream mode, only when receive from
+        WAL archive finishes (no more WAL left there) or fails for any reason.
+        This parameter can only be set in
+        the <filename>postgresql.conf</filename> file or on the server
+        command line.
+       </para>
+       <para>
+        Reading WAL from archive may not always be efficient and cheaper
+        because network latencies, disk IO cost might differ on the archive as
+        compared to primary and often the archive may sit far from standby
+        impacting recovery performance on the standby. Hence reading WAL
+        from the primary, by setting this parameter, as opposed to the archive
+        enables the standby to catch up with the primary sooner thus reducing
+        replication lag and avoiding WAL files accumulation on the primary.
+      </para>
+       <para>
+        Note that the standby may not always attempt to switch source from
+        WAL archive to streaming replication at exact
+        <varname>streaming_replication_retry_interval</varname> intervals.
+        For example, if the parameter is set to <literal>1min</literal> and
+        fetching from WAL archive takes <literal>5min</literal>, then the
+        source switch attempt happens for the next WAL after current WAL is
+        fetched from WAL archive and applied.
+      </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 9a80084a68..4b25cdae0d 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -65,6 +65,12 @@
 #define RECOVERY_COMMAND_FILE	"recovery.conf"
 #define RECOVERY_COMMAND_DONE	"recovery.done"
 
+
+#define StreamingReplRetryEnabled() \
+	(streaming_replication_retry_interval > 0 && \
+	 StandbyMode && \
+	 currentSource == XLOG_FROM_ARCHIVE)
+
 /*
  * GUC support
  */
@@ -88,6 +94,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 300000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -295,6 +302,11 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/*
+ * Holds the timestamp at which WaitForWALToBecomeAvailable()'s state machine
+ * switches to XLOG_FROM_ARCHIVE.
+ */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -437,6 +449,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static bool ShouldSwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3413,6 +3427,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	bool	sourceSwitched = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
 
@@ -3434,6 +3449,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval milliseconds. If
+	 * successful, the state machine moves to XLOG_FROM_STREAM state, otherwise
+	 * it falls back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3457,19 +3477,20 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
-		 * advance the state machine if so. The failure to read might've
-		 * happened outside this function, e.g when a CRC check fails on a
-		 * record, or within this loop.
+		 * First check if we failed to read from the current source or we
+		 * intentionally would want to switch the source from archive to
+		 * primary, and advance the state machine if so. The failure to read
+		 * might've happened outside this function, e.g when a CRC check fails
+		 * on a record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed || sourceSwitched)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
-			 * readahead.  Let the caller process everything that has been
-			 * decoded already first.
+			 * readahead if we failed to read from the current source. Let the
+			 * caller process everything that has been decoded already first.
 			 */
-			if (nonblocking)
+			if (nonblocking && lastSourceFailed)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3599,15 +3620,34 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we're switching to archive. */
+			if (StreamingReplRetryEnabled())
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			if (sourceSwitched)
+			{
+				elog(DEBUG2,
+					 "switched WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+					 xlogSourceNames[currentSource],
+					 xlogSourceNames[oldSource],
+					 streaming_replication_retry_interval);
+			}
+			else
+			{
+				elog(DEBUG2, "switched WAL source from %s to %s after %s",
+					 xlogSourceNames[oldSource],
+					 xlogSourceNames[currentSource],
+					 lastSourceFailed ? "failure" : "success");
+			}
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
 		 * source.
 		 */
 		lastSourceFailed = false;
+		sourceSwitched = false;
 
 		switch (currentSource)
 		{
@@ -3630,6 +3670,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				sourceSwitched = ShouldSwitchWALSourceToPrimary();
+
+				if (sourceSwitched)
+					break;
+
 				/*
 				 * Try to restore the file from archive, or read an existing
 				 * file from pg_wal.
@@ -3872,6 +3917,63 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * This function tells if a standby should make an attempt to read WAL from
+ * primary after reading from archive for at least
+ * streaming_replication_retry_interval milliseconds. Reading WAL from the
+ * archive may not always be efficient and cheaper because network latencies,
+ * disk IO cost might differ on the archive as compared to the primary and
+ * often the archive may sit far from the standby - all adding to recovery
+ * performance on the standby. Hence reading WAL from the primary as opposed to
+ * the archive enables the standby to catch up with the primary sooner thus
+ * reducing replication lag and avoiding WAL files accumulation on the primary.
+ *
+ * We are here for any of the following reasons:
+ * 1) standby in initial recovery after start/restart.
+ * 2) standby stopped streaming from primary because of connectivity issues
+ * with the primary (either due to network issues or crash in the primary or
+ * something else) or walreceiver got killed or crashed for whatever reasons.
+ */
+static bool
+ShouldSwitchWALSourceToPrimary(void)
+{
+	bool sourceSwitched;
+
+	if (!StreamingReplRetryEnabled())
+		return false;
+
+	if (switched_to_archive_at > 0)
+	{
+		TimestampTz curr_time;
+
+		curr_time = GetCurrentTimestamp();
+
+		if (TimestampDifferenceExceeds(switched_to_archive_at, curr_time,
+									   streaming_replication_retry_interval))
+		{
+			elog(DEBUG2,
+				 "trying to switch WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+				 xlogSourceNames[XLOG_FROM_STREAM],
+				 xlogSourceNames[currentSource],
+				 streaming_replication_retry_interval);
+
+			sourceSwitched = true;
+		}
+		else
+			sourceSwitched = false;
+	}
+	else if (switched_to_archive_at == 0)
+	{
+		/*
+		 * Save the timestamp if we're about to fetch WAL from archive for the
+		 * first time.
+		 */
+		switched_to_archive_at = GetCurrentTimestamp();
+		sourceSwitched = false;
+	}
+
+	return sourceSwitched;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 55bf998511..587cee5bb8 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -3321,6 +3321,18 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&streaming_replication_retry_interval,
+		300000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 90bec0502c..6e083c72da 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -351,6 +351,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 5min	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication
+					# in milliseconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index 0aa85d90e8..2d5c815246 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/t/034_wal_source_switch.pl b/src/test/recovery/t/034_wal_source_switch.pl
new file mode 100644
index 0000000000..d33ca9635c
--- /dev/null
+++ b/src/test/recovery/t/034_wal_source_switch.pl
@@ -0,0 +1,126 @@
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Test for WAL source switch feature
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+$ENV{PGDATABASE} = 'postgres';
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(
+    allows_streaming => 1,
+    has_archiving => 1,
+    extra => ['--wal-segsize=1']);
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf(
+	'postgresql.conf', qq(
+    min_wal_size = 2MB
+    max_wal_size = 1GB
+    checkpoint_timeout = 1h
+    wal_recycle = off
+));
+$node_primary->start;
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('rep1')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create a standby linking to it using the replication slot
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup($node_primary, $backup_name,
+	has_streaming => 1,
+    has_restoring => 1);
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+primary_slot_name = 'rep1'
+min_wal_size = 2MB
+max_wal_size = 1GB
+checkpoint_timeout = 1h
+streaming_replication_retry_interval = 100ms
+wal_recycle = off
+log_min_messages = 'debug2'
+));
+
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+# Stop standby
+$node_standby->stop;
+
+# Advance WAL by 100 segments (= 100MB) on primary
+advance_wal($node_primary, 100);
+
+# Wait for primary to generate requested WAL files
+$node_primary->poll_query_until('postgres',
+	q|SELECT COUNT(*) >= 100 FROM pg_ls_waldir()|, 't');
+
+# Standby now connects to primary during inital recovery after fetching WAL
+# from archive for about streaming_replication_retry_interval milliseconds.
+$node_standby->start;
+
+$node_primary->wait_for_catchup($node_standby);
+
+ok(find_in_log(
+		$node_standby,
+        qr/restored log file ".*" from archive/),
+	    'check that some of WAL segments were fetched from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/trying to switch WAL source to .* after fetching WAL from .* for at least .* milliseconds/),
+	    'check that standby tried to switch WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/switched WAL source to .* after fetching WAL from .* for at least .* milliseconds/),
+	    'check that standby actually switched WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/started streaming WAL from primary at .* on timeline .*/),
+	    'check that standby strated streaming from primary');
+
+# Stop standby
+$node_standby->stop;
+
+# Stop primary
+$node_primary->stop;
+#####################################
+# Advance WAL of $node by $n segments
+sub advance_wal
+{
+	my ($node, $n) = @_;
+
+	# Advance by $n segments (= (wal_segment_size * $n) bytes) on primary.
+	for (my $i = 0; $i < $n; $i++)
+	{
+		$node->safe_psql('postgres',
+			"CREATE TABLE t (); DROP TABLE t; SELECT pg_switch_wal();");
+	}
+	return;
+}
+
+# find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
+
+done_testing();
-- 
2.34.1

From 857415757ec21bd8b0195c17694618a1cbf22a57 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Mon, 12 Sep 2022 04:44:35 +0000
Subject: [PATCH v5] Allow standby to switch WAL source from archive to
 streaming replication

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be efficient and cheaper because network
latencies, disk IO cost might differ on the archive as compared
to primary and often the archive may sit far from standby
impacting recovery performance on the standby. Hence reading WAL
from the primary, by setting this parameter, as opposed to the
archive enables the standby to catch up with the primary sooner
thus reducing replication lag and avoiding WAL files accumulation
on the primary.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication. If the standby fails to switch to stream
mode, it falls back to archive mode.

Reported-by: SATYANARAYANA NARLAPURAM
Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  45 +++++++
 src/backend/access/transam/xlogrecovery.c     | 124 +++++++++++++++--
 src/backend/utils/misc/guc.c                  |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/t/034_wal_source_switch.pl  | 126 ++++++++++++++++++
 6 files changed, 301 insertions(+), 11 deletions(-)
 create mode 100644 src/test/recovery/t/034_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index a5cd4e44c7..694da93b8c 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4840,6 +4840,51 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+    <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from WAL archive to streaming replication (get WAL from
+        primary). If the standby fails to switch to stream mode, it falls back
+        to archive mode.
+        If this value is specified without units, it is taken as milliseconds.
+        The default is five minutes (<literal>5min</literal>).
+        With a lower setting of this parameter, the standby makes frequent
+        WAL source switch attempts when the primary is lost for quite longer.
+        To avoid this, set a reasonable value.
+        A setting of <literal>0</literal> disables the feature. When disabled,
+        the standby typically switches to stream mode, only when receive from
+        WAL archive finishes (no more WAL left there) or fails for any reason.
+        This parameter can only be set in
+        the <filename>postgresql.conf</filename> file or on the server
+        command line.
+       </para>
+       <para>
+        Reading WAL from archive may not always be efficient and cheaper
+        because network latencies, disk IO cost might differ on the archive as
+        compared to primary and often the archive may sit far from standby
+        impacting recovery performance on the standby. Hence reading WAL
+        from the primary, by setting this parameter, as opposed to the archive
+        enables the standby to catch up with the primary sooner thus reducing
+        replication lag and avoiding WAL files accumulation on the primary.
+      </para>
+       <para>
+        Note that the standby may not always attempt to switch source from
+        WAL archive to streaming replication at exact
+        <varname>streaming_replication_retry_interval</varname> intervals.
+        For example, if the parameter is set to <literal>1min</literal> and
+        fetching from WAL archive takes <literal>5min</literal>, then the
+        source switch attempt happens for the next WAL after current WAL is
+        fetched from WAL archive and applied.
+      </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 9a80084a68..35f7985e65 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -65,6 +65,12 @@
 #define RECOVERY_COMMAND_FILE	"recovery.conf"
 #define RECOVERY_COMMAND_DONE	"recovery.done"
 
+
+#define StreamingReplRetryEnabled() \
+	(streaming_replication_retry_interval > 0 && \
+	 StandbyMode && \
+	 currentSource == XLOG_FROM_ARCHIVE)
+
 /*
  * GUC support
  */
@@ -88,6 +94,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 300000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -295,6 +302,11 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/*
+ * Holds the timestamp at which WaitForWALToBecomeAvailable()'s state machine
+ * switches to XLOG_FROM_ARCHIVE.
+ */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -437,6 +449,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static bool ShouldSwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3413,6 +3427,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	bool	switchSource = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
 
@@ -3434,6 +3449,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval milliseconds. If
+	 * successful, the state machine moves to XLOG_FROM_STREAM state, otherwise
+	 * it falls back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3457,19 +3477,20 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
-		 * advance the state machine if so. The failure to read might've
-		 * happened outside this function, e.g when a CRC check fails on a
-		 * record, or within this loop.
+		 * First check if we failed to read from the current source or we
+		 * intentionally would want to switch the source from archive to
+		 * primary, and advance the state machine if so. The failure to read
+		 * might've happened outside this function, e.g when a CRC check fails
+		 * on a record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed || switchSource)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
-			 * readahead.  Let the caller process everything that has been
-			 * decoded already first.
+			 * readahead if we failed to read from the current source. Let the
+			 * caller process everything that has been decoded already first.
 			 */
-			if (nonblocking)
+			if (nonblocking && lastSourceFailed)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3599,15 +3620,34 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we're switching to archive. */
+			if (StreamingReplRetryEnabled())
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			if (switchSource)
+			{
+				elog(DEBUG2,
+					 "switched WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+					 xlogSourceNames[currentSource],
+					 xlogSourceNames[oldSource],
+					 streaming_replication_retry_interval);
+			}
+			else
+			{
+				elog(DEBUG2, "switched WAL source from %s to %s after %s",
+					 xlogSourceNames[oldSource],
+					 xlogSourceNames[currentSource],
+					 lastSourceFailed ? "failure" : "success");
+			}
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
 		 * source.
 		 */
 		lastSourceFailed = false;
+		switchSource = false;
 
 		switch (currentSource)
 		{
@@ -3630,6 +3670,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				switchSource = ShouldSwitchWALSourceToPrimary();
+
+				if (switchSource)
+					break;
+
 				/*
 				 * Try to restore the file from archive, or read an existing
 				 * file from pg_wal.
@@ -3872,6 +3917,63 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * This function tells if a standby should make an attempt to read WAL from
+ * primary after reading from archive for at least
+ * streaming_replication_retry_interval milliseconds. Reading WAL from the
+ * archive may not always be efficient and cheaper because network latencies,
+ * disk IO cost might differ on the archive as compared to the primary and
+ * often the archive may sit far from the standby - all adding to recovery
+ * performance on the standby. Hence reading WAL from the primary as opposed to
+ * the archive enables the standby to catch up with the primary sooner thus
+ * reducing replication lag and avoiding WAL files accumulation on the primary.
+ *
+ * We are here for any of the following reasons:
+ * 1) standby in initial recovery after start/restart.
+ * 2) standby stopped streaming from primary because of connectivity issues
+ * with the primary (either due to network issues or crash in the primary or
+ * something else) or walreceiver got killed or crashed for whatever reasons.
+ */
+static bool
+ShouldSwitchWALSourceToPrimary(void)
+{
+	bool shouldSwitchSource = false;
+
+	if (!StreamingReplRetryEnabled())
+		return shouldSwitchSource;
+
+	if (switched_to_archive_at > 0)
+	{
+		TimestampTz curr_time;
+
+		curr_time = GetCurrentTimestamp();
+
+		if (TimestampDifferenceExceeds(switched_to_archive_at, curr_time,
+									   streaming_replication_retry_interval))
+		{
+			elog(DEBUG2,
+				 "trying to switch WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+				 xlogSourceNames[XLOG_FROM_STREAM],
+				 xlogSourceNames[currentSource],
+				 streaming_replication_retry_interval);
+
+			shouldSwitchSource = true;
+		}
+		else
+			shouldSwitchSource = false;
+	}
+	else if (switched_to_archive_at == 0)
+	{
+		/*
+		 * Save the timestamp if we're about to fetch WAL from archive for the
+		 * first time.
+		 */
+		switched_to_archive_at = GetCurrentTimestamp();
+		shouldSwitchSource = false;
+	}
+
+	return shouldSwitchSource;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 55bf998511..587cee5bb8 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -3321,6 +3321,18 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&streaming_replication_retry_interval,
+		300000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 90bec0502c..6e083c72da 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -351,6 +351,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 5min	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication
+					# in milliseconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index 0aa85d90e8..2d5c815246 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/t/034_wal_source_switch.pl b/src/test/recovery/t/034_wal_source_switch.pl
new file mode 100644
index 0000000000..d33ca9635c
--- /dev/null
+++ b/src/test/recovery/t/034_wal_source_switch.pl
@@ -0,0 +1,126 @@
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Test for WAL source switch feature
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+$ENV{PGDATABASE} = 'postgres';
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(
+    allows_streaming => 1,
+    has_archiving => 1,
+    extra => ['--wal-segsize=1']);
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf(
+	'postgresql.conf', qq(
+    min_wal_size = 2MB
+    max_wal_size = 1GB
+    checkpoint_timeout = 1h
+    wal_recycle = off
+));
+$node_primary->start;
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('rep1')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create a standby linking to it using the replication slot
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup($node_primary, $backup_name,
+	has_streaming => 1,
+    has_restoring => 1);
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+primary_slot_name = 'rep1'
+min_wal_size = 2MB
+max_wal_size = 1GB
+checkpoint_timeout = 1h
+streaming_replication_retry_interval = 100ms
+wal_recycle = off
+log_min_messages = 'debug2'
+));
+
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+# Stop standby
+$node_standby->stop;
+
+# Advance WAL by 100 segments (= 100MB) on primary
+advance_wal($node_primary, 100);
+
+# Wait for primary to generate requested WAL files
+$node_primary->poll_query_until('postgres',
+	q|SELECT COUNT(*) >= 100 FROM pg_ls_waldir()|, 't');
+
+# Standby now connects to primary during inital recovery after fetching WAL
+# from archive for about streaming_replication_retry_interval milliseconds.
+$node_standby->start;
+
+$node_primary->wait_for_catchup($node_standby);
+
+ok(find_in_log(
+		$node_standby,
+        qr/restored log file ".*" from archive/),
+	    'check that some of WAL segments were fetched from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/trying to switch WAL source to .* after fetching WAL from .* for at least .* milliseconds/),
+	    'check that standby tried to switch WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/switched WAL source to .* after fetching WAL from .* for at least .* milliseconds/),
+	    'check that standby actually switched WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/started streaming WAL from primary at .* on timeline .*/),
+	    'check that standby strated streaming from primary');
+
+# Stop standby
+$node_standby->stop;
+
+# Stop primary
+$node_primary->stop;
+#####################################
+# Advance WAL of $node by $n segments
+sub advance_wal
+{
+	my ($node, $n) = @_;
+
+	# Advance by $n segments (= (wal_segment_size * $n) bytes) on primary.
+	for (my $i = 0; $i < $n; $i++)
+	{
+		$node->safe_psql('postgres',
+			"CREATE TABLE t (); DROP TABLE t; SELECT pg_switch_wal();");
+	}
+	return;
+}
+
+# find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
+
+done_testing();
-- 
2.34.1

From b1181f681718a7ca90453a0ad2bc80500983e699 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Thu, 15 Sep 2022 04:55:23 +0000
Subject: [PATCH v6] Allow standby to switch WAL source from archive to
 streaming replication

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be efficient and cheaper because network
latencies, disk IO cost might differ on the archive as compared
to primary and often the archive may sit far from standby
impacting recovery performance on the standby. Hence reading WAL
from the primary, by setting this parameter, as opposed to the
archive enables the standby to catch up with the primary sooner
thus reducing replication lag and avoiding WAL files accumulation
on the primary.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication. If the standby fails to switch to stream
mode, it falls back to archive mode.

Reported-by: SATYANARAYANA NARLAPURAM
Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  45 +++++++
 src/backend/access/transam/xlogrecovery.c     | 124 +++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/t/034_wal_source_switch.pl  | 126 ++++++++++++++++++
 6 files changed, 301 insertions(+), 11 deletions(-)
 create mode 100644 src/test/recovery/t/034_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 700914684d..892442a053 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4840,6 +4840,51 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+    <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from WAL archive to streaming replication (get WAL from
+        primary). If the standby fails to switch to stream mode, it falls back
+        to archive mode.
+        If this value is specified without units, it is taken as milliseconds.
+        The default is five minutes (<literal>5min</literal>).
+        With a lower setting of this parameter, the standby makes frequent
+        WAL source switch attempts when the primary is lost for quite longer.
+        To avoid this, set a reasonable value.
+        A setting of <literal>0</literal> disables the feature. When disabled,
+        the standby typically switches to stream mode, only when receive from
+        WAL archive finishes (no more WAL left there) or fails for any reason.
+        This parameter can only be set in
+        the <filename>postgresql.conf</filename> file or on the server
+        command line.
+       </para>
+       <para>
+        Reading WAL from archive may not always be efficient and cheaper
+        because network latencies, disk IO cost might differ on the archive as
+        compared to primary and often the archive may sit far from standby
+        impacting recovery performance on the standby. Hence reading WAL
+        from the primary, by setting this parameter, as opposed to the archive
+        enables the standby to catch up with the primary sooner thus reducing
+        replication lag and avoiding WAL files accumulation on the primary.
+      </para>
+       <para>
+        Note that the standby may not always attempt to switch source from
+        WAL archive to streaming replication at exact
+        <varname>streaming_replication_retry_interval</varname> intervals.
+        For example, if the parameter is set to <literal>1min</literal> and
+        fetching from WAL archive takes <literal>5min</literal>, then the
+        source switch attempt happens for the next WAL after current WAL is
+        fetched from WAL archive and applied.
+      </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 30661bdad6..e737575bf5 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -68,6 +68,12 @@
 #define RECOVERY_COMMAND_FILE	"recovery.conf"
 #define RECOVERY_COMMAND_DONE	"recovery.done"
 
+
+#define StreamingReplRetryEnabled() \
+	(streaming_replication_retry_interval > 0 && \
+	 StandbyMode && \
+	 currentSource == XLOG_FROM_ARCHIVE)
+
 /*
  * GUC support
  */
@@ -91,6 +97,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 300000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -298,6 +305,11 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/*
+ * Holds the timestamp at which WaitForWALToBecomeAvailable()'s state machine
+ * switches to XLOG_FROM_ARCHIVE.
+ */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +452,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static bool ShouldSwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3416,6 +3430,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	bool	switchSource = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
 
@@ -3437,6 +3452,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval milliseconds. If
+	 * successful, the state machine moves to XLOG_FROM_STREAM state, otherwise
+	 * it falls back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3460,19 +3480,20 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
-		 * advance the state machine if so. The failure to read might've
-		 * happened outside this function, e.g when a CRC check fails on a
-		 * record, or within this loop.
+		 * First check if we failed to read from the current source or we
+		 * intentionally would want to switch the source from archive to
+		 * primary, and advance the state machine if so. The failure to read
+		 * might've happened outside this function, e.g when a CRC check fails
+		 * on a record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed || switchSource)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
-			 * readahead.  Let the caller process everything that has been
-			 * decoded already first.
+			 * readahead if we failed to read from the current source. Let the
+			 * caller process everything that has been decoded already first.
 			 */
-			if (nonblocking)
+			if (nonblocking && lastSourceFailed)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3602,15 +3623,34 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we're switching to archive. */
+			if (StreamingReplRetryEnabled())
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			if (switchSource)
+			{
+				elog(DEBUG2,
+					 "switched WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+					 xlogSourceNames[currentSource],
+					 xlogSourceNames[oldSource],
+					 streaming_replication_retry_interval);
+			}
+			else
+			{
+				elog(DEBUG2, "switched WAL source from %s to %s after %s",
+					 xlogSourceNames[oldSource],
+					 xlogSourceNames[currentSource],
+					 lastSourceFailed ? "failure" : "success");
+			}
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
 		 * source.
 		 */
 		lastSourceFailed = false;
+		switchSource = false;
 
 		switch (currentSource)
 		{
@@ -3633,6 +3673,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				switchSource = ShouldSwitchWALSourceToPrimary();
+
+				if (switchSource)
+					break;
+
 				/*
 				 * Try to restore the file from archive, or read an existing
 				 * file from pg_wal.
@@ -3875,6 +3920,63 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * This function tells if a standby should make an attempt to read WAL from
+ * primary after reading from archive for at least
+ * streaming_replication_retry_interval milliseconds. Reading WAL from the
+ * archive may not always be efficient and cheaper because network latencies,
+ * disk IO cost might differ on the archive as compared to the primary and
+ * often the archive may sit far from the standby - all adding to recovery
+ * performance on the standby. Hence reading WAL from the primary as opposed to
+ * the archive enables the standby to catch up with the primary sooner thus
+ * reducing replication lag and avoiding WAL files accumulation on the primary.
+ *
+ * We are here for any of the following reasons:
+ * 1) standby in initial recovery after start/restart.
+ * 2) standby stopped streaming from primary because of connectivity issues
+ * with the primary (either due to network issues or crash in the primary or
+ * something else) or walreceiver got killed or crashed for whatever reasons.
+ */
+static bool
+ShouldSwitchWALSourceToPrimary(void)
+{
+	bool shouldSwitchSource = false;
+
+	if (!StreamingReplRetryEnabled())
+		return shouldSwitchSource;
+
+	if (switched_to_archive_at > 0)
+	{
+		TimestampTz curr_time;
+
+		curr_time = GetCurrentTimestamp();
+
+		if (TimestampDifferenceExceeds(switched_to_archive_at, curr_time,
+									   streaming_replication_retry_interval))
+		{
+			elog(DEBUG2,
+				 "trying to switch WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+				 xlogSourceNames[XLOG_FROM_STREAM],
+				 xlogSourceNames[currentSource],
+				 streaming_replication_retry_interval);
+
+			shouldSwitchSource = true;
+		}
+		else
+			shouldSwitchSource = false;
+	}
+	else if (switched_to_archive_at == 0)
+	{
+		/*
+		 * Save the timestamp if we're about to fetch WAL from archive for the
+		 * first time.
+		 */
+		switched_to_archive_at = GetCurrentTimestamp();
+		shouldSwitchSource = false;
+	}
+
+	return shouldSwitchSource;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 550e95056c..cbb9cfca51 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3075,6 +3075,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&streaming_replication_retry_interval,
+		300000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 2ae76e5cfb..84e52f3688 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -351,6 +351,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 5min	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication
+					# in milliseconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index 0aa85d90e8..2d5c815246 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/t/034_wal_source_switch.pl b/src/test/recovery/t/034_wal_source_switch.pl
new file mode 100644
index 0000000000..d33ca9635c
--- /dev/null
+++ b/src/test/recovery/t/034_wal_source_switch.pl
@@ -0,0 +1,126 @@
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Test for WAL source switch feature
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+$ENV{PGDATABASE} = 'postgres';
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(
+    allows_streaming => 1,
+    has_archiving => 1,
+    extra => ['--wal-segsize=1']);
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf(
+	'postgresql.conf', qq(
+    min_wal_size = 2MB
+    max_wal_size = 1GB
+    checkpoint_timeout = 1h
+    wal_recycle = off
+));
+$node_primary->start;
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('rep1')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create a standby linking to it using the replication slot
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup($node_primary, $backup_name,
+	has_streaming => 1,
+    has_restoring => 1);
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+primary_slot_name = 'rep1'
+min_wal_size = 2MB
+max_wal_size = 1GB
+checkpoint_timeout = 1h
+streaming_replication_retry_interval = 100ms
+wal_recycle = off
+log_min_messages = 'debug2'
+));
+
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+# Stop standby
+$node_standby->stop;
+
+# Advance WAL by 100 segments (= 100MB) on primary
+advance_wal($node_primary, 100);
+
+# Wait for primary to generate requested WAL files
+$node_primary->poll_query_until('postgres',
+	q|SELECT COUNT(*) >= 100 FROM pg_ls_waldir()|, 't');
+
+# Standby now connects to primary during inital recovery after fetching WAL
+# from archive for about streaming_replication_retry_interval milliseconds.
+$node_standby->start;
+
+$node_primary->wait_for_catchup($node_standby);
+
+ok(find_in_log(
+		$node_standby,
+        qr/restored log file ".*" from archive/),
+	    'check that some of WAL segments were fetched from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/trying to switch WAL source to .* after fetching WAL from .* for at least .* milliseconds/),
+	    'check that standby tried to switch WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/switched WAL source to .* after fetching WAL from .* for at least .* milliseconds/),
+	    'check that standby actually switched WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/started streaming WAL from primary at .* on timeline .*/),
+	    'check that standby strated streaming from primary');
+
+# Stop standby
+$node_standby->stop;
+
+# Stop primary
+$node_primary->stop;
+#####################################
+# Advance WAL of $node by $n segments
+sub advance_wal
+{
+	my ($node, $n) = @_;
+
+	# Advance by $n segments (= (wal_segment_size * $n) bytes) on primary.
+	for (my $i = 0; $i < $n; $i++)
+	{
+		$node->safe_psql('postgres',
+			"CREATE TABLE t (); DROP TABLE t; SELECT pg_switch_wal();");
+	}
+	return;
+}
+
+# find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
+
+done_testing();
-- 
2.34.1

From b94327d9af60a44f252251be97ee1efabc964f97 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Mon, 19 Sep 2022 14:04:49 +0000
Subject: [PATCH v7] Allow standby to switch WAL source from archive to
 streaming replication

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be efficient and cheaper because network
latencies, disk IO cost might differ on the archive as compared
to primary and often the archive may sit far from standby
impacting recovery performance on the standby. Hence reading WAL
from the primary, by setting this parameter, as opposed to the
archive enables the standby to catch up with the primary sooner
thus reducing replication lag and avoiding WAL files accumulation
on the primary.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication. If the standby fails to switch to stream
mode, it falls back to archive mode.

Reported-by: SATYANARAYANA NARLAPURAM
Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  45 +++++++
 src/backend/access/transam/xlogrecovery.c     | 124 +++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/t/034_wal_source_switch.pl  | 126 ++++++++++++++++++
 6 files changed, 301 insertions(+), 11 deletions(-)
 create mode 100644 src/test/recovery/t/034_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 700914684d..892442a053 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4840,6 +4840,51 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+    <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from WAL archive to streaming replication (get WAL from
+        primary). If the standby fails to switch to stream mode, it falls back
+        to archive mode.
+        If this value is specified without units, it is taken as milliseconds.
+        The default is five minutes (<literal>5min</literal>).
+        With a lower setting of this parameter, the standby makes frequent
+        WAL source switch attempts when the primary is lost for quite longer.
+        To avoid this, set a reasonable value.
+        A setting of <literal>0</literal> disables the feature. When disabled,
+        the standby typically switches to stream mode, only when receive from
+        WAL archive finishes (no more WAL left there) or fails for any reason.
+        This parameter can only be set in
+        the <filename>postgresql.conf</filename> file or on the server
+        command line.
+       </para>
+       <para>
+        Reading WAL from archive may not always be efficient and cheaper
+        because network latencies, disk IO cost might differ on the archive as
+        compared to primary and often the archive may sit far from standby
+        impacting recovery performance on the standby. Hence reading WAL
+        from the primary, by setting this parameter, as opposed to the archive
+        enables the standby to catch up with the primary sooner thus reducing
+        replication lag and avoiding WAL files accumulation on the primary.
+      </para>
+       <para>
+        Note that the standby may not always attempt to switch source from
+        WAL archive to streaming replication at exact
+        <varname>streaming_replication_retry_interval</varname> intervals.
+        For example, if the parameter is set to <literal>1min</literal> and
+        fetching from WAL archive takes <literal>5min</literal>, then the
+        source switch attempt happens for the next WAL after current WAL is
+        fetched from WAL archive and applied.
+      </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index b41e682664..afc769cf16 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -68,6 +68,12 @@
 #define RECOVERY_COMMAND_FILE	"recovery.conf"
 #define RECOVERY_COMMAND_DONE	"recovery.done"
 
+
+#define SwitchFromArchiveToStreamEnabled() \
+	(streaming_replication_retry_interval > 0 && \
+	 StandbyMode && \
+	 currentSource == XLOG_FROM_ARCHIVE)
+
 /*
  * GUC support
  */
@@ -91,6 +97,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 300000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -298,6 +305,11 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/*
+ * Holds the timestamp at which WaitForWALToBecomeAvailable()'s state machine
+ * switches to XLOG_FROM_ARCHIVE.
+ */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +452,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static bool ShouldSwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3416,6 +3430,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	bool	switchSource = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
 
@@ -3437,6 +3452,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval milliseconds. If
+	 * successful, the state machine moves to XLOG_FROM_STREAM state, otherwise
+	 * it falls back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3460,19 +3480,20 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
-		 * advance the state machine if so. The failure to read might've
-		 * happened outside this function, e.g when a CRC check fails on a
-		 * record, or within this loop.
+		 * First check if we failed to read from the current source or we
+		 * intentionally would want to switch the source from archive to
+		 * primary, and advance the state machine if so. The failure to read
+		 * might've happened outside this function, e.g when a CRC check fails
+		 * on a record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed || switchSource)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
-			 * readahead.  Let the caller process everything that has been
-			 * decoded already first.
+			 * readahead if we failed to read from the current source. Let the
+			 * caller process everything that has been decoded already first.
 			 */
-			if (nonblocking)
+			if (nonblocking && lastSourceFailed)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3601,15 +3622,34 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we're switching to archive. */
+			if (SwitchFromArchiveToStreamEnabled())
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			if (switchSource)
+			{
+				elog(DEBUG2,
+					 "switched WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+					 xlogSourceNames[currentSource],
+					 xlogSourceNames[oldSource],
+					 streaming_replication_retry_interval);
+			}
+			else
+			{
+				elog(DEBUG2, "switched WAL source from %s to %s after %s",
+					 xlogSourceNames[oldSource],
+					 xlogSourceNames[currentSource],
+					 lastSourceFailed ? "failure" : "success");
+			}
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
 		 * source.
 		 */
 		lastSourceFailed = false;
+		switchSource = false;
 
 		switch (currentSource)
 		{
@@ -3632,6 +3672,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				switchSource = ShouldSwitchWALSourceToPrimary();
+
+				if (switchSource)
+					break;
+
 				/*
 				 * Try to restore the file from archive, or read an existing
 				 * file from pg_wal.
@@ -3874,6 +3919,63 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * This function tells if a standby should make an attempt to read WAL from
+ * primary after reading from archive for at least
+ * streaming_replication_retry_interval milliseconds. Reading WAL from the
+ * archive may not always be efficient and cheaper because network latencies,
+ * disk IO cost might differ on the archive as compared to the primary and
+ * often the archive may sit far from the standby - all adding to recovery
+ * performance on the standby. Hence reading WAL from the primary as opposed to
+ * the archive enables the standby to catch up with the primary sooner thus
+ * reducing replication lag and avoiding WAL files accumulation on the primary.
+ *
+ * We are here for any of the following reasons:
+ * 1) standby in initial recovery after start/restart.
+ * 2) standby stopped streaming from primary because of connectivity issues
+ * with the primary (either due to network issues or crash in the primary or
+ * something else) or walreceiver got killed or crashed for whatever reasons.
+ */
+static bool
+ShouldSwitchWALSourceToPrimary(void)
+{
+	bool shouldSwitchSource = false;
+
+	if (!SwitchFromArchiveToStreamEnabled())
+		return shouldSwitchSource;
+
+	if (switched_to_archive_at > 0)
+	{
+		TimestampTz curr_time;
+
+		curr_time = GetCurrentTimestamp();
+
+		if (TimestampDifferenceExceeds(switched_to_archive_at, curr_time,
+									   streaming_replication_retry_interval))
+		{
+			elog(DEBUG2,
+				 "trying to switch WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+				 xlogSourceNames[XLOG_FROM_STREAM],
+				 xlogSourceNames[currentSource],
+				 streaming_replication_retry_interval);
+
+			shouldSwitchSource = true;
+		}
+		else
+			shouldSwitchSource = false;
+	}
+	else if (switched_to_archive_at == 0)
+	{
+		/*
+		 * Save the timestamp if we're about to fetch WAL from archive for the
+		 * first time.
+		 */
+		switched_to_archive_at = GetCurrentTimestamp();
+		shouldSwitchSource = false;
+	}
+
+	return shouldSwitchSource;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 550e95056c..cbb9cfca51 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3075,6 +3075,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&streaming_replication_retry_interval,
+		300000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 2ae76e5cfb..84e52f3688 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -351,6 +351,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 5min	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication
+					# in milliseconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index 0aa85d90e8..2d5c815246 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/t/034_wal_source_switch.pl b/src/test/recovery/t/034_wal_source_switch.pl
new file mode 100644
index 0000000000..d33ca9635c
--- /dev/null
+++ b/src/test/recovery/t/034_wal_source_switch.pl
@@ -0,0 +1,126 @@
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Test for WAL source switch feature
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+$ENV{PGDATABASE} = 'postgres';
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(
+    allows_streaming => 1,
+    has_archiving => 1,
+    extra => ['--wal-segsize=1']);
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf(
+	'postgresql.conf', qq(
+    min_wal_size = 2MB
+    max_wal_size = 1GB
+    checkpoint_timeout = 1h
+    wal_recycle = off
+));
+$node_primary->start;
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('rep1')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create a standby linking to it using the replication slot
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup($node_primary, $backup_name,
+	has_streaming => 1,
+    has_restoring => 1);
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+primary_slot_name = 'rep1'
+min_wal_size = 2MB
+max_wal_size = 1GB
+checkpoint_timeout = 1h
+streaming_replication_retry_interval = 100ms
+wal_recycle = off
+log_min_messages = 'debug2'
+));
+
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+# Stop standby
+$node_standby->stop;
+
+# Advance WAL by 100 segments (= 100MB) on primary
+advance_wal($node_primary, 100);
+
+# Wait for primary to generate requested WAL files
+$node_primary->poll_query_until('postgres',
+	q|SELECT COUNT(*) >= 100 FROM pg_ls_waldir()|, 't');
+
+# Standby now connects to primary during inital recovery after fetching WAL
+# from archive for about streaming_replication_retry_interval milliseconds.
+$node_standby->start;
+
+$node_primary->wait_for_catchup($node_standby);
+
+ok(find_in_log(
+		$node_standby,
+        qr/restored log file ".*" from archive/),
+	    'check that some of WAL segments were fetched from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/trying to switch WAL source to .* after fetching WAL from .* for at least .* milliseconds/),
+	    'check that standby tried to switch WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/switched WAL source to .* after fetching WAL from .* for at least .* milliseconds/),
+	    'check that standby actually switched WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/started streaming WAL from primary at .* on timeline .*/),
+	    'check that standby strated streaming from primary');
+
+# Stop standby
+$node_standby->stop;
+
+# Stop primary
+$node_primary->stop;
+#####################################
+# Advance WAL of $node by $n segments
+sub advance_wal
+{
+	my ($node, $n) = @_;
+
+	# Advance by $n segments (= (wal_segment_size * $n) bytes) on primary.
+	for (my $i = 0; $i < $n; $i++)
+	{
+		$node->safe_psql('postgres',
+			"CREATE TABLE t (); DROP TABLE t; SELECT pg_switch_wal();");
+	}
+	return;
+}
+
+# find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
+
+done_testing();
-- 
2.34.1

From 171e11088cca63e99726b49b1b9b408eed81f299 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Sun, 9 Oct 2022 08:11:10 +0000
Subject: [PATCH v8] Allow standby to switch WAL source from archive to
 streaming replication

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be efficient and cheaper because network
latencies, disk IO cost might differ on the archive as compared
to primary and often the archive may sit far from standby
impacting recovery performance on the standby. Hence reading WAL
from the primary, by setting this parameter, as opposed to the
archive enables the standby to catch up with the primary sooner
thus reducing replication lag and avoiding WAL files accumulation
on the primary.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication. If the standby fails to switch to stream
mode, it falls back to archive mode.

Reported-by: SATYANARAYANA NARLAPURAM
Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  45 ++++++
 doc/src/sgml/high-availability.sgml           |   7 +
 src/backend/access/transam/xlogrecovery.c     | 132 ++++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/t/034_wal_source_switch.pl  | 126 +++++++++++++++++
 7 files changed, 312 insertions(+), 15 deletions(-)
 create mode 100644 src/test/recovery/t/034_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 66312b53b8..85baac9bbb 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4847,6 +4847,51 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+    <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from WAL archive to streaming replication (get WAL from
+        primary). If the standby fails to switch to stream mode, it falls back
+        to archive mode.
+        If this value is specified without units, it is taken as milliseconds.
+        The default is five minutes (<literal>5min</literal>).
+        With a lower setting of this parameter, the standby makes frequent
+        WAL source switch attempts when the primary is lost for quite longer.
+        To avoid this, set a reasonable value.
+        A setting of <literal>0</literal> disables the feature. When disabled,
+        the standby typically switches to stream mode, only when receive from
+        WAL archive finishes (no more WAL left there) or fails for any reason.
+        This parameter can only be set in
+        the <filename>postgresql.conf</filename> file or on the server
+        command line.
+       </para>
+       <para>
+        Reading WAL from archive may not always be efficient and cheaper
+        because network latencies, disk IO cost might differ on the archive as
+        compared to primary and often the archive may sit far from standby
+        impacting recovery performance on the standby. Hence reading WAL
+        from the primary, by setting this parameter, as opposed to the archive
+        enables the standby to catch up with the primary sooner thus reducing
+        replication lag and avoiding WAL files accumulation on the primary.
+      </para>
+       <para>
+        Note that the standby may not always attempt to switch source from
+        WAL archive to streaming replication at exact
+        <varname>streaming_replication_retry_interval</varname> intervals.
+        For example, if the parameter is set to <literal>1min</literal> and
+        fetching from WAL archive takes <literal>5min</literal>, then the
+        source switch attempt happens for the next WAL after current WAL is
+        fetched from WAL archive and applied.
+      </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index b2b3129397..e38ce258e7 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -636,6 +636,13 @@ protocol to make nodes agree on a serializable transactional order.
     <filename>pg_wal</filename> at any time to have them replayed.
    </para>
 
+   <para>
+    The standby server can attempt to switch to streaming replication after
+    reading WAL from archive, see
+    <xref linkend="guc-streaming-replication-retry-interval"/> for more
+    details.
+   </para>
+
    <para>
     At startup, the standby begins by restoring all WAL available in the
     archive location, calling <varname>restore_command</varname>. Once it
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index cb07694aea..d0939f9b0b 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -68,6 +68,12 @@
 #define RECOVERY_COMMAND_FILE	"recovery.conf"
 #define RECOVERY_COMMAND_DONE	"recovery.done"
 
+
+#define SwitchFromArchiveToStreamEnabled() \
+	(streaming_replication_retry_interval > 0 && \
+	 StandbyMode && \
+	 currentSource == XLOG_FROM_ARCHIVE)
+
 /*
  * GUC support
  */
@@ -91,6 +97,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 300000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -298,6 +305,11 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/*
+ * Holds the timestamp at which WaitForWALToBecomeAvailable()'s state machine
+ * switches to XLOG_FROM_ARCHIVE.
+ */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +452,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static bool ShouldSwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3416,8 +3430,10 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	bool	switchSource = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
+	XLogSource	readFrom;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -3437,6 +3453,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval milliseconds. If
+	 * successful, the state machine moves to XLOG_FROM_STREAM state, otherwise
+	 * it falls back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3460,19 +3481,20 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
-		 * advance the state machine if so. The failure to read might've
-		 * happened outside this function, e.g when a CRC check fails on a
-		 * record, or within this loop.
+		 * First check if we failed to read from the current source or we
+		 * intentionally would want to switch the source from archive to
+		 * primary, and advance the state machine if so. The failure to read
+		 * might've happened outside this function, e.g when a CRC check fails
+		 * on a record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed || switchSource)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
-			 * readahead.  Let the caller process everything that has been
-			 * decoded already first.
+			 * readahead if we failed to read from the current source. Let the
+			 * caller process everything that has been decoded already first.
 			 */
-			if (nonblocking)
+			if (nonblocking && lastSourceFailed)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3601,15 +3623,30 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we're switching to archive. */
+			if (SwitchFromArchiveToStreamEnabled())
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			if (switchSource)
+				elog(DEBUG2,
+					 "switched WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+					 xlogSourceNames[currentSource],
+					 xlogSourceNames[oldSource],
+					 streaming_replication_retry_interval);
+			else
+				elog(DEBUG2, "switched WAL source from %s to %s after %s",
+					 xlogSourceNames[oldSource],
+					 xlogSourceNames[currentSource],
+					 lastSourceFailed ? "failure" : "success");
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
 		 * source.
 		 */
 		lastSourceFailed = false;
+		switchSource = false;
 
 		switch (currentSource)
 		{
@@ -3632,13 +3669,21 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				switchSource = ShouldSwitchWALSourceToPrimary();
+
 				/*
 				 * Try to restore the file from archive, or read an existing
-				 * file from pg_wal.
+				 * file from pg_wal. However, before switching the source to
+				 * stream mode, give it a chance to read from pg_wal.
 				 */
-				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2,
-											  currentSource == XLOG_FROM_ARCHIVE ? XLOG_FROM_ANY :
-											  currentSource);
+				if (switchSource)
+					readFrom = XLOG_FROM_PG_WAL;
+				else if (currentSource == XLOG_FROM_ARCHIVE)
+					readFrom = XLOG_FROM_ANY;
+				else
+					readFrom = currentSource;
+
+				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2, readFrom);
 				if (readFile >= 0)
 					return XLREAD_SUCCESS;	/* success! */
 
@@ -3874,6 +3919,63 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * This function tells if a standby should make an attempt to read WAL from
+ * primary after reading from archive for at least
+ * streaming_replication_retry_interval milliseconds. Reading WAL from the
+ * archive may not always be efficient and cheaper because network latencies,
+ * disk IO cost might differ on the archive as compared to the primary and
+ * often the archive may sit far from the standby - all adding to recovery
+ * performance on the standby. Hence reading WAL from the primary as opposed to
+ * the archive enables the standby to catch up with the primary sooner thus
+ * reducing replication lag and avoiding WAL files accumulation on the primary.
+ *
+ * We are here for any of the following reasons:
+ * 1) standby in initial recovery after start/restart.
+ * 2) standby stopped streaming from primary because of connectivity issues
+ * with the primary (either due to network issues or crash in the primary or
+ * something else) or walreceiver got killed or crashed for whatever reasons.
+ */
+static bool
+ShouldSwitchWALSourceToPrimary(void)
+{
+	bool shouldSwitchSource = false;
+
+	if (!SwitchFromArchiveToStreamEnabled())
+		return shouldSwitchSource;
+
+	if (switched_to_archive_at > 0)
+	{
+		TimestampTz curr_time;
+
+		curr_time = GetCurrentTimestamp();
+
+		if (TimestampDifferenceExceeds(switched_to_archive_at, curr_time,
+									   streaming_replication_retry_interval))
+		{
+			elog(DEBUG2,
+				 "trying to switch WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+				 xlogSourceNames[XLOG_FROM_STREAM],
+				 xlogSourceNames[currentSource],
+				 streaming_replication_retry_interval);
+
+			shouldSwitchSource = true;
+		}
+		else
+			shouldSwitchSource = false;
+	}
+	else if (switched_to_archive_at == 0)
+	{
+		/*
+		 * Save the timestamp if we're about to fetch WAL from archive for the
+		 * first time.
+		 */
+		switched_to_archive_at = GetCurrentTimestamp();
+		shouldSwitchSource = false;
+	}
+
+	return shouldSwitchSource;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 05ab087934..df09125611 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3065,6 +3065,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&streaming_replication_retry_interval,
+		300000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 868d21c351..97bc00d5e6 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -351,6 +351,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 5min	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication
+					# in milliseconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index 0e3e246bd2..8c5be66946 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/t/034_wal_source_switch.pl b/src/test/recovery/t/034_wal_source_switch.pl
new file mode 100644
index 0000000000..d33ca9635c
--- /dev/null
+++ b/src/test/recovery/t/034_wal_source_switch.pl
@@ -0,0 +1,126 @@
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Test for WAL source switch feature
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+$ENV{PGDATABASE} = 'postgres';
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(
+    allows_streaming => 1,
+    has_archiving => 1,
+    extra => ['--wal-segsize=1']);
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf(
+	'postgresql.conf', qq(
+    min_wal_size = 2MB
+    max_wal_size = 1GB
+    checkpoint_timeout = 1h
+    wal_recycle = off
+));
+$node_primary->start;
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('rep1')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create a standby linking to it using the replication slot
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup($node_primary, $backup_name,
+	has_streaming => 1,
+    has_restoring => 1);
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+primary_slot_name = 'rep1'
+min_wal_size = 2MB
+max_wal_size = 1GB
+checkpoint_timeout = 1h
+streaming_replication_retry_interval = 100ms
+wal_recycle = off
+log_min_messages = 'debug2'
+));
+
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+# Stop standby
+$node_standby->stop;
+
+# Advance WAL by 100 segments (= 100MB) on primary
+advance_wal($node_primary, 100);
+
+# Wait for primary to generate requested WAL files
+$node_primary->poll_query_until('postgres',
+	q|SELECT COUNT(*) >= 100 FROM pg_ls_waldir()|, 't');
+
+# Standby now connects to primary during inital recovery after fetching WAL
+# from archive for about streaming_replication_retry_interval milliseconds.
+$node_standby->start;
+
+$node_primary->wait_for_catchup($node_standby);
+
+ok(find_in_log(
+		$node_standby,
+        qr/restored log file ".*" from archive/),
+	    'check that some of WAL segments were fetched from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/trying to switch WAL source to .* after fetching WAL from .* for at least .* milliseconds/),
+	    'check that standby tried to switch WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/switched WAL source to .* after fetching WAL from .* for at least .* milliseconds/),
+	    'check that standby actually switched WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/started streaming WAL from primary at .* on timeline .*/),
+	    'check that standby strated streaming from primary');
+
+# Stop standby
+$node_standby->stop;
+
+# Stop primary
+$node_primary->stop;
+#####################################
+# Advance WAL of $node by $n segments
+sub advance_wal
+{
+	my ($node, $n) = @_;
+
+	# Advance by $n segments (= (wal_segment_size * $n) bytes) on primary.
+	for (my $i = 0; $i < $n; $i++)
+	{
+		$node->safe_psql('postgres',
+			"CREATE TABLE t (); DROP TABLE t; SELECT pg_switch_wal();");
+	}
+	return;
+}
+
+# find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
+
+done_testing();
-- 
2.34.1

From 72f64424863575d5bb90af977566b58b33e81aa3 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Mon, 17 Oct 2022 15:32:17 +0000
Subject: [PATCH v9] Allow standby to switch WAL source from archive to
 streaming replication

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be efficient and cheaper because network
latencies, disk IO cost might differ on the archive as compared
to primary and often the archive may sit far from standby
impacting recovery performance on the standby. Hence reading WAL
from the primary, by setting this parameter, as opposed to the
archive enables the standby to catch up with the primary sooner
thus reducing replication lag and avoiding WAL files accumulation
on the primary.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication. However, exhaust all the WAL present in
pg_wal before switching. If the standby fails to switch to stream
mode, it falls back to archive mode.

Reported-by: SATYANARAYANA NARLAPURAM
Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  46 +++++
 doc/src/sgml/high-availability.sgml           |   7 +
 src/backend/access/transam/xlogrecovery.c     | 158 ++++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/t/034_wal_source_switch.pl  | 126 ++++++++++++++
 7 files changed, 337 insertions(+), 17 deletions(-)
 create mode 100644 src/test/recovery/t/034_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 66312b53b8..17dc961300 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4847,6 +4847,52 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+    <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from WAL archive to streaming replication (get WAL from
+        primary). However, exhaust all the WAL present in pg_wal before
+        switching. If the standby fails to switch to stream mode, it falls
+        back to archive mode.
+        If this value is specified without units, it is taken as milliseconds.
+        The default is five minutes (<literal>5min</literal>).
+        With a lower setting of this parameter, the standby makes frequent
+        WAL source switch attempts when the primary is lost for quite longer.
+        To avoid this, set a reasonable value.
+        A setting of <literal>0</literal> disables the feature. When disabled,
+        the standby typically switches to stream mode, only when receive from
+        WAL archive finishes (no more WAL left there) or fails for any reason.
+        This parameter can only be set in
+        the <filename>postgresql.conf</filename> file or on the server
+        command line.
+       </para>
+       <para>
+        Reading WAL from archive may not always be efficient and cheaper
+        because network latencies, disk IO cost might differ on the archive as
+        compared to primary and often the archive may sit far from standby
+        impacting recovery performance on the standby. Hence reading WAL
+        from the primary, by setting this parameter, as opposed to the archive
+        enables the standby to catch up with the primary sooner thus reducing
+        replication lag and avoiding WAL files accumulation on the primary.
+      </para>
+       <para>
+        Note that the standby may not always attempt to switch source from
+        WAL archive to streaming replication at exact
+        <varname>streaming_replication_retry_interval</varname> intervals.
+        For example, if the parameter is set to <literal>1min</literal> and
+        fetching from WAL archive takes <literal>5min</literal>, then the
+        source switch attempt happens for the next WAL after current WAL is
+        fetched from WAL archive and applied.
+      </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index b2b3129397..e38ce258e7 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -636,6 +636,13 @@ protocol to make nodes agree on a serializable transactional order.
     <filename>pg_wal</filename> at any time to have them replayed.
    </para>
 
+   <para>
+    The standby server can attempt to switch to streaming replication after
+    reading WAL from archive, see
+    <xref linkend="guc-streaming-replication-retry-interval"/> for more
+    details.
+   </para>
+
    <para>
     At startup, the standby begins by restoring all WAL available in the
     archive location, calling <varname>restore_command</varname>. Once it
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index cb07694aea..6faadf4c13 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -68,6 +68,12 @@
 #define RECOVERY_COMMAND_FILE	"recovery.conf"
 #define RECOVERY_COMMAND_DONE	"recovery.done"
 
+
+#define SwitchFromArchiveToStreamEnabled() \
+	(streaming_replication_retry_interval > 0 && \
+	 StandbyMode && \
+	 currentSource == XLOG_FROM_ARCHIVE)
+
 /*
  * GUC support
  */
@@ -91,6 +97,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 300000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -298,6 +305,11 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/*
+ * Holds the timestamp at which WaitForWALToBecomeAvailable()'s state machine
+ * switches to XLOG_FROM_ARCHIVE.
+ */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +452,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static bool ShouldSwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3416,8 +3430,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	static bool canSwitchSource = false;
+	bool	switchSource = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
+	XLogSource	readFrom;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -3437,6 +3454,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval milliseconds. However,
+	 * exhaust all the WAL present in pg_wal before switching. If successful,
+	 * the state machine moves to XLOG_FROM_STREAM state, otherwise it falls
+	 * back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3460,19 +3483,20 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
-		 * advance the state machine if so. The failure to read might've
-		 * happened outside this function, e.g when a CRC check fails on a
-		 * record, or within this loop.
+		 * First check if we failed to read from the current source or we
+		 * intentionally would want to switch the source from archive to
+		 * primary, and advance the state machine if so. The failure to read
+		 * might've happened outside this function, e.g when a CRC check fails
+		 * on a record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed || switchSource)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
-			 * readahead.  Let the caller process everything that has been
-			 * decoded already first.
+			 * readahead if we failed to read from the current source. Let the
+			 * caller process everything that has been decoded already first.
 			 */
-			if (nonblocking)
+			if (nonblocking && lastSourceFailed)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3601,9 +3625,24 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we're switching to archive. */
+			if (SwitchFromArchiveToStreamEnabled())
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			if (switchSource)
+				ereport(DEBUG2,
+						(errmsg("switched WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+								xlogSourceNames[currentSource],
+								xlogSourceNames[oldSource],
+								streaming_replication_retry_interval)));
+			else
+				ereport(DEBUG2,
+						(errmsg("switched WAL source from %s to %s after %s",
+								xlogSourceNames[oldSource],
+								xlogSourceNames[currentSource],
+								lastSourceFailed ? "failure" : "success")));
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
@@ -3611,6 +3650,13 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		 */
 		lastSourceFailed = false;
 
+		if (switchSource)
+		{
+			Assert(canSwitchSource == true);
+			switchSource = false;
+			canSwitchSource = false;
+		}
+
 		switch (currentSource)
 		{
 			case XLOG_FROM_ARCHIVE:
@@ -3632,13 +3678,24 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/*
+				 * See if we can switch the source to streaming from archive.
+				 */
+				if (!canSwitchSource)
+					canSwitchSource = ShouldSwitchWALSourceToPrimary();
+
 				/*
 				 * Try to restore the file from archive, or read an existing
-				 * file from pg_wal.
+				 * file from pg_wal. However, before switching the source to
+				 * stream mode, give it a chance to read all the WAL from
+				 * pg_wal.
 				 */
-				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2,
-											  currentSource == XLOG_FROM_ARCHIVE ? XLOG_FROM_ANY :
-											  currentSource);
+				if (canSwitchSource)
+					readFrom = XLOG_FROM_PG_WAL;
+				else
+					readFrom = currentSource;
+
+				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2, readFrom);
 				if (readFile >= 0)
 					return XLREAD_SUCCESS;	/* success! */
 
@@ -3646,6 +3703,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				 * Nope, not found in archive or pg_wal.
 				 */
 				lastSourceFailed = true;
+
+				/*
+				 * We have exhausted the WAL in pg_wal, ready to switch to
+				 * streaming.
+				 */
+				if (canSwitchSource)
+					switchSource = true;
+
 				break;
 
 			case XLOG_FROM_STREAM:
@@ -3874,6 +3939,61 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * This function tells if a standby should make an attempt to read WAL from
+ * primary after reading from archive for at least
+ * streaming_replication_retry_interval milliseconds. Reading WAL from the
+ * archive may not always be efficient and cheaper because network latencies,
+ * disk IO cost might differ on the archive as compared to the primary and
+ * often the archive may sit far from the standby - all adding to recovery
+ * performance on the standby. Hence reading WAL from the primary as opposed to
+ * the archive enables the standby to catch up with the primary sooner thus
+ * reducing replication lag and avoiding WAL files accumulation on the primary.
+ *
+ * We are here for any of the following reasons:
+ * 1) standby in initial recovery after start/restart.
+ * 2) standby stopped streaming from primary because of connectivity issues
+ * with the primary (either due to network issues or crash in the primary or
+ * something else) or walreceiver got killed or crashed for whatever reasons.
+ */
+static bool
+ShouldSwitchWALSourceToPrimary(void)
+{
+	bool shouldSwitchSource = false;
+
+	if (!SwitchFromArchiveToStreamEnabled())
+		return shouldSwitchSource;
+
+	if (switched_to_archive_at > 0)
+	{
+		TimestampTz curr_time;
+
+		curr_time = GetCurrentTimestamp();
+
+		if (TimestampDifferenceExceeds(switched_to_archive_at, curr_time,
+									   streaming_replication_retry_interval))
+		{
+			ereport(DEBUG2,
+					(errmsg("trying to switch WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+							xlogSourceNames[XLOG_FROM_STREAM],
+							xlogSourceNames[currentSource],
+							streaming_replication_retry_interval),
+					 errdetail("However, all the WAL present in pg_wal is exhausted before switching.")));
+
+			shouldSwitchSource = true;
+		}
+	}
+	else if (switched_to_archive_at == 0)
+	{
+		/*
+		 * Save the timestamp if we're about to fetch WAL from archive for the
+		 * first time.
+		 */
+		switched_to_archive_at = GetCurrentTimestamp();
+	}
+
+	return shouldSwitchSource;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
@@ -4199,7 +4319,11 @@ XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source)
 				continue;
 		}
 
-		if (source == XLOG_FROM_ANY || source == XLOG_FROM_ARCHIVE)
+		/*
+		 * When failed to read from archive, try reading from pg_wal, see
+		 * below.
+		 */
+		if (source == XLOG_FROM_ARCHIVE)
 		{
 			fd = XLogFileRead(segno, emode, tli,
 							  XLOG_FROM_ARCHIVE, true);
@@ -4212,7 +4336,7 @@ XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source)
 			}
 		}
 
-		if (source == XLOG_FROM_ANY || source == XLOG_FROM_PG_WAL)
+		if (source == XLOG_FROM_ARCHIVE || source == XLOG_FROM_PG_WAL)
 		{
 			fd = XLogFileRead(segno, emode, tli,
 							  XLOG_FROM_PG_WAL, true);
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 05ab087934..df09125611 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3065,6 +3065,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&streaming_replication_retry_interval,
+		300000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 868d21c351..97bc00d5e6 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -351,6 +351,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 5min	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication
+					# in milliseconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index 0e3e246bd2..8c5be66946 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/t/034_wal_source_switch.pl b/src/test/recovery/t/034_wal_source_switch.pl
new file mode 100644
index 0000000000..0bc988df19
--- /dev/null
+++ b/src/test/recovery/t/034_wal_source_switch.pl
@@ -0,0 +1,126 @@
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Test for WAL source switch feature
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+$ENV{PGDATABASE} = 'postgres';
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(
+    allows_streaming => 1,
+    has_archiving => 1,
+    extra => ['--wal-segsize=1']);
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf(
+	'postgresql.conf', qq(
+    min_wal_size = 2MB
+    max_wal_size = 1GB
+    checkpoint_timeout = 1h
+    wal_recycle = off
+));
+$node_primary->start;
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('rep1')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create a standby linking to it using the replication slot
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup($node_primary, $backup_name,
+	has_streaming => 1,
+    has_restoring => 1);
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+primary_slot_name = 'rep1'
+min_wal_size = 2MB
+max_wal_size = 1GB
+checkpoint_timeout = 1h
+streaming_replication_retry_interval = 100ms
+wal_recycle = off
+log_min_messages = 'debug2'
+));
+
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+# Stop standby
+$node_standby->stop;
+
+# Advance WAL by 100 segments (= 100MB) on primary
+advance_wal($node_primary, 100);
+
+# Wait for primary to generate requested WAL files
+$node_primary->poll_query_until('postgres',
+	q|SELECT COUNT(*) >= 100 FROM pg_ls_waldir()|, 't');
+
+# Standby now connects to primary during inital recovery after fetching WAL
+# from archive for about streaming_replication_retry_interval milliseconds.
+$node_standby->start;
+
+$node_primary->wait_for_catchup($node_standby);
+
+ok(find_in_log(
+		$node_standby,
+        qr/restored log file ".*" from archive/),
+	    'check that some of WAL segments were fetched from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/trying to switch WAL source to stream after fetching WAL from archive for at least .* milliseconds/),
+	    'check that standby tried to switch WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/switched WAL source to stream after fetching WAL from archive for at least .* milliseconds/),
+	    'check that standby actually switched WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/started streaming WAL from primary at .* on timeline .*/),
+	    'check that standby strated streaming from primary');
+
+# Stop standby
+$node_standby->stop;
+
+# Stop primary
+$node_primary->stop;
+#####################################
+# Advance WAL of $node by $n segments
+sub advance_wal
+{
+	my ($node, $n) = @_;
+
+	# Advance by $n segments (= (wal_segment_size * $n) bytes) on primary.
+	for (my $i = 0; $i < $n; $i++)
+	{
+		$node->safe_psql('postgres',
+			"CREATE TABLE t (); DROP TABLE t; SELECT pg_switch_wal();");
+	}
+	return;
+}
+
+# find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
+
+done_testing();
-- 
2.34.1

From 70323a5c96a889a21a5339c531193b6440d2e270 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Wed, 16 Nov 2022 05:34:41 +0000
Subject: [PATCH v10] Allow standby to switch WAL source from archive to
 streaming replication

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be efficient and cheaper because network
latencies, disk IO cost might differ on the archive as compared
to primary and often the archive may sit far from standby
impacting recovery performance on the standby. Hence reading WAL
from the primary, by setting this parameter, as opposed to the
archive enables the standby to catch up with the primary sooner
thus reducing replication lag and avoiding WAL files accumulation
on the primary.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication. However, exhaust all the WAL present in
pg_wal before switching. If the standby fails to switch to stream
mode, it falls back to archive mode.

Reported-by: SATYANARAYANA NARLAPURAM
Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  46 +++++
 doc/src/sgml/high-availability.sgml           |   7 +
 src/backend/access/transam/xlogrecovery.c     | 158 ++++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/meson.build                 |   1 +
 src/test/recovery/t/034_wal_source_switch.pl  | 126 ++++++++++++++
 8 files changed, 338 insertions(+), 17 deletions(-)
 create mode 100644 src/test/recovery/t/034_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index bd50ea8e48..8a3a9ff296 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4852,6 +4852,52 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+    <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from WAL archive to streaming replication (get WAL from
+        primary). However, exhaust all the WAL present in pg_wal before
+        switching. If the standby fails to switch to stream mode, it falls
+        back to archive mode.
+        If this value is specified without units, it is taken as milliseconds.
+        The default is five minutes (<literal>5min</literal>).
+        With a lower setting of this parameter, the standby makes frequent
+        WAL source switch attempts when the primary is lost for quite longer.
+        To avoid this, set a reasonable value.
+        A setting of <literal>0</literal> disables the feature. When disabled,
+        the standby typically switches to stream mode, only when receive from
+        WAL archive finishes (no more WAL left there) or fails for any reason.
+        This parameter can only be set in
+        the <filename>postgresql.conf</filename> file or on the server
+        command line.
+       </para>
+       <para>
+        Reading WAL from archive may not always be efficient and cheaper
+        because network latencies, disk IO cost might differ on the archive as
+        compared to primary and often the archive may sit far from standby
+        impacting recovery performance on the standby. Hence reading WAL
+        from the primary, by setting this parameter, as opposed to the archive
+        enables the standby to catch up with the primary sooner thus reducing
+        replication lag and avoiding WAL files accumulation on the primary.
+      </para>
+       <para>
+        Note that the standby may not always attempt to switch source from
+        WAL archive to streaming replication at exact
+        <varname>streaming_replication_retry_interval</varname> intervals.
+        For example, if the parameter is set to <literal>1min</literal> and
+        fetching from WAL archive takes <literal>5min</literal>, then the
+        source switch attempt happens for the next WAL after current WAL is
+        fetched from WAL archive and applied.
+      </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index b2b3129397..e38ce258e7 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -636,6 +636,13 @@ protocol to make nodes agree on a serializable transactional order.
     <filename>pg_wal</filename> at any time to have them replayed.
    </para>
 
+   <para>
+    The standby server can attempt to switch to streaming replication after
+    reading WAL from archive, see
+    <xref linkend="guc-streaming-replication-retry-interval"/> for more
+    details.
+   </para>
+
    <para>
     At startup, the standby begins by restoring all WAL available in the
     archive location, calling <varname>restore_command</varname>. Once it
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index cb07694aea..6faadf4c13 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -68,6 +68,12 @@
 #define RECOVERY_COMMAND_FILE	"recovery.conf"
 #define RECOVERY_COMMAND_DONE	"recovery.done"
 
+
+#define SwitchFromArchiveToStreamEnabled() \
+	(streaming_replication_retry_interval > 0 && \
+	 StandbyMode && \
+	 currentSource == XLOG_FROM_ARCHIVE)
+
 /*
  * GUC support
  */
@@ -91,6 +97,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 300000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -298,6 +305,11 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/*
+ * Holds the timestamp at which WaitForWALToBecomeAvailable()'s state machine
+ * switches to XLOG_FROM_ARCHIVE.
+ */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +452,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static bool ShouldSwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3416,8 +3430,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	static bool canSwitchSource = false;
+	bool	switchSource = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
+	XLogSource	readFrom;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -3437,6 +3454,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval milliseconds. However,
+	 * exhaust all the WAL present in pg_wal before switching. If successful,
+	 * the state machine moves to XLOG_FROM_STREAM state, otherwise it falls
+	 * back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3460,19 +3483,20 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
-		 * advance the state machine if so. The failure to read might've
-		 * happened outside this function, e.g when a CRC check fails on a
-		 * record, or within this loop.
+		 * First check if we failed to read from the current source or we
+		 * intentionally would want to switch the source from archive to
+		 * primary, and advance the state machine if so. The failure to read
+		 * might've happened outside this function, e.g when a CRC check fails
+		 * on a record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed || switchSource)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
-			 * readahead.  Let the caller process everything that has been
-			 * decoded already first.
+			 * readahead if we failed to read from the current source. Let the
+			 * caller process everything that has been decoded already first.
 			 */
-			if (nonblocking)
+			if (nonblocking && lastSourceFailed)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3601,9 +3625,24 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we're switching to archive. */
+			if (SwitchFromArchiveToStreamEnabled())
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			if (switchSource)
+				ereport(DEBUG2,
+						(errmsg("switched WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+								xlogSourceNames[currentSource],
+								xlogSourceNames[oldSource],
+								streaming_replication_retry_interval)));
+			else
+				ereport(DEBUG2,
+						(errmsg("switched WAL source from %s to %s after %s",
+								xlogSourceNames[oldSource],
+								xlogSourceNames[currentSource],
+								lastSourceFailed ? "failure" : "success")));
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
@@ -3611,6 +3650,13 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		 */
 		lastSourceFailed = false;
 
+		if (switchSource)
+		{
+			Assert(canSwitchSource == true);
+			switchSource = false;
+			canSwitchSource = false;
+		}
+
 		switch (currentSource)
 		{
 			case XLOG_FROM_ARCHIVE:
@@ -3632,13 +3678,24 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/*
+				 * See if we can switch the source to streaming from archive.
+				 */
+				if (!canSwitchSource)
+					canSwitchSource = ShouldSwitchWALSourceToPrimary();
+
 				/*
 				 * Try to restore the file from archive, or read an existing
-				 * file from pg_wal.
+				 * file from pg_wal. However, before switching the source to
+				 * stream mode, give it a chance to read all the WAL from
+				 * pg_wal.
 				 */
-				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2,
-											  currentSource == XLOG_FROM_ARCHIVE ? XLOG_FROM_ANY :
-											  currentSource);
+				if (canSwitchSource)
+					readFrom = XLOG_FROM_PG_WAL;
+				else
+					readFrom = currentSource;
+
+				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2, readFrom);
 				if (readFile >= 0)
 					return XLREAD_SUCCESS;	/* success! */
 
@@ -3646,6 +3703,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				 * Nope, not found in archive or pg_wal.
 				 */
 				lastSourceFailed = true;
+
+				/*
+				 * We have exhausted the WAL in pg_wal, ready to switch to
+				 * streaming.
+				 */
+				if (canSwitchSource)
+					switchSource = true;
+
 				break;
 
 			case XLOG_FROM_STREAM:
@@ -3874,6 +3939,61 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * This function tells if a standby should make an attempt to read WAL from
+ * primary after reading from archive for at least
+ * streaming_replication_retry_interval milliseconds. Reading WAL from the
+ * archive may not always be efficient and cheaper because network latencies,
+ * disk IO cost might differ on the archive as compared to the primary and
+ * often the archive may sit far from the standby - all adding to recovery
+ * performance on the standby. Hence reading WAL from the primary as opposed to
+ * the archive enables the standby to catch up with the primary sooner thus
+ * reducing replication lag and avoiding WAL files accumulation on the primary.
+ *
+ * We are here for any of the following reasons:
+ * 1) standby in initial recovery after start/restart.
+ * 2) standby stopped streaming from primary because of connectivity issues
+ * with the primary (either due to network issues or crash in the primary or
+ * something else) or walreceiver got killed or crashed for whatever reasons.
+ */
+static bool
+ShouldSwitchWALSourceToPrimary(void)
+{
+	bool shouldSwitchSource = false;
+
+	if (!SwitchFromArchiveToStreamEnabled())
+		return shouldSwitchSource;
+
+	if (switched_to_archive_at > 0)
+	{
+		TimestampTz curr_time;
+
+		curr_time = GetCurrentTimestamp();
+
+		if (TimestampDifferenceExceeds(switched_to_archive_at, curr_time,
+									   streaming_replication_retry_interval))
+		{
+			ereport(DEBUG2,
+					(errmsg("trying to switch WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+							xlogSourceNames[XLOG_FROM_STREAM],
+							xlogSourceNames[currentSource],
+							streaming_replication_retry_interval),
+					 errdetail("However, all the WAL present in pg_wal is exhausted before switching.")));
+
+			shouldSwitchSource = true;
+		}
+	}
+	else if (switched_to_archive_at == 0)
+	{
+		/*
+		 * Save the timestamp if we're about to fetch WAL from archive for the
+		 * first time.
+		 */
+		switched_to_archive_at = GetCurrentTimestamp();
+	}
+
+	return shouldSwitchSource;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
@@ -4199,7 +4319,11 @@ XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source)
 				continue;
 		}
 
-		if (source == XLOG_FROM_ANY || source == XLOG_FROM_ARCHIVE)
+		/*
+		 * When failed to read from archive, try reading from pg_wal, see
+		 * below.
+		 */
+		if (source == XLOG_FROM_ARCHIVE)
 		{
 			fd = XLogFileRead(segno, emode, tli,
 							  XLOG_FROM_ARCHIVE, true);
@@ -4212,7 +4336,7 @@ XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source)
 			}
 		}
 
-		if (source == XLOG_FROM_ANY || source == XLOG_FROM_PG_WAL)
+		if (source == XLOG_FROM_ARCHIVE || source == XLOG_FROM_PG_WAL)
 		{
 			fd = XLogFileRead(segno, emode, tli,
 							  XLOG_FROM_PG_WAL, true);
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 836b49484a..6492248d19 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3063,6 +3063,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&streaming_replication_retry_interval,
+		300000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 868d21c351..97bc00d5e6 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -351,6 +351,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 5min	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication
+					# in milliseconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index 0e3e246bd2..8c5be66946 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build
index b0e398363f..3bd4ec4b37 100644
--- a/src/test/recovery/meson.build
+++ b/src/test/recovery/meson.build
@@ -38,6 +38,7 @@ tests += {
       't/031_recovery_conflict.pl',
       't/032_relfilenode_reuse.pl',
       't/033_replay_tsp_drops.pl',
+      't/034_wal_source_switch.pl',
     ],
   },
 }
diff --git a/src/test/recovery/t/034_wal_source_switch.pl b/src/test/recovery/t/034_wal_source_switch.pl
new file mode 100644
index 0000000000..0bc988df19
--- /dev/null
+++ b/src/test/recovery/t/034_wal_source_switch.pl
@@ -0,0 +1,126 @@
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Test for WAL source switch feature
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+$ENV{PGDATABASE} = 'postgres';
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(
+    allows_streaming => 1,
+    has_archiving => 1,
+    extra => ['--wal-segsize=1']);
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf(
+	'postgresql.conf', qq(
+    min_wal_size = 2MB
+    max_wal_size = 1GB
+    checkpoint_timeout = 1h
+    wal_recycle = off
+));
+$node_primary->start;
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('rep1')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create a standby linking to it using the replication slot
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup($node_primary, $backup_name,
+	has_streaming => 1,
+    has_restoring => 1);
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+primary_slot_name = 'rep1'
+min_wal_size = 2MB
+max_wal_size = 1GB
+checkpoint_timeout = 1h
+streaming_replication_retry_interval = 100ms
+wal_recycle = off
+log_min_messages = 'debug2'
+));
+
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+# Stop standby
+$node_standby->stop;
+
+# Advance WAL by 100 segments (= 100MB) on primary
+advance_wal($node_primary, 100);
+
+# Wait for primary to generate requested WAL files
+$node_primary->poll_query_until('postgres',
+	q|SELECT COUNT(*) >= 100 FROM pg_ls_waldir()|, 't');
+
+# Standby now connects to primary during inital recovery after fetching WAL
+# from archive for about streaming_replication_retry_interval milliseconds.
+$node_standby->start;
+
+$node_primary->wait_for_catchup($node_standby);
+
+ok(find_in_log(
+		$node_standby,
+        qr/restored log file ".*" from archive/),
+	    'check that some of WAL segments were fetched from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/trying to switch WAL source to stream after fetching WAL from archive for at least .* milliseconds/),
+	    'check that standby tried to switch WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/switched WAL source to stream after fetching WAL from archive for at least .* milliseconds/),
+	    'check that standby actually switched WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/started streaming WAL from primary at .* on timeline .*/),
+	    'check that standby strated streaming from primary');
+
+# Stop standby
+$node_standby->stop;
+
+# Stop primary
+$node_primary->stop;
+#####################################
+# Advance WAL of $node by $n segments
+sub advance_wal
+{
+	my ($node, $n) = @_;
+
+	# Advance by $n segments (= (wal_segment_size * $n) bytes) on primary.
+	for (my $i = 0; $i < $n; $i++)
+	{
+		$node->safe_psql('postgres',
+			"CREATE TABLE t (); DROP TABLE t; SELECT pg_switch_wal();");
+	}
+	return;
+}
+
+# Find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
+
+done_testing();
-- 
2.34.1

From 6cf6c9bc0a6d6c4325459cddc22fd6fcdc970a03 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Fri, 24 Feb 2023 04:38:13 +0000
Subject: [PATCH v11] Allow standby to switch WAL source from archive to
 streaming replication

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be efficient and cheaper because network
latencies, disk IO cost might differ on the archive as compared
to primary and often the archive may sit far from standby
impacting recovery performance on the standby. Hence reading WAL
from the primary, by setting this parameter, as opposed to the
archive enables the standby to catch up with the primary sooner
thus reducing replication lag and avoiding WAL files accumulation
on the primary.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication. However, exhaust all the WAL present in
pg_wal before switching. If the standby fails to switch to stream
mode, it falls back to archive mode.

Reported-by: SATYANARAYANA NARLAPURAM
Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  46 +++++
 doc/src/sgml/high-availability.sgml           |   7 +
 src/backend/access/transam/xlogrecovery.c     | 158 ++++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/meson.build                 |   1 +
 src/test/recovery/t/035_wal_source_switch.pl  | 126 ++++++++++++++
 8 files changed, 338 insertions(+), 17 deletions(-)
 create mode 100644 src/test/recovery/t/035_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index e5c41cc6c6..fd400ac662 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4884,6 +4884,52 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+    <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from WAL archive to streaming replication (get WAL from
+        primary). However, exhaust all the WAL present in pg_wal before
+        switching. If the standby fails to switch to stream mode, it falls
+        back to archive mode.
+        If this value is specified without units, it is taken as milliseconds.
+        The default is five minutes (<literal>5min</literal>).
+        With a lower setting of this parameter, the standby makes frequent
+        WAL source switch attempts when the primary is lost for quite longer.
+        To avoid this, set a reasonable value.
+        A setting of <literal>0</literal> disables the feature. When disabled,
+        the standby typically switches to stream mode, only when receive from
+        WAL archive finishes (no more WAL left there) or fails for any reason.
+        This parameter can only be set in
+        the <filename>postgresql.conf</filename> file or on the server
+        command line.
+       </para>
+       <para>
+        Reading WAL from archive may not always be efficient and cheaper
+        because network latencies, disk IO cost might differ on the archive as
+        compared to primary and often the archive may sit far from standby
+        impacting recovery performance on the standby. Hence reading WAL
+        from the primary, by setting this parameter, as opposed to the archive
+        enables the standby to catch up with the primary sooner thus reducing
+        replication lag and avoiding WAL files accumulation on the primary.
+      </para>
+       <para>
+        Note that the standby may not always attempt to switch source from
+        WAL archive to streaming replication at exact
+        <varname>streaming_replication_retry_interval</varname> intervals.
+        For example, if the parameter is set to <literal>1min</literal> and
+        fetching from WAL archive takes <literal>5min</literal>, then the
+        source switch attempt happens for the next WAL after current WAL is
+        fetched from WAL archive and applied.
+      </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index f180607528..0323f4cc43 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -636,6 +636,13 @@ protocol to make nodes agree on a serializable transactional order.
     <filename>pg_wal</filename> at any time to have them replayed.
    </para>
 
+   <para>
+    The standby server can attempt to switch to streaming replication after
+    reading WAL from archive, see
+    <xref linkend="guc-streaming-replication-retry-interval"/> for more
+    details.
+   </para>
+
    <para>
     At startup, the standby begins by restoring all WAL available in the
     archive location, calling <varname>restore_command</varname>. Once it
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index dbe9394762..199b313dc1 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -68,6 +68,12 @@
 #define RECOVERY_COMMAND_FILE	"recovery.conf"
 #define RECOVERY_COMMAND_DONE	"recovery.done"
 
+
+#define SwitchFromArchiveToStreamEnabled() \
+	(streaming_replication_retry_interval > 0 && \
+	 StandbyMode && \
+	 currentSource == XLOG_FROM_ARCHIVE)
+
 /*
  * GUC support
  */
@@ -91,6 +97,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 300000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -297,6 +304,11 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/*
+ * Holds the timestamp at which WaitForWALToBecomeAvailable()'s state machine
+ * switches to XLOG_FROM_ARCHIVE.
+ */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +452,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static bool ShouldSwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3441,8 +3455,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	static bool canSwitchSource = false;
+	bool	switchSource = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
+	XLogSource	readFrom;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -3462,6 +3479,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval milliseconds. However,
+	 * exhaust all the WAL present in pg_wal before switching. If successful,
+	 * the state machine moves to XLOG_FROM_STREAM state, otherwise it falls
+	 * back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3485,19 +3508,20 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
-		 * advance the state machine if so. The failure to read might've
-		 * happened outside this function, e.g when a CRC check fails on a
-		 * record, or within this loop.
+		 * First check if we failed to read from the current source or we
+		 * intentionally would want to switch the source from archive to
+		 * primary, and advance the state machine if so. The failure to read
+		 * might've happened outside this function, e.g when a CRC check fails
+		 * on a record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed || switchSource)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
-			 * readahead.  Let the caller process everything that has been
-			 * decoded already first.
+			 * readahead if we failed to read from the current source. Let the
+			 * caller process everything that has been decoded already first.
 			 */
-			if (nonblocking)
+			if (nonblocking && lastSourceFailed)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3629,9 +3653,24 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we're switching to archive. */
+			if (SwitchFromArchiveToStreamEnabled())
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			if (switchSource)
+				ereport(DEBUG2,
+						(errmsg("switched WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+								xlogSourceNames[currentSource],
+								xlogSourceNames[oldSource],
+								streaming_replication_retry_interval)));
+			else
+				ereport(DEBUG2,
+						(errmsg("switched WAL source from %s to %s after %s",
+								xlogSourceNames[oldSource],
+								xlogSourceNames[currentSource],
+								lastSourceFailed ? "failure" : "success")));
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
@@ -3639,6 +3678,13 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		 */
 		lastSourceFailed = false;
 
+		if (switchSource)
+		{
+			Assert(canSwitchSource == true);
+			switchSource = false;
+			canSwitchSource = false;
+		}
+
 		switch (currentSource)
 		{
 			case XLOG_FROM_ARCHIVE:
@@ -3660,13 +3706,24 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/*
+				 * See if we can switch the source to streaming from archive.
+				 */
+				if (!canSwitchSource)
+					canSwitchSource = ShouldSwitchWALSourceToPrimary();
+
 				/*
 				 * Try to restore the file from archive, or read an existing
-				 * file from pg_wal.
+				 * file from pg_wal. However, before switching the source to
+				 * stream mode, give it a chance to read all the WAL from
+				 * pg_wal.
 				 */
-				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2,
-											  currentSource == XLOG_FROM_ARCHIVE ? XLOG_FROM_ANY :
-											  currentSource);
+				if (canSwitchSource)
+					readFrom = XLOG_FROM_PG_WAL;
+				else
+					readFrom = currentSource;
+
+				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2, readFrom);
 				if (readFile >= 0)
 					return XLREAD_SUCCESS;	/* success! */
 
@@ -3674,6 +3731,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				 * Nope, not found in archive or pg_wal.
 				 */
 				lastSourceFailed = true;
+
+				/*
+				 * We have exhausted the WAL in pg_wal, ready to switch to
+				 * streaming.
+				 */
+				if (canSwitchSource)
+					switchSource = true;
+
 				break;
 
 			case XLOG_FROM_STREAM:
@@ -3904,6 +3969,61 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * This function tells if a standby should make an attempt to read WAL from
+ * primary after reading from archive for at least
+ * streaming_replication_retry_interval milliseconds. Reading WAL from the
+ * archive may not always be efficient and cheaper because network latencies,
+ * disk IO cost might differ on the archive as compared to the primary and
+ * often the archive may sit far from the standby - all adding to recovery
+ * performance on the standby. Hence reading WAL from the primary as opposed to
+ * the archive enables the standby to catch up with the primary sooner thus
+ * reducing replication lag and avoiding WAL files accumulation on the primary.
+ *
+ * We are here for any of the following reasons:
+ * 1) standby in initial recovery after start/restart.
+ * 2) standby stopped streaming from primary because of connectivity issues
+ * with the primary (either due to network issues or crash in the primary or
+ * something else) or walreceiver got killed or crashed for whatever reasons.
+ */
+static bool
+ShouldSwitchWALSourceToPrimary(void)
+{
+	bool shouldSwitchSource = false;
+
+	if (!SwitchFromArchiveToStreamEnabled())
+		return shouldSwitchSource;
+
+	if (switched_to_archive_at > 0)
+	{
+		TimestampTz curr_time;
+
+		curr_time = GetCurrentTimestamp();
+
+		if (TimestampDifferenceExceeds(switched_to_archive_at, curr_time,
+									   streaming_replication_retry_interval))
+		{
+			ereport(DEBUG2,
+					(errmsg("trying to switch WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+							xlogSourceNames[XLOG_FROM_STREAM],
+							xlogSourceNames[currentSource],
+							streaming_replication_retry_interval),
+					 errdetail("However, all the WAL present in pg_wal is exhausted before switching.")));
+
+			shouldSwitchSource = true;
+		}
+	}
+	else if (switched_to_archive_at == 0)
+	{
+		/*
+		 * Save the timestamp if we're about to fetch WAL from archive for the
+		 * first time.
+		 */
+		switched_to_archive_at = GetCurrentTimestamp();
+	}
+
+	return shouldSwitchSource;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
@@ -4229,7 +4349,11 @@ XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source)
 				continue;
 		}
 
-		if (source == XLOG_FROM_ANY || source == XLOG_FROM_ARCHIVE)
+		/*
+		 * When failed to read from archive, try reading from pg_wal, see
+		 * below.
+		 */
+		if (source == XLOG_FROM_ARCHIVE)
 		{
 			fd = XLogFileRead(segno, emode, tli,
 							  XLOG_FROM_ARCHIVE, true);
@@ -4242,7 +4366,7 @@ XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source)
 			}
 		}
 
-		if (source == XLOG_FROM_ANY || source == XLOG_FROM_PG_WAL)
+		if (source == XLOG_FROM_ARCHIVE || source == XLOG_FROM_PG_WAL)
 		{
 			fd = XLogFileRead(segno, emode, tli,
 							  XLOG_FROM_PG_WAL, true);
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 1c0583fe26..c741b02297 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3128,6 +3128,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&streaming_replication_retry_interval,
+		300000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index d06074b86f..ebb73e14de 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -351,6 +351,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 5min	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication
+					# in milliseconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index 47c29350f5..dfa0301d61 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build
index 59465b97f3..ad964c4cb0 100644
--- a/src/test/recovery/meson.build
+++ b/src/test/recovery/meson.build
@@ -40,6 +40,7 @@ tests += {
       't/032_relfilenode_reuse.pl',
       't/033_replay_tsp_drops.pl',
       't/034_create_database.pl',
+      't/035_wal_source_switch.pl',
     ],
   },
 }
diff --git a/src/test/recovery/t/035_wal_source_switch.pl b/src/test/recovery/t/035_wal_source_switch.pl
new file mode 100644
index 0000000000..c28fba5d88
--- /dev/null
+++ b/src/test/recovery/t/035_wal_source_switch.pl
@@ -0,0 +1,126 @@
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Test for WAL source switch feature
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+$ENV{PGDATABASE} = 'postgres';
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(
+    allows_streaming => 1,
+    has_archiving => 1,
+    extra => ['--wal-segsize=1']);
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf(
+	'postgresql.conf', qq(
+    min_wal_size = 2MB
+    max_wal_size = 1GB
+    checkpoint_timeout = 1h
+    wal_recycle = off
+));
+$node_primary->start;
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('rep1')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create a standby linking to it using the replication slot
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup($node_primary, $backup_name,
+	has_streaming => 1,
+    has_restoring => 1);
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+primary_slot_name = 'rep1'
+min_wal_size = 2MB
+max_wal_size = 1GB
+checkpoint_timeout = 1h
+streaming_replication_retry_interval = 100ms
+wal_recycle = off
+log_min_messages = 'debug2'
+));
+
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+# Stop standby
+$node_standby->stop;
+
+# Advance WAL by 100 segments (= 100MB) on primary
+advance_wal($node_primary, 100);
+
+# Wait for primary to generate requested WAL files
+$node_primary->poll_query_until('postgres',
+	q|SELECT COUNT(*) >= 100 FROM pg_ls_waldir()|, 't');
+
+# Standby now connects to primary during inital recovery after fetching WAL
+# from archive for about streaming_replication_retry_interval milliseconds.
+$node_standby->start;
+
+$node_primary->wait_for_catchup($node_standby);
+
+ok(find_in_log(
+		$node_standby,
+        qr/restored log file ".*" from archive/),
+	    'check that some of WAL segments were fetched from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/trying to switch WAL source to stream after fetching WAL from archive for at least .* milliseconds/),
+	    'check that standby tried to switch WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/switched WAL source to stream after fetching WAL from archive for at least .* milliseconds/),
+	    'check that standby actually switched WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/started streaming WAL from primary at .* on timeline .*/),
+	    'check that standby strated streaming from primary');
+
+# Stop standby
+$node_standby->stop;
+
+# Stop primary
+$node_primary->stop;
+#####################################
+# Advance WAL of $node by $n segments
+sub advance_wal
+{
+	my ($node, $n) = @_;
+
+	# Advance by $n segments (= (wal_segment_size * $n) bytes) on primary.
+	for (my $i = 0; $i < $n; $i++)
+	{
+		$node->safe_psql('postgres',
+			"CREATE TABLE t (); DROP TABLE t; SELECT pg_switch_wal();");
+	}
+	return;
+}
+
+# Find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
+
+done_testing();
-- 
2.34.1

From 80a982fb1c0d0737c5144d8ef50bb5e3ff845d07 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Tue, 25 Apr 2023 08:36:48 +0000
Subject: [PATCH v12] Allow standby to switch WAL source from archive to
 streaming replication

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be as efficient and fast as reading from
primary. This can be due to the differences in disk types, IO
costs, network latencies etc., all impacting recovery
performance on standby. And, while standby is reading WAL from
archive, primary accumulates WAL because the standby's replication
slot stays inactive. To avoid these problems, one can use this
parameter to make standby switch to stream mode sooner.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication (getting WAL from primary). However, standby
exhausts all the WAL present in pg_wal before switching. If
standby fails to switch to stream mode, it falls back to archive
mode.

Reported-by: SATYANARAYANA NARLAPURAM
Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  45 ++++++
 doc/src/sgml/high-availability.sgml           |  15 +-
 src/backend/access/transam/xlogrecovery.c     | 144 ++++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/meson.build                 |   1 +
 src/test/recovery/t/037_wal_source_switch.pl  | 126 +++++++++++++++
 8 files changed, 328 insertions(+), 20 deletions(-)
 create mode 100644 src/test/recovery/t/037_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index b56f073a91..8050f981e9 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4923,6 +4923,51 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from WAL archive to streaming replication (getting WAL from
+        primary). However, standby exhausts all the WAL present in pg_wal
+        before switching. If standby fails to switch to stream mode, it falls
+        back to archive mode. If this parameter's value is specified without
+        units, it is taken as milliseconds. Default is five minutes
+        (<literal>5min</literal>). With a lower value for this parameter,
+        standby makes frequent WAL source switch attempts. To avoid this, it is
+        recommended to set a reasonable value. A setting of <literal>0</literal>
+        disables the feature. When disabled, standby typically switches to
+        stream mode only when receive from WAL archive finishes (no more WAL
+        left there) or fails for any reason. This parameter can only be set in
+        the <filename>postgresql.conf</filename> file or on the server command
+        line.
+       </para>
+       <note>
+        <para>
+         Standby may not always attempt to switch source from WAL archive to
+         streaming replication at exact <varname>streaming_replication_retry_interval</varname>
+         intervals. For example, if the parameter is set to <literal>1min</literal>
+         and fetching WAL file from archive takes about <literal>2min</literal>,
+         then the source switch attempt happens for the next WAL file after
+         current WAL file fetched from archive is fully applied.
+        </para>
+       </note>
+       <para>
+        Reading WAL from archive may not always be as efficient and fast as
+        reading from primary. This can be due to the differences in disk types,
+        IO costs, network latencies etc., all impacting recovery performance on
+        standby. And, while standby is reading WAL from archive, primary
+        accumulates WAL because the standby's replication slot stays inactive.
+        To avoid these problems, one can use this parameter to make standby
+        switch to stream mode sooner.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index cf61b2ed2a..3b592f48c8 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -628,12 +628,15 @@ protocol to make nodes agree on a serializable transactional order.
     In standby mode, the server continuously applies WAL received from the
     primary server. The standby server can read WAL from a WAL archive
     (see <xref linkend="guc-restore-command"/>) or directly from the primary
-    over a TCP connection (streaming replication). The standby server will
-    also attempt to restore any WAL found in the standby cluster's
-    <filename>pg_wal</filename> directory. That typically happens after a server
-    restart, when the standby replays again WAL that was streamed from the
-    primary before the restart, but you can also manually copy files to
-    <filename>pg_wal</filename> at any time to have them replayed.
+    over a TCP connection (streaming replication) or attempt to switch to
+    streaming replication after reading from archive when
+    <xref linkend="guc-streaming-replication-retry-interval"/> parameter is
+    set. The standby server will also attempt to restore any WAL found in the
+    standby cluster's <filename>pg_wal</filename> directory. That typically
+    happens after a server restart, when the standby replays again WAL that was
+    streamed from the primary before the restart, but you can also manually
+    copy files to <filename>pg_wal</filename> at any time to have them
+    replayed.
    </para>
 
    <para>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 188f6d6f85..dbe2520dcc 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -68,6 +68,11 @@
 #define RECOVERY_COMMAND_FILE	"recovery.conf"
 #define RECOVERY_COMMAND_DONE	"recovery.done"
 
+#define SwitchFromArchiveToStreamEnabled() \
+	(streaming_replication_retry_interval > 0 && \
+	 StandbyMode && \
+	 currentSource == XLOG_FROM_ARCHIVE)
+
 /*
  * GUC support
  */
@@ -91,6 +96,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 300000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -297,6 +303,11 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/*
+ * Holds the timestamp at which WaitForWALToBecomeAvailable()'s state machine
+ * switches to XLOG_FROM_ARCHIVE.
+ */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +451,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static bool ShouldSwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3460,8 +3473,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	static bool canSwitchSource = false;
+	bool	switchSource = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
+	XLogSource	readFrom;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -3481,6 +3497,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval milliseconds. However,
+	 * exhaust all the WAL present in pg_wal before switching. If successful,
+	 * the state machine moves to XLOG_FROM_STREAM state, otherwise it falls
+	 * back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3504,19 +3526,20 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
+		 * First check if we failed to read from the current source or we
+		 * intentionally want to switch the source from archive to primary, and
 		 * advance the state machine if so. The failure to read might've
 		 * happened outside this function, e.g when a CRC check fails on a
 		 * record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed || switchSource)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
-			 * readahead.  Let the caller process everything that has been
-			 * decoded already first.
+			 * readahead if we failed to read from the current source. Let the
+			 * caller process everything that has been decoded already first.
 			 */
-			if (nonblocking)
+			if (nonblocking && lastSourceFailed)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3648,9 +3671,24 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we're switching to archive. */
+			if (SwitchFromArchiveToStreamEnabled())
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			if (switchSource)
+				ereport(DEBUG2,
+						(errmsg("switched WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+								xlogSourceNames[currentSource],
+								xlogSourceNames[oldSource],
+								streaming_replication_retry_interval)));
+			else
+				ereport(DEBUG2,
+						(errmsg("switched WAL source from %s to %s after %s",
+								xlogSourceNames[oldSource],
+								xlogSourceNames[currentSource],
+								lastSourceFailed ? "failure" : "success")));
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
@@ -3658,6 +3696,13 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		 */
 		lastSourceFailed = false;
 
+		if (switchSource)
+		{
+			Assert(canSwitchSource == true);
+			switchSource = false;
+			canSwitchSource = false;
+		}
+
 		switch (currentSource)
 		{
 			case XLOG_FROM_ARCHIVE:
@@ -3679,13 +3724,24 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/*
+				 * See if we can switch the source to streaming from archive.
+				 */
+				if (!canSwitchSource)
+					canSwitchSource = ShouldSwitchWALSourceToPrimary();
+
 				/*
 				 * Try to restore the file from archive, or read an existing
-				 * file from pg_wal.
+				 * file from pg_wal. However, before switching the source to
+				 * stream mode, give it a chance to read all the WAL from
+				 * pg_wal.
 				 */
-				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2,
-											  currentSource == XLOG_FROM_ARCHIVE ? XLOG_FROM_ANY :
-											  currentSource);
+				if (canSwitchSource)
+					readFrom = XLOG_FROM_PG_WAL;
+				else
+					readFrom = currentSource;
+
+				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2, readFrom);
 				if (readFile >= 0)
 					return XLREAD_SUCCESS;	/* success! */
 
@@ -3693,6 +3749,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				 * Nope, not found in archive or pg_wal.
 				 */
 				lastSourceFailed = true;
+
+				/*
+				 * Exhausted all the WAL in pg_wal, now ready to switch to
+				 * streaming.
+				 */
+				if (canSwitchSource)
+					switchSource = true;
+
 				break;
 
 			case XLOG_FROM_STREAM:
@@ -3923,6 +3987,54 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * This function tells if a standby can make an attempt to read WAL from
+ * primary after reading from archive for at least
+ * streaming_replication_retry_interval milliseconds. Reading WAL from archive
+ * may not always be as efficient and fast as reading from primary. This can be
+ * due to the differences in disk types, IO costs, network latencies etc., all
+ * impacting recovery performance on standby. And, while standby is reading WAL
+ * from archive, primary accumulates WAL because the standby's replication slot
+ * stays inactive. To avoid these problems, we try to make standby switch to
+ * stream mode sooner.
+ */
+static bool
+ShouldSwitchWALSourceToPrimary(void)
+{
+	bool shouldSwitchSource = false;
+
+	if (!SwitchFromArchiveToStreamEnabled())
+		return shouldSwitchSource;
+
+	if (switched_to_archive_at > 0)
+	{
+		TimestampTz curr_time;
+
+		curr_time = GetCurrentTimestamp();
+
+		if (TimestampDifferenceExceeds(switched_to_archive_at, curr_time,
+									   streaming_replication_retry_interval))
+		{
+			ereport(DEBUG2,
+					(errmsg("trying to switch WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+							xlogSourceNames[XLOG_FROM_STREAM],
+							xlogSourceNames[currentSource],
+							streaming_replication_retry_interval)));
+
+			shouldSwitchSource = true;
+		}
+	}
+	else if (switched_to_archive_at == 0)
+	{
+		/*
+		 * Save the timestamp if we're about to fetch WAL from archive for the
+		 * first time.
+		 */
+		switched_to_archive_at = GetCurrentTimestamp();
+	}
+
+	return shouldSwitchSource;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
@@ -4248,7 +4360,11 @@ XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source)
 				continue;
 		}
 
-		if (source == XLOG_FROM_ANY || source == XLOG_FROM_ARCHIVE)
+		/*
+		 * When failed to read from archive, try reading from pg_wal, see
+		 * below.
+		 */
+		if (source == XLOG_FROM_ARCHIVE)
 		{
 			fd = XLogFileRead(segno, emode, tli,
 							  XLOG_FROM_ARCHIVE, true);
@@ -4261,7 +4377,7 @@ XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source)
 			}
 		}
 
-		if (source == XLOG_FROM_ANY || source == XLOG_FROM_PG_WAL)
+		if (source == XLOG_FROM_ARCHIVE || source == XLOG_FROM_PG_WAL)
 		{
 			fd = XLogFileRead(segno, emode, tli,
 							  XLOG_FROM_PG_WAL, true);
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 2f42cebaf6..a2116501a3 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3159,6 +3159,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&streaming_replication_retry_interval,
+		300000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 0609853995..6f6b95a996 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -355,6 +355,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 5min	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication
+					# in milliseconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index 47c29350f5..dfa0301d61 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build
index 2008958010..aaf71b9323 100644
--- a/src/test/recovery/meson.build
+++ b/src/test/recovery/meson.build
@@ -42,6 +42,7 @@ tests += {
       't/034_create_database.pl',
       't/035_standby_logical_decoding.pl',
       't/036_truncated_dropped.pl',
+      't/037_wal_source_switch.pl',
     ],
   },
 }
diff --git a/src/test/recovery/t/037_wal_source_switch.pl b/src/test/recovery/t/037_wal_source_switch.pl
new file mode 100644
index 0000000000..c28fba5d88
--- /dev/null
+++ b/src/test/recovery/t/037_wal_source_switch.pl
@@ -0,0 +1,126 @@
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Test for WAL source switch feature
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+$ENV{PGDATABASE} = 'postgres';
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(
+    allows_streaming => 1,
+    has_archiving => 1,
+    extra => ['--wal-segsize=1']);
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf(
+	'postgresql.conf', qq(
+    min_wal_size = 2MB
+    max_wal_size = 1GB
+    checkpoint_timeout = 1h
+    wal_recycle = off
+));
+$node_primary->start;
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('rep1')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create a standby linking to it using the replication slot
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup($node_primary, $backup_name,
+	has_streaming => 1,
+    has_restoring => 1);
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+primary_slot_name = 'rep1'
+min_wal_size = 2MB
+max_wal_size = 1GB
+checkpoint_timeout = 1h
+streaming_replication_retry_interval = 100ms
+wal_recycle = off
+log_min_messages = 'debug2'
+));
+
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+# Stop standby
+$node_standby->stop;
+
+# Advance WAL by 100 segments (= 100MB) on primary
+advance_wal($node_primary, 100);
+
+# Wait for primary to generate requested WAL files
+$node_primary->poll_query_until('postgres',
+	q|SELECT COUNT(*) >= 100 FROM pg_ls_waldir()|, 't');
+
+# Standby now connects to primary during inital recovery after fetching WAL
+# from archive for about streaming_replication_retry_interval milliseconds.
+$node_standby->start;
+
+$node_primary->wait_for_catchup($node_standby);
+
+ok(find_in_log(
+		$node_standby,
+        qr/restored log file ".*" from archive/),
+	    'check that some of WAL segments were fetched from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/trying to switch WAL source to stream after fetching WAL from archive for at least .* milliseconds/),
+	    'check that standby tried to switch WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/switched WAL source to stream after fetching WAL from archive for at least .* milliseconds/),
+	    'check that standby actually switched WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/started streaming WAL from primary at .* on timeline .*/),
+	    'check that standby strated streaming from primary');
+
+# Stop standby
+$node_standby->stop;
+
+# Stop primary
+$node_primary->stop;
+#####################################
+# Advance WAL of $node by $n segments
+sub advance_wal
+{
+	my ($node, $n) = @_;
+
+	# Advance by $n segments (= (wal_segment_size * $n) bytes) on primary.
+	for (my $i = 0; $i < $n; $i++)
+	{
+		$node->safe_psql('postgres',
+			"CREATE TABLE t (); DROP TABLE t; SELECT pg_switch_wal();");
+	}
+	return;
+}
+
+# Find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
+
+done_testing();
-- 
2.34.1

From 0086efe82c2da6a696cf84cb1dbfdf33c297df74 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Fri, 21 Jul 2023 07:06:52 +0000
Subject: [PATCH v13] Allow standby to switch WAL source from archive to 
 streaming replication

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be as efficient and fast as reading from
primary. This can be due to the differences in disk types, IO
costs, network latencies etc., all impacting recovery
performance on standby. And, while standby is reading WAL from
archive, primary accumulates WAL because the standby's replication
slot stays inactive. To avoid these problems, one can use this
parameter to make standby switch to stream mode sooner.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication (getting WAL from primary). However, standby
exhausts all the WAL present in pg_wal before switching. If
standby fails to switch to stream mode, it falls back to archive
mode.

Reported-by: SATYANARAYANA NARLAPURAM
Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  45 ++++++
 doc/src/sgml/high-availability.sgml           |  15 +-
 src/backend/access/transam/xlogrecovery.c     | 144 ++++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/meson.build                 |   1 +
 src/test/recovery/t/038_wal_source_switch.pl  | 126 +++++++++++++++
 8 files changed, 328 insertions(+), 20 deletions(-)
 create mode 100644 src/test/recovery/t/038_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 11251fa05e..474a215bdc 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4854,6 +4854,51 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from WAL archive to streaming replication (getting WAL from
+        primary). However, standby exhausts all the WAL present in pg_wal
+        before switching. If standby fails to switch to stream mode, it falls
+        back to archive mode. If this parameter's value is specified without
+        units, it is taken as milliseconds. Default is five minutes
+        (<literal>5min</literal>). With a lower value for this parameter,
+        standby makes frequent WAL source switch attempts. To avoid this, it is
+        recommended to set a reasonable value. A setting of <literal>0</literal>
+        disables the feature. When disabled, standby typically switches to
+        stream mode only when receive from WAL archive finishes (no more WAL
+        left there) or fails for any reason. This parameter can only be set in
+        the <filename>postgresql.conf</filename> file or on the server command
+        line.
+       </para>
+       <note>
+        <para>
+         Standby may not always attempt to switch source from WAL archive to
+         streaming replication at exact <varname>streaming_replication_retry_interval</varname>
+         intervals. For example, if the parameter is set to <literal>1min</literal>
+         and fetching WAL file from archive takes about <literal>2min</literal>,
+         then the source switch attempt happens for the next WAL file after
+         current WAL file fetched from archive is fully applied.
+        </para>
+       </note>
+       <para>
+        Reading WAL from archive may not always be as efficient and fast as
+        reading from primary. This can be due to the differences in disk types,
+        IO costs, network latencies etc., all impacting recovery performance on
+        standby. And, while standby is reading WAL from archive, primary
+        accumulates WAL because the standby's replication slot stays inactive.
+        To avoid these problems, one can use this parameter to make standby
+        switch to stream mode sooner.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index 5f9257313a..b04dae7ce0 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -628,12 +628,15 @@ protocol to make nodes agree on a serializable transactional order.
     In standby mode, the server continuously applies WAL received from the
     primary server. The standby server can read WAL from a WAL archive
     (see <xref linkend="guc-restore-command"/>) or directly from the primary
-    over a TCP connection (streaming replication). The standby server will
-    also attempt to restore any WAL found in the standby cluster's
-    <filename>pg_wal</filename> directory. That typically happens after a server
-    restart, when the standby replays again WAL that was streamed from the
-    primary before the restart, but you can also manually copy files to
-    <filename>pg_wal</filename> at any time to have them replayed.
+    over a TCP connection (streaming replication) or attempt to switch to
+    streaming replication after reading from archive when
+    <xref linkend="guc-streaming-replication-retry-interval"/> parameter is
+    set. The standby server will also attempt to restore any WAL found in the
+    standby cluster's <filename>pg_wal</filename> directory. That typically
+    happens after a server restart, when the standby replays again WAL that was
+    streamed from the primary before the restart, but you can also manually
+    copy files to <filename>pg_wal</filename> at any time to have them
+    replayed.
    </para>
 
    <para>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index becc2bda62..49c67723b5 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -68,6 +68,11 @@
 #define RECOVERY_COMMAND_FILE	"recovery.conf"
 #define RECOVERY_COMMAND_DONE	"recovery.done"
 
+#define SwitchFromArchiveToStreamEnabled() \
+	(streaming_replication_retry_interval > 0 && \
+	 StandbyMode && \
+	 currentSource == XLOG_FROM_ARCHIVE)
+
 /*
  * GUC support
  */
@@ -91,6 +96,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 300000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -297,6 +303,11 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/*
+ * Holds the timestamp at which WaitForWALToBecomeAvailable()'s state machine
+ * switches to XLOG_FROM_ARCHIVE.
+ */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +451,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static bool ShouldSwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3460,8 +3473,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	static bool canSwitchSource = false;
+	bool	switchSource = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
+	XLogSource	readFrom;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -3481,6 +3497,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval milliseconds. However,
+	 * exhaust all the WAL present in pg_wal before switching. If successful,
+	 * the state machine moves to XLOG_FROM_STREAM state, otherwise it falls
+	 * back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3504,19 +3526,20 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
+		 * First check if we failed to read from the current source or we
+		 * intentionally want to switch the source from archive to primary, and
 		 * advance the state machine if so. The failure to read might've
 		 * happened outside this function, e.g when a CRC check fails on a
 		 * record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed || switchSource)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
-			 * readahead.  Let the caller process everything that has been
-			 * decoded already first.
+			 * readahead if we failed to read from the current source. Let the
+			 * caller process everything that has been decoded already first.
 			 */
-			if (nonblocking)
+			if (nonblocking && lastSourceFailed)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3648,9 +3671,24 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we're switching to archive. */
+			if (SwitchFromArchiveToStreamEnabled())
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			if (switchSource)
+				ereport(DEBUG2,
+						(errmsg("switched WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+								xlogSourceNames[currentSource],
+								xlogSourceNames[oldSource],
+								streaming_replication_retry_interval)));
+			else
+				ereport(DEBUG2,
+						(errmsg("switched WAL source from %s to %s after %s",
+								xlogSourceNames[oldSource],
+								xlogSourceNames[currentSource],
+								lastSourceFailed ? "failure" : "success")));
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
@@ -3658,6 +3696,13 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		 */
 		lastSourceFailed = false;
 
+		if (switchSource)
+		{
+			Assert(canSwitchSource == true);
+			switchSource = false;
+			canSwitchSource = false;
+		}
+
 		switch (currentSource)
 		{
 			case XLOG_FROM_ARCHIVE:
@@ -3679,13 +3724,24 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/*
+				 * See if we can switch the source to streaming from archive.
+				 */
+				if (!canSwitchSource)
+					canSwitchSource = ShouldSwitchWALSourceToPrimary();
+
 				/*
 				 * Try to restore the file from archive, or read an existing
-				 * file from pg_wal.
+				 * file from pg_wal. However, before switching the source to
+				 * stream mode, give it a chance to read all the WAL from
+				 * pg_wal.
 				 */
-				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2,
-											  currentSource == XLOG_FROM_ARCHIVE ? XLOG_FROM_ANY :
-											  currentSource);
+				if (canSwitchSource)
+					readFrom = XLOG_FROM_PG_WAL;
+				else
+					readFrom = currentSource;
+
+				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2, readFrom);
 				if (readFile >= 0)
 					return XLREAD_SUCCESS;	/* success! */
 
@@ -3693,6 +3749,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				 * Nope, not found in archive or pg_wal.
 				 */
 				lastSourceFailed = true;
+
+				/*
+				 * Exhausted all the WAL in pg_wal, now ready to switch to
+				 * streaming.
+				 */
+				if (canSwitchSource)
+					switchSource = true;
+
 				break;
 
 			case XLOG_FROM_STREAM:
@@ -3923,6 +3987,54 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * This function tells if a standby can make an attempt to read WAL from
+ * primary after reading from archive for at least
+ * streaming_replication_retry_interval milliseconds. Reading WAL from archive
+ * may not always be as efficient and fast as reading from primary. This can be
+ * due to the differences in disk types, IO costs, network latencies etc., all
+ * impacting recovery performance on standby. And, while standby is reading WAL
+ * from archive, primary accumulates WAL because the standby's replication slot
+ * stays inactive. To avoid these problems, we try to make standby switch to
+ * stream mode sooner.
+ */
+static bool
+ShouldSwitchWALSourceToPrimary(void)
+{
+	bool shouldSwitchSource = false;
+
+	if (!SwitchFromArchiveToStreamEnabled())
+		return shouldSwitchSource;
+
+	if (switched_to_archive_at > 0)
+	{
+		TimestampTz curr_time;
+
+		curr_time = GetCurrentTimestamp();
+
+		if (TimestampDifferenceExceeds(switched_to_archive_at, curr_time,
+									   streaming_replication_retry_interval))
+		{
+			ereport(DEBUG2,
+					(errmsg("trying to switch WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+							xlogSourceNames[XLOG_FROM_STREAM],
+							xlogSourceNames[currentSource],
+							streaming_replication_retry_interval)));
+
+			shouldSwitchSource = true;
+		}
+	}
+	else if (switched_to_archive_at == 0)
+	{
+		/*
+		 * Save the timestamp if we're about to fetch WAL from archive for the
+		 * first time.
+		 */
+		switched_to_archive_at = GetCurrentTimestamp();
+	}
+
+	return shouldSwitchSource;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
@@ -4248,7 +4360,11 @@ XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source)
 				continue;
 		}
 
-		if (source == XLOG_FROM_ANY || source == XLOG_FROM_ARCHIVE)
+		/*
+		 * When failed to read from archive, try reading from pg_wal, see
+		 * below.
+		 */
+		if (source == XLOG_FROM_ARCHIVE)
 		{
 			fd = XLogFileRead(segno, emode, tli,
 							  XLOG_FROM_ARCHIVE, true);
@@ -4261,7 +4377,7 @@ XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source)
 			}
 		}
 
-		if (source == XLOG_FROM_ANY || source == XLOG_FROM_PG_WAL)
+		if (source == XLOG_FROM_ARCHIVE || source == XLOG_FROM_PG_WAL)
 		{
 			fd = XLogFileRead(segno, emode, tli,
 							  XLOG_FROM_PG_WAL, true);
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index f9dba43b8c..9322e5d694 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3156,6 +3156,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&streaming_replication_retry_interval,
+		300000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index c768af9a73..87d66ae81e 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -354,6 +354,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 5min	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication
+					# in milliseconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index 47c29350f5..dfa0301d61 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build
index e7328e4894..634fc9a2b2 100644
--- a/src/test/recovery/meson.build
+++ b/src/test/recovery/meson.build
@@ -43,6 +43,7 @@ tests += {
       't/035_standby_logical_decoding.pl',
       't/036_truncated_dropped.pl',
       't/037_invalid_database.pl',
+      't/038_wal_source_switch.pl',
     ],
   },
 }
diff --git a/src/test/recovery/t/038_wal_source_switch.pl b/src/test/recovery/t/038_wal_source_switch.pl
new file mode 100644
index 0000000000..c28fba5d88
--- /dev/null
+++ b/src/test/recovery/t/038_wal_source_switch.pl
@@ -0,0 +1,126 @@
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Test for WAL source switch feature
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+$ENV{PGDATABASE} = 'postgres';
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(
+    allows_streaming => 1,
+    has_archiving => 1,
+    extra => ['--wal-segsize=1']);
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf(
+	'postgresql.conf', qq(
+    min_wal_size = 2MB
+    max_wal_size = 1GB
+    checkpoint_timeout = 1h
+    wal_recycle = off
+));
+$node_primary->start;
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('rep1')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create a standby linking to it using the replication slot
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup($node_primary, $backup_name,
+	has_streaming => 1,
+    has_restoring => 1);
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+primary_slot_name = 'rep1'
+min_wal_size = 2MB
+max_wal_size = 1GB
+checkpoint_timeout = 1h
+streaming_replication_retry_interval = 100ms
+wal_recycle = off
+log_min_messages = 'debug2'
+));
+
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+# Stop standby
+$node_standby->stop;
+
+# Advance WAL by 100 segments (= 100MB) on primary
+advance_wal($node_primary, 100);
+
+# Wait for primary to generate requested WAL files
+$node_primary->poll_query_until('postgres',
+	q|SELECT COUNT(*) >= 100 FROM pg_ls_waldir()|, 't');
+
+# Standby now connects to primary during inital recovery after fetching WAL
+# from archive for about streaming_replication_retry_interval milliseconds.
+$node_standby->start;
+
+$node_primary->wait_for_catchup($node_standby);
+
+ok(find_in_log(
+		$node_standby,
+        qr/restored log file ".*" from archive/),
+	    'check that some of WAL segments were fetched from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/trying to switch WAL source to stream after fetching WAL from archive for at least .* milliseconds/),
+	    'check that standby tried to switch WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/switched WAL source to stream after fetching WAL from archive for at least .* milliseconds/),
+	    'check that standby actually switched WAL source to primary from archive');
+
+ok(find_in_log(
+		$node_standby,
+        qr/started streaming WAL from primary at .* on timeline .*/),
+	    'check that standby strated streaming from primary');
+
+# Stop standby
+$node_standby->stop;
+
+# Stop primary
+$node_primary->stop;
+#####################################
+# Advance WAL of $node by $n segments
+sub advance_wal
+{
+	my ($node, $n) = @_;
+
+	# Advance by $n segments (= (wal_segment_size * $n) bytes) on primary.
+	for (my $i = 0; $i < $n; $i++)
+	{
+		$node->safe_psql('postgres',
+			"CREATE TABLE t (); DROP TABLE t; SELECT pg_switch_wal();");
+	}
+	return;
+}
+
+# Find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
+
+done_testing();
-- 
2.34.1

From e6010e3b2e4c52d32a5d2f3eb2d59954617b221b Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Sat, 21 Oct 2023 13:41:46 +0000
Subject: [PATCH v14] Allow standby to switch WAL source from archive to
 streaming

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be as efficient and fast as reading from
primary. This can be due to the differences in disk types, IO
costs, network latencies etc., all impacting recovery
performance on standby. And, while standby is reading WAL from
archive, primary accumulates WAL because the standby's replication
slot stays inactive. To avoid these problems, one can use this
parameter to make standby switch to stream mode sooner.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication (getting WAL from primary). However, standby
exhausts all the WAL present in pg_wal before switching. If
standby fails to switch to stream mode, it falls back to archive
mode.

Reported-by: SATYANARAYANA NARLAPURAM
Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  45 ++++++
 doc/src/sgml/high-availability.sgml           |  15 +-
 src/backend/access/transam/xlogrecovery.c     | 146 ++++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/meson.build                 |   1 +
 src/test/recovery/t/040_wal_source_switch.pl  | 130 ++++++++++++++++
 8 files changed, 333 insertions(+), 21 deletions(-)
 create mode 100644 src/test/recovery/t/040_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 3839c72c86..3a18ba9b26 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4791,6 +4791,51 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from WAL archive to streaming replication (getting WAL from
+        primary). However, standby exhausts all the WAL present in pg_wal
+        before switching. If standby fails to switch to stream mode, it falls
+        back to archive mode. If this parameter's value is specified without
+        units, it is taken as milliseconds. Default is five minutes
+        (<literal>5min</literal>). With a lower value for this parameter,
+        standby makes frequent WAL source switch attempts. To avoid this, it is
+        recommended to set a reasonable value. A setting of <literal>0</literal>
+        disables the feature. When disabled, standby typically switches to
+        stream mode only when receive from WAL archive finishes (no more WAL
+        left there) or fails for any reason. This parameter can only be set in
+        the <filename>postgresql.conf</filename> file or on the server command
+        line.
+       </para>
+       <note>
+        <para>
+         Standby may not always attempt to switch source from WAL archive to
+         streaming replication at exact <varname>streaming_replication_retry_interval</varname>
+         intervals. For example, if the parameter is set to <literal>1min</literal>
+         and fetching WAL file from archive takes about <literal>2min</literal>,
+         then the source switch attempt happens for the next WAL file after
+         current WAL file fetched from archive is fully applied.
+        </para>
+       </note>
+       <para>
+        Reading WAL from archive may not always be as efficient and fast as
+        reading from primary. This can be due to the differences in disk types,
+        IO costs, network latencies etc., all impacting recovery performance on
+        standby. And, while standby is reading WAL from archive, primary
+        accumulates WAL because the standby's replication slot stays inactive.
+        To avoid these problems, one can use this parameter to make standby
+        switch to stream mode sooner.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index 5f9257313a..b04dae7ce0 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -628,12 +628,15 @@ protocol to make nodes agree on a serializable transactional order.
     In standby mode, the server continuously applies WAL received from the
     primary server. The standby server can read WAL from a WAL archive
     (see <xref linkend="guc-restore-command"/>) or directly from the primary
-    over a TCP connection (streaming replication). The standby server will
-    also attempt to restore any WAL found in the standby cluster's
-    <filename>pg_wal</filename> directory. That typically happens after a server
-    restart, when the standby replays again WAL that was streamed from the
-    primary before the restart, but you can also manually copy files to
-    <filename>pg_wal</filename> at any time to have them replayed.
+    over a TCP connection (streaming replication) or attempt to switch to
+    streaming replication after reading from archive when
+    <xref linkend="guc-streaming-replication-retry-interval"/> parameter is
+    set. The standby server will also attempt to restore any WAL found in the
+    standby cluster's <filename>pg_wal</filename> directory. That typically
+    happens after a server restart, when the standby replays again WAL that was
+    streamed from the primary before the restart, but you can also manually
+    copy files to <filename>pg_wal</filename> at any time to have them
+    replayed.
    </para>
 
    <para>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index d6f2bb8286..c1a2a83a0b 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -68,6 +68,11 @@
 #define RECOVERY_COMMAND_FILE	"recovery.conf"
 #define RECOVERY_COMMAND_DONE	"recovery.done"
 
+#define SwitchFromArchiveToStreamEnabled() \
+	(streaming_replication_retry_interval > 0 && \
+	 StandbyMode && \
+	 currentSource == XLOG_FROM_ARCHIVE)
+
 /*
  * GUC support
  */
@@ -91,6 +96,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 300000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -297,6 +303,11 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/*
+ * Holds the timestamp at which WaitForWALToBecomeAvailable()'s state machine
+ * switches to XLOG_FROM_ARCHIVE.
+ */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +451,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static bool ShouldSwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3471,8 +3484,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	static bool canSwitchSource = false;
+	bool		switchSource = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
+	XLogSource	readFrom;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -3492,6 +3508,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval milliseconds. However,
+	 * exhaust all the WAL present in pg_wal before switching. If successful,
+	 * the state machine moves to XLOG_FROM_STREAM state, otherwise it falls
+	 * back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3515,19 +3537,20 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
-		 * advance the state machine if so. The failure to read might've
+		 * First check if we failed to read from the current source or we
+		 * intentionally want to switch the source from archive to primary,
+		 * and advance the state machine if so. The failure to read might've
 		 * happened outside this function, e.g when a CRC check fails on a
 		 * record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed || switchSource)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
-			 * readahead.  Let the caller process everything that has been
-			 * decoded already first.
+			 * readahead if we failed to read from the current source. Let the
+			 * caller process everything that has been decoded already first.
 			 */
-			if (nonblocking)
+			if (nonblocking && lastSourceFailed)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3659,9 +3682,24 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we're switching to archive. */
+			if (SwitchFromArchiveToStreamEnabled())
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			if (switchSource)
+				ereport(DEBUG1,
+						(errmsg_internal("switched WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+										 xlogSourceNames[currentSource],
+										 xlogSourceNames[oldSource],
+										 streaming_replication_retry_interval)));
+			else
+				ereport(DEBUG1,
+						(errmsg_internal("switched WAL source from %s to %s after %s",
+										 xlogSourceNames[oldSource],
+										 xlogSourceNames[currentSource],
+										 lastSourceFailed ? "failure" : "success")));
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
@@ -3669,6 +3707,13 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		 */
 		lastSourceFailed = false;
 
+		if (switchSource)
+		{
+			Assert(canSwitchSource == true);
+			switchSource = false;
+			canSwitchSource = false;
+		}
+
 		switch (currentSource)
 		{
 			case XLOG_FROM_ARCHIVE:
@@ -3690,13 +3735,24 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/*
+				 * See if we can switch the source to streaming from archive.
+				 */
+				if (!canSwitchSource)
+					canSwitchSource = ShouldSwitchWALSourceToPrimary();
+
 				/*
 				 * Try to restore the file from archive, or read an existing
-				 * file from pg_wal.
+				 * file from pg_wal. However, before switching the source to
+				 * stream mode, give it a chance to read all the WAL from
+				 * pg_wal.
 				 */
-				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2,
-											  currentSource == XLOG_FROM_ARCHIVE ? XLOG_FROM_ANY :
-											  currentSource);
+				if (canSwitchSource)
+					readFrom = XLOG_FROM_PG_WAL;
+				else
+					readFrom = currentSource;
+
+				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2, readFrom);
 				if (readFile >= 0)
 					return XLREAD_SUCCESS;	/* success! */
 
@@ -3704,6 +3760,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				 * Nope, not found in archive or pg_wal.
 				 */
 				lastSourceFailed = true;
+
+				/*
+				 * Exhausted all the WAL in pg_wal, now ready to switch to
+				 * streaming.
+				 */
+				if (canSwitchSource)
+					switchSource = true;
+
 				break;
 
 			case XLOG_FROM_STREAM:
@@ -3934,6 +3998,54 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * This function tells if a standby can make an attempt to read WAL from
+ * primary after reading from archive for at least
+ * streaming_replication_retry_interval milliseconds. Reading WAL from archive
+ * may not always be as efficient and fast as reading from primary. This can be
+ * due to the differences in disk types, IO costs, network latencies etc., all
+ * impacting recovery performance on standby. And, while standby is reading WAL
+ * from archive, primary accumulates WAL because the standby's replication slot
+ * stays inactive. To avoid these problems, we try to make standby switch to
+ * stream mode sooner.
+ */
+static bool
+ShouldSwitchWALSourceToPrimary(void)
+{
+	bool		shouldSwitchSource = false;
+
+	if (!SwitchFromArchiveToStreamEnabled())
+		return shouldSwitchSource;
+
+	if (switched_to_archive_at > 0)
+	{
+		TimestampTz curr_time;
+
+		curr_time = GetCurrentTimestamp();
+
+		if (TimestampDifferenceExceeds(switched_to_archive_at, curr_time,
+									   streaming_replication_retry_interval))
+		{
+			ereport(DEBUG1,
+					(errmsg_internal("trying to switch WAL source to %s after fetching WAL from %s for at least %d milliseconds",
+									 xlogSourceNames[XLOG_FROM_STREAM],
+									 xlogSourceNames[currentSource],
+									 streaming_replication_retry_interval)));
+
+			shouldSwitchSource = true;
+		}
+	}
+	else if (switched_to_archive_at == 0)
+	{
+		/*
+		 * Save the timestamp if we're about to fetch WAL from archive for the
+		 * first time.
+		 */
+		switched_to_archive_at = GetCurrentTimestamp();
+	}
+
+	return shouldSwitchSource;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
@@ -4259,7 +4371,11 @@ XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source)
 				continue;
 		}
 
-		if (source == XLOG_FROM_ANY || source == XLOG_FROM_ARCHIVE)
+		/*
+		 * When failed to read from archive, try reading from pg_wal, see
+		 * below.
+		 */
+		if (source == XLOG_FROM_ARCHIVE)
 		{
 			fd = XLogFileRead(segno, emode, tli,
 							  XLOG_FROM_ARCHIVE, true);
@@ -4272,7 +4388,7 @@ XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source)
 			}
 		}
 
-		if (source == XLOG_FROM_ANY || source == XLOG_FROM_PG_WAL)
+		if (source == XLOG_FROM_ARCHIVE || source == XLOG_FROM_PG_WAL)
 		{
 			fd = XLogFileRead(segno, emode, tli,
 							  XLOG_FROM_PG_WAL, true);
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 4c58574166..b6dfa74e3f 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3168,6 +3168,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&streaming_replication_retry_interval,
+		300000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index d08d55c3fe..0e03887683 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -352,6 +352,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 5min	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication
+					# in milliseconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index 47c29350f5..dfa0301d61 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build
index 9d8039684a..06041a5f74 100644
--- a/src/test/recovery/meson.build
+++ b/src/test/recovery/meson.build
@@ -45,6 +45,7 @@ tests += {
       't/037_invalid_database.pl',
       't/038_save_logical_slots_shutdown.pl',
       't/039_end_of_wal.pl',
+      't/040_wal_source_switch.pl',
     ],
   },
 }
diff --git a/src/test/recovery/t/040_wal_source_switch.pl b/src/test/recovery/t/040_wal_source_switch.pl
new file mode 100644
index 0000000000..9388fa3a39
--- /dev/null
+++ b/src/test/recovery/t/040_wal_source_switch.pl
@@ -0,0 +1,130 @@
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Test for WAL source switch feature
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+$ENV{PGDATABASE} = 'postgres';
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(
+	allows_streaming => 1,
+	has_archiving    => 1,
+	extra            => ['--wal-segsize=1']);
+
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf(
+	'postgresql.conf', qq(
+    min_wal_size = 2MB
+    max_wal_size = 1GB
+    checkpoint_timeout = 1h
+    wal_recycle = off
+));
+$node_primary->start;
+
+# Create an inactive replication slot to keep the WAL
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('rep1')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create a standby linking to it using the replication slot
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup(
+	$node_primary, $backup_name,
+	has_streaming => 1,
+	has_restoring => 1);
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+primary_slot_name = 'rep1'
+min_wal_size = 2MB
+max_wal_size = 1GB
+checkpoint_timeout = 1h
+streaming_replication_retry_interval = 10ms
+wal_recycle = off
+log_min_messages = 'debug1'
+));
+
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+$node_standby->stop;
+
+# Advance WAL by 10 segments (= 10MB) on primary
+advance_wal($node_primary, 10);
+
+# Wait for primary to generate requested WAL files
+$node_primary->poll_query_until('postgres',
+	"SELECT COUNT(*) >= 10 FROM pg_ls_waldir();")
+  or die "Timed out while waiting for primary to generate WAL";
+
+# Wait until generated WAL files have been stored on the archives of the
+# primary. This ensures that the standby created below will be able to restore
+# the WAL files.
+my $primary_archive = $node_primary->archive_dir;
+$node_primary->poll_query_until('postgres',
+	"SELECT COUNT(*) >= 10 FROM pg_ls_dir('$primary_archive', false, false) a WHERE a ~ '^[0-9A-F]{24}\$';"
+) or die "Timed out while waiting for archiving of WAL by primary";
+
+# Generate some data on the primary
+$node_primary->safe_psql('postgres',
+	"CREATE TABLE test_tbl AS SELECT a FROM generate_series(1,10) AS a;");
+
+my $offset = -s $node_standby->logfile;
+
+# Standby now connects to primary during inital recovery after fetching WAL
+# from archive for about streaming_replication_retry_interval milliseconds.
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+$node_standby->wait_for_log(
+	qr/LOG: ( [A-Z0-9]+:)? restored log file ".*" from archive/, $offset);
+
+$node_standby->wait_for_log(
+	qr/DEBUG: ( [A-Z0-9]+:)? trying to switch WAL source to stream after fetching WAL from archive for at least 10 milliseconds/,
+	$offset);
+
+$node_standby->wait_for_log(
+	qr/DEBUG: ( [A-Z0-9]+:)? switched WAL source to stream after fetching WAL from archive for at least 10 milliseconds/,
+	$offset);
+
+$node_standby->wait_for_log(
+	qr/LOG: ( [A-Z0-9]+:)? started streaming WAL from primary at .* on timeline .*/,
+	$offset);
+
+# Check the streamed data on the standby
+my $result =
+  $node_standby->safe_psql('postgres', "SELECT COUNT(*) FROM test_tbl;");
+
+is($result, '10', 'check streamed data on standby');
+
+$node_standby->stop;
+$node_primary->stop;
+
+#####################################
+# Advance WAL of $node by $n segments
+sub advance_wal
+{
+	my ($node, $n) = @_;
+
+	# Advance by $n segments (= (wal_segment_size * $n) bytes) on primary.
+	for (my $i = 0; $i < $n; $i++)
+	{
+		$node->safe_psql('postgres',
+			"CREATE TABLE t (); DROP TABLE t; SELECT pg_switch_wal();");
+	}
+	return;
+}
+
+done_testing();
-- 
2.34.1

From 8bdd3b999343c02ae01a7d15ef7fdd0be25623bd Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Thu, 28 Dec 2023 11:14:05 +0000
Subject: [PATCH v15] Allow standby to switch WAL source from archive to
 streaming

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be as efficient and fast as reading from
primary. This can be due to the differences in disk types, IO
costs, network latencies etc.. All of these can impact the
recovery performance on standby and increase the replication lag
on primary. In addition, the primary keeps accumulating WAL needed
for the standby while the standby reads WAL from archive because
the standby replication slot stays inactive. To avoid these
problems, one can use this parameter to make standby switch to
stream mode sooner.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication (getting WAL from primary). However, standby
exhausts all the WAL present in pg_wal before switching. If
standby fails to switch to stream mode, it falls back to archive
mode.

Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi, SATYANARAYANA NARLAPURAM
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  47 +++++++
 doc/src/sgml/high-availability.sgml           |  15 ++-
 src/backend/access/transam/xlogrecovery.c     | 115 ++++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/meson.build                 |   1 +
 src/test/recovery/t/040_wal_source_switch.pl  |  93 ++++++++++++++
 8 files changed, 269 insertions(+), 19 deletions(-)
 create mode 100644 src/test/recovery/t/040_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index b5624ca884..04aa2fa8d2 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4866,6 +4866,53 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from archive to streaming replication (i.e., getting WAL from
+        primary). However, the standby exhausts all the WAL present in pg_wal
+        before switching. If the standby fails to switch to stream mode, it
+        falls back to archive mode. If this parameter value is specified
+        without units, it is taken as milliseconds. Default is
+        <literal>5min</literal>. With a lower value for this parameter, the
+        standby makes frequent WAL source switch attempts. To avoid this, it is
+        recommended to set a reasonable value. A setting of <literal>0</literal>
+        disables the feature. When disabled, the standby typically switches to
+        stream mode only after receiving WAL from archive finishes (i.e., no
+        more WAL left there) or fails for any reason. This parameter can only
+        be set in the <filename>postgresql.conf</filename> file or on the
+        server command line.
+       </para>
+       <note>
+        <para>
+         Standby may not always attempt to switch source from WAL archive to
+         streaming replication at exact
+         <varname>streaming_replication_retry_interval</varname> intervals. For
+         example, if the parameter is set to <literal>1min</literal> and
+         fetching WAL file from archive takes about <literal>2min</literal>,
+         then the source switch attempt happens for the next WAL file after
+         current WAL file fetched from archive is fully applied.
+        </para>
+       </note>
+       <para>
+        Reading WAL from archive may not always be as efficient and fast as
+        reading from primary. This can be due to the differences in disk types,
+        IO costs, network latencies etc.. All of these can impact the recovery
+        performance on standby and increase the replication lag on primary. In
+        addition, the primary keeps accumulating WAL needed for the standby
+        while the standby reads WAL from archive because the standby
+        replication slot stays inactive. To avoid these problems, one can use
+        this parameter to make standby switch to stream mode sooner.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index 9dd52ff275..35926e1df3 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -628,12 +628,15 @@ protocol to make nodes agree on a serializable transactional order.
     In standby mode, the server continuously applies WAL received from the
     primary server. The standby server can read WAL from a WAL archive
     (see <xref linkend="guc-restore-command"/>) or directly from the primary
-    over a TCP connection (streaming replication). The standby server will
-    also attempt to restore any WAL found in the standby cluster's
-    <filename>pg_wal</filename> directory. That typically happens after a server
-    restart, when the standby replays again WAL that was streamed from the
-    primary before the restart, but you can also manually copy files to
-    <filename>pg_wal</filename> at any time to have them replayed.
+    over a TCP connection (streaming replication) or attempt to switch to
+    streaming replication after reading from archive when
+    <xref linkend="guc-streaming-replication-retry-interval"/> parameter is
+    set. The standby server will also attempt to restore any WAL found in the
+    standby cluster's <filename>pg_wal</filename> directory. That typically
+    happens after a server restart, when the standby replays again WAL that was
+    streamed from the primary before the restart, but you can also manually
+    copy files to <filename>pg_wal</filename> at any time to have them
+    replayed.
    </para>
 
    <para>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 6f4f81f992..e59057558e 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -91,6 +91,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 300000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -297,6 +298,8 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/* Holds the timestamp at which standby switched WAL source to archive */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +443,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static bool SwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3492,8 +3497,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	static bool canSwitchSource = false;
+	bool		switchSource = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
+	XLogSource	readFrom;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -3513,6 +3521,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval milliseconds. However,
+	 * exhaust all the WAL present in pg_wal before switching. If successful,
+	 * the state machine moves to XLOG_FROM_STREAM state, otherwise it falls
+	 * back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3536,19 +3550,20 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
-		 * advance the state machine if so. The failure to read might've
+		 * First check if we failed to read from the current source or we
+		 * intentionally want to switch the source from archive to primary,
+		 * and advance the state machine if so. The failure to read might've
 		 * happened outside this function, e.g when a CRC check fails on a
 		 * record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed || switchSource)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
-			 * readahead.  Let the caller process everything that has been
-			 * decoded already first.
+			 * readahead if we failed to read from the current source. Let the
+			 * caller process everything that has been decoded already first.
 			 */
-			if (nonblocking)
+			if (nonblocking && lastSourceFailed)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3680,9 +3695,27 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we are switching to archive */
+			if (currentSource == XLOG_FROM_ARCHIVE)
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			ereport(DEBUG1,
+					errmsg_internal("switched WAL source from %s to %s after %s",
+									xlogSourceNames[oldSource],
+									xlogSourceNames[currentSource],
+									(switchSource ? "timeout" : (lastSourceFailed ? "failure" : "success"))));
+
+			/* Reset the WAL source switch state */
+			if (switchSource)
+			{
+				Assert(canSwitchSource);
+				Assert(currentSource == XLOG_FROM_STREAM);
+				Assert(oldSource == XLOG_FROM_ARCHIVE);
+				switchSource = false;
+				canSwitchSource = false;
+			}
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
@@ -3711,13 +3744,23 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/* See if we can switch WAL source to streaming */
+				if (!canSwitchSource)
+					canSwitchSource = SwitchWALSourceToPrimary();
+
 				/*
 				 * Try to restore the file from archive, or read an existing
-				 * file from pg_wal.
+				 * file from pg_wal. However, before switching WAL source to
+				 * streaming, give it a chance to read all the WAL from
+				 * pg_wal.
 				 */
-				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2,
-											  currentSource == XLOG_FROM_ARCHIVE ? XLOG_FROM_ANY :
-											  currentSource);
+				if (canSwitchSource)
+					readFrom = XLOG_FROM_PG_WAL;
+				else
+					readFrom = currentSource == XLOG_FROM_ARCHIVE ?
+						XLOG_FROM_ANY : currentSource;
+
+				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2, readFrom);
 				if (readFile >= 0)
 					return XLREAD_SUCCESS;	/* success! */
 
@@ -3725,6 +3768,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				 * Nope, not found in archive or pg_wal.
 				 */
 				lastSourceFailed = true;
+
+				/*
+				 * Read all the WAL in pg_wal. Now ready to switch to
+				 * streaming.
+				 */
+				if (canSwitchSource)
+					switchSource = true;
+
 				break;
 
 			case XLOG_FROM_STREAM:
@@ -3955,6 +4006,44 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * Check if standby can make an attempt to read WAL from primary after reading
+ * from archive for at least a configurable duration.
+ *
+ * Reading WAL from archive may not always be as efficient and fast as reading
+ * from primary. This can be due to the differences in disk types, IO costs,
+ * network latencies etc.. All of these can impact the recovery performance on
+ * standby and increase the replication lag on primary. In addition, the
+ * primary keeps accumulating WAL needed for the standby while the standby
+ * reads WAL from archive because the standby replication slot stays inactive.
+ * To avoid these problems, the standby will try to switch to stream mode
+ * sooner.
+ */
+static bool
+SwitchWALSourceToPrimary(void)
+{
+	TimestampTz now;
+
+	if (streaming_replication_retry_interval <= 0 ||
+		!StandbyMode ||
+		currentSource != XLOG_FROM_ARCHIVE)
+		return false;
+
+	now = GetCurrentTimestamp();
+
+	/* First time through */
+	if (switched_to_archive_at == 0)
+	{
+		switched_to_archive_at = now;
+		return false;
+	}
+
+	if (TimestampDifferenceExceeds(switched_to_archive_at, now,
+								   streaming_replication_retry_interval))
+		return true;
+
+	return false;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 9f59440526..127029e3e4 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3200,6 +3200,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&streaming_replication_retry_interval,
+		300000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index b2809c711a..4dd27554aa 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -360,6 +360,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 5min	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication
+					# in milliseconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index ee0bc74278..2e9fb8dfe6 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build
index 9d8039684a..06041a5f74 100644
--- a/src/test/recovery/meson.build
+++ b/src/test/recovery/meson.build
@@ -45,6 +45,7 @@ tests += {
       't/037_invalid_database.pl',
       't/038_save_logical_slots_shutdown.pl',
       't/039_end_of_wal.pl',
+      't/040_wal_source_switch.pl',
     ],
   },
 }
diff --git a/src/test/recovery/t/040_wal_source_switch.pl b/src/test/recovery/t/040_wal_source_switch.pl
new file mode 100644
index 0000000000..5586019eae
--- /dev/null
+++ b/src/test/recovery/t/040_wal_source_switch.pl
@@ -0,0 +1,93 @@
+# Copyright (c) 2023, PostgreSQL Global Development Group
+#
+# Test for WAL source switch feature.
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(
+	allows_streaming => 1,
+	has_archiving    => 1,
+	extra            => ['--wal-segsize=1']);
+
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf('postgresql.conf', qq(
+    min_wal_size = 2MB
+    max_wal_size = 1GB
+    checkpoint_timeout = 1h
+	autovacuum = off
+));
+$node_primary->start;
+
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('standby_slot')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create a streaming standby
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup(
+	$node_primary, $backup_name,
+	has_streaming => 1,
+	has_restoring => 1);
+$node_standby->append_conf('postgresql.conf', qq(
+	primary_slot_name = 'standby_slot'
+	streaming_replication_retry_interval = 1ms
+	log_min_messages = 'debug1'
+));
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+$node_standby->stop;
+
+# Advance WAL by 5 segments (= 5MB) on primary
+$node_primary->advance_wal(5);
+
+# Wait for primary to generate requested WAL files
+$node_primary->poll_query_until('postgres',
+	"SELECT COUNT(*) >= 5 FROM pg_ls_waldir();")
+  or die "Timed out while waiting for primary to generate WAL";
+
+# Wait until generated WAL files have been stored on the archives of the
+# primary. This ensures that the standby created below will be able to restore
+# the WAL files.
+my $primary_archive = $node_primary->archive_dir;
+$node_primary->poll_query_until('postgres',
+	"SELECT COUNT(*) >= 5 FROM pg_ls_dir('$primary_archive', false, false) a WHERE a ~ '^[0-9A-F]{24}\$';"
+) or die "Timed out while waiting for archiving of WAL by primary";
+
+# Generate some data on the primary
+$node_primary->safe_psql('postgres',
+	"CREATE TABLE test_tbl AS SELECT a FROM generate_series(1,5) AS a;");
+
+my $offset = -s $node_standby->logfile;
+
+# Standby now connects to primary during inital recovery after fetching WAL
+# from archive for about streaming_replication_retry_interval milliseconds.
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+$node_standby->wait_for_log(
+	qr/DEBUG: ( [A-Z0-9]+:)? switched WAL source from archive to stream after timeout/,
+	$offset);
+$node_standby->wait_for_log(
+	qr/LOG: ( [A-Z0-9]+:)? started streaming WAL from primary at .* on timeline .*/,
+	$offset);
+
+# Check that the data from primary is streamed to standby
+my $result =
+  $node_standby->safe_psql('postgres', "SELECT COUNT(*) FROM test_tbl;");
+is($result, '5', 'data from primary is streamed to standby');
+
+done_testing();
-- 
2.34.1

From e1464a4c8f5c7b7baf65e3856aa262bee10c86f2 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Wed, 3 Jan 2024 11:24:34 +0000
Subject: [PATCH v16] Allow standby to switch WAL source from archive to
 streaming

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be as efficient and fast as reading from
primary. This can be due to the differences in disk types, IO
costs, network latencies etc.. All of these can impact the
recovery performance on standby and increase the replication lag
on primary. In addition, the primary keeps accumulating WAL needed
for the standby while the standby reads WAL from archive because
the standby replication slot stays inactive. To avoid these
problems, one can use this parameter to make standby switch to
stream mode sooner.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication (getting WAL from primary). However, standby
exhausts all the WAL present in pg_wal before switching. If
standby fails to switch to stream mode, it falls back to archive
mode.

Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi, SATYANARAYANA NARLAPURAM
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  47 +++++++
 doc/src/sgml/high-availability.sgml           |  15 ++-
 src/backend/access/transam/xlogrecovery.c     | 115 ++++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/meson.build                 |   1 +
 src/test/recovery/t/040_wal_source_switch.pl  |  93 ++++++++++++++
 8 files changed, 269 insertions(+), 19 deletions(-)
 create mode 100644 src/test/recovery/t/040_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index f323bba018..5d507029d5 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4866,6 +4866,53 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from archive to streaming replication (i.e., getting WAL from
+        primary). However, the standby exhausts all the WAL present in pg_wal
+        before switching. If the standby fails to switch to stream mode, it
+        falls back to archive mode. If this parameter value is specified
+        without units, it is taken as milliseconds. Default is
+        <literal>5min</literal>. With a lower value for this parameter, the
+        standby makes frequent WAL source switch attempts. To avoid this, it is
+        recommended to set a reasonable value. A setting of <literal>0</literal>
+        disables the feature. When disabled, the standby typically switches to
+        stream mode only after receiving WAL from archive finishes (i.e., no
+        more WAL left there) or fails for any reason. This parameter can only
+        be set in the <filename>postgresql.conf</filename> file or on the
+        server command line.
+       </para>
+       <note>
+        <para>
+         Standby may not always attempt to switch source from WAL archive to
+         streaming replication at exact
+         <varname>streaming_replication_retry_interval</varname> intervals. For
+         example, if the parameter is set to <literal>1min</literal> and
+         fetching WAL file from archive takes about <literal>2min</literal>,
+         then the source switch attempt happens for the next WAL file after
+         current WAL file fetched from archive is fully applied.
+        </para>
+       </note>
+       <para>
+        Reading WAL from archive may not always be as efficient and fast as
+        reading from primary. This can be due to the differences in disk types,
+        IO costs, network latencies etc.. All of these can impact the recovery
+        performance on standby and increase the replication lag on primary. In
+        addition, the primary keeps accumulating WAL needed for the standby
+        while the standby reads WAL from archive because the standby
+        replication slot stays inactive. To avoid these problems, one can use
+        this parameter to make standby switch to stream mode sooner.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index 9dd52ff275..35926e1df3 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -628,12 +628,15 @@ protocol to make nodes agree on a serializable transactional order.
     In standby mode, the server continuously applies WAL received from the
     primary server. The standby server can read WAL from a WAL archive
     (see <xref linkend="guc-restore-command"/>) or directly from the primary
-    over a TCP connection (streaming replication). The standby server will
-    also attempt to restore any WAL found in the standby cluster's
-    <filename>pg_wal</filename> directory. That typically happens after a server
-    restart, when the standby replays again WAL that was streamed from the
-    primary before the restart, but you can also manually copy files to
-    <filename>pg_wal</filename> at any time to have them replayed.
+    over a TCP connection (streaming replication) or attempt to switch to
+    streaming replication after reading from archive when
+    <xref linkend="guc-streaming-replication-retry-interval"/> parameter is
+    set. The standby server will also attempt to restore any WAL found in the
+    standby cluster's <filename>pg_wal</filename> directory. That typically
+    happens after a server restart, when the standby replays again WAL that was
+    streamed from the primary before the restart, but you can also manually
+    copy files to <filename>pg_wal</filename> at any time to have them
+    replayed.
    </para>
 
    <para>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 6f4f81f992..e59057558e 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -91,6 +91,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 300000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -297,6 +298,8 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/* Holds the timestamp at which standby switched WAL source to archive */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +443,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static bool SwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3492,8 +3497,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	static bool canSwitchSource = false;
+	bool		switchSource = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
+	XLogSource	readFrom;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -3513,6 +3521,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval milliseconds. However,
+	 * exhaust all the WAL present in pg_wal before switching. If successful,
+	 * the state machine moves to XLOG_FROM_STREAM state, otherwise it falls
+	 * back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3536,19 +3550,20 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
-		 * advance the state machine if so. The failure to read might've
+		 * First check if we failed to read from the current source or we
+		 * intentionally want to switch the source from archive to primary,
+		 * and advance the state machine if so. The failure to read might've
 		 * happened outside this function, e.g when a CRC check fails on a
 		 * record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed || switchSource)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
-			 * readahead.  Let the caller process everything that has been
-			 * decoded already first.
+			 * readahead if we failed to read from the current source. Let the
+			 * caller process everything that has been decoded already first.
 			 */
-			if (nonblocking)
+			if (nonblocking && lastSourceFailed)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3680,9 +3695,27 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we are switching to archive */
+			if (currentSource == XLOG_FROM_ARCHIVE)
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			ereport(DEBUG1,
+					errmsg_internal("switched WAL source from %s to %s after %s",
+									xlogSourceNames[oldSource],
+									xlogSourceNames[currentSource],
+									(switchSource ? "timeout" : (lastSourceFailed ? "failure" : "success"))));
+
+			/* Reset the WAL source switch state */
+			if (switchSource)
+			{
+				Assert(canSwitchSource);
+				Assert(currentSource == XLOG_FROM_STREAM);
+				Assert(oldSource == XLOG_FROM_ARCHIVE);
+				switchSource = false;
+				canSwitchSource = false;
+			}
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
@@ -3711,13 +3744,23 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/* See if we can switch WAL source to streaming */
+				if (!canSwitchSource)
+					canSwitchSource = SwitchWALSourceToPrimary();
+
 				/*
 				 * Try to restore the file from archive, or read an existing
-				 * file from pg_wal.
+				 * file from pg_wal. However, before switching WAL source to
+				 * streaming, give it a chance to read all the WAL from
+				 * pg_wal.
 				 */
-				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2,
-											  currentSource == XLOG_FROM_ARCHIVE ? XLOG_FROM_ANY :
-											  currentSource);
+				if (canSwitchSource)
+					readFrom = XLOG_FROM_PG_WAL;
+				else
+					readFrom = currentSource == XLOG_FROM_ARCHIVE ?
+						XLOG_FROM_ANY : currentSource;
+
+				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2, readFrom);
 				if (readFile >= 0)
 					return XLREAD_SUCCESS;	/* success! */
 
@@ -3725,6 +3768,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				 * Nope, not found in archive or pg_wal.
 				 */
 				lastSourceFailed = true;
+
+				/*
+				 * Read all the WAL in pg_wal. Now ready to switch to
+				 * streaming.
+				 */
+				if (canSwitchSource)
+					switchSource = true;
+
 				break;
 
 			case XLOG_FROM_STREAM:
@@ -3955,6 +4006,44 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * Check if standby can make an attempt to read WAL from primary after reading
+ * from archive for at least a configurable duration.
+ *
+ * Reading WAL from archive may not always be as efficient and fast as reading
+ * from primary. This can be due to the differences in disk types, IO costs,
+ * network latencies etc.. All of these can impact the recovery performance on
+ * standby and increase the replication lag on primary. In addition, the
+ * primary keeps accumulating WAL needed for the standby while the standby
+ * reads WAL from archive because the standby replication slot stays inactive.
+ * To avoid these problems, the standby will try to switch to stream mode
+ * sooner.
+ */
+static bool
+SwitchWALSourceToPrimary(void)
+{
+	TimestampTz now;
+
+	if (streaming_replication_retry_interval <= 0 ||
+		!StandbyMode ||
+		currentSource != XLOG_FROM_ARCHIVE)
+		return false;
+
+	now = GetCurrentTimestamp();
+
+	/* First time through */
+	if (switched_to_archive_at == 0)
+	{
+		switched_to_archive_at = now;
+		return false;
+	}
+
+	if (TimestampDifferenceExceeds(switched_to_archive_at, now,
+								   streaming_replication_retry_interval))
+		return true;
+
+	return false;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 3945a92ddd..74f58fcae4 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3211,6 +3211,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&streaming_replication_retry_interval,
+		300000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index b2809c711a..4dd27554aa 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -360,6 +360,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 5min	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication
+					# in milliseconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index ee0bc74278..2e9fb8dfe6 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build
index 9d8039684a..06041a5f74 100644
--- a/src/test/recovery/meson.build
+++ b/src/test/recovery/meson.build
@@ -45,6 +45,7 @@ tests += {
       't/037_invalid_database.pl',
       't/038_save_logical_slots_shutdown.pl',
       't/039_end_of_wal.pl',
+      't/040_wal_source_switch.pl',
     ],
   },
 }
diff --git a/src/test/recovery/t/040_wal_source_switch.pl b/src/test/recovery/t/040_wal_source_switch.pl
new file mode 100644
index 0000000000..b1ee9d6242
--- /dev/null
+++ b/src/test/recovery/t/040_wal_source_switch.pl
@@ -0,0 +1,93 @@
+# Copyright (c) 2023, PostgreSQL Global Development Group
+#
+# Test for WAL source switch feature.
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(
+	allows_streaming => 1,
+	has_archiving    => 1,
+	extra            => ['--wal-segsize=1']);
+
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf('postgresql.conf', qq(
+    min_wal_size = 2MB
+    max_wal_size = 1GB
+    checkpoint_timeout = 1h
+	autovacuum = off
+));
+$node_primary->start;
+
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('standby_slot')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create a streaming standby
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup(
+	$node_primary, $backup_name,
+	has_streaming => 1,
+	has_restoring => 1);
+$node_standby->append_conf('postgresql.conf', qq(
+	primary_slot_name = 'standby_slot'
+	streaming_replication_retry_interval = 1ms
+	log_min_messages = 'debug1'
+));
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+$node_standby->stop;
+
+# Advance WAL by 5 segments (= 5MB) on primary
+$node_primary->advance_wal(5);
+
+# Wait for primary to generate requested WAL files
+$node_primary->poll_query_until('postgres',
+	"SELECT COUNT(*) >= 5 FROM pg_ls_waldir();")
+  or die "Timed out while waiting for primary to generate WAL";
+
+# Wait until generated WAL files have been stored on the archives of the
+# primary. This ensures that the standby created below will be able to restore
+# the WAL files.
+my $primary_archive = $node_primary->archive_dir;
+$node_primary->poll_query_until('postgres',
+	"SELECT COUNT(*) >= 5 FROM pg_ls_dir('$primary_archive', false, false) a WHERE a ~ '^[0-9A-F]{24}\$';"
+) or die "Timed out while waiting for archiving of WAL by primary";
+
+# Generate some data on the primary
+$node_primary->safe_psql('postgres',
+	"CREATE TABLE test_tbl AS SELECT a FROM generate_series(1,5) AS a;");
+
+my $offset = -s $node_standby->logfile;
+
+# Standby now connects to primary during inital recovery after fetching WAL
+# from archive for about streaming_replication_retry_interval milliseconds.
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+$node_standby->wait_for_log(
+	qr/DEBUG: ( [A-Z0-9]+:)? switched WAL source from archive to stream after timeout/,
+	$offset);
+$node_standby->wait_for_log(
+	qr/LOG: ( [A-Z0-9]+:)? started streaming WAL from primary at .* on timeline .*/,
+	$offset);
+
+# Check that the data from primary is streamed to standby
+my $result =
+  $node_standby->safe_psql('postgres', "SELECT COUNT(*) FROM test_tbl;");
+is($result, '5', 'data from primary is streamed to standby');
+
+done_testing();
-- 
2.34.1

From 95a6f6c484dcff68f01ff90d9011abff2f15ad89 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Wed, 31 Jan 2024 11:59:02 +0000
Subject: [PATCH v17] Allow standby to switch WAL source from archive to
 streaming

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be as efficient and fast as reading from
primary. This can be due to the differences in disk types, IO
costs, network latencies etc.. All of these can impact the
recovery performance on standby and increase the replication lag
on primary. In addition, the primary keeps accumulating WAL needed
for the standby while the standby reads WAL from archive because
the standby replication slot stays inactive. To avoid these
problems, one can use this parameter to make standby switch to
stream mode sooner.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication (getting WAL from primary). However, standby
exhausts all the WAL present in pg_wal before switching. If
standby fails to switch to stream mode, it falls back to archive
mode.

Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi, SATYANARAYANA NARLAPURAM
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  47 +++++++
 doc/src/sgml/high-availability.sgml           |  15 ++-
 src/backend/access/transam/xlogrecovery.c     | 115 ++++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/meson.build                 |   1 +
 src/test/recovery/t/041_wal_source_switch.pl  |  93 ++++++++++++++
 8 files changed, 269 insertions(+), 19 deletions(-)
 create mode 100644 src/test/recovery/t/041_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 61038472c5..f0e45cf49d 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4867,6 +4867,53 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from archive to streaming replication (i.e., getting WAL from
+        primary). However, the standby exhausts all the WAL present in pg_wal
+        before switching. If the standby fails to switch to stream mode, it
+        falls back to archive mode. If this parameter value is specified
+        without units, it is taken as milliseconds. Default is
+        <literal>5min</literal>. With a lower value for this parameter, the
+        standby makes frequent WAL source switch attempts. To avoid this, it is
+        recommended to set a reasonable value. A setting of <literal>0</literal>
+        disables the feature. When disabled, the standby typically switches to
+        stream mode only after receiving WAL from archive finishes (i.e., no
+        more WAL left there) or fails for any reason. This parameter can only
+        be set in the <filename>postgresql.conf</filename> file or on the
+        server command line.
+       </para>
+       <note>
+        <para>
+         Standby may not always attempt to switch source from WAL archive to
+         streaming replication at exact
+         <varname>streaming_replication_retry_interval</varname> intervals. For
+         example, if the parameter is set to <literal>1min</literal> and
+         fetching WAL file from archive takes about <literal>2min</literal>,
+         then the source switch attempt happens for the next WAL file after
+         current WAL file fetched from archive is fully applied.
+        </para>
+       </note>
+       <para>
+        Reading WAL from archive may not always be as efficient and fast as
+        reading from primary. This can be due to the differences in disk types,
+        IO costs, network latencies etc.. All of these can impact the recovery
+        performance on standby and increase the replication lag on primary. In
+        addition, the primary keeps accumulating WAL needed for the standby
+        while the standby reads WAL from archive because the standby
+        replication slot stays inactive. To avoid these problems, one can use
+        this parameter to make standby switch to stream mode sooner.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index 236c0af65f..ab2e4293bf 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -628,12 +628,15 @@ protocol to make nodes agree on a serializable transactional order.
     In standby mode, the server continuously applies WAL received from the
     primary server. The standby server can read WAL from a WAL archive
     (see <xref linkend="guc-restore-command"/>) or directly from the primary
-    over a TCP connection (streaming replication). The standby server will
-    also attempt to restore any WAL found in the standby cluster's
-    <filename>pg_wal</filename> directory. That typically happens after a server
-    restart, when the standby replays again WAL that was streamed from the
-    primary before the restart, but you can also manually copy files to
-    <filename>pg_wal</filename> at any time to have them replayed.
+    over a TCP connection (streaming replication) or attempt to switch to
+    streaming replication after reading from archive when
+    <xref linkend="guc-streaming-replication-retry-interval"/> parameter is
+    set. The standby server will also attempt to restore any WAL found in the
+    standby cluster's <filename>pg_wal</filename> directory. That typically
+    happens after a server restart, when the standby replays again WAL that was
+    streamed from the primary before the restart, but you can also manually
+    copy files to <filename>pg_wal</filename> at any time to have them
+    replayed.
    </para>
 
    <para>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 0bb472da27..7f83ea22a1 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -91,6 +91,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 300000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -297,6 +298,8 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/* Holds the timestamp at which standby switched WAL source to archive */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +443,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static bool SwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3526,8 +3531,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	static bool canSwitchSource = false;
+	bool		switchSource = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
+	XLogSource	readFrom;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -3547,6 +3555,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval milliseconds. However,
+	 * exhaust all the WAL present in pg_wal before switching. If successful,
+	 * the state machine moves to XLOG_FROM_STREAM state, otherwise it falls
+	 * back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3570,19 +3584,20 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
-		 * advance the state machine if so. The failure to read might've
+		 * First check if we failed to read from the current source or we
+		 * intentionally want to switch the source from archive to primary,
+		 * and advance the state machine if so. The failure to read might've
 		 * happened outside this function, e.g when a CRC check fails on a
 		 * record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed || switchSource)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
-			 * readahead.  Let the caller process everything that has been
-			 * decoded already first.
+			 * readahead if we failed to read from the current source. Let the
+			 * caller process everything that has been decoded already first.
 			 */
-			if (nonblocking)
+			if (nonblocking && lastSourceFailed)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3714,9 +3729,27 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we are switching to archive */
+			if (currentSource == XLOG_FROM_ARCHIVE)
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			ereport(DEBUG1,
+					errmsg_internal("switched WAL source from %s to %s after %s",
+									xlogSourceNames[oldSource],
+									xlogSourceNames[currentSource],
+									(switchSource ? "timeout" : (lastSourceFailed ? "failure" : "success"))));
+
+			/* Reset the WAL source switch state */
+			if (switchSource)
+			{
+				Assert(canSwitchSource);
+				Assert(currentSource == XLOG_FROM_STREAM);
+				Assert(oldSource == XLOG_FROM_ARCHIVE);
+				switchSource = false;
+				canSwitchSource = false;
+			}
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
@@ -3745,13 +3778,23 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/* See if we can switch WAL source to streaming */
+				if (!canSwitchSource)
+					canSwitchSource = SwitchWALSourceToPrimary();
+
 				/*
 				 * Try to restore the file from archive, or read an existing
-				 * file from pg_wal.
+				 * file from pg_wal. However, before switching WAL source to
+				 * streaming, give it a chance to read all the WAL from
+				 * pg_wal.
 				 */
-				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2,
-											  currentSource == XLOG_FROM_ARCHIVE ? XLOG_FROM_ANY :
-											  currentSource);
+				if (canSwitchSource)
+					readFrom = XLOG_FROM_PG_WAL;
+				else
+					readFrom = currentSource == XLOG_FROM_ARCHIVE ?
+						XLOG_FROM_ANY : currentSource;
+
+				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2, readFrom);
 				if (readFile >= 0)
 					return XLREAD_SUCCESS;	/* success! */
 
@@ -3759,6 +3802,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				 * Nope, not found in archive or pg_wal.
 				 */
 				lastSourceFailed = true;
+
+				/*
+				 * Read all the WAL in pg_wal. Now ready to switch to
+				 * streaming.
+				 */
+				if (canSwitchSource)
+					switchSource = true;
+
 				break;
 
 			case XLOG_FROM_STREAM:
@@ -3989,6 +4040,44 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * Check if standby can make an attempt to read WAL from primary after reading
+ * from archive for at least a configurable duration.
+ *
+ * Reading WAL from archive may not always be as efficient and fast as reading
+ * from primary. This can be due to the differences in disk types, IO costs,
+ * network latencies etc.. All of these can impact the recovery performance on
+ * standby and increase the replication lag on primary. In addition, the
+ * primary keeps accumulating WAL needed for the standby while the standby
+ * reads WAL from archive because the standby replication slot stays inactive.
+ * To avoid these problems, the standby will try to switch to stream mode
+ * sooner.
+ */
+static bool
+SwitchWALSourceToPrimary(void)
+{
+	TimestampTz now;
+
+	if (streaming_replication_retry_interval <= 0 ||
+		!StandbyMode ||
+		currentSource != XLOG_FROM_ARCHIVE)
+		return false;
+
+	now = GetCurrentTimestamp();
+
+	/* First time through */
+	if (switched_to_archive_at == 0)
+	{
+		switched_to_archive_at = now;
+		return false;
+	}
+
+	if (TimestampDifferenceExceeds(switched_to_archive_at, now,
+								   streaming_replication_retry_interval))
+		return true;
+
+	return false;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 7fe58518d7..6179107371 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3221,6 +3221,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&streaming_replication_retry_interval,
+		300000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index da10b43dac..19e8f8f5be 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -360,6 +360,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 5min	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication
+					# in milliseconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index c423464e8b..73c5a86f4c 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build
index bf087ac2a9..d891199462 100644
--- a/src/test/recovery/meson.build
+++ b/src/test/recovery/meson.build
@@ -46,6 +46,7 @@ tests += {
       't/038_save_logical_slots_shutdown.pl',
       't/039_end_of_wal.pl',
       't/040_standby_failover_slots_sync.pl',
+      't/041_wal_source_switch.pl',
     ],
   },
 }
diff --git a/src/test/recovery/t/041_wal_source_switch.pl b/src/test/recovery/t/041_wal_source_switch.pl
new file mode 100644
index 0000000000..b1ee9d6242
--- /dev/null
+++ b/src/test/recovery/t/041_wal_source_switch.pl
@@ -0,0 +1,93 @@
+# Copyright (c) 2024, PostgreSQL Global Development Group
+#
+# Test for WAL source switch feature.
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(
+	allows_streaming => 1,
+	has_archiving    => 1,
+	extra            => ['--wal-segsize=1']);
+
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf('postgresql.conf', qq(
+    min_wal_size = 2MB
+    max_wal_size = 1GB
+    checkpoint_timeout = 1h
+	autovacuum = off
+));
+$node_primary->start;
+
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('standby_slot')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create a streaming standby
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup(
+	$node_primary, $backup_name,
+	has_streaming => 1,
+	has_restoring => 1);
+$node_standby->append_conf('postgresql.conf', qq(
+	primary_slot_name = 'standby_slot'
+	streaming_replication_retry_interval = 1ms
+	log_min_messages = 'debug1'
+));
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+$node_standby->stop;
+
+# Advance WAL by 5 segments (= 5MB) on primary
+$node_primary->advance_wal(5);
+
+# Wait for primary to generate requested WAL files
+$node_primary->poll_query_until('postgres',
+	"SELECT COUNT(*) >= 5 FROM pg_ls_waldir();")
+  or die "Timed out while waiting for primary to generate WAL";
+
+# Wait until generated WAL files have been stored on the archives of the
+# primary. This ensures that the standby created below will be able to restore
+# the WAL files.
+my $primary_archive = $node_primary->archive_dir;
+$node_primary->poll_query_until('postgres',
+	"SELECT COUNT(*) >= 5 FROM pg_ls_dir('$primary_archive', false, false) a WHERE a ~ '^[0-9A-F]{24}\$';"
+) or die "Timed out while waiting for archiving of WAL by primary";
+
+# Generate some data on the primary
+$node_primary->safe_psql('postgres',
+	"CREATE TABLE test_tbl AS SELECT a FROM generate_series(1,5) AS a;");
+
+my $offset = -s $node_standby->logfile;
+
+# Standby now connects to primary during inital recovery after fetching WAL
+# from archive for about streaming_replication_retry_interval milliseconds.
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+$node_standby->wait_for_log(
+	qr/DEBUG: ( [A-Z0-9]+:)? switched WAL source from archive to stream after timeout/,
+	$offset);
+$node_standby->wait_for_log(
+	qr/LOG: ( [A-Z0-9]+:)? started streaming WAL from primary at .* on timeline .*/,
+	$offset);
+
+# Check that the data from primary is streamed to standby
+my $result =
+  $node_standby->safe_psql('postgres', "SELECT COUNT(*) FROM test_tbl;");
+is($result, '5', 'data from primary is streamed to standby');
+
+done_testing();
-- 
2.34.1

From 6eb23bc2527a4457fe5b7272240b5e3d7f8a3949 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Mon, 19 Feb 2024 09:09:33 +0000
Subject: [PATCH v18] Allow standby to switch WAL source from archive to
 streaming

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be as efficient and fast as reading from
primary. This can be due to the differences in disk types, IO
costs, network latencies etc.. All of these can impact the
recovery performance on standby and increase the replication lag
on primary. In addition, the primary keeps accumulating WAL needed
for the standby while the standby reads WAL from archive because
the standby replication slot stays inactive. To avoid these
problems, one can use this parameter to make standby switch to
stream mode sooner.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication (getting WAL from primary). However, standby
exhausts all the WAL present in pg_wal before switching. If
standby fails to switch to stream mode, it falls back to archive
mode.

Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi, SATYANARAYANA NARLAPURAM
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  47 +++++++
 doc/src/sgml/high-availability.sgml           |  15 ++-
 src/backend/access/transam/xlogrecovery.c     | 115 ++++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/meson.build                 |   1 +
 src/test/recovery/t/041_wal_source_switch.pl  | 108 ++++++++++++++++
 8 files changed, 284 insertions(+), 19 deletions(-)
 create mode 100644 src/test/recovery/t/041_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index ffd711b7f2..e9a4f3062c 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4872,6 +4872,53 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from archive to streaming replication (i.e., getting WAL from
+        primary). However, the standby exhausts all the WAL present in pg_wal
+        before switching. If the standby fails to switch to stream mode, it
+        falls back to archive mode. If this parameter value is specified
+        without units, it is taken as milliseconds. Default is
+        <literal>5min</literal>. With a lower value for this parameter, the
+        standby makes frequent WAL source switch attempts. To avoid this, it is
+        recommended to set a reasonable value. A setting of <literal>0</literal>
+        disables the feature. When disabled, the standby typically switches to
+        stream mode only after receiving WAL from archive finishes (i.e., no
+        more WAL left there) or fails for any reason. This parameter can only
+        be set in the <filename>postgresql.conf</filename> file or on the
+        server command line.
+       </para>
+       <note>
+        <para>
+         Standby may not always attempt to switch source from WAL archive to
+         streaming replication at exact
+         <varname>streaming_replication_retry_interval</varname> intervals. For
+         example, if the parameter is set to <literal>1min</literal> and
+         fetching WAL file from archive takes about <literal>2min</literal>,
+         then the source switch attempt happens for the next WAL file after
+         current WAL file fetched from archive is fully applied.
+        </para>
+       </note>
+       <para>
+        Reading WAL from archive may not always be as efficient and fast as
+        reading from primary. This can be due to the differences in disk types,
+        IO costs, network latencies etc.. All of these can impact the recovery
+        performance on standby and increase the replication lag on primary. In
+        addition, the primary keeps accumulating WAL needed for the standby
+        while the standby reads WAL from archive because the standby
+        replication slot stays inactive. To avoid these problems, one can use
+        this parameter to make standby switch to stream mode sooner.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index 236c0af65f..ab2e4293bf 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -628,12 +628,15 @@ protocol to make nodes agree on a serializable transactional order.
     In standby mode, the server continuously applies WAL received from the
     primary server. The standby server can read WAL from a WAL archive
     (see <xref linkend="guc-restore-command"/>) or directly from the primary
-    over a TCP connection (streaming replication). The standby server will
-    also attempt to restore any WAL found in the standby cluster's
-    <filename>pg_wal</filename> directory. That typically happens after a server
-    restart, when the standby replays again WAL that was streamed from the
-    primary before the restart, but you can also manually copy files to
-    <filename>pg_wal</filename> at any time to have them replayed.
+    over a TCP connection (streaming replication) or attempt to switch to
+    streaming replication after reading from archive when
+    <xref linkend="guc-streaming-replication-retry-interval"/> parameter is
+    set. The standby server will also attempt to restore any WAL found in the
+    standby cluster's <filename>pg_wal</filename> directory. That typically
+    happens after a server restart, when the standby replays again WAL that was
+    streamed from the primary before the restart, but you can also manually
+    copy files to <filename>pg_wal</filename> at any time to have them
+    replayed.
    </para>
 
    <para>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 0bb472da27..7f83ea22a1 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -91,6 +91,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 300000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -297,6 +298,8 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/* Holds the timestamp at which standby switched WAL source to archive */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +443,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static bool SwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3526,8 +3531,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	static bool canSwitchSource = false;
+	bool		switchSource = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
+	XLogSource	readFrom;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -3547,6 +3555,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval milliseconds. However,
+	 * exhaust all the WAL present in pg_wal before switching. If successful,
+	 * the state machine moves to XLOG_FROM_STREAM state, otherwise it falls
+	 * back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3570,19 +3584,20 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
-		 * advance the state machine if so. The failure to read might've
+		 * First check if we failed to read from the current source or we
+		 * intentionally want to switch the source from archive to primary,
+		 * and advance the state machine if so. The failure to read might've
 		 * happened outside this function, e.g when a CRC check fails on a
 		 * record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed || switchSource)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
-			 * readahead.  Let the caller process everything that has been
-			 * decoded already first.
+			 * readahead if we failed to read from the current source. Let the
+			 * caller process everything that has been decoded already first.
 			 */
-			if (nonblocking)
+			if (nonblocking && lastSourceFailed)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3714,9 +3729,27 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we are switching to archive */
+			if (currentSource == XLOG_FROM_ARCHIVE)
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			ereport(DEBUG1,
+					errmsg_internal("switched WAL source from %s to %s after %s",
+									xlogSourceNames[oldSource],
+									xlogSourceNames[currentSource],
+									(switchSource ? "timeout" : (lastSourceFailed ? "failure" : "success"))));
+
+			/* Reset the WAL source switch state */
+			if (switchSource)
+			{
+				Assert(canSwitchSource);
+				Assert(currentSource == XLOG_FROM_STREAM);
+				Assert(oldSource == XLOG_FROM_ARCHIVE);
+				switchSource = false;
+				canSwitchSource = false;
+			}
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
@@ -3745,13 +3778,23 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/* See if we can switch WAL source to streaming */
+				if (!canSwitchSource)
+					canSwitchSource = SwitchWALSourceToPrimary();
+
 				/*
 				 * Try to restore the file from archive, or read an existing
-				 * file from pg_wal.
+				 * file from pg_wal. However, before switching WAL source to
+				 * streaming, give it a chance to read all the WAL from
+				 * pg_wal.
 				 */
-				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2,
-											  currentSource == XLOG_FROM_ARCHIVE ? XLOG_FROM_ANY :
-											  currentSource);
+				if (canSwitchSource)
+					readFrom = XLOG_FROM_PG_WAL;
+				else
+					readFrom = currentSource == XLOG_FROM_ARCHIVE ?
+						XLOG_FROM_ANY : currentSource;
+
+				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2, readFrom);
 				if (readFile >= 0)
 					return XLREAD_SUCCESS;	/* success! */
 
@@ -3759,6 +3802,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				 * Nope, not found in archive or pg_wal.
 				 */
 				lastSourceFailed = true;
+
+				/*
+				 * Read all the WAL in pg_wal. Now ready to switch to
+				 * streaming.
+				 */
+				if (canSwitchSource)
+					switchSource = true;
+
 				break;
 
 			case XLOG_FROM_STREAM:
@@ -3989,6 +4040,44 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * Check if standby can make an attempt to read WAL from primary after reading
+ * from archive for at least a configurable duration.
+ *
+ * Reading WAL from archive may not always be as efficient and fast as reading
+ * from primary. This can be due to the differences in disk types, IO costs,
+ * network latencies etc.. All of these can impact the recovery performance on
+ * standby and increase the replication lag on primary. In addition, the
+ * primary keeps accumulating WAL needed for the standby while the standby
+ * reads WAL from archive because the standby replication slot stays inactive.
+ * To avoid these problems, the standby will try to switch to stream mode
+ * sooner.
+ */
+static bool
+SwitchWALSourceToPrimary(void)
+{
+	TimestampTz now;
+
+	if (streaming_replication_retry_interval <= 0 ||
+		!StandbyMode ||
+		currentSource != XLOG_FROM_ARCHIVE)
+		return false;
+
+	now = GetCurrentTimestamp();
+
+	/* First time through */
+	if (switched_to_archive_at == 0)
+	{
+		switched_to_archive_at = now;
+		return false;
+	}
+
+	if (TimestampDifferenceExceeds(switched_to_archive_at, now,
+								   streaming_replication_retry_interval))
+		return true;
+
+	return false;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 70652f0a3f..8c835a37a3 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3232,6 +3232,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&streaming_replication_retry_interval,
+		300000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index e10755972a..1aec929dbc 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -360,6 +360,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 5min	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication
+					# in milliseconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index c423464e8b..73c5a86f4c 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build
index bf087ac2a9..d891199462 100644
--- a/src/test/recovery/meson.build
+++ b/src/test/recovery/meson.build
@@ -46,6 +46,7 @@ tests += {
       't/038_save_logical_slots_shutdown.pl',
       't/039_end_of_wal.pl',
       't/040_standby_failover_slots_sync.pl',
+      't/041_wal_source_switch.pl',
     ],
   },
 }
diff --git a/src/test/recovery/t/041_wal_source_switch.pl b/src/test/recovery/t/041_wal_source_switch.pl
new file mode 100644
index 0000000000..a664547948
--- /dev/null
+++ b/src/test/recovery/t/041_wal_source_switch.pl
@@ -0,0 +1,108 @@
+# Copyright (c) 2024, PostgreSQL Global Development Group
+#
+# Test for WAL source switch feature.
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(
+	allows_streaming => 1,
+	has_archiving    => 1,
+	extra            => ['--wal-segsize=1']);
+
+# Ensure checkpoint doesn't come in our way
+$primary->append_conf('postgresql.conf', qq(
+    min_wal_size = 2MB
+    max_wal_size = 1GB
+    checkpoint_timeout = 1h
+	autovacuum = off
+));
+$primary->start;
+
+$primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('standby_slot')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$primary->backup($backup_name);
+
+# Create a streaming standby
+my $standby = PostgreSQL::Test::Cluster->new('standby');
+$standby->init_from_backup(
+	$primary, $backup_name,
+	has_streaming => 1,
+	has_restoring => 1);
+
+my $retry_interval = 1;
+$standby->append_conf('postgresql.conf', qq(
+	primary_slot_name = 'standby_slot'
+	streaming_replication_retry_interval = '${retry_interval}ms'
+	log_min_messages = 'debug1'
+));
+$standby->start;
+
+# Wait until standby has replayed enough data
+$primary->wait_for_catchup($standby);
+
+$standby->stop;
+
+# Advance WAL by 5 segments (= 5MB) on primary
+$primary->advance_wal(5);
+
+# Wait for primary to generate requested WAL files
+$primary->poll_query_until('postgres',
+	"SELECT COUNT(*) >= 5 FROM pg_ls_waldir();")
+  or die "Timed out while waiting for primary to generate WAL";
+
+# Wait until generated WAL files have been stored on the archives of the
+# primary. This ensures that the standby created below will be able to restore
+# the WAL files.
+my $primary_archive = $primary->archive_dir;
+$primary->poll_query_until('postgres',
+	"SELECT COUNT(*) >= 5 FROM pg_ls_dir('$primary_archive', false, false) a WHERE a ~ '^[0-9A-F]{24}\$';"
+) or die "Timed out while waiting for archiving of WAL by primary";
+
+# Generate some data on the primary
+$primary->safe_psql('postgres',
+	"CREATE TABLE test_tbl AS SELECT a FROM generate_series(1,5) AS a;");
+
+my $offset = -s $standby->logfile;
+
+# Standby initially fetches WAL from archive after the restart. Since it is
+# asked to retry fetching from primary after retry interval
+# (i.e. streaming_replication_retry_interval), it will do so. To mimic the
+# standby spending some time fetching from archive, we use apply delay
+# (i.e. recovery_min_apply_delay) greater than the retry interval, so that for
+# fetching the next WAL file the standby honours retry interval and fetches it
+# from primary.
+my $apply_delay = $retry_interval * 5;
+$standby->append_conf(
+	'postgresql.conf', qq(
+recovery_min_apply_delay = '${apply_delay}ms'
+));
+$standby->start;
+
+# Wait until standby has replayed enough data
+$primary->wait_for_catchup($standby);
+
+$standby->wait_for_log(
+	qr/DEBUG: ( [A-Z0-9]+:)? switched WAL source from archive to stream after timeout/,
+	$offset);
+$standby->wait_for_log(
+	qr/LOG: ( [A-Z0-9]+:)? started streaming WAL from primary at .* on timeline .*/,
+	$offset);
+
+# Check that the data from primary is streamed to standby
+my $result =
+  $standby->safe_psql('postgres', "SELECT COUNT(*) FROM test_tbl;");
+is($result, '5', 'data from primary is streamed to standby');
+
+$standby->stop;
+$primary->stop;
+
+done_testing();
-- 
2.34.1

From 910220fcd8e186c2e79990158bff9972cc9c604a Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Tue, 20 Feb 2024 05:37:24 +0000
Subject: [PATCH v19] Allow standby to switch WAL source from archive to
 streaming

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be as efficient and fast as reading from
primary. This can be due to the differences in disk types, IO
costs, network latencies etc.. All of these can impact the
recovery performance on standby and increase the replication lag
on primary. In addition, the primary keeps accumulating WAL needed
for the standby while the standby reads WAL from archive because
the standby replication slot stays inactive. To avoid these
problems, one can use this parameter to make standby switch to
stream mode sooner.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication (getting WAL from primary). However, standby
exhausts all the WAL present in pg_wal before switching. If
standby fails to switch to stream mode, it falls back to archive
mode.

Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi, SATYANARAYANA NARLAPURAM
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  48 ++++++++
 doc/src/sgml/high-availability.sgml           |  15 ++-
 src/backend/access/transam/xlogrecovery.c     | 115 ++++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/meson.build                 |   1 +
 src/test/recovery/t/041_wal_source_switch.pl  | 110 +++++++++++++++++
 8 files changed, 287 insertions(+), 19 deletions(-)
 create mode 100644 src/test/recovery/t/041_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index ffd711b7f2..2fbf1ad6e1 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4872,6 +4872,54 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from archive to streaming replication (i.e., getting WAL from
+        primary). However, the standby exhausts all the WAL present in
+        <filename>pg_wal</filename> directory before switching. If the standby
+        fails to switch to stream mode, it falls back to archive mode. If this
+        parameter value is specified without units, it is taken as
+        milliseconds. Default is <literal>5min</literal>. With a lower value
+        for this parameter, the standby makes frequent WAL source switch
+        attempts. To avoid this, it is recommended to set a reasonable value.
+        A setting of <literal>0</literal> disables the feature. When disabled,
+        the standby typically switches to stream mode only after receiving WAL
+        from archive finishes (i.e., no more WAL left there) or fails for any
+        reason. This parameter can only be set in the
+        <filename>postgresql.conf</filename> file or on the server command
+        line.
+       </para>
+       <note>
+        <para>
+         Standby may not always attempt to switch source from WAL archive to
+         streaming replication at exact
+         <varname>streaming_replication_retry_interval</varname> intervals. For
+         example, if the parameter is set to <literal>1min</literal> and
+         fetching WAL file from archive takes about <literal>2min</literal>,
+         then the source switch attempt happens for the next WAL file after
+         current WAL file fetched from archive is fully applied.
+        </para>
+       </note>
+       <para>
+        Reading WAL from archive may not always be as efficient and fast as
+        reading from primary. This can be due to the differences in disk types,
+        IO costs, network latencies etc.. All of these can impact the recovery
+        performance on standby and increase the replication lag on primary. In
+        addition, the primary keeps accumulating WAL needed for the standby
+        while the standby reads WAL from archive because the standby
+        replication slot stays inactive. To avoid these problems, one can use
+        this parameter to make standby switch to stream mode sooner.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index 236c0af65f..ab2e4293bf 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -628,12 +628,15 @@ protocol to make nodes agree on a serializable transactional order.
     In standby mode, the server continuously applies WAL received from the
     primary server. The standby server can read WAL from a WAL archive
     (see <xref linkend="guc-restore-command"/>) or directly from the primary
-    over a TCP connection (streaming replication). The standby server will
-    also attempt to restore any WAL found in the standby cluster's
-    <filename>pg_wal</filename> directory. That typically happens after a server
-    restart, when the standby replays again WAL that was streamed from the
-    primary before the restart, but you can also manually copy files to
-    <filename>pg_wal</filename> at any time to have them replayed.
+    over a TCP connection (streaming replication) or attempt to switch to
+    streaming replication after reading from archive when
+    <xref linkend="guc-streaming-replication-retry-interval"/> parameter is
+    set. The standby server will also attempt to restore any WAL found in the
+    standby cluster's <filename>pg_wal</filename> directory. That typically
+    happens after a server restart, when the standby replays again WAL that was
+    streamed from the primary before the restart, but you can also manually
+    copy files to <filename>pg_wal</filename> at any time to have them
+    replayed.
    </para>
 
    <para>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 0bb472da27..7f83ea22a1 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -91,6 +91,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 300000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -297,6 +298,8 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/* Holds the timestamp at which standby switched WAL source to archive */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +443,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static bool SwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3526,8 +3531,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	static bool canSwitchSource = false;
+	bool		switchSource = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
+	XLogSource	readFrom;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -3547,6 +3555,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval milliseconds. However,
+	 * exhaust all the WAL present in pg_wal before switching. If successful,
+	 * the state machine moves to XLOG_FROM_STREAM state, otherwise it falls
+	 * back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3570,19 +3584,20 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
-		 * advance the state machine if so. The failure to read might've
+		 * First check if we failed to read from the current source or we
+		 * intentionally want to switch the source from archive to primary,
+		 * and advance the state machine if so. The failure to read might've
 		 * happened outside this function, e.g when a CRC check fails on a
 		 * record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed || switchSource)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
-			 * readahead.  Let the caller process everything that has been
-			 * decoded already first.
+			 * readahead if we failed to read from the current source. Let the
+			 * caller process everything that has been decoded already first.
 			 */
-			if (nonblocking)
+			if (nonblocking && lastSourceFailed)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3714,9 +3729,27 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we are switching to archive */
+			if (currentSource == XLOG_FROM_ARCHIVE)
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			ereport(DEBUG1,
+					errmsg_internal("switched WAL source from %s to %s after %s",
+									xlogSourceNames[oldSource],
+									xlogSourceNames[currentSource],
+									(switchSource ? "timeout" : (lastSourceFailed ? "failure" : "success"))));
+
+			/* Reset the WAL source switch state */
+			if (switchSource)
+			{
+				Assert(canSwitchSource);
+				Assert(currentSource == XLOG_FROM_STREAM);
+				Assert(oldSource == XLOG_FROM_ARCHIVE);
+				switchSource = false;
+				canSwitchSource = false;
+			}
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
@@ -3745,13 +3778,23 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/* See if we can switch WAL source to streaming */
+				if (!canSwitchSource)
+					canSwitchSource = SwitchWALSourceToPrimary();
+
 				/*
 				 * Try to restore the file from archive, or read an existing
-				 * file from pg_wal.
+				 * file from pg_wal. However, before switching WAL source to
+				 * streaming, give it a chance to read all the WAL from
+				 * pg_wal.
 				 */
-				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2,
-											  currentSource == XLOG_FROM_ARCHIVE ? XLOG_FROM_ANY :
-											  currentSource);
+				if (canSwitchSource)
+					readFrom = XLOG_FROM_PG_WAL;
+				else
+					readFrom = currentSource == XLOG_FROM_ARCHIVE ?
+						XLOG_FROM_ANY : currentSource;
+
+				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2, readFrom);
 				if (readFile >= 0)
 					return XLREAD_SUCCESS;	/* success! */
 
@@ -3759,6 +3802,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				 * Nope, not found in archive or pg_wal.
 				 */
 				lastSourceFailed = true;
+
+				/*
+				 * Read all the WAL in pg_wal. Now ready to switch to
+				 * streaming.
+				 */
+				if (canSwitchSource)
+					switchSource = true;
+
 				break;
 
 			case XLOG_FROM_STREAM:
@@ -3989,6 +4040,44 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * Check if standby can make an attempt to read WAL from primary after reading
+ * from archive for at least a configurable duration.
+ *
+ * Reading WAL from archive may not always be as efficient and fast as reading
+ * from primary. This can be due to the differences in disk types, IO costs,
+ * network latencies etc.. All of these can impact the recovery performance on
+ * standby and increase the replication lag on primary. In addition, the
+ * primary keeps accumulating WAL needed for the standby while the standby
+ * reads WAL from archive because the standby replication slot stays inactive.
+ * To avoid these problems, the standby will try to switch to stream mode
+ * sooner.
+ */
+static bool
+SwitchWALSourceToPrimary(void)
+{
+	TimestampTz now;
+
+	if (streaming_replication_retry_interval <= 0 ||
+		!StandbyMode ||
+		currentSource != XLOG_FROM_ARCHIVE)
+		return false;
+
+	now = GetCurrentTimestamp();
+
+	/* First time through */
+	if (switched_to_archive_at == 0)
+	{
+		switched_to_archive_at = now;
+		return false;
+	}
+
+	if (TimestampDifferenceExceeds(switched_to_archive_at, now,
+								   streaming_replication_retry_interval))
+		return true;
+
+	return false;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 70652f0a3f..8c835a37a3 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3232,6 +3232,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&streaming_replication_retry_interval,
+		300000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index e10755972a..1aec929dbc 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -360,6 +360,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 5min	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication
+					# in milliseconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index c423464e8b..73c5a86f4c 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build
index bf087ac2a9..d891199462 100644
--- a/src/test/recovery/meson.build
+++ b/src/test/recovery/meson.build
@@ -46,6 +46,7 @@ tests += {
       't/038_save_logical_slots_shutdown.pl',
       't/039_end_of_wal.pl',
       't/040_standby_failover_slots_sync.pl',
+      't/041_wal_source_switch.pl',
     ],
   },
 }
diff --git a/src/test/recovery/t/041_wal_source_switch.pl b/src/test/recovery/t/041_wal_source_switch.pl
new file mode 100644
index 0000000000..082680bf4a
--- /dev/null
+++ b/src/test/recovery/t/041_wal_source_switch.pl
@@ -0,0 +1,110 @@
+# Copyright (c) 2024, PostgreSQL Global Development Group
+#
+# Test for WAL source switch feature.
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(
+	allows_streaming => 1,
+	has_archiving => 1,
+	extra => ['--wal-segsize=1']);
+
+# Ensure checkpoint doesn't come in our way
+$primary->append_conf(
+	'postgresql.conf', qq(
+    min_wal_size = 2MB
+    max_wal_size = 1GB
+    checkpoint_timeout = 1h
+	autovacuum = off
+));
+$primary->start;
+
+$primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('standby_slot')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$primary->backup($backup_name);
+
+# Create a streaming standby
+my $standby = PostgreSQL::Test::Cluster->new('standby');
+$standby->init_from_backup(
+	$primary, $backup_name,
+	has_streaming => 1,
+	has_restoring => 1);
+
+my $retry_interval = 1;
+$standby->append_conf(
+	'postgresql.conf', qq(
+	primary_slot_name = 'standby_slot'
+	streaming_replication_retry_interval = '${retry_interval}ms'
+	log_min_messages = 'debug1'
+));
+$standby->start;
+
+# Wait until standby has replayed enough data
+$primary->wait_for_catchup($standby);
+
+$standby->stop;
+
+# Advance WAL by 5 segments (= 5MB) on primary
+$primary->advance_wal(5);
+
+# Wait for primary to generate requested WAL files
+$primary->poll_query_until('postgres',
+	"SELECT COUNT(*) >= 5 FROM pg_ls_waldir();")
+  or die "Timed out while waiting for primary to generate WAL";
+
+# Wait until generated WAL files have been stored on the archives of the
+# primary. This ensures that the standby created below will be able to restore
+# the WAL files.
+my $primary_archive = $primary->archive_dir;
+$primary->poll_query_until('postgres',
+	"SELECT COUNT(*) >= 5 FROM pg_ls_dir('$primary_archive', false, false) a WHERE a ~ '^[0-9A-F]{24}\$';"
+) or die "Timed out while waiting for archiving of WAL by primary";
+
+# Generate some data on the primary
+$primary->safe_psql('postgres',
+	"CREATE TABLE test_tbl AS SELECT a FROM generate_series(1,5) AS a;");
+
+my $offset = -s $standby->logfile;
+
+# Standby initially fetches WAL from archive after the restart. Since it is
+# asked to retry fetching from primary after retry interval
+# (i.e. streaming_replication_retry_interval), it will do so. To mimic the
+# standby spending some time fetching from archive, we use apply delay
+# (i.e. recovery_min_apply_delay) greater than the retry interval, so that for
+# fetching the next WAL file the standby honours retry interval and fetches it
+# from primary.
+my $apply_delay = $retry_interval * 5;
+$standby->append_conf(
+	'postgresql.conf', qq(
+recovery_min_apply_delay = '${apply_delay}ms'
+));
+$standby->start;
+
+# Wait until standby has replayed enough data
+$primary->wait_for_catchup($standby);
+
+$standby->wait_for_log(
+	qr/DEBUG: ( [A-Z0-9]+:)? switched WAL source from archive to stream after timeout/,
+	$offset);
+$standby->wait_for_log(
+	qr/LOG: ( [A-Z0-9]+:)? started streaming WAL from primary at .* on timeline .*/,
+	$offset);
+
+# Check that the data from primary is streamed to standby
+my $result =
+  $standby->safe_psql('postgres', "SELECT COUNT(*) FROM test_tbl;");
+is($result, '5', 'data from primary is streamed to standby');
+
+$standby->stop;
+$primary->stop;
+
+done_testing();
-- 
2.34.1

diff --git a/src/test/recovery/t/041_wal_source_switch.pl b/src/test/recovery/t/041_wal_source_switch.pl
index 082680bf4a..b5eddba1d5 100644
--- a/src/test/recovery/t/041_wal_source_switch.pl
+++ b/src/test/recovery/t/041_wal_source_switch.pl
@@ -18,9 +18,9 @@ $primary->init(
 # Ensure checkpoint doesn't come in our way
 $primary->append_conf(
 	'postgresql.conf', qq(
-    min_wal_size = 2MB
-    max_wal_size = 1GB
-    checkpoint_timeout = 1h
+	min_wal_size = 2MB
+	max_wal_size = 1GB
+	checkpoint_timeout = 1h
 	autovacuum = off
 ));
 $primary->start;
@@ -85,7 +85,7 @@ my $offset = -s $standby->logfile;
 my $apply_delay = $retry_interval * 5;
 $standby->append_conf(
 	'postgresql.conf', qq(
-recovery_min_apply_delay = '${apply_delay}ms'
+	recovery_min_apply_delay = '${apply_delay}ms'
 ));
 $standby->start;
 

From beea9f0b8bbc76bb48dd1a5d64a5b52bafd09e6f Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Tue, 20 Feb 2024 07:31:29 +0000
Subject: [PATCH v20] Allow standby to switch WAL source from archive to
 streaming

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be as efficient and fast as reading from
primary. This can be due to the differences in disk types, IO
costs, network latencies etc.. All of these can impact the
recovery performance on standby and increase the replication lag
on primary. In addition, the primary keeps accumulating WAL needed
for the standby while the standby reads WAL from archive because
the standby replication slot stays inactive. To avoid these
problems, one can use this parameter to make standby switch to
stream mode sooner.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication (getting WAL from primary). However, standby
exhausts all the WAL present in pg_wal before switching. If
standby fails to switch to stream mode, it falls back to archive
mode.

Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi, SATYANARAYANA NARLAPURAM
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  48 ++++++++
 doc/src/sgml/high-availability.sgml           |  15 ++-
 src/backend/access/transam/xlogrecovery.c     | 115 ++++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   4 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/meson.build                 |   1 +
 src/test/recovery/t/041_wal_source_switch.pl  | 110 +++++++++++++++++
 8 files changed, 287 insertions(+), 19 deletions(-)
 create mode 100644 src/test/recovery/t/041_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index ffd711b7f2..2fbf1ad6e1 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4872,6 +4872,54 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from archive to streaming replication (i.e., getting WAL from
+        primary). However, the standby exhausts all the WAL present in
+        <filename>pg_wal</filename> directory before switching. If the standby
+        fails to switch to stream mode, it falls back to archive mode. If this
+        parameter value is specified without units, it is taken as
+        milliseconds. Default is <literal>5min</literal>. With a lower value
+        for this parameter, the standby makes frequent WAL source switch
+        attempts. To avoid this, it is recommended to set a reasonable value.
+        A setting of <literal>0</literal> disables the feature. When disabled,
+        the standby typically switches to stream mode only after receiving WAL
+        from archive finishes (i.e., no more WAL left there) or fails for any
+        reason. This parameter can only be set in the
+        <filename>postgresql.conf</filename> file or on the server command
+        line.
+       </para>
+       <note>
+        <para>
+         Standby may not always attempt to switch source from WAL archive to
+         streaming replication at exact
+         <varname>streaming_replication_retry_interval</varname> intervals. For
+         example, if the parameter is set to <literal>1min</literal> and
+         fetching WAL file from archive takes about <literal>2min</literal>,
+         then the source switch attempt happens for the next WAL file after
+         current WAL file fetched from archive is fully applied.
+        </para>
+       </note>
+       <para>
+        Reading WAL from archive may not always be as efficient and fast as
+        reading from primary. This can be due to the differences in disk types,
+        IO costs, network latencies etc.. All of these can impact the recovery
+        performance on standby and increase the replication lag on primary. In
+        addition, the primary keeps accumulating WAL needed for the standby
+        while the standby reads WAL from archive because the standby
+        replication slot stays inactive. To avoid these problems, one can use
+        this parameter to make standby switch to stream mode sooner.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index 236c0af65f..ab2e4293bf 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -628,12 +628,15 @@ protocol to make nodes agree on a serializable transactional order.
     In standby mode, the server continuously applies WAL received from the
     primary server. The standby server can read WAL from a WAL archive
     (see <xref linkend="guc-restore-command"/>) or directly from the primary
-    over a TCP connection (streaming replication). The standby server will
-    also attempt to restore any WAL found in the standby cluster's
-    <filename>pg_wal</filename> directory. That typically happens after a server
-    restart, when the standby replays again WAL that was streamed from the
-    primary before the restart, but you can also manually copy files to
-    <filename>pg_wal</filename> at any time to have them replayed.
+    over a TCP connection (streaming replication) or attempt to switch to
+    streaming replication after reading from archive when
+    <xref linkend="guc-streaming-replication-retry-interval"/> parameter is
+    set. The standby server will also attempt to restore any WAL found in the
+    standby cluster's <filename>pg_wal</filename> directory. That typically
+    happens after a server restart, when the standby replays again WAL that was
+    streamed from the primary before the restart, but you can also manually
+    copy files to <filename>pg_wal</filename> at any time to have them
+    replayed.
    </para>
 
    <para>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 0bb472da27..7f83ea22a1 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -91,6 +91,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 300000;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -297,6 +298,8 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/* Holds the timestamp at which standby switched WAL source to archive */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +443,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static bool SwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3526,8 +3531,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	static bool canSwitchSource = false;
+	bool		switchSource = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
+	XLogSource	readFrom;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -3547,6 +3555,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval milliseconds. However,
+	 * exhaust all the WAL present in pg_wal before switching. If successful,
+	 * the state machine moves to XLOG_FROM_STREAM state, otherwise it falls
+	 * back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3570,19 +3584,20 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
-		 * advance the state machine if so. The failure to read might've
+		 * First check if we failed to read from the current source or we
+		 * intentionally want to switch the source from archive to primary,
+		 * and advance the state machine if so. The failure to read might've
 		 * happened outside this function, e.g when a CRC check fails on a
 		 * record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed || switchSource)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
-			 * readahead.  Let the caller process everything that has been
-			 * decoded already first.
+			 * readahead if we failed to read from the current source. Let the
+			 * caller process everything that has been decoded already first.
 			 */
-			if (nonblocking)
+			if (nonblocking && lastSourceFailed)
 				return XLREAD_WOULDBLOCK;
 
 			switch (currentSource)
@@ -3714,9 +3729,27 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we are switching to archive */
+			if (currentSource == XLOG_FROM_ARCHIVE)
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			ereport(DEBUG1,
+					errmsg_internal("switched WAL source from %s to %s after %s",
+									xlogSourceNames[oldSource],
+									xlogSourceNames[currentSource],
+									(switchSource ? "timeout" : (lastSourceFailed ? "failure" : "success"))));
+
+			/* Reset the WAL source switch state */
+			if (switchSource)
+			{
+				Assert(canSwitchSource);
+				Assert(currentSource == XLOG_FROM_STREAM);
+				Assert(oldSource == XLOG_FROM_ARCHIVE);
+				switchSource = false;
+				canSwitchSource = false;
+			}
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
@@ -3745,13 +3778,23 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/* See if we can switch WAL source to streaming */
+				if (!canSwitchSource)
+					canSwitchSource = SwitchWALSourceToPrimary();
+
 				/*
 				 * Try to restore the file from archive, or read an existing
-				 * file from pg_wal.
+				 * file from pg_wal. However, before switching WAL source to
+				 * streaming, give it a chance to read all the WAL from
+				 * pg_wal.
 				 */
-				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2,
-											  currentSource == XLOG_FROM_ARCHIVE ? XLOG_FROM_ANY :
-											  currentSource);
+				if (canSwitchSource)
+					readFrom = XLOG_FROM_PG_WAL;
+				else
+					readFrom = currentSource == XLOG_FROM_ARCHIVE ?
+						XLOG_FROM_ANY : currentSource;
+
+				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2, readFrom);
 				if (readFile >= 0)
 					return XLREAD_SUCCESS;	/* success! */
 
@@ -3759,6 +3802,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				 * Nope, not found in archive or pg_wal.
 				 */
 				lastSourceFailed = true;
+
+				/*
+				 * Read all the WAL in pg_wal. Now ready to switch to
+				 * streaming.
+				 */
+				if (canSwitchSource)
+					switchSource = true;
+
 				break;
 
 			case XLOG_FROM_STREAM:
@@ -3989,6 +4040,44 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * Check if standby can make an attempt to read WAL from primary after reading
+ * from archive for at least a configurable duration.
+ *
+ * Reading WAL from archive may not always be as efficient and fast as reading
+ * from primary. This can be due to the differences in disk types, IO costs,
+ * network latencies etc.. All of these can impact the recovery performance on
+ * standby and increase the replication lag on primary. In addition, the
+ * primary keeps accumulating WAL needed for the standby while the standby
+ * reads WAL from archive because the standby replication slot stays inactive.
+ * To avoid these problems, the standby will try to switch to stream mode
+ * sooner.
+ */
+static bool
+SwitchWALSourceToPrimary(void)
+{
+	TimestampTz now;
+
+	if (streaming_replication_retry_interval <= 0 ||
+		!StandbyMode ||
+		currentSource != XLOG_FROM_ARCHIVE)
+		return false;
+
+	now = GetCurrentTimestamp();
+
+	/* First time through */
+	if (switched_to_archive_at == 0)
+	{
+		switched_to_archive_at = now;
+		return false;
+	}
+
+	if (TimestampDifferenceExceeds(switched_to_archive_at, now,
+								   streaming_replication_retry_interval))
+		return true;
+
+	return false;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 70652f0a3f..8c835a37a3 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3232,6 +3232,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_MS
+		},
+		&streaming_replication_retry_interval,
+		300000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index e10755972a..1aec929dbc 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -360,6 +360,10 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 5min	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication
+					# in milliseconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 
 # - Subscribers -
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index c423464e8b..73c5a86f4c 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build
index bf087ac2a9..d891199462 100644
--- a/src/test/recovery/meson.build
+++ b/src/test/recovery/meson.build
@@ -46,6 +46,7 @@ tests += {
       't/038_save_logical_slots_shutdown.pl',
       't/039_end_of_wal.pl',
       't/040_standby_failover_slots_sync.pl',
+      't/041_wal_source_switch.pl',
     ],
   },
 }
diff --git a/src/test/recovery/t/041_wal_source_switch.pl b/src/test/recovery/t/041_wal_source_switch.pl
new file mode 100644
index 0000000000..1a9637ec86
--- /dev/null
+++ b/src/test/recovery/t/041_wal_source_switch.pl
@@ -0,0 +1,110 @@
+# Copyright (c) 2024, PostgreSQL Global Development Group
+#
+# Test for WAL source switch feature.
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+# Initialize primary node, setting wal-segsize to 1MB
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(
+	allows_streaming => 1,
+	has_archiving => 1,
+	extra => ['--wal-segsize=1']);
+
+# Ensure checkpoint doesn't come in our way
+$primary->append_conf(
+	'postgresql.conf', qq(
+min_wal_size = 2MB
+max_wal_size = 1GB
+checkpoint_timeout = 1h
+autovacuum = off
+));
+$primary->start;
+
+$primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('standby_slot')");
+
+# Take backup
+my $backup_name = 'my_backup';
+$primary->backup($backup_name);
+
+# Create a streaming standby
+my $standby = PostgreSQL::Test::Cluster->new('standby');
+$standby->init_from_backup(
+	$primary, $backup_name,
+	has_streaming => 1,
+	has_restoring => 1);
+
+my $retry_interval = 1;
+$standby->append_conf(
+	'postgresql.conf', qq(
+primary_slot_name = 'standby_slot'
+streaming_replication_retry_interval = '${retry_interval}ms'
+log_min_messages = 'debug1'
+));
+$standby->start;
+
+# Wait until standby has replayed enough data
+$primary->wait_for_catchup($standby);
+
+$standby->stop;
+
+# Advance WAL by 5 segments (= 5MB) on primary
+$primary->advance_wal(5);
+
+# Wait for primary to generate requested WAL files
+$primary->poll_query_until('postgres',
+	"SELECT COUNT(*) >= 5 FROM pg_ls_waldir();")
+  or die "Timed out while waiting for primary to generate WAL";
+
+# Wait until generated WAL files have been stored on the archives of the
+# primary. This ensures that the standby created below will be able to restore
+# the WAL files.
+my $primary_archive = $primary->archive_dir;
+$primary->poll_query_until('postgres',
+	"SELECT COUNT(*) >= 5 FROM pg_ls_dir('$primary_archive', false, false) a WHERE a ~ '^[0-9A-F]{24}\$';"
+) or die "Timed out while waiting for archiving of WAL by primary";
+
+# Generate some data on the primary
+$primary->safe_psql('postgres',
+	"CREATE TABLE test_tbl AS SELECT a FROM generate_series(1,5) AS a;");
+
+my $offset = -s $standby->logfile;
+
+# Standby initially fetches WAL from archive after the restart. Since it is
+# asked to retry fetching from primary after retry interval
+# (i.e. streaming_replication_retry_interval), it will do so. To mimic the
+# standby spending some time fetching from archive, we use apply delay
+# (i.e. recovery_min_apply_delay) greater than the retry interval, so that for
+# fetching the next WAL file the standby honours retry interval and fetches it
+# from primary.
+my $apply_delay = $retry_interval * 5;
+$standby->append_conf(
+	'postgresql.conf', qq(
+recovery_min_apply_delay = '${apply_delay}ms'
+));
+$standby->start;
+
+# Wait until standby has replayed enough data
+$primary->wait_for_catchup($standby);
+
+$standby->wait_for_log(
+	qr/DEBUG: ( [A-Z0-9]+:)? switched WAL source from archive to stream after timeout/,
+	$offset);
+$standby->wait_for_log(
+	qr/LOG: ( [A-Z0-9]+:)? started streaming WAL from primary at .* on timeline .*/,
+	$offset);
+
+# Check that the data from primary is streamed to standby
+my $result =
+  $standby->safe_psql('postgres', "SELECT COUNT(*) FROM test_tbl;");
+is($result, '5', 'data from primary is streamed to standby');
+
+$standby->stop;
+$primary->stop;
+
+done_testing();
-- 
2.34.1

From 82eb49593a563295a7370aa1d87db94c8aa313db Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Tue, 5 Mar 2024 17:37:50 +0000
Subject: [PATCH v21] Allow standby to switch WAL source from archive to
 streaming

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be as efficient and fast as reading from
primary. This can be due to the differences in disk types, IO
costs, network latencies etc.. All of these can impact the
recovery performance on standby and increase the replication lag
on primary. In addition, the primary keeps accumulating WAL needed
for the standby while the standby reads WAL from archive because
the standby replication slot stays inactive. To avoid these
problems, one can use this parameter to make standby switch to
stream mode sooner.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication (getting WAL from primary). However, standby
exhausts all the WAL present in pg_wal before switching. If
standby fails to switch to stream mode, it falls back to archive
mode.

Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi, SATYANARAYANA NARLAPURAM
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  49 ++++++++
 doc/src/sgml/high-availability.sgml           |  15 ++-
 src/backend/access/transam/xlogrecovery.c     | 107 +++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   3 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/meson.build                 |   1 +
 src/test/recovery/t/042_wal_source_switch.pl  | 113 ++++++++++++++++++
 8 files changed, 286 insertions(+), 15 deletions(-)
 create mode 100644 src/test/recovery/t/042_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index b38cbd714a..02e79f32fb 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -5011,6 +5011,55 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from archive to streaming replication (i.e., getting WAL from
+        primary). However, the standby exhausts all the WAL present in
+        <filename>pg_wal</filename> directory before switching. If the standby
+        fails to switch to stream mode, it falls back to archive mode. If this
+        parameter value is specified without units, it is taken as seconds.
+        With a lower value for this parameter, the standby makes frequent WAL
+        source switch attempts. To avoid this, it is recommended to set a
+        value depending on the rate of WAL generation on the primary. If the
+        WAL files grow faster, the disk runs out of space sooner, so setting a
+        value to make frequent WAL source switch attempts can help. The default
+        is zero, disabling this feature. When disabled, the standby typically
+        switches to stream mode only after receiving WAL from archive finishes
+        (i.e., no more WAL left there) or fails for any reason. This parameter
+        can only be set in the <filename>postgresql.conf</filename> file or on
+        the server command line.
+       </para>
+       <note>
+        <para>
+         Standby may not always attempt to switch source from WAL archive to
+         streaming replication at exact
+         <varname>streaming_replication_retry_interval</varname> intervals. For
+         example, if the parameter is set to <literal>1min</literal> and
+         fetching WAL file from archive takes about <literal>2min</literal>,
+         then the source switch attempt happens for the next WAL file after
+         the current WAL file fetched from archive is fully applied.
+        </para>
+       </note>
+       <para>
+        Reading WAL from archive may not always be as efficient and fast as
+        reading from primary. This can be due to the differences in disk types,
+        IO costs, network latencies etc. All of these can impact the recovery
+        performance on standby, and can increase the replication lag on
+        primary. In addition, the primary keeps accumulating WAL needed for the
+        standby while the standby reads WAL from archive, because the standby
+        replication slot stays inactive. To avoid these problems, one can use
+        this parameter to make standby switch to stream mode sooner.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index 236c0af65f..ab2e4293bf 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -628,12 +628,15 @@ protocol to make nodes agree on a serializable transactional order.
     In standby mode, the server continuously applies WAL received from the
     primary server. The standby server can read WAL from a WAL archive
     (see <xref linkend="guc-restore-command"/>) or directly from the primary
-    over a TCP connection (streaming replication). The standby server will
-    also attempt to restore any WAL found in the standby cluster's
-    <filename>pg_wal</filename> directory. That typically happens after a server
-    restart, when the standby replays again WAL that was streamed from the
-    primary before the restart, but you can also manually copy files to
-    <filename>pg_wal</filename> at any time to have them replayed.
+    over a TCP connection (streaming replication) or attempt to switch to
+    streaming replication after reading from archive when
+    <xref linkend="guc-streaming-replication-retry-interval"/> parameter is
+    set. The standby server will also attempt to restore any WAL found in the
+    standby cluster's <filename>pg_wal</filename> directory. That typically
+    happens after a server restart, when the standby replays again WAL that was
+    streamed from the primary before the restart, but you can also manually
+    copy files to <filename>pg_wal</filename> at any time to have them
+    replayed.
    </para>
 
    <para>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 853b540945..ca73234695 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -91,6 +91,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 0;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -297,6 +298,8 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/* Holds the timestamp at which standby switched WAL source to archive */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +443,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static bool SwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3541,8 +3546,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool nonblocking)
 {
 	static TimestampTz last_fail_time = 0;
+	static bool canSwitchSource = false;
+	bool		switchSource = false;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
+	XLogSource	readFrom;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -3562,6 +3570,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval seconds. However,
+	 * exhaust all the WAL present in pg_wal before switching. If successful,
+	 * the state machine moves to XLOG_FROM_STREAM state, otherwise it falls
+	 * back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3585,12 +3599,13 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
+		 * First check if we failed to read from the current source or we
+		 * intentionally want to switch the source from archive to stream, and
 		 * advance the state machine if so. The failure to read might've
 		 * happened outside this function, e.g when a CRC check fails on a
 		 * record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed || switchSource)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
@@ -3729,9 +3744,27 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we are switching to archive */
+			if (currentSource == XLOG_FROM_ARCHIVE)
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			ereport(DEBUG1,
+					errmsg_internal("switched WAL source from %s to %s after %s",
+									xlogSourceNames[oldSource],
+									xlogSourceNames[currentSource],
+									(switchSource ? "timeout" : (lastSourceFailed ? "failure" : "success"))));
+
+			/* Reset the WAL source switch state */
+			if (switchSource)
+			{
+				Assert(canSwitchSource);
+				Assert(currentSource == XLOG_FROM_STREAM);
+				Assert(oldSource == XLOG_FROM_ARCHIVE);
+				switchSource = false;
+				canSwitchSource = false;
+			}
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
@@ -3760,13 +3793,23 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/* See if we can switch WAL source to streaming */
+				if (!canSwitchSource)
+					canSwitchSource = SwitchWALSourceToPrimary();
+
 				/*
 				 * Try to restore the file from archive, or read an existing
-				 * file from pg_wal.
+				 * file from pg_wal. However, before switching WAL source to
+				 * streaming, give it a chance to read all the WAL from
+				 * pg_wal.
 				 */
-				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2,
-											  currentSource == XLOG_FROM_ARCHIVE ? XLOG_FROM_ANY :
-											  currentSource);
+				if (canSwitchSource)
+					readFrom = XLOG_FROM_PG_WAL;
+				else
+					readFrom = currentSource == XLOG_FROM_ARCHIVE ?
+						XLOG_FROM_ANY : currentSource;
+
+				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2, readFrom);
 				if (readFile >= 0)
 					return XLREAD_SUCCESS;	/* success! */
 
@@ -3774,6 +3817,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				 * Nope, not found in archive or pg_wal.
 				 */
 				lastSourceFailed = true;
+
+				/*
+				 * Read all the WAL in pg_wal. Now ready to switch to
+				 * streaming.
+				 */
+				if (canSwitchSource)
+					switchSource = true;
+
 				break;
 
 			case XLOG_FROM_STREAM:
@@ -4004,6 +4055,44 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * Check if standby can make an attempt to read WAL from primary after reading
+ * from archive for at least a configurable duration.
+ *
+ * Reading WAL from archive may not always be as efficient and fast as reading
+ * from primary. This can be due to the differences in disk types, IO costs,
+ * network latencies etc. All of these can impact the recovery performance on
+ * standby and increase the replication lag on primary. In addition, the
+ * primary keeps accumulating WAL needed for the standby while the standby
+ * reads WAL from archive because the standby replication slot stays inactive.
+ * To avoid these problems, the standby will try to switch to stream mode
+ * sooner.
+ */
+static bool
+SwitchWALSourceToPrimary(void)
+{
+	TimestampTz now;
+
+	if (streaming_replication_retry_interval <= 0 ||
+		!StandbyMode ||
+		currentSource != XLOG_FROM_ARCHIVE)
+		return false;
+
+	now = GetCurrentTimestamp();
+
+	/* First time through */
+	if (switched_to_archive_at == 0)
+	{
+		switched_to_archive_at = now;
+		return false;
+	}
+
+	if (TimestampDifferenceExceeds(switched_to_archive_at, now,
+								   streaming_replication_retry_interval * 1000))
+		return true;
+
+	return false;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 45013582a7..e54d82dd1c 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3273,6 +3273,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_S
+		},
+		&streaming_replication_retry_interval,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index edcc0282b2..6f87209d62 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -369,6 +369,9 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 0	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication in seconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 #sync_replication_slots = off			# enables slot synchronization on the physical standby from the primary
 
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index c423464e8b..73c5a86f4c 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build
index c67249500e..3a8ecd5e54 100644
--- a/src/test/recovery/meson.build
+++ b/src/test/recovery/meson.build
@@ -50,6 +50,7 @@ tests += {
       't/039_end_of_wal.pl',
       't/040_standby_failover_slots_sync.pl',
       't/041_checkpoint_at_promote.pl',
+      't/042_wal_source_switch.pl',
     ],
   },
 }
diff --git a/src/test/recovery/t/042_wal_source_switch.pl b/src/test/recovery/t/042_wal_source_switch.pl
new file mode 100644
index 0000000000..b00ed29f73
--- /dev/null
+++ b/src/test/recovery/t/042_wal_source_switch.pl
@@ -0,0 +1,113 @@
+
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+# Checks for WAL source switch feature.
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# Initialize primary node
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(allows_streaming => 1, has_archiving => 1);
+
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf(
+	'postgresql.conf', qq(
+checkpoint_timeout = 1h
+autovacuum = off
+));
+$node_primary->start;
+
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('standby_slot')");
+
+# And some content
+$node_primary->safe_psql('postgres',
+	"CREATE TABLE tab_int AS SELECT generate_series(1, 10) AS a");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create streaming standby from backup
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup(
+	$node_primary, $backup_name,
+	has_streaming => 1,
+	has_restoring => 1);
+
+my $retry_interval = 1;
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+primary_slot_name = 'standby_slot'
+streaming_replication_retry_interval = '${retry_interval}s'
+log_min_messages = 'debug2'
+));
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+# Generate some data on the primary while the standby is down
+$node_standby->stop;
+for my $i (1 .. 10)
+{
+	$node_primary->safe_psql('postgres',
+		"INSERT INTO tab_int VALUES (generate_series(11, 20));");
+	$node_primary->safe_psql('postgres', "SELECT pg_switch_wal();");
+}
+
+# Now wait for replay to complete on standby. We're done waiting when the
+# standby has replayed up to the previously saved primary LSN.
+my $cur_lsn =
+  $node_primary->safe_psql('postgres', "SELECT pg_current_wal_lsn()");
+
+# Generate 1 more WAL file so that we wait predictably for the archiving of
+# all WAL files.
+$node_primary->advance_wal(1);
+
+my $walfile_name =
+  $node_primary->safe_psql('postgres', "SELECT pg_walfile_name('$cur_lsn')");
+
+$node_primary->poll_query_until('postgres',
+	"SELECT count(*) = 1 FROM pg_stat_archiver WHERE last_archived_wal = '$walfile_name';"
+) or die "Timed out while waiting for archiving of WAL by primary";
+
+my $offset = -s $node_standby->logfile;
+
+# Standby initially fetches WAL from archive after the restart. Since it is
+# asked to retry fetching from primary after retry interval
+# (i.e. streaming_replication_retry_interval), it will do so. To mimic the
+# standby spending some time fetching from archive, we use apply delay
+# (i.e. recovery_min_apply_delay) greater than the retry interval, so that for
+# fetching the next WAL file the standby honours retry interval and fetches it
+# from primary.
+my $delay = $retry_interval * 5;
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+recovery_min_apply_delay = '${delay}s'
+));
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+$node_standby->wait_for_log(
+	qr/DEBUG: ( [A-Z0-9]+:)? switched WAL source from archive to stream after timeout/,
+	$offset);
+$node_standby->wait_for_log(
+	qr/LOG: ( [A-Z0-9]+:)? started streaming WAL from primary at .* on timeline .*/,
+	$offset);
+
+# Check that the data from primary is streamed to standby
+my $row_cnt1 =
+  $node_primary->safe_psql('postgres', "SELECT count(*) FROM tab_int;");
+
+my $row_cnt2 =
+  $node_standby->safe_psql('postgres', "SELECT count(*) FROM tab_int;");
+is($row_cnt1, $row_cnt2, 'data from primary is streamed to standby');
+
+done_testing();
-- 
2.34.1

From f2c9d1a170ce4cc536ce9139d7a2e0fdc268be69 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Wed, 6 Mar 2024 04:12:26 +0000
Subject: [PATCH v22] Allow standby to switch WAL source from archive to
 streaming

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be as efficient and fast as reading from
primary. This can be due to the differences in disk types, IO
costs, network latencies etc.. All of these can impact the
recovery performance on standby and increase the replication lag
on primary. In addition, the primary keeps accumulating WAL needed
for the standby while the standby reads WAL from archive because
the standby replication slot stays inactive. To avoid these
problems, one can use this parameter to make standby switch to
stream mode sooner.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication (getting WAL from primary). However, standby
exhausts all the WAL present in pg_wal before switching. If
standby fails to switch to stream mode, it falls back to archive
mode.

Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi, SATYANARAYANA NARLAPURAM
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  49 ++++++++
 doc/src/sgml/high-availability.sgml           |  15 ++-
 src/backend/access/transam/xlogrecovery.c     | 117 ++++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   3 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/meson.build                 |   1 +
 src/test/recovery/t/042_wal_source_switch.pl  | 113 +++++++++++++++++
 8 files changed, 296 insertions(+), 15 deletions(-)
 create mode 100644 src/test/recovery/t/042_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index b38cbd714a..02e79f32fb 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -5011,6 +5011,55 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from archive to streaming replication (i.e., getting WAL from
+        primary). However, the standby exhausts all the WAL present in
+        <filename>pg_wal</filename> directory before switching. If the standby
+        fails to switch to stream mode, it falls back to archive mode. If this
+        parameter value is specified without units, it is taken as seconds.
+        With a lower value for this parameter, the standby makes frequent WAL
+        source switch attempts. To avoid this, it is recommended to set a
+        value depending on the rate of WAL generation on the primary. If the
+        WAL files grow faster, the disk runs out of space sooner, so setting a
+        value to make frequent WAL source switch attempts can help. The default
+        is zero, disabling this feature. When disabled, the standby typically
+        switches to stream mode only after receiving WAL from archive finishes
+        (i.e., no more WAL left there) or fails for any reason. This parameter
+        can only be set in the <filename>postgresql.conf</filename> file or on
+        the server command line.
+       </para>
+       <note>
+        <para>
+         Standby may not always attempt to switch source from WAL archive to
+         streaming replication at exact
+         <varname>streaming_replication_retry_interval</varname> intervals. For
+         example, if the parameter is set to <literal>1min</literal> and
+         fetching WAL file from archive takes about <literal>2min</literal>,
+         then the source switch attempt happens for the next WAL file after
+         the current WAL file fetched from archive is fully applied.
+        </para>
+       </note>
+       <para>
+        Reading WAL from archive may not always be as efficient and fast as
+        reading from primary. This can be due to the differences in disk types,
+        IO costs, network latencies etc. All of these can impact the recovery
+        performance on standby, and can increase the replication lag on
+        primary. In addition, the primary keeps accumulating WAL needed for the
+        standby while the standby reads WAL from archive, because the standby
+        replication slot stays inactive. To avoid these problems, one can use
+        this parameter to make standby switch to stream mode sooner.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index 236c0af65f..ab2e4293bf 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -628,12 +628,15 @@ protocol to make nodes agree on a serializable transactional order.
     In standby mode, the server continuously applies WAL received from the
     primary server. The standby server can read WAL from a WAL archive
     (see <xref linkend="guc-restore-command"/>) or directly from the primary
-    over a TCP connection (streaming replication). The standby server will
-    also attempt to restore any WAL found in the standby cluster's
-    <filename>pg_wal</filename> directory. That typically happens after a server
-    restart, when the standby replays again WAL that was streamed from the
-    primary before the restart, but you can also manually copy files to
-    <filename>pg_wal</filename> at any time to have them replayed.
+    over a TCP connection (streaming replication) or attempt to switch to
+    streaming replication after reading from archive when
+    <xref linkend="guc-streaming-replication-retry-interval"/> parameter is
+    set. The standby server will also attempt to restore any WAL found in the
+    standby cluster's <filename>pg_wal</filename> directory. That typically
+    happens after a server restart, when the standby replays again WAL that was
+    streamed from the primary before the restart, but you can also manually
+    copy files to <filename>pg_wal</filename> at any time to have them
+    replayed.
    </para>
 
    <para>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 853b540945..14e846883e 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -91,6 +91,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 0;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -297,6 +298,20 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/*
+ * Represents WAL source switch states.
+ */
+typedef enum
+{
+	SWITCH_TO_STREAMING_NONE,	/* no switch necessary */
+	SWITCH_TO_STREAMING_PENDING,	/* exhausting pg_wal */
+	SWITCH_TO_STREAMING,		/* switch to streaming now */
+}			WALSourceSwitchState;
+
+static WALSourceSwitchState wal_source_switch_state = SWITCH_TO_STREAMING_NONE;
+
+/* Holds the timestamp at which standby switched WAL source to archive */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +455,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static WALSourceSwitchState SwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3543,6 +3560,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	static TimestampTz last_fail_time = 0;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
+	XLogSource	readFrom;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -3562,6 +3580,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval seconds. However,
+	 * exhaust all the WAL present in pg_wal before switching. If successful,
+	 * the state machine moves to XLOG_FROM_STREAM state, otherwise it falls
+	 * back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3585,12 +3609,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
+		 * First check if we failed to read from the current source or we
+		 * intentionally want to switch the source from archive to stream, and
 		 * advance the state machine if so. The failure to read might've
 		 * happened outside this function, e.g when a CRC check fails on a
 		 * record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed ||
+			wal_source_switch_state == SWITCH_TO_STREAMING)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
@@ -3729,9 +3755,26 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we are switching to archive */
+			if (currentSource == XLOG_FROM_ARCHIVE)
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			ereport(DEBUG1,
+					errmsg_internal("switched WAL source from %s to %s after %s",
+									xlogSourceNames[oldSource],
+									xlogSourceNames[currentSource],
+									((wal_source_switch_state == SWITCH_TO_STREAMING) ? "timeout" : (lastSourceFailed ? "failure" : "success"))));
+
+			/* Reset the WAL source switch state */
+			if (wal_source_switch_state == SWITCH_TO_STREAMING)
+			{
+				Assert(oldSource == XLOG_FROM_ARCHIVE);
+				Assert(currentSource == XLOG_FROM_STREAM);
+
+				wal_source_switch_state = SWITCH_TO_STREAMING_NONE;
+			}
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
@@ -3760,13 +3803,23 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/* See if we can switch WAL source to streaming */
+				if (wal_source_switch_state == SWITCH_TO_STREAMING_NONE)
+					wal_source_switch_state = SwitchWALSourceToPrimary();
+
 				/*
 				 * Try to restore the file from archive, or read an existing
-				 * file from pg_wal.
+				 * file from pg_wal. However, before switching WAL source to
+				 * streaming, give it a chance to read all the WAL from
+				 * pg_wal.
 				 */
-				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2,
-											  currentSource == XLOG_FROM_ARCHIVE ? XLOG_FROM_ANY :
-											  currentSource);
+				if (wal_source_switch_state == SWITCH_TO_STREAMING_PENDING)
+					readFrom = XLOG_FROM_PG_WAL;
+				else
+					readFrom = currentSource == XLOG_FROM_ARCHIVE ?
+						XLOG_FROM_ANY : currentSource;
+
+				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2, readFrom);
 				if (readFile >= 0)
 					return XLREAD_SUCCESS;	/* success! */
 
@@ -3774,6 +3827,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				 * Nope, not found in archive or pg_wal.
 				 */
 				lastSourceFailed = true;
+
+				/*
+				 * Read all the WAL in pg_wal. Now ready to switch to
+				 * streaming.
+				 */
+				if (wal_source_switch_state == SWITCH_TO_STREAMING_PENDING)
+					wal_source_switch_state = SWITCH_TO_STREAMING;
+
 				break;
 
 			case XLOG_FROM_STREAM:
@@ -4004,6 +4065,44 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * Check if standby can make an attempt to read WAL from primary after reading
+ * from archive for at least a configurable duration.
+ *
+ * Reading WAL from archive may not always be as efficient and fast as reading
+ * from primary. This can be due to the differences in disk types, IO costs,
+ * network latencies etc. All of these can impact the recovery performance on
+ * standby and increase the replication lag on primary. In addition, the
+ * primary keeps accumulating WAL needed for the standby while the standby
+ * reads WAL from archive because the standby replication slot stays inactive.
+ * To avoid these problems, the standby will try to switch to stream mode
+ * sooner.
+ */
+static WALSourceSwitchState
+SwitchWALSourceToPrimary(void)
+{
+	TimestampTz now;
+
+	if (streaming_replication_retry_interval <= 0 ||
+		!StandbyMode ||
+		currentSource != XLOG_FROM_ARCHIVE)
+		return SWITCH_TO_STREAMING_NONE;
+
+	now = GetCurrentTimestamp();
+
+	/* First time through */
+	if (switched_to_archive_at == 0)
+	{
+		switched_to_archive_at = now;
+		return SWITCH_TO_STREAMING_NONE;
+	}
+
+	if (TimestampDifferenceExceeds(switched_to_archive_at, now,
+								   streaming_replication_retry_interval * 1000))
+		return SWITCH_TO_STREAMING_PENDING;
+
+	return SWITCH_TO_STREAMING_NONE;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 45013582a7..e54d82dd1c 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3273,6 +3273,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_S
+		},
+		&streaming_replication_retry_interval,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index edcc0282b2..6f87209d62 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -369,6 +369,9 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 0	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication in seconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 #sync_replication_slots = off			# enables slot synchronization on the physical standby from the primary
 
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index c423464e8b..73c5a86f4c 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build
index c67249500e..3a8ecd5e54 100644
--- a/src/test/recovery/meson.build
+++ b/src/test/recovery/meson.build
@@ -50,6 +50,7 @@ tests += {
       't/039_end_of_wal.pl',
       't/040_standby_failover_slots_sync.pl',
       't/041_checkpoint_at_promote.pl',
+      't/042_wal_source_switch.pl',
     ],
   },
 }
diff --git a/src/test/recovery/t/042_wal_source_switch.pl b/src/test/recovery/t/042_wal_source_switch.pl
new file mode 100644
index 0000000000..b00ed29f73
--- /dev/null
+++ b/src/test/recovery/t/042_wal_source_switch.pl
@@ -0,0 +1,113 @@
+
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+# Checks for WAL source switch feature.
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# Initialize primary node
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(allows_streaming => 1, has_archiving => 1);
+
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf(
+	'postgresql.conf', qq(
+checkpoint_timeout = 1h
+autovacuum = off
+));
+$node_primary->start;
+
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('standby_slot')");
+
+# And some content
+$node_primary->safe_psql('postgres',
+	"CREATE TABLE tab_int AS SELECT generate_series(1, 10) AS a");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create streaming standby from backup
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup(
+	$node_primary, $backup_name,
+	has_streaming => 1,
+	has_restoring => 1);
+
+my $retry_interval = 1;
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+primary_slot_name = 'standby_slot'
+streaming_replication_retry_interval = '${retry_interval}s'
+log_min_messages = 'debug2'
+));
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+# Generate some data on the primary while the standby is down
+$node_standby->stop;
+for my $i (1 .. 10)
+{
+	$node_primary->safe_psql('postgres',
+		"INSERT INTO tab_int VALUES (generate_series(11, 20));");
+	$node_primary->safe_psql('postgres', "SELECT pg_switch_wal();");
+}
+
+# Now wait for replay to complete on standby. We're done waiting when the
+# standby has replayed up to the previously saved primary LSN.
+my $cur_lsn =
+  $node_primary->safe_psql('postgres', "SELECT pg_current_wal_lsn()");
+
+# Generate 1 more WAL file so that we wait predictably for the archiving of
+# all WAL files.
+$node_primary->advance_wal(1);
+
+my $walfile_name =
+  $node_primary->safe_psql('postgres', "SELECT pg_walfile_name('$cur_lsn')");
+
+$node_primary->poll_query_until('postgres',
+	"SELECT count(*) = 1 FROM pg_stat_archiver WHERE last_archived_wal = '$walfile_name';"
+) or die "Timed out while waiting for archiving of WAL by primary";
+
+my $offset = -s $node_standby->logfile;
+
+# Standby initially fetches WAL from archive after the restart. Since it is
+# asked to retry fetching from primary after retry interval
+# (i.e. streaming_replication_retry_interval), it will do so. To mimic the
+# standby spending some time fetching from archive, we use apply delay
+# (i.e. recovery_min_apply_delay) greater than the retry interval, so that for
+# fetching the next WAL file the standby honours retry interval and fetches it
+# from primary.
+my $delay = $retry_interval * 5;
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+recovery_min_apply_delay = '${delay}s'
+));
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+$node_standby->wait_for_log(
+	qr/DEBUG: ( [A-Z0-9]+:)? switched WAL source from archive to stream after timeout/,
+	$offset);
+$node_standby->wait_for_log(
+	qr/LOG: ( [A-Z0-9]+:)? started streaming WAL from primary at .* on timeline .*/,
+	$offset);
+
+# Check that the data from primary is streamed to standby
+my $row_cnt1 =
+  $node_primary->safe_psql('postgres', "SELECT count(*) FROM tab_int;");
+
+my $row_cnt2 =
+  $node_standby->safe_psql('postgres', "SELECT count(*) FROM tab_int;");
+is($row_cnt1, $row_cnt2, 'data from primary is streamed to standby');
+
+done_testing();
-- 
2.34.1

From b451e18b7b1f91eb3a6857efe552e7fe97cc7f39 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Sun, 17 Mar 2024 05:48:38 +0000
Subject: [PATCH v23] Allow standby to switch WAL source from archive to
 streaming

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be as efficient and fast as reading from
primary. This can be due to the differences in disk types, IO
costs, network latencies etc.. All of these can impact the
recovery performance on standby and increase the replication lag
on primary. In addition, the primary keeps accumulating WAL needed
for the standby while the standby reads WAL from archive because
the standby replication slot stays inactive. To avoid these
problems, one can use this parameter to make standby switch to
stream mode sooner.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication (getting WAL from primary). However, standby
exhausts all the WAL present in pg_wal before switching. If
standby fails to switch to stream mode, it falls back to archive
mode.

Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi, SATYANARAYANA NARLAPURAM
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  49 ++++++++
 doc/src/sgml/high-availability.sgml           |  15 ++-
 src/backend/access/transam/xlogrecovery.c     | 117 ++++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   3 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/meson.build                 |   1 +
 src/test/recovery/t/043_wal_source_switch.pl  | 113 +++++++++++++++++
 8 files changed, 296 insertions(+), 15 deletions(-)
 create mode 100644 src/test/recovery/t/043_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 65a6e6c408..40c2ae93d3 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -5050,6 +5050,55 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from archive to streaming replication (i.e., getting WAL from
+        primary). However, the standby exhausts all the WAL present in
+        <filename>pg_wal</filename> directory before switching. If the standby
+        fails to switch to stream mode, it falls back to archive mode. If this
+        parameter value is specified without units, it is taken as seconds.
+        With a lower value for this parameter, the standby makes frequent WAL
+        source switch attempts. To avoid this, it is recommended to set a
+        value depending on the rate of WAL generation on the primary. If the
+        WAL files grow faster, the disk runs out of space sooner, so setting a
+        value to make frequent WAL source switch attempts can help. The default
+        is zero, disabling this feature. When disabled, the standby typically
+        switches to stream mode only after receiving WAL from archive finishes
+        (i.e., no more WAL left there) or fails for any reason. This parameter
+        can only be set in the <filename>postgresql.conf</filename> file or on
+        the server command line.
+       </para>
+       <note>
+        <para>
+         Standby may not always attempt to switch source from WAL archive to
+         streaming replication at exact
+         <varname>streaming_replication_retry_interval</varname> intervals. For
+         example, if the parameter is set to <literal>1min</literal> and
+         fetching WAL file from archive takes about <literal>2min</literal>,
+         then the source switch attempt happens for the next WAL file after
+         the current WAL file fetched from archive is fully applied.
+        </para>
+       </note>
+       <para>
+        Reading WAL from archive may not always be as efficient and fast as
+        reading from primary. This can be due to the differences in disk types,
+        IO costs, network latencies etc. All of these can impact the recovery
+        performance on standby, and can increase the replication lag on
+        primary. In addition, the primary keeps accumulating WAL needed for the
+        standby while the standby reads WAL from archive, because the standby
+        replication slot stays inactive. To avoid these problems, one can use
+        this parameter to make standby switch to stream mode sooner.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index b48209fc2f..a4e555c6a1 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -628,12 +628,15 @@ protocol to make nodes agree on a serializable transactional order.
     In standby mode, the server continuously applies WAL received from the
     primary server. The standby server can read WAL from a WAL archive
     (see <xref linkend="guc-restore-command"/>) or directly from the primary
-    over a TCP connection (streaming replication). The standby server will
-    also attempt to restore any WAL found in the standby cluster's
-    <filename>pg_wal</filename> directory. That typically happens after a server
-    restart, when the standby replays again WAL that was streamed from the
-    primary before the restart, but you can also manually copy files to
-    <filename>pg_wal</filename> at any time to have them replayed.
+    over a TCP connection (streaming replication) or attempt to switch to
+    streaming replication after reading from archive when
+    <xref linkend="guc-streaming-replication-retry-interval"/> parameter is
+    set. The standby server will also attempt to restore any WAL found in the
+    standby cluster's <filename>pg_wal</filename> directory. That typically
+    happens after a server restart, when the standby replays again WAL that was
+    streamed from the primary before the restart, but you can also manually
+    copy files to <filename>pg_wal</filename> at any time to have them
+    replayed.
    </para>
 
    <para>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 29c5bec084..6f696b3f7b 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -91,6 +91,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 0;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -297,6 +298,20 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/*
+ * Represents WAL source switch states.
+ */
+typedef enum
+{
+	SWITCH_TO_STREAMING_NONE,	/* no switch necessary */
+	SWITCH_TO_STREAMING_PENDING,	/* exhausting pg_wal */
+	SWITCH_TO_STREAMING,		/* switch to streaming now */
+}			WALSourceSwitchState;
+
+static WALSourceSwitchState wal_source_switch_state = SWITCH_TO_STREAMING_NONE;
+
+/* Holds the timestamp at which standby switched WAL source to archive */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +455,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static WALSourceSwitchState SwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3546,6 +3563,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	static TimestampTz last_fail_time = 0;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
+	XLogSource	readFrom;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -3565,6 +3583,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval seconds. However,
+	 * exhaust all the WAL present in pg_wal before switching. If successful,
+	 * the state machine moves to XLOG_FROM_STREAM state, otherwise it falls
+	 * back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3588,12 +3612,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
+		 * First check if we failed to read from the current source or we
+		 * intentionally want to switch the source from archive to stream, and
 		 * advance the state machine if so. The failure to read might've
 		 * happened outside this function, e.g when a CRC check fails on a
 		 * record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed ||
+			wal_source_switch_state == SWITCH_TO_STREAMING)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
@@ -3732,9 +3758,26 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we are switching to archive */
+			if (currentSource == XLOG_FROM_ARCHIVE)
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			ereport(DEBUG1,
+					errmsg_internal("switched WAL source from %s to %s after %s",
+									xlogSourceNames[oldSource],
+									xlogSourceNames[currentSource],
+									((wal_source_switch_state == SWITCH_TO_STREAMING) ? "timeout" : (lastSourceFailed ? "failure" : "success"))));
+
+			/* Reset the WAL source switch state */
+			if (wal_source_switch_state == SWITCH_TO_STREAMING)
+			{
+				Assert(oldSource == XLOG_FROM_ARCHIVE);
+				Assert(currentSource == XLOG_FROM_STREAM);
+
+				wal_source_switch_state = SWITCH_TO_STREAMING_NONE;
+			}
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
@@ -3763,13 +3806,23 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/* See if we can switch WAL source to streaming */
+				if (wal_source_switch_state == SWITCH_TO_STREAMING_NONE)
+					wal_source_switch_state = SwitchWALSourceToPrimary();
+
 				/*
 				 * Try to restore the file from archive, or read an existing
-				 * file from pg_wal.
+				 * file from pg_wal. However, before switching WAL source to
+				 * streaming, give it a chance to read all the WAL from
+				 * pg_wal.
 				 */
-				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2,
-											  currentSource == XLOG_FROM_ARCHIVE ? XLOG_FROM_ANY :
-											  currentSource);
+				if (wal_source_switch_state == SWITCH_TO_STREAMING_PENDING)
+					readFrom = XLOG_FROM_PG_WAL;
+				else
+					readFrom = currentSource == XLOG_FROM_ARCHIVE ?
+						XLOG_FROM_ANY : currentSource;
+
+				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2, readFrom);
 				if (readFile >= 0)
 					return XLREAD_SUCCESS;	/* success! */
 
@@ -3777,6 +3830,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				 * Nope, not found in archive or pg_wal.
 				 */
 				lastSourceFailed = true;
+
+				/*
+				 * Read all the WAL in pg_wal. Now ready to switch to
+				 * streaming.
+				 */
+				if (wal_source_switch_state == SWITCH_TO_STREAMING_PENDING)
+					wal_source_switch_state = SWITCH_TO_STREAMING;
+
 				break;
 
 			case XLOG_FROM_STREAM:
@@ -4007,6 +4068,44 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * Check if standby can make an attempt to read WAL from primary after reading
+ * from archive for at least a configurable duration.
+ *
+ * Reading WAL from archive may not always be as efficient and fast as reading
+ * from primary. This can be due to the differences in disk types, IO costs,
+ * network latencies etc. All of these can impact the recovery performance on
+ * standby and increase the replication lag on primary. In addition, the
+ * primary keeps accumulating WAL needed for the standby while the standby
+ * reads WAL from archive because the standby replication slot stays inactive.
+ * To avoid these problems, the standby will try to switch to stream mode
+ * sooner.
+ */
+static WALSourceSwitchState
+SwitchWALSourceToPrimary(void)
+{
+	TimestampTz now;
+
+	if (streaming_replication_retry_interval <= 0 ||
+		!StandbyMode ||
+		currentSource != XLOG_FROM_ARCHIVE)
+		return SWITCH_TO_STREAMING_NONE;
+
+	now = GetCurrentTimestamp();
+
+	/* First time through */
+	if (switched_to_archive_at == 0)
+	{
+		switched_to_archive_at = now;
+		return SWITCH_TO_STREAMING_NONE;
+	}
+
+	if (TimestampDifferenceExceeds(switched_to_archive_at, now,
+								   streaming_replication_retry_interval * 1000))
+		return SWITCH_TO_STREAMING_PENDING;
+
+	return SWITCH_TO_STREAMING_NONE;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 57d9de4dd9..d206cb0849 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3273,6 +3273,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_S
+		},
+		&streaming_replication_retry_interval,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 2244ee52f7..deb4be7809 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -371,6 +371,9 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 0	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication in seconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 #sync_replication_slots = off			# enables slot synchronization on the physical standby from the primary
 
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index c423464e8b..73c5a86f4c 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build
index b1eb77b1ec..fdf5814a22 100644
--- a/src/test/recovery/meson.build
+++ b/src/test/recovery/meson.build
@@ -51,6 +51,7 @@ tests += {
       't/040_standby_failover_slots_sync.pl',
       't/041_checkpoint_at_promote.pl',
       't/042_low_level_backup.pl',
+      't/043_wal_source_switch.pl',
     ],
   },
 }
diff --git a/src/test/recovery/t/043_wal_source_switch.pl b/src/test/recovery/t/043_wal_source_switch.pl
new file mode 100644
index 0000000000..b00ed29f73
--- /dev/null
+++ b/src/test/recovery/t/043_wal_source_switch.pl
@@ -0,0 +1,113 @@
+
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+# Checks for WAL source switch feature.
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# Initialize primary node
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(allows_streaming => 1, has_archiving => 1);
+
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf(
+	'postgresql.conf', qq(
+checkpoint_timeout = 1h
+autovacuum = off
+));
+$node_primary->start;
+
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('standby_slot')");
+
+# And some content
+$node_primary->safe_psql('postgres',
+	"CREATE TABLE tab_int AS SELECT generate_series(1, 10) AS a");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create streaming standby from backup
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup(
+	$node_primary, $backup_name,
+	has_streaming => 1,
+	has_restoring => 1);
+
+my $retry_interval = 1;
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+primary_slot_name = 'standby_slot'
+streaming_replication_retry_interval = '${retry_interval}s'
+log_min_messages = 'debug2'
+));
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+# Generate some data on the primary while the standby is down
+$node_standby->stop;
+for my $i (1 .. 10)
+{
+	$node_primary->safe_psql('postgres',
+		"INSERT INTO tab_int VALUES (generate_series(11, 20));");
+	$node_primary->safe_psql('postgres', "SELECT pg_switch_wal();");
+}
+
+# Now wait for replay to complete on standby. We're done waiting when the
+# standby has replayed up to the previously saved primary LSN.
+my $cur_lsn =
+  $node_primary->safe_psql('postgres', "SELECT pg_current_wal_lsn()");
+
+# Generate 1 more WAL file so that we wait predictably for the archiving of
+# all WAL files.
+$node_primary->advance_wal(1);
+
+my $walfile_name =
+  $node_primary->safe_psql('postgres', "SELECT pg_walfile_name('$cur_lsn')");
+
+$node_primary->poll_query_until('postgres',
+	"SELECT count(*) = 1 FROM pg_stat_archiver WHERE last_archived_wal = '$walfile_name';"
+) or die "Timed out while waiting for archiving of WAL by primary";
+
+my $offset = -s $node_standby->logfile;
+
+# Standby initially fetches WAL from archive after the restart. Since it is
+# asked to retry fetching from primary after retry interval
+# (i.e. streaming_replication_retry_interval), it will do so. To mimic the
+# standby spending some time fetching from archive, we use apply delay
+# (i.e. recovery_min_apply_delay) greater than the retry interval, so that for
+# fetching the next WAL file the standby honours retry interval and fetches it
+# from primary.
+my $delay = $retry_interval * 5;
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+recovery_min_apply_delay = '${delay}s'
+));
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+$node_standby->wait_for_log(
+	qr/DEBUG: ( [A-Z0-9]+:)? switched WAL source from archive to stream after timeout/,
+	$offset);
+$node_standby->wait_for_log(
+	qr/LOG: ( [A-Z0-9]+:)? started streaming WAL from primary at .* on timeline .*/,
+	$offset);
+
+# Check that the data from primary is streamed to standby
+my $row_cnt1 =
+  $node_primary->safe_psql('postgres', "SELECT count(*) FROM tab_int;");
+
+my $row_cnt2 =
+  $node_standby->safe_psql('postgres', "SELECT count(*) FROM tab_int;");
+is($row_cnt1, $row_cnt2, 'data from primary is streamed to standby');
+
+done_testing();
-- 
2.34.1

From 2fb59e8edfec6c35322f2abfd31c5cb967234093 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Sat, 23 Mar 2024 09:21:50 +0000
Subject: [PATCH v24] Allow standby to switch WAL source from archive to
 streaming

A standby typically switches to streaming replication (get WAL
from primary), only when receive from WAL archive finishes (no
more WAL left there) or fails for any reason. Reading WAL from
archive may not always be as efficient and fast as reading from
primary. This can be due to the differences in disk types, IO
costs, network latencies etc.. All of these can impact the
recovery performance on standby and increase the replication lag
on primary. In addition, the primary keeps accumulating WAL needed
for the standby while the standby reads WAL from archive because
the standby replication slot stays inactive. To avoid these
problems, one can use this parameter to make standby switch to
stream mode sooner.

This feature adds a new GUC that specifies amount of time after
which standby attempts to switch WAL source from WAL archive to
streaming replication (getting WAL from primary). However, standby
exhausts all the WAL present in pg_wal before switching. If
standby fails to switch to stream mode, it falls back to archive
mode.

Author: Bharath Rupireddy
Reviewed-by: Cary Huang, Nathan Bossart
Reviewed-by: Kyotaro Horiguchi, SATYANARAYANA NARLAPURAM
Reviewed-by: Michael Paquier
Discussion: https://www.postgresql.org/message-id/CAHg+QDdLmfpS0n0U3U+e+dw7X7jjEOsJJ0aLEsrtxs-tUyf5Ag@mail.gmail.com
---
 doc/src/sgml/config.sgml                      |  49 +++++++
 doc/src/sgml/high-availability.sgml           |  15 ++-
 src/backend/access/transam/xlogrecovery.c     | 124 ++++++++++++++++--
 src/backend/utils/misc/guc_tables.c           |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   3 +
 src/include/access/xlogrecovery.h             |   1 +
 src/test/recovery/meson.build                 |   1 +
 src/test/recovery/t/043_wal_source_switch.pl  | 113 ++++++++++++++++
 src/tools/pgindent/typedefs.list              |   1 +
 9 files changed, 304 insertions(+), 15 deletions(-)
 create mode 100644 src/test/recovery/t/043_wal_source_switch.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 65a6e6c408..40c2ae93d3 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -5050,6 +5050,55 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-streaming-replication-retry-interval" xreflabel="streaming_replication_retry_interval">
+      <term><varname>streaming_replication_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>streaming_replication_retry_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies amount of time after which standby attempts to switch WAL
+        source from archive to streaming replication (i.e., getting WAL from
+        primary). However, the standby exhausts all the WAL present in
+        <filename>pg_wal</filename> directory before switching. If the standby
+        fails to switch to stream mode, it falls back to archive mode. If this
+        parameter value is specified without units, it is taken as seconds.
+        With a lower value for this parameter, the standby makes frequent WAL
+        source switch attempts. To avoid this, it is recommended to set a
+        value depending on the rate of WAL generation on the primary. If the
+        WAL files grow faster, the disk runs out of space sooner, so setting a
+        value to make frequent WAL source switch attempts can help. The default
+        is zero, disabling this feature. When disabled, the standby typically
+        switches to stream mode only after receiving WAL from archive finishes
+        (i.e., no more WAL left there) or fails for any reason. This parameter
+        can only be set in the <filename>postgresql.conf</filename> file or on
+        the server command line.
+       </para>
+       <note>
+        <para>
+         Standby may not always attempt to switch source from WAL archive to
+         streaming replication at exact
+         <varname>streaming_replication_retry_interval</varname> intervals. For
+         example, if the parameter is set to <literal>1min</literal> and
+         fetching WAL file from archive takes about <literal>2min</literal>,
+         then the source switch attempt happens for the next WAL file after
+         the current WAL file fetched from archive is fully applied.
+        </para>
+       </note>
+       <para>
+        Reading WAL from archive may not always be as efficient and fast as
+        reading from primary. This can be due to the differences in disk types,
+        IO costs, network latencies etc. All of these can impact the recovery
+        performance on standby, and can increase the replication lag on
+        primary. In addition, the primary keeps accumulating WAL needed for the
+        standby while the standby reads WAL from archive, because the standby
+        replication slot stays inactive. To avoid these problems, one can use
+        this parameter to make standby switch to stream mode sooner.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-recovery-min-apply-delay" xreflabel="recovery_min_apply_delay">
       <term><varname>recovery_min_apply_delay</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index b48209fc2f..a4e555c6a1 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -628,12 +628,15 @@ protocol to make nodes agree on a serializable transactional order.
     In standby mode, the server continuously applies WAL received from the
     primary server. The standby server can read WAL from a WAL archive
     (see <xref linkend="guc-restore-command"/>) or directly from the primary
-    over a TCP connection (streaming replication). The standby server will
-    also attempt to restore any WAL found in the standby cluster's
-    <filename>pg_wal</filename> directory. That typically happens after a server
-    restart, when the standby replays again WAL that was streamed from the
-    primary before the restart, but you can also manually copy files to
-    <filename>pg_wal</filename> at any time to have them replayed.
+    over a TCP connection (streaming replication) or attempt to switch to
+    streaming replication after reading from archive when
+    <xref linkend="guc-streaming-replication-retry-interval"/> parameter is
+    set. The standby server will also attempt to restore any WAL found in the
+    standby cluster's <filename>pg_wal</filename> directory. That typically
+    happens after a server restart, when the standby replays again WAL that was
+    streamed from the primary before the restart, but you can also manually
+    copy files to <filename>pg_wal</filename> at any time to have them
+    replayed.
    </para>
 
    <para>
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 29c5bec084..013a28503d 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -91,6 +91,7 @@ TimestampTz recoveryTargetTime;
 const char *recoveryTargetName;
 XLogRecPtr	recoveryTargetLSN;
 int			recovery_min_apply_delay = 0;
+int			streaming_replication_retry_interval = 0;
 
 /* options formerly taken from recovery.conf for XLOG streaming */
 char	   *PrimaryConnInfo = NULL;
@@ -297,6 +298,20 @@ bool		reachedConsistency = false;
 static char *replay_image_masked = NULL;
 static char *primary_image_masked = NULL;
 
+/*
+ * Represents WAL source switch states.
+ */
+typedef enum
+{
+	SWITCH_TO_STREAMING_NONE,	/* no switch necessary */
+	SWITCH_TO_STREAMING_PENDING,	/* exhausting pg_wal */
+	SWITCH_TO_STREAMING,		/* switch to streaming now */
+} WALSourceSwitchState;
+
+static WALSourceSwitchState wal_source_switch_state = SWITCH_TO_STREAMING_NONE;
+
+/* Holds the timestamp at which standby switched WAL source to archive */
+static TimestampTz switched_to_archive_at = 0;
 
 /*
  * Shared-memory state for WAL recovery.
@@ -440,6 +455,8 @@ static bool HotStandbyActiveInReplay(void);
 static void SetCurrentChunkStartTime(TimestampTz xtime);
 static void SetLatestXTime(TimestampTz xtime);
 
+static WALSourceSwitchState SwitchWALSourceToPrimary(void);
+
 /*
  * Initialization of shared memory for WAL recovery
  */
@@ -3546,6 +3563,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	static TimestampTz last_fail_time = 0;
 	TimestampTz now;
 	bool		streaming_reply_sent = false;
+	XLogSource	readFrom;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -3565,6 +3583,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * those actions are taken when reading from the previous source fails, as
 	 * part of advancing to the next state.
 	 *
+	 * Try reading WAL from primary after being in XLOG_FROM_ARCHIVE state for
+	 * at least streaming_replication_retry_interval seconds. However,
+	 * exhaust all the WAL present in pg_wal before switching. If successful,
+	 * the state machine moves to XLOG_FROM_STREAM state, otherwise it falls
+	 * back to XLOG_FROM_ARCHIVE state.
+	 *
 	 * If standby mode is turned off while reading WAL from stream, we move
 	 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
 	 * the files (which would be required at end of recovery, e.g., timeline
@@ -3588,12 +3612,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		bool		startWalReceiver = false;
 
 		/*
-		 * First check if we failed to read from the current source, and
+		 * First check if we failed to read from the current source or we
+		 * intentionally want to switch the source from archive to stream, and
 		 * advance the state machine if so. The failure to read might've
 		 * happened outside this function, e.g when a CRC check fails on a
 		 * record, or within this loop.
 		 */
-		if (lastSourceFailed)
+		if (lastSourceFailed ||
+			wal_source_switch_state == SWITCH_TO_STREAMING)
 		{
 			/*
 			 * Don't allow any retry loops to occur during nonblocking
@@ -3732,9 +3758,26 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 		}
 
 		if (currentSource != oldSource)
-			elog(DEBUG2, "switched WAL source from %s to %s after %s",
-				 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
-				 lastSourceFailed ? "failure" : "success");
+		{
+			/* Save the timestamp at which we are switching to archive */
+			if (currentSource == XLOG_FROM_ARCHIVE)
+				switched_to_archive_at = GetCurrentTimestamp();
+
+			ereport(DEBUG1,
+					errmsg_internal("switched WAL source from %s to %s after %s",
+									xlogSourceNames[oldSource],
+									xlogSourceNames[currentSource],
+									((wal_source_switch_state == SWITCH_TO_STREAMING) ? "timeout" : (lastSourceFailed ? "failure" : "success"))));
+
+			/* Reset the WAL source switch state */
+			if (wal_source_switch_state == SWITCH_TO_STREAMING)
+			{
+				Assert(oldSource == XLOG_FROM_ARCHIVE);
+				Assert(currentSource == XLOG_FROM_STREAM);
+
+				wal_source_switch_state = SWITCH_TO_STREAMING_NONE;
+			}
+		}
 
 		/*
 		 * We've now handled possible failure. Try to read from the chosen
@@ -3763,13 +3806,30 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				if (randAccess)
 					curFileTLI = 0;
 
+				/* See if we can switch WAL source to streaming */
+				if (wal_source_switch_state == SWITCH_TO_STREAMING_NONE)
+					wal_source_switch_state = SwitchWALSourceToPrimary();
+
 				/*
 				 * Try to restore the file from archive, or read an existing
-				 * file from pg_wal.
+				 * file from pg_wal. However, before switching WAL source to
+				 * streaming, give it a chance to read all the WAL from
+				 * pg_wal.
 				 */
-				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2,
-											  currentSource == XLOG_FROM_ARCHIVE ? XLOG_FROM_ANY :
-											  currentSource);
+				if (wal_source_switch_state == SWITCH_TO_STREAMING_PENDING)
+				{
+					/*
+					 * We already are in archive when we are trying to switch
+					 * to streaming, see SwitchWALSourceToPrimary.
+					 */
+					Assert(currentSource == XLOG_FROM_ARCHIVE);
+					readFrom = XLOG_FROM_PG_WAL;
+				}
+				else
+					readFrom = currentSource == XLOG_FROM_ARCHIVE ?
+						XLOG_FROM_ANY : currentSource;
+
+				readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2, readFrom);
 				if (readFile >= 0)
 					return XLREAD_SUCCESS;	/* success! */
 
@@ -3777,6 +3837,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 				 * Nope, not found in archive or pg_wal.
 				 */
 				lastSourceFailed = true;
+
+				/*
+				 * Read all the WAL in pg_wal. Now ready to switch to
+				 * streaming.
+				 */
+				if (wal_source_switch_state == SWITCH_TO_STREAMING_PENDING)
+					wal_source_switch_state = SWITCH_TO_STREAMING;
+
 				break;
 
 			case XLOG_FROM_STREAM:
@@ -4007,6 +4075,44 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	return XLREAD_FAIL;			/* not reached */
 }
 
+/*
+ * Check if standby can make an attempt to read WAL from primary after reading
+ * from archive for at least a configurable duration.
+ *
+ * Reading WAL from archive may not always be as efficient and fast as reading
+ * from primary. This can be due to the differences in disk types, IO costs,
+ * network latencies etc. All of these can impact the recovery performance on
+ * standby and increase the replication lag on primary. In addition, the
+ * primary keeps accumulating WAL needed for the standby while the standby
+ * reads WAL from archive because the standby replication slot stays inactive.
+ * To avoid these problems, the standby will try to switch to stream mode
+ * sooner.
+ */
+static WALSourceSwitchState
+SwitchWALSourceToPrimary(void)
+{
+	TimestampTz now;
+
+	if (streaming_replication_retry_interval <= 0 ||
+		!StandbyMode ||
+		currentSource != XLOG_FROM_ARCHIVE)
+		return SWITCH_TO_STREAMING_NONE;
+
+	now = GetCurrentTimestamp();
+
+	/* First time through */
+	if (switched_to_archive_at == 0)
+	{
+		switched_to_archive_at = now;
+		return SWITCH_TO_STREAMING_NONE;
+	}
+
+	if (TimestampDifferenceExceeds(switched_to_archive_at, now,
+								   streaming_replication_retry_interval * 1000))
+		return SWITCH_TO_STREAMING_PENDING;
+
+	return SWITCH_TO_STREAMING_NONE;
+}
 
 /*
  * Determine what log level should be used to report a corrupt WAL record
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 1e71e7db4a..7bd7f4674f 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3273,6 +3273,18 @@ struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"streaming_replication_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time after which standby attempts to switch WAL "
+						 "source from archive to streaming replication."),
+			gettext_noop("0 turns this feature off."),
+			GUC_UNIT_S
+		},
+		&streaming_replication_retry_interval,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 2244ee52f7..deb4be7809 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -371,6 +371,9 @@
 					# in milliseconds; 0 disables
 #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
 					# retrieve WAL after a failed attempt
+#streaming_replication_retry_interval = 0	# time after which standby
+					# attempts to switch WAL source from archive to
+					# streaming replication in seconds; 0 disables
 #recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
 #sync_replication_slots = off			# enables slot synchronization on the physical standby from the primary
 
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index c423464e8b..73c5a86f4c 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -57,6 +57,7 @@ extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
 extern PGDLLIMPORT char *recoveryEndCommand;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT int streaming_replication_retry_interval;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build
index b1eb77b1ec..fdf5814a22 100644
--- a/src/test/recovery/meson.build
+++ b/src/test/recovery/meson.build
@@ -51,6 +51,7 @@ tests += {
       't/040_standby_failover_slots_sync.pl',
       't/041_checkpoint_at_promote.pl',
       't/042_low_level_backup.pl',
+      't/043_wal_source_switch.pl',
     ],
   },
 }
diff --git a/src/test/recovery/t/043_wal_source_switch.pl b/src/test/recovery/t/043_wal_source_switch.pl
new file mode 100644
index 0000000000..b00ed29f73
--- /dev/null
+++ b/src/test/recovery/t/043_wal_source_switch.pl
@@ -0,0 +1,113 @@
+
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+# Checks for WAL source switch feature.
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# Initialize primary node
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(allows_streaming => 1, has_archiving => 1);
+
+# Ensure checkpoint doesn't come in our way
+$node_primary->append_conf(
+	'postgresql.conf', qq(
+checkpoint_timeout = 1h
+autovacuum = off
+));
+$node_primary->start;
+
+$node_primary->safe_psql('postgres',
+	"SELECT pg_create_physical_replication_slot('standby_slot')");
+
+# And some content
+$node_primary->safe_psql('postgres',
+	"CREATE TABLE tab_int AS SELECT generate_series(1, 10) AS a");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+# Create streaming standby from backup
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup(
+	$node_primary, $backup_name,
+	has_streaming => 1,
+	has_restoring => 1);
+
+my $retry_interval = 1;
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+primary_slot_name = 'standby_slot'
+streaming_replication_retry_interval = '${retry_interval}s'
+log_min_messages = 'debug2'
+));
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+# Generate some data on the primary while the standby is down
+$node_standby->stop;
+for my $i (1 .. 10)
+{
+	$node_primary->safe_psql('postgres',
+		"INSERT INTO tab_int VALUES (generate_series(11, 20));");
+	$node_primary->safe_psql('postgres', "SELECT pg_switch_wal();");
+}
+
+# Now wait for replay to complete on standby. We're done waiting when the
+# standby has replayed up to the previously saved primary LSN.
+my $cur_lsn =
+  $node_primary->safe_psql('postgres', "SELECT pg_current_wal_lsn()");
+
+# Generate 1 more WAL file so that we wait predictably for the archiving of
+# all WAL files.
+$node_primary->advance_wal(1);
+
+my $walfile_name =
+  $node_primary->safe_psql('postgres', "SELECT pg_walfile_name('$cur_lsn')");
+
+$node_primary->poll_query_until('postgres',
+	"SELECT count(*) = 1 FROM pg_stat_archiver WHERE last_archived_wal = '$walfile_name';"
+) or die "Timed out while waiting for archiving of WAL by primary";
+
+my $offset = -s $node_standby->logfile;
+
+# Standby initially fetches WAL from archive after the restart. Since it is
+# asked to retry fetching from primary after retry interval
+# (i.e. streaming_replication_retry_interval), it will do so. To mimic the
+# standby spending some time fetching from archive, we use apply delay
+# (i.e. recovery_min_apply_delay) greater than the retry interval, so that for
+# fetching the next WAL file the standby honours retry interval and fetches it
+# from primary.
+my $delay = $retry_interval * 5;
+$node_standby->append_conf(
+	'postgresql.conf', qq(
+recovery_min_apply_delay = '${delay}s'
+));
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+$node_primary->wait_for_catchup($node_standby);
+
+$node_standby->wait_for_log(
+	qr/DEBUG: ( [A-Z0-9]+:)? switched WAL source from archive to stream after timeout/,
+	$offset);
+$node_standby->wait_for_log(
+	qr/LOG: ( [A-Z0-9]+:)? started streaming WAL from primary at .* on timeline .*/,
+	$offset);
+
+# Check that the data from primary is streamed to standby
+my $row_cnt1 =
+  $node_primary->safe_psql('postgres', "SELECT count(*) FROM tab_int;");
+
+my $row_cnt2 =
+  $node_standby->safe_psql('postgres', "SELECT count(*) FROM tab_int;");
+is($row_cnt1, $row_cnt2, 'data from primary is streamed to standby');
+
+done_testing();
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index e2a0525dd4..3a710dcb90 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3031,6 +3031,7 @@ WALReadError
 WALSegmentCloseCB
 WALSegmentContext
 WALSegmentOpenCB
+WALSourceSwitchState
 WCHAR
 WCOKind
 WFW_WaitOption
-- 
2.34.1