Patch: add recovery_timeout option to control timeout of restore_command nonzero status code

From a5d941717df71e4c16941b8a02f0dca40a1107a0 Mon Sep 17 00:00:00 2001
From: Alexey Vasiliev <leopard.not.a@gmail.com>
Date: Mon, 3 Nov 2014 00:21:14 +0200
Subject: [PATCH] add recovery_timeout to controll timeout between
 restore_command nonzero

---
 doc/src/sgml/recovery-config.sgml               | 24 ++++++++++++++++++++++++
 src/backend/access/transam/recovery.conf.sample |  5 +++++
 src/backend/access/transam/xlog.c               | 20 ++++++++++++++++++--
 3 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/doc/src/sgml/recovery-config.sgml b/doc/src/sgml/recovery-config.sgml
index 0f1ff34..98eb451 100644
--- a/doc/src/sgml/recovery-config.sgml
+++ b/doc/src/sgml/recovery-config.sgml
@@ -145,6 +145,30 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
       </listitem>
      </varlistentry>
 
+     <varlistentry id="restore-command-retry-interval" xreflabel="restore_command_retry_interval">
+      <term><varname>restore_command_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+        <primary><varname>restore_command_retry_interval</> recovery parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        By default, if <varname>restore_command</> return nonzero status,
+        server will retry command again after 5 seconds. This parameter
+        allow to change this time. This parameter is optional. This can
+        be useful to increase/decrease number of a restore_command calls.
+       </para>
+       <para>
+        This is useful, if I using for restore of wal logs some
+        external storage (like AWS S3) and no matter what the slave database
+        will lag behind the master. The problem, what for each request to
+        AWS S3 need to pay, what is why for N nodes, which try to get next
+        wal log each 5 seconds will be bigger price, than for example each
+        30 seconds.
+       </para>
+      </listitem>
+     </varlistentry>
+
     </variablelist>
 
   </sect1>
diff --git a/src/backend/access/transam/recovery.conf.sample b/src/backend/access/transam/recovery.conf.sample
index 7657df3..4eee8f4 100644
--- a/src/backend/access/transam/recovery.conf.sample
+++ b/src/backend/access/transam/recovery.conf.sample
@@ -58,6 +58,11 @@
 #
 #recovery_end_command = ''
 #
+# specifies an optional timeout after nonzero code of restore_command.
+# This can be useful to increase/decrease number of a restore_command calls.
+#
+#restore_command_retry_interval = 5s
+#
 #---------------------------------------------------------------------------
 # RECOVERY TARGET PARAMETERS
 #---------------------------------------------------------------------------
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 3c9aeae..c26101e 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -233,6 +233,7 @@ static TimestampTz recoveryTargetTime;
 static char *recoveryTargetName;
 static int	recovery_min_apply_delay = 0;
 static TimestampTz recoveryDelayUntilTime;
+static int 	restore_command_retry_interval = 0;
 
 /* options taken from recovery.conf for XLOG streaming */
 static bool StandbyModeRequested = false;
@@ -5245,6 +5246,20 @@ readRecoveryCommandFile(void)
 					(errmsg_internal("trigger_file = '%s'",
 									 TriggerFile)));
 		}
+		else if (strcmp(item->name, "restore_command_retry_interval") == 0)
+		{
+			const char *hintmsg;
+
+			if (!parse_int(item->value, &restore_command_retry_interval, GUC_UNIT_MS,
+						   &hintmsg))
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("parameter \"%s\" requires a temporal value",
+								"restore_command_retry_interval"),
+						 hintmsg ? errhint("%s", _(hintmsg)) : 0));
+			ereport(DEBUG2,
+					(errmsg_internal("restore_command_retry_interval = '%s'", item->value)));
+		}
 		else if (strcmp(item->name, "recovery_min_apply_delay") == 0)
 		{
 			const char *hintmsg;
@@ -11104,13 +11119,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					}
 
 					/*
-					 * Wait for more WAL to arrive. Time out after 5 seconds,
+					 * Wait for more WAL to arrive. Time out after
+					 * restore_command_retry_interval (5 seconds by default),
 					 * like when polling the archive, to react to a trigger
 					 * file promptly.
 					 */
 					WaitLatch(&XLogCtl->recoveryWakeupLatch,
 							  WL_LATCH_SET | WL_TIMEOUT,
-							  5000L);
+							  restore_command_retry_interval > 0 ? restore_command_retry_interval : 5000L);
 					ResetLatch(&XLogCtl->recoveryWakeupLatch);
 					break;
 				}
-- 
1.9.3 (Apple Git-50)

diff --git a/doc/src/sgml/recovery-config.sgml b/doc/src/sgml/recovery-config.sgml
index ef78bc0..53ccf13 100644
--- a/doc/src/sgml/recovery-config.sgml
+++ b/doc/src/sgml/recovery-config.sgml
@@ -145,6 +145,29 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
       </listitem>
      </varlistentry>
 
+     <varlistentry id="restore-command-retry-interval" xreflabel="restore_command_retry_interval">
+      <term><varname>restore_command_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+        <primary><varname>restore_command_retry_interval</> recovery parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        If <varname>restore_command</> returns nonzero exit status code, retry
+        command after the interval of time specified by this parameter.
+        Default value is <literal>5s</>.
+       </para>
+       <para>
+        This is useful, if I using for restore of wal logs some
+        external storage (like AWS S3) and no matter what the slave database
+        will lag behind the master. The problem, what for each request to
+        AWS S3 need to pay, what is why for N nodes, which try to get next
+        wal log each 5 seconds will be bigger price, than for example each
+        30 seconds.
+       </para>
+      </listitem>
+     </varlistentry>
+
     </variablelist>
 
   </sect1>
diff --git a/src/backend/access/transam/recovery.conf.sample b/src/backend/access/transam/recovery.conf.sample
index b777400..7ed6f3b 100644
--- a/src/backend/access/transam/recovery.conf.sample
+++ b/src/backend/access/transam/recovery.conf.sample
@@ -58,6 +58,11 @@
 #
 #recovery_end_command = ''
 #
+# specifies an optional timeout after nonzero code of restore_command.
+# This can be useful to increase/decrease number of a restore_command calls.
+#
+#restore_command_retry_interval = 5s
+#
 #---------------------------------------------------------------------------
 # RECOVERY TARGET PARAMETERS
 #---------------------------------------------------------------------------
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e5dddd4..02c55a8 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -235,6 +235,7 @@ static TimestampTz recoveryTargetTime;
 static char *recoveryTargetName;
 static int	recovery_min_apply_delay = 0;
 static TimestampTz recoveryDelayUntilTime;
+static int 	restore_command_retry_interval = 5000L;
 
 /* options taken from recovery.conf for XLOG streaming */
 static bool StandbyModeRequested = false;
@@ -4881,6 +4882,28 @@ readRecoveryCommandFile(void)
 					(errmsg_internal("trigger_file = '%s'",
 									 TriggerFile)));
 		}
+		else if (strcmp(item->name, "restore_command_retry_interval") == 0)
+		{
+			const char *hintmsg;
+
+			if (!parse_int(item->value, &restore_command_retry_interval, GUC_UNIT_MS,
+						   &hintmsg))
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("parameter \"%s\" requires a temporal value",
+								"restore_command_retry_interval"),
+						 hintmsg ? errhint("%s", _(hintmsg)) : 0));
+			ereport(DEBUG2,
+					(errmsg_internal("restore_command_retry_interval = '%s'", item->value)));
+
+			if (restore_command_retry_interval <= 0)
+			{
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("\"%s\" must be bigger zero",
+					"restore_command_retry_interval")));
+			}
+		}
 		else if (strcmp(item->name, "recovery_min_apply_delay") == 0)
 		{
 			const char *hintmsg;
@@ -10658,13 +10681,14 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					}
 
 					/*
-					 * Wait for more WAL to arrive. Time out after 5 seconds,
+					 * Wait for more WAL to arrive. Time out after
+					 * restore_command_retry_interval (5 seconds by default),
 					 * like when polling the archive, to react to a trigger
 					 * file promptly.
 					 */
 					WaitLatch(&XLogCtl->recoveryWakeupLatch,
 							  WL_LATCH_SET | WL_TIMEOUT,
-							  5000L);
+							  restore_command_retry_interval);
 					ResetLatch(&XLogCtl->recoveryWakeupLatch);
 					break;
 				}

diff --git a/doc/src/sgml/recovery-config.sgml b/doc/src/sgml/recovery-config.sgml
index ef78bc0..38420a5 100644
--- a/doc/src/sgml/recovery-config.sgml
+++ b/doc/src/sgml/recovery-config.sgml
@@ -145,6 +145,26 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
       </listitem>
      </varlistentry>
 
+     <varlistentry id="restore-command-retry-interval" xreflabel="restore_command_retry_interval">
+      <term><varname>restore_command_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+        <primary><varname>restore_command_retry_interval</> recovery parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        If <varname>restore_command</> returns nonzero exit status code, retry
+        command after the interval of time specified by this parameter.
+        Default value is <literal>5s</>.
+       </para>
+       <para>
+        This is useful, if I using for restore of wal logs some
+        external storage and no matter what the slave database
+        will lag behind the master.
+       </para>
+      </listitem>
+     </varlistentry>
+
     </variablelist>
 
   </sect1>
diff --git a/src/backend/access/transam/recovery.conf.sample b/src/backend/access/transam/recovery.conf.sample
index b777400..5b63f60 100644
--- a/src/backend/access/transam/recovery.conf.sample
+++ b/src/backend/access/transam/recovery.conf.sample
@@ -58,6 +58,11 @@
 #
 #recovery_end_command = ''
 #
+# specifies an optional retry interval of restore_command command, if previous return nonzero exit status code.
+# This can be useful to increase/decrease number of a restore_command calls.
+#
+#restore_command_retry_interval = 5s
+#
 #---------------------------------------------------------------------------
 # RECOVERY TARGET PARAMETERS
 #---------------------------------------------------------------------------
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e5dddd4..83a6db0 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -235,6 +235,7 @@ static TimestampTz recoveryTargetTime;
 static char *recoveryTargetName;
 static int	recovery_min_apply_delay = 0;
 static TimestampTz recoveryDelayUntilTime;
+static int 	restore_command_retry_interval = 5;
 
 /* options taken from recovery.conf for XLOG streaming */
 static bool StandbyModeRequested = false;
@@ -4881,6 +4882,28 @@ readRecoveryCommandFile(void)
 					(errmsg_internal("trigger_file = '%s'",
 									 TriggerFile)));
 		}
+		else if (strcmp(item->name, "restore_command_retry_interval") == 0)
+		{
+			const char *hintmsg;
+
+			if (!parse_int(item->value, &restore_command_retry_interval, GUC_UNIT_S,
+						   &hintmsg))
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("parameter \"%s\" requires a temporal value",
+								"restore_command_retry_interval"),
+						 hintmsg ? errhint("%s", _(hintmsg)) : 0));
+			ereport(DEBUG2,
+					(errmsg_internal("restore_command_retry_interval = '%s'", item->value)));
+
+			if (restore_command_retry_interval < 1)
+			{
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("\"%s\" must have a strictly positive value",
+					"restore_command_retry_interval")));
+			}
+		}
 		else if (strcmp(item->name, "recovery_min_apply_delay") == 0)
 		{
 			const char *hintmsg;
@@ -10495,13 +10518,13 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					 * machine, so we've exhausted all the options for
 					 * obtaining the requested WAL. We're going to loop back
 					 * and retry from the archive, but if it hasn't been long
-					 * since last attempt, sleep 5 seconds to avoid
-					 * busy-waiting.
+					 * since last attempt, sleep restore_command_retry_interval
+					 * (by default 5 seconds) to avoid busy-waiting.
 					 */
 					now = (pg_time_t) time(NULL);
-					if ((now - last_fail_time) < 5)
+					if ((now - last_fail_time) < restore_command_retry_interval)
 					{
-						pg_usleep(1000000L * (5 - (now - last_fail_time)));
+						pg_usleep(1000000L * (restore_command_retry_interval - (now - last_fail_time)));
 						now = (pg_time_t) time(NULL);
 					}
 					last_fail_time = now;

diff --git a/doc/src/sgml/recovery-config.sgml b/doc/src/sgml/recovery-config.sgml
index ef78bc0..52cb47d 100644
--- a/doc/src/sgml/recovery-config.sgml
+++ b/doc/src/sgml/recovery-config.sgml
@@ -145,6 +145,29 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
       </listitem>
      </varlistentry>
 
+     <varlistentry id="restore-command-retry-interval" xreflabel="restore_command_retry_interval">
+      <term><varname>restore_command_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+        <primary><varname>restore_command_retry_interval</> recovery parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        If <varname>restore_command</> returns nonzero exit status code, retry
+        command after the interval of time specified by this parameter.
+        Default value is <literal>5s</>.
+       </para>
+       <para>
+        This is useful, if I using for restore of wal logs some
+        external storage and request each 5 seconds for wal logs
+        can be useless and cost additional money.
+        Increasing this parameter allow to increase retry interval of time,
+        if not found new wal logs inside external storage and no matter
+        what the slave database will lag behind the master.
+       </para>
+      </listitem>
+     </varlistentry>
+
     </variablelist>
 
   </sect1>
diff --git a/src/backend/access/transam/recovery.conf.sample b/src/backend/access/transam/recovery.conf.sample
index b777400..5b63f60 100644
--- a/src/backend/access/transam/recovery.conf.sample
+++ b/src/backend/access/transam/recovery.conf.sample
@@ -58,6 +58,11 @@
 #
 #recovery_end_command = ''
 #
+# specifies an optional retry interval of restore_command command, if previous return nonzero exit status code.
+# This can be useful to increase/decrease number of a restore_command calls.
+#
+#restore_command_retry_interval = 5s
+#
 #---------------------------------------------------------------------------
 # RECOVERY TARGET PARAMETERS
 #---------------------------------------------------------------------------
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e5dddd4..67a873c 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -235,6 +235,7 @@ static TimestampTz recoveryTargetTime;
 static char *recoveryTargetName;
 static int	recovery_min_apply_delay = 0;
 static TimestampTz recoveryDelayUntilTime;
+static int 	restore_command_retry_interval = 5000;
 
 /* options taken from recovery.conf for XLOG streaming */
 static bool StandbyModeRequested = false;
@@ -4881,6 +4882,28 @@ readRecoveryCommandFile(void)
 					(errmsg_internal("trigger_file = '%s'",
 									 TriggerFile)));
 		}
+		else if (strcmp(item->name, "restore_command_retry_interval") == 0)
+		{
+			const char *hintmsg;
+
+			if (!parse_int(item->value, &restore_command_retry_interval, GUC_UNIT_MS,
+						   &hintmsg))
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("parameter \"%s\" requires a temporal value",
+								"restore_command_retry_interval"),
+						 hintmsg ? errhint("%s", _(hintmsg)) : 0));
+			ereport(DEBUG2,
+					(errmsg_internal("restore_command_retry_interval = '%s'", item->value)));
+
+			if (restore_command_retry_interval < 100)
+			{
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("\"%s\" must have a bigger 100 milliseconds value",
+					"restore_command_retry_interval")));
+			}
+		}
 		else if (strcmp(item->name, "recovery_min_apply_delay") == 0)
 		{
 			const char *hintmsg;
@@ -10495,13 +10518,13 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					 * machine, so we've exhausted all the options for
 					 * obtaining the requested WAL. We're going to loop back
 					 * and retry from the archive, but if it hasn't been long
-					 * since last attempt, sleep 5 seconds to avoid
-					 * busy-waiting.
+					 * since last attempt, sleep restore_command_retry_interval
+					 * (by default 5 seconds) to avoid busy-waiting.
 					 */
 					now = (pg_time_t) time(NULL);
-					if ((now - last_fail_time) < 5)
+					if (((now - last_fail_time) * 1000) < restore_command_retry_interval)
 					{
-						pg_usleep(1000000L * (5 - (now - last_fail_time)));
+						pg_usleep(1000L * (restore_command_retry_interval - ((now - last_fail_time) * 1000)));
 						now = (pg_time_t) time(NULL);
 					}
 					last_fail_time = now;

From 9bb71e39b218885466a53c005a87361d4ae889fa Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@otacoo.com>
Date: Mon, 5 Jan 2015 17:10:13 +0900
Subject: [PATCH] Add restore_command_retry_interval to control retries of
 restore_command

restore_command_retry_interval can be specified in recovery.conf to control
the interval of time between retries of restore_command when failures occur.
---
 doc/src/sgml/recovery-config.sgml               | 21 +++++++++++++
 src/backend/access/transam/recovery.conf.sample |  9 ++++++
 src/backend/access/transam/xlog.c               | 42 ++++++++++++++++++++-----
 3 files changed, 64 insertions(+), 8 deletions(-)

diff --git a/doc/src/sgml/recovery-config.sgml b/doc/src/sgml/recovery-config.sgml
index 0c64ff2..0488ae3 100644
--- a/doc/src/sgml/recovery-config.sgml
+++ b/doc/src/sgml/recovery-config.sgml
@@ -145,6 +145,27 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
       </listitem>
      </varlistentry>
 
+     <varlistentry id="restore-command-retry-interval" xreflabel="restore_command_retry_interval">
+      <term><varname>restore_command_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+        <primary><varname>restore_command_retry_interval</> recovery parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        If <varname>restore_command</> returns nonzero exit status code, retry
+        command after the interval of time specified by this parameter,
+        measured in milliseconds if no unit is specified. Default value is
+        <literal>5s</>.
+       </para>
+       <para>
+        This command is particularly useful for warm standby configurations
+        to leverage the amount of retries done to restore a WAL segment file
+        depending on the activity of a master node.
+       </para>
+      </listitem>
+     </varlistentry>
+
     </variablelist>
 
   </sect1>
diff --git a/src/backend/access/transam/recovery.conf.sample b/src/backend/access/transam/recovery.conf.sample
index b777400..8699a33 100644
--- a/src/backend/access/transam/recovery.conf.sample
+++ b/src/backend/access/transam/recovery.conf.sample
@@ -58,6 +58,15 @@
 #
 #recovery_end_command = ''
 #
+#
+# restore_command_retry_interval
+#
+# specifies an optional retry interval for restore_command command, if
+# previous command returned nonzero exit status code. This can be useful
+# to increase or decrease the number of calls of restore_command.
+#
+#restore_command_retry_interval = '5s'
+#
 #---------------------------------------------------------------------------
 # RECOVERY TARGET PARAMETERS
 #---------------------------------------------------------------------------
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 5cc7e47..1944e6f 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -235,6 +235,7 @@ static TimestampTz recoveryTargetTime;
 static char *recoveryTargetName;
 static int	recovery_min_apply_delay = 0;
 static TimestampTz recoveryDelayUntilTime;
+static int	restore_command_retry_interval = 5000;
 
 /* options taken from recovery.conf for XLOG streaming */
 static bool StandbyModeRequested = false;
@@ -4881,6 +4882,26 @@ readRecoveryCommandFile(void)
 					(errmsg_internal("trigger_file = '%s'",
 									 TriggerFile)));
 		}
+		else if (strcmp(item->name, "restore_command_retry_interval") == 0)
+		{
+			const char *hintmsg;
+
+			if (!parse_int(item->value, &restore_command_retry_interval, GUC_UNIT_MS,
+						   &hintmsg))
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("parameter \"%s\" requires a temporal value",
+								"restore_command_retry_interval"),
+						 hintmsg ? errhint("%s", _(hintmsg)) : 0));
+			ereport(DEBUG2,
+					(errmsg_internal("restore_command_retry_interval = '%s'", item->value)));
+
+			if (restore_command_retry_interval <= 0)
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("\"%s\" must have a value strictly positive",
+								"restore_command_retry_interval")));
+		}
 		else if (strcmp(item->name, "recovery_min_apply_delay") == 0)
 		{
 			const char *hintmsg;
@@ -10345,8 +10366,8 @@ static bool
 WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool fetching_ckpt, XLogRecPtr tliRecPtr)
 {
-	static pg_time_t last_fail_time = 0;
-	pg_time_t	now;
+	TimestampTz now = GetCurrentTimestamp();
+	TimestampTz	last_fail_time = now;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -10495,14 +10516,19 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					 * machine, so we've exhausted all the options for
 					 * obtaining the requested WAL. We're going to loop back
 					 * and retry from the archive, but if it hasn't been long
-					 * since last attempt, sleep 5 seconds to avoid
-					 * busy-waiting.
+					 * since last attempt, sleep the amount of time specified
+					 * by restore_command_retry_interval to avoid busy-waiting.
 					 */
-					now = (pg_time_t) time(NULL);
-					if ((now - last_fail_time) < 5)
+					now = GetCurrentTimestamp();
+					if (!TimestampDifferenceExceeds(last_fail_time, now,
+													restore_command_retry_interval))
 					{
-						pg_usleep(1000000L * (5 - (now - last_fail_time)));
-						now = (pg_time_t) time(NULL);
+						long		secs;
+						int			microsecs;
+
+						TimestampDifference(last_fail_time, now, &secs, &microsecs);
+						pg_usleep(restore_command_retry_interval * 1000L - (1000000L * secs + 1L * microsecs));
+						now = GetCurrentTimestamp();
 					}
 					last_fail_time = now;
 					currentSource = XLOG_FROM_ARCHIVE;
-- 
2.2.1

From 0206c417f5b28eadb5f9d67ee0643320d7c0b621 Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@otacoo.com>
Date: Mon, 19 Jan 2015 16:08:48 +0900
Subject: [PATCH] Add wal_availability_check_interval to control WAL fetching
 on failure

Default value is 5s.
---
 doc/src/sgml/recovery-config.sgml               | 21 +++++++++++++
 src/backend/access/transam/recovery.conf.sample | 10 +++++++
 src/backend/access/transam/xlog.c               | 39 ++++++++++++++++++++-----
 src/backend/utils/adt/timestamp.c               | 19 ++++++++++++
 src/include/utils/timestamp.h                   |  2 ++
 5 files changed, 83 insertions(+), 8 deletions(-)

diff --git a/doc/src/sgml/recovery-config.sgml b/doc/src/sgml/recovery-config.sgml
index 0c64ff2..7fd51ce 100644
--- a/doc/src/sgml/recovery-config.sgml
+++ b/doc/src/sgml/recovery-config.sgml
@@ -145,6 +145,27 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
       </listitem>
      </varlistentry>
 
+     <varlistentry id="wal-availability-check-interval" xreflabel="wal_availability_check_interval">
+      <term><varname>wal_availability_check_interval</varname> (<type>integer</type>)
+      <indexterm>
+        <primary><varname>wal_availability_check_interval</> recovery parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        This parameter specifies the amount of time to wait when
+        WAL is not available for a node in recovery. Default value is
+        <literal>5s</>.
+       </para>
+       <para>
+        A node in recovery will wait for this amount of time if
+        <varname>restore_command</> returns nonzero exit status code when
+        fetching new WAL segment files from archive or when a WAL receiver
+        is not able to fetch a WAL record when using streaming replication.
+       </para>
+      </listitem>
+     </varlistentry>
+
     </variablelist>
 
   </sect1>
diff --git a/src/backend/access/transam/recovery.conf.sample b/src/backend/access/transam/recovery.conf.sample
index b777400..70d3946 100644
--- a/src/backend/access/transam/recovery.conf.sample
+++ b/src/backend/access/transam/recovery.conf.sample
@@ -58,6 +58,16 @@
 #
 #recovery_end_command = ''
 #
+#
+# wal_availability_check_interval
+#
+# specifies an optional interval to check for availability of WAL when
+# recovering a node. This interval of time represents the frequency of
+# retries if a previous command of restore_command returned nonzero exit
+# status code or if a walreceiver did not stream completely a WAL record.
+#
+#wal_availability_check_interval = '5s'
+#
 #---------------------------------------------------------------------------
 # RECOVERY TARGET PARAMETERS
 #---------------------------------------------------------------------------
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 629a457..0da9f5c 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -235,6 +235,7 @@ static TimestampTz recoveryTargetTime;
 static char *recoveryTargetName;
 static int	recovery_min_apply_delay = 0;
 static TimestampTz recoveryDelayUntilTime;
+static int	wal_availability_check_interval = 5000;
 
 /* options taken from recovery.conf for XLOG streaming */
 static bool StandbyModeRequested = false;
@@ -4881,6 +4882,26 @@ readRecoveryCommandFile(void)
 					(errmsg_internal("trigger_file = '%s'",
 									 TriggerFile)));
 		}
+		else if (strcmp(item->name, "wal_availability_check_interval") == 0)
+		{
+			const char *hintmsg;
+
+			if (!parse_int(item->value, &wal_availability_check_interval, GUC_UNIT_MS,
+						   &hintmsg))
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("parameter \"%s\" requires a temporal value",
+								"wal_availability_check_interval"),
+						 hintmsg ? errhint("%s", _(hintmsg)) : 0));
+			ereport(DEBUG2,
+					(errmsg_internal("wal_availability_check_interval = '%s'", item->value)));
+
+			if (wal_availability_check_interval <= 0)
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("\"%s\" must have a value strictly positive",
+								"wal_availability_check_interval")));
+		}
 		else if (strcmp(item->name, "recovery_min_apply_delay") == 0)
 		{
 			const char *hintmsg;
@@ -10340,8 +10361,8 @@ static bool
 WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool fetching_ckpt, XLogRecPtr tliRecPtr)
 {
-	static pg_time_t last_fail_time = 0;
-	pg_time_t	now;
+	TimestampTz now = GetCurrentTimestamp();
+	TimestampTz	last_fail_time = now;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -10490,14 +10511,16 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					 * machine, so we've exhausted all the options for
 					 * obtaining the requested WAL. We're going to loop back
 					 * and retry from the archive, but if it hasn't been long
-					 * since last attempt, sleep 5 seconds to avoid
-					 * busy-waiting.
+					 * since last attempt, sleep the amount of time specified
+					 * by wal_availability_check_interval to avoid busy-waiting.
 					 */
-					now = (pg_time_t) time(NULL);
-					if ((now - last_fail_time) < 5)
+					now = GetCurrentTimestamp();
+					if (!TimestampDifferenceExceeds(last_fail_time, now,
+													wal_availability_check_interval))
 					{
-						pg_usleep(1000000L * (5 - (now - last_fail_time)));
-						now = (pg_time_t) time(NULL);
+						TimestampSleepDifference(last_fail_time, now,
+												 wal_availability_check_interval);
+						now = GetCurrentTimestamp();
 					}
 					last_fail_time = now;
 					currentSource = XLOG_FROM_ARCHIVE;
diff --git a/src/backend/utils/adt/timestamp.c b/src/backend/utils/adt/timestamp.c
index 67e0cf9..1098fc9 100644
--- a/src/backend/utils/adt/timestamp.c
+++ b/src/backend/utils/adt/timestamp.c
@@ -1674,6 +1674,25 @@ TimestampDifferenceExceeds(TimestampTz start_time,
 }
 
 /*
+ * TimestampSleepDifference -- sleep for the amout of interval time
+ *		specified, reduced by the difference between two timestamps.
+ *
+ * Both inputs must be ordinary finite timestamps (in current usage,
+ * they'll be results from GetCurrentTimestamp()).
+ */
+void
+TimestampSleepDifference(TimestampTz start_time,
+						 TimestampTz stop_time,
+						 int interval_msec)
+{
+	long        secs;
+	int         microsecs;
+
+	TimestampDifference(start_time, stop_time, &secs, &microsecs);
+	pg_usleep(interval_msec * 1000L - (1000000L * secs + 1L * microsecs));
+}
+
+/*
  * Convert a time_t to TimestampTz.
  *
  * We do not use time_t internally in Postgres, but this is provided for use
diff --git a/src/include/utils/timestamp.h b/src/include/utils/timestamp.h
index 70118f5..e271d0b 100644
--- a/src/include/utils/timestamp.h
+++ b/src/include/utils/timestamp.h
@@ -217,6 +217,8 @@ extern void TimestampDifference(TimestampTz start_time, TimestampTz stop_time,
 extern bool TimestampDifferenceExceeds(TimestampTz start_time,
 						   TimestampTz stop_time,
 						   int msec);
+extern void TimestampSleepDifference(TimestampTz start_time,
+					TimestampTz stop_time, int interval_msec);
 
 /*
  * Prototypes for functions to deal with integer timestamps, when the native
-- 
2.2.2

From d77429197838c0ad9d378aba08137d4ca0b384fd Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@otacoo.com>
Date: Mon, 19 Jan 2015 16:08:48 +0900
Subject: [PATCH] Add wal_availability_check_interval to control WAL fetching
 on failure

Default value is 5s.
---
 doc/src/sgml/recovery-config.sgml               | 21 +++++++++++++
 src/backend/access/transam/recovery.conf.sample | 10 +++++++
 src/backend/access/transam/xlog.c               | 39 ++++++++++++++++++-------
 src/backend/utils/adt/timestamp.c               | 24 +++++++++++++++
 src/include/utils/timestamp.h                   |  2 ++
 5 files changed, 86 insertions(+), 10 deletions(-)

diff --git a/doc/src/sgml/recovery-config.sgml b/doc/src/sgml/recovery-config.sgml
index 0c64ff2..7fd51ce 100644
--- a/doc/src/sgml/recovery-config.sgml
+++ b/doc/src/sgml/recovery-config.sgml
@@ -145,6 +145,27 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
       </listitem>
      </varlistentry>
 
+     <varlistentry id="wal-availability-check-interval" xreflabel="wal_availability_check_interval">
+      <term><varname>wal_availability_check_interval</varname> (<type>integer</type>)
+      <indexterm>
+        <primary><varname>wal_availability_check_interval</> recovery parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        This parameter specifies the amount of time to wait when
+        WAL is not available for a node in recovery. Default value is
+        <literal>5s</>.
+       </para>
+       <para>
+        A node in recovery will wait for this amount of time if
+        <varname>restore_command</> returns nonzero exit status code when
+        fetching new WAL segment files from archive or when a WAL receiver
+        is not able to fetch a WAL record when using streaming replication.
+       </para>
+      </listitem>
+     </varlistentry>
+
     </variablelist>
 
   </sect1>
diff --git a/src/backend/access/transam/recovery.conf.sample b/src/backend/access/transam/recovery.conf.sample
index b777400..70d3946 100644
--- a/src/backend/access/transam/recovery.conf.sample
+++ b/src/backend/access/transam/recovery.conf.sample
@@ -58,6 +58,16 @@
 #
 #recovery_end_command = ''
 #
+#
+# wal_availability_check_interval
+#
+# specifies an optional interval to check for availability of WAL when
+# recovering a node. This interval of time represents the frequency of
+# retries if a previous command of restore_command returned nonzero exit
+# status code or if a walreceiver did not stream completely a WAL record.
+#
+#wal_availability_check_interval = '5s'
+#
 #---------------------------------------------------------------------------
 # RECOVERY TARGET PARAMETERS
 #---------------------------------------------------------------------------
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 629a457..39b4e1c 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -235,6 +235,7 @@ static TimestampTz recoveryTargetTime;
 static char *recoveryTargetName;
 static int	recovery_min_apply_delay = 0;
 static TimestampTz recoveryDelayUntilTime;
+static int	wal_availability_check_interval = 5000;
 
 /* options taken from recovery.conf for XLOG streaming */
 static bool StandbyModeRequested = false;
@@ -4881,6 +4882,26 @@ readRecoveryCommandFile(void)
 					(errmsg_internal("trigger_file = '%s'",
 									 TriggerFile)));
 		}
+		else if (strcmp(item->name, "wal_availability_check_interval") == 0)
+		{
+			const char *hintmsg;
+
+			if (!parse_int(item->value, &wal_availability_check_interval, GUC_UNIT_MS,
+						   &hintmsg))
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("parameter \"%s\" requires a temporal value",
+								"wal_availability_check_interval"),
+						 hintmsg ? errhint("%s", _(hintmsg)) : 0));
+			ereport(DEBUG2,
+					(errmsg_internal("wal_availability_check_interval = '%s'", item->value)));
+
+			if (wal_availability_check_interval <= 0)
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("\"%s\" must have a value strictly positive",
+								"wal_availability_check_interval")));
+		}
 		else if (strcmp(item->name, "recovery_min_apply_delay") == 0)
 		{
 			const char *hintmsg;
@@ -10340,8 +10361,8 @@ static bool
 WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool fetching_ckpt, XLogRecPtr tliRecPtr)
 {
-	static pg_time_t last_fail_time = 0;
-	pg_time_t	now;
+	TimestampTz now = GetCurrentTimestamp();
+	TimestampTz	last_fail_time = now;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -10490,15 +10511,13 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					 * machine, so we've exhausted all the options for
 					 * obtaining the requested WAL. We're going to loop back
 					 * and retry from the archive, but if it hasn't been long
-					 * since last attempt, sleep 5 seconds to avoid
-					 * busy-waiting.
+					 * since last attempt, sleep the amount of time specified
+					 * by wal_availability_check_interval to avoid busy-waiting.
 					 */
-					now = (pg_time_t) time(NULL);
-					if ((now - last_fail_time) < 5)
-					{
-						pg_usleep(1000000L * (5 - (now - last_fail_time)));
-						now = (pg_time_t) time(NULL);
-					}
+					now = GetCurrentTimestamp();
+					if (TimestampSleepDifference(last_fail_time, now,
+											wal_availability_check_interval))
+						now = GetCurrentTimestamp();
 					last_fail_time = now;
 					currentSource = XLOG_FROM_ARCHIVE;
 					break;
diff --git a/src/backend/utils/adt/timestamp.c b/src/backend/utils/adt/timestamp.c
index 67e0cf9..2b11c42 100644
--- a/src/backend/utils/adt/timestamp.c
+++ b/src/backend/utils/adt/timestamp.c
@@ -1674,6 +1674,30 @@ TimestampDifferenceExceeds(TimestampTz start_time,
 }
 
 /*
+ * TimestampSleepDifference -- sleep for the amout of interval time
+ *		specified, reduced by the difference between two timestamps.
+ *		Returns true if sleep is done, false otherwise.
+ *
+ * Both inputs must be ordinary finite timestamps (in current usage,
+ * they'll be results from GetCurrentTimestamp()).
+ */
+bool
+TimestampSleepDifference(TimestampTz start_time,
+						 TimestampTz stop_time,
+						 int interval_msec)
+{
+	long        secs;
+	int         microsecs;
+
+	if (TimestampDifferenceExceeds(start_time, stop_time, interval_msec))
+		return false;
+
+	TimestampDifference(start_time, stop_time, &secs, &microsecs);
+	pg_usleep(interval_msec * 1000L - (1000000L * secs + 1L * microsecs));
+	return true;
+}
+
+/*
  * Convert a time_t to TimestampTz.
  *
  * We do not use time_t internally in Postgres, but this is provided for use
diff --git a/src/include/utils/timestamp.h b/src/include/utils/timestamp.h
index 70118f5..e4a7673 100644
--- a/src/include/utils/timestamp.h
+++ b/src/include/utils/timestamp.h
@@ -217,6 +217,8 @@ extern void TimestampDifference(TimestampTz start_time, TimestampTz stop_time,
 extern bool TimestampDifferenceExceeds(TimestampTz start_time,
 						   TimestampTz stop_time,
 						   int msec);
+extern bool TimestampSleepDifference(TimestampTz start_time,
+					TimestampTz stop_time, int interval_msec);
 
 /*
  * Prototypes for functions to deal with integer timestamps, when the native
-- 
2.2.2

From e07949030a676b033f4a563cfaac4687bcc37dca Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@otacoo.com>
Date: Mon, 19 Jan 2015 16:08:48 +0900
Subject: [PATCH] Add wal_retrieve_retry_interval to control WAL retrieval at
 recovery

This parameter allows to control the amount of time process will wait
for WAL from a source when a failure occurred for a standby node, or
for the amount of time to wait when WAL is expected from a source.
Default value is 5s.
---
 doc/src/sgml/recovery-config.sgml               | 17 +++++++++
 src/backend/access/transam/recovery.conf.sample |  9 +++++
 src/backend/access/transam/xlog.c               | 50 +++++++++++++++++--------
 src/backend/utils/adt/timestamp.c               | 24 ++++++++++++
 src/include/utils/timestamp.h                   |  2 +
 5 files changed, 87 insertions(+), 15 deletions(-)

diff --git a/doc/src/sgml/recovery-config.sgml b/doc/src/sgml/recovery-config.sgml
index 0c64ff2..becb2d3 100644
--- a/doc/src/sgml/recovery-config.sgml
+++ b/doc/src/sgml/recovery-config.sgml
@@ -145,6 +145,23 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
       </listitem>
      </varlistentry>
 
+     <varlistentry id="wal-retrieve-retry-interval" xreflabel="wal_retrieve_retry_interval">
+      <term><varname>wal_retrieve_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+        <primary><varname>wal_retrieve_retry_interval</> recovery parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        This parameter specifies the amount of time to wait for WAL to become
+        available when a failure occurred when reading WAL from a source (be
+        it via streaming replication, local <filename>pg_xlog</> or WAL archive)
+        for a node in standby mode, or when WAL is expected from a source.
+        Default value is <literal>5s</>.
+       </para>
+      </listitem>
+     </varlistentry>
+
     </variablelist>
 
   </sect1>
diff --git a/src/backend/access/transam/recovery.conf.sample b/src/backend/access/transam/recovery.conf.sample
index b777400..458308c 100644
--- a/src/backend/access/transam/recovery.conf.sample
+++ b/src/backend/access/transam/recovery.conf.sample
@@ -58,6 +58,15 @@
 #
 #recovery_end_command = ''
 #
+#
+# wal_retrieve_retry_interval
+#
+# specifies an optional internal to wait for WAL to become available when
+# a failure occurred when reading WAL from a source for a node in standby
+# mode, or when WAL is expected from a source.
+#
+#wal_retrieve_retry_interval = '5s'
+#
 #---------------------------------------------------------------------------
 # RECOVERY TARGET PARAMETERS
 #---------------------------------------------------------------------------
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 629a457..76bd5e2 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -235,6 +235,7 @@ static TimestampTz recoveryTargetTime;
 static char *recoveryTargetName;
 static int	recovery_min_apply_delay = 0;
 static TimestampTz recoveryDelayUntilTime;
+static int	wal_retrieve_retry_interval = 5000;
 
 /* options taken from recovery.conf for XLOG streaming */
 static bool StandbyModeRequested = false;
@@ -4881,6 +4882,26 @@ readRecoveryCommandFile(void)
 					(errmsg_internal("trigger_file = '%s'",
 									 TriggerFile)));
 		}
+		else if (strcmp(item->name, "wal_retrieve_retry_interval") == 0)
+		{
+			const char *hintmsg;
+
+			if (!parse_int(item->value, &wal_retrieve_retry_interval, GUC_UNIT_MS,
+						   &hintmsg))
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("parameter \"%s\" requires a temporal value",
+								"wal_retrieve_retry_interval"),
+						 hintmsg ? errhint("%s", _(hintmsg)) : 0));
+			ereport(DEBUG2,
+					(errmsg_internal("wal_retrieve_retry_interval = '%s'", item->value)));
+
+			if (wal_retrieve_retry_interval <= 0)
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("\"%s\" must have a value strictly positive",
+								"wal_retrieve_retry_interval")));
+		}
 		else if (strcmp(item->name, "recovery_min_apply_delay") == 0)
 		{
 			const char *hintmsg;
@@ -10340,8 +10361,8 @@ static bool
 WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool fetching_ckpt, XLogRecPtr tliRecPtr)
 {
-	static pg_time_t last_fail_time = 0;
-	pg_time_t	now;
+	TimestampTz now = GetCurrentTimestamp();
+	TimestampTz	last_fail_time = now;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -10351,7 +10372,8 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * 2. Check trigger file
 	 * 3. Read from primary server via walreceiver (XLOG_FROM_STREAM)
 	 * 4. Rescan timelines
-	 * 5. Sleep 5 seconds, and loop back to 1.
+	 * 5. Sleep for the internal of time specified by
+	 *    wal_retrieve_retry_interval, and loop back to 1.
 	 *
 	 * Failure to read from the current source advances the state machine to
 	 * the next state.
@@ -10490,15 +10512,13 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					 * machine, so we've exhausted all the options for
 					 * obtaining the requested WAL. We're going to loop back
 					 * and retry from the archive, but if it hasn't been long
-					 * since last attempt, sleep 5 seconds to avoid
-					 * busy-waiting.
+					 * since last attempt, sleep the amount of time specified
+					 * by wal_retrieve_retry_interval to avoid busy-waiting.
 					 */
-					now = (pg_time_t) time(NULL);
-					if ((now - last_fail_time) < 5)
-					{
-						pg_usleep(1000000L * (5 - (now - last_fail_time)));
-						now = (pg_time_t) time(NULL);
-					}
+					now = GetCurrentTimestamp();
+					if (TimestampSleepDifference(last_fail_time, now,
+											wal_retrieve_retry_interval))
+						now = GetCurrentTimestamp();
 					last_fail_time = now;
 					currentSource = XLOG_FROM_ARCHIVE;
 					break;
@@ -10653,13 +10673,13 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					}
 
 					/*
-					 * Wait for more WAL to arrive. Time out after 5 seconds,
-					 * like when polling the archive, to react to a trigger
-					 * file promptly.
+					 * Wait for more WAL to arrive. Time out after
+					 * wal_retrieve_retry_interval ms, like when polling the archive
+					 * to react to a trigger file promptly.
 					 */
 					WaitLatch(&XLogCtl->recoveryWakeupLatch,
 							  WL_LATCH_SET | WL_TIMEOUT,
-							  5000L);
+							  wal_retrieve_retry_interval * 1L);
 					ResetLatch(&XLogCtl->recoveryWakeupLatch);
 					break;
 				}
diff --git a/src/backend/utils/adt/timestamp.c b/src/backend/utils/adt/timestamp.c
index 67e0cf9..2b11c42 100644
--- a/src/backend/utils/adt/timestamp.c
+++ b/src/backend/utils/adt/timestamp.c
@@ -1674,6 +1674,30 @@ TimestampDifferenceExceeds(TimestampTz start_time,
 }
 
 /*
+ * TimestampSleepDifference -- sleep for the amout of interval time
+ *		specified, reduced by the difference between two timestamps.
+ *		Returns true if sleep is done, false otherwise.
+ *
+ * Both inputs must be ordinary finite timestamps (in current usage,
+ * they'll be results from GetCurrentTimestamp()).
+ */
+bool
+TimestampSleepDifference(TimestampTz start_time,
+						 TimestampTz stop_time,
+						 int interval_msec)
+{
+	long        secs;
+	int         microsecs;
+
+	if (TimestampDifferenceExceeds(start_time, stop_time, interval_msec))
+		return false;
+
+	TimestampDifference(start_time, stop_time, &secs, &microsecs);
+	pg_usleep(interval_msec * 1000L - (1000000L * secs + 1L * microsecs));
+	return true;
+}
+
+/*
  * Convert a time_t to TimestampTz.
  *
  * We do not use time_t internally in Postgres, but this is provided for use
diff --git a/src/include/utils/timestamp.h b/src/include/utils/timestamp.h
index 70118f5..e4a7673 100644
--- a/src/include/utils/timestamp.h
+++ b/src/include/utils/timestamp.h
@@ -217,6 +217,8 @@ extern void TimestampDifference(TimestampTz start_time, TimestampTz stop_time,
 extern bool TimestampDifferenceExceeds(TimestampTz start_time,
 						   TimestampTz stop_time,
 						   int msec);
+extern bool TimestampSleepDifference(TimestampTz start_time,
+					TimestampTz stop_time, int interval_msec);
 
 /*
  * Prototypes for functions to deal with integer timestamps, when the native
-- 
2.2.2

From 9c3a7fa35538993c1345a40fe0973332a76bdb81 Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@otacoo.com>
Date: Mon, 19 Jan 2015 16:08:48 +0900
Subject: [PATCH] Add wal_availability_check_interval

This parameter aids to control at which timing WAL availability is checked
when a node is in recovery, particularly  when successive failures happen when
fetching WAL archives, or when fetching WAL records from a streaming source.

Default value is 5s.
---
 doc/src/sgml/recovery-config.sgml               | 21 +++++++++++
 src/backend/access/transam/recovery.conf.sample | 10 ++++++
 src/backend/access/transam/xlog.c               | 47 +++++++++++++++++--------
 src/backend/utils/adt/timestamp.c               | 38 ++++++++++++++++++++
 src/include/utils/timestamp.h                   |  2 ++
 5 files changed, 104 insertions(+), 14 deletions(-)

diff --git a/doc/src/sgml/recovery-config.sgml b/doc/src/sgml/recovery-config.sgml
index 0c64ff2..7fd51ce 100644
--- a/doc/src/sgml/recovery-config.sgml
+++ b/doc/src/sgml/recovery-config.sgml
@@ -145,6 +145,27 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
       </listitem>
      </varlistentry>
 
+     <varlistentry id="wal-availability-check-interval" xreflabel="wal_availability_check_interval">
+      <term><varname>wal_availability_check_interval</varname> (<type>integer</type>)
+      <indexterm>
+        <primary><varname>wal_availability_check_interval</> recovery parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        This parameter specifies the amount of time to wait when
+        WAL is not available for a node in recovery. Default value is
+        <literal>5s</>.
+       </para>
+       <para>
+        A node in recovery will wait for this amount of time if
+        <varname>restore_command</> returns nonzero exit status code when
+        fetching new WAL segment files from archive or when a WAL receiver
+        is not able to fetch a WAL record when using streaming replication.
+       </para>
+      </listitem>
+     </varlistentry>
+
     </variablelist>
 
   </sect1>
diff --git a/src/backend/access/transam/recovery.conf.sample b/src/backend/access/transam/recovery.conf.sample
index b777400..70d3946 100644
--- a/src/backend/access/transam/recovery.conf.sample
+++ b/src/backend/access/transam/recovery.conf.sample
@@ -58,6 +58,16 @@
 #
 #recovery_end_command = ''
 #
+#
+# wal_availability_check_interval
+#
+# specifies an optional interval to check for availability of WAL when
+# recovering a node. This interval of time represents the frequency of
+# retries if a previous command of restore_command returned nonzero exit
+# status code or if a walreceiver did not stream completely a WAL record.
+#
+#wal_availability_check_interval = '5s'
+#
 #---------------------------------------------------------------------------
 # RECOVERY TARGET PARAMETERS
 #---------------------------------------------------------------------------
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 629a457..4f4efca 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -235,6 +235,7 @@ static TimestampTz recoveryTargetTime;
 static char *recoveryTargetName;
 static int	recovery_min_apply_delay = 0;
 static TimestampTz recoveryDelayUntilTime;
+static int	wal_availability_check_interval = 5000;
 
 /* options taken from recovery.conf for XLOG streaming */
 static bool StandbyModeRequested = false;
@@ -4881,6 +4882,26 @@ readRecoveryCommandFile(void)
 					(errmsg_internal("trigger_file = '%s'",
 									 TriggerFile)));
 		}
+		else if (strcmp(item->name, "wal_availability_check_interval") == 0)
+		{
+			const char *hintmsg;
+
+			if (!parse_int(item->value, &wal_availability_check_interval, GUC_UNIT_MS,
+						   &hintmsg))
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("parameter \"%s\" requires a temporal value",
+								"wal_availability_check_interval"),
+						 hintmsg ? errhint("%s", _(hintmsg)) : 0));
+			ereport(DEBUG2,
+					(errmsg_internal("wal_availability_check_interval = '%s'", item->value)));
+
+			if (wal_availability_check_interval <= 0)
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("\"%s\" must have a value strictly positive",
+								"wal_availability_check_interval")));
+		}
 		else if (strcmp(item->name, "recovery_min_apply_delay") == 0)
 		{
 			const char *hintmsg;
@@ -10340,8 +10361,8 @@ static bool
 WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool fetching_ckpt, XLogRecPtr tliRecPtr)
 {
-	static pg_time_t last_fail_time = 0;
-	pg_time_t	now;
+	TimestampTz now = GetCurrentTimestamp();
+	TimestampTz	last_fail_time = now;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -10490,15 +10511,13 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					 * machine, so we've exhausted all the options for
 					 * obtaining the requested WAL. We're going to loop back
 					 * and retry from the archive, but if it hasn't been long
-					 * since last attempt, sleep 5 seconds to avoid
-					 * busy-waiting.
+					 * since last attempt, sleep the amount of time specified
+					 * by wal_availability_check_interval to avoid busy-waiting.
 					 */
-					now = (pg_time_t) time(NULL);
-					if ((now - last_fail_time) < 5)
-					{
-						pg_usleep(1000000L * (5 - (now - last_fail_time)));
-						now = (pg_time_t) time(NULL);
-					}
+					now = GetCurrentTimestamp();
+					if (TimestampSleepDifference(last_fail_time, now,
+											wal_availability_check_interval))
+						now = GetCurrentTimestamp();
 					last_fail_time = now;
 					currentSource = XLOG_FROM_ARCHIVE;
 					break;
@@ -10653,13 +10672,13 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					}
 
 					/*
-					 * Wait for more WAL to arrive. Time out after 5 seconds,
-					 * like when polling the archive, to react to a trigger
-					 * file promptly.
+					 * Wait for more WAL to arrive. Time out after the amount of
+					 * time specified by wal_availability_check_interval, like
+					 * when polling the archive, to react to a trigger file promptly.
 					 */
 					WaitLatch(&XLogCtl->recoveryWakeupLatch,
 							  WL_LATCH_SET | WL_TIMEOUT,
-							  5000L);
+							  wal_availability_check_interval * 1000L);
 					ResetLatch(&XLogCtl->recoveryWakeupLatch);
 					break;
 				}
diff --git a/src/backend/utils/adt/timestamp.c b/src/backend/utils/adt/timestamp.c
index 67e0cf9..a4cf717 100644
--- a/src/backend/utils/adt/timestamp.c
+++ b/src/backend/utils/adt/timestamp.c
@@ -1674,6 +1674,44 @@ TimestampDifferenceExceeds(TimestampTz start_time,
 }
 
 /*
+ * TimestampSleepDifference -- sleep for the amout of interval time
+ *		specified, reduced by the difference between two timestamps.
+ *		Returns true if sleep is done, false otherwise.
+ *
+ * Both inputs must be ordinary finite timestamps (in current usage,
+ * they'll be results from GetCurrentTimestamp()). Sleep is done
+ * per steps of 1s to be able to handle process interruptions without
+ * having to wait for a too long time.
+ */
+bool
+TimestampSleepDifference(TimestampTz start_time,
+						 TimestampTz stop_time,
+						 int interval_msec)
+{
+	long        secs, total_time;
+	int         microsecs;
+
+	if (TimestampDifferenceExceeds(start_time, stop_time, interval_msec))
+		return false;
+
+	TimestampDifference(start_time, stop_time, &secs, &microsecs);
+	total_time = interval_msec * 1000L - (1000000L * secs + 1L * microsecs);
+
+	while (total_time > 0)
+	{
+		long wait_time = 1000000L;	/* 1s */
+
+		if (total_time < wait_time)
+			wait_time = total_time;
+
+		pg_usleep(wait_time);
+		HandleStartupProcInterrupts();
+		total_time -= wait_time;
+	}
+	return true;
+}
+
+/*
  * Convert a time_t to TimestampTz.
  *
  * We do not use time_t internally in Postgres, but this is provided for use
diff --git a/src/include/utils/timestamp.h b/src/include/utils/timestamp.h
index 70118f5..e4a7673 100644
--- a/src/include/utils/timestamp.h
+++ b/src/include/utils/timestamp.h
@@ -217,6 +217,8 @@ extern void TimestampDifference(TimestampTz start_time, TimestampTz stop_time,
 extern bool TimestampDifferenceExceeds(TimestampTz start_time,
 						   TimestampTz stop_time,
 						   int msec);
+extern bool TimestampSleepDifference(TimestampTz start_time,
+					TimestampTz stop_time, int interval_msec);
 
 /*
  * Prototypes for functions to deal with integer timestamps, when the native
-- 
2.2.2

From 06a4d3d1f5fe4362d7be1404cbf0b45b74fea69f Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@otacoo.com>
Date: Mon, 19 Jan 2015 16:08:48 +0900
Subject: [PATCH] Add wal_retrieve_retry_interval

This parameter aids to control at which timing WAL availability is checked
when a node is in recovery, particularly  when successive failures happen when
fetching WAL archives, or when fetching WAL records from a streaming source.

Default value is 5s.
---
 doc/src/sgml/recovery-config.sgml               | 17 +++++++++
 src/backend/access/transam/recovery.conf.sample |  9 +++++
 src/backend/access/transam/xlog.c               | 47 +++++++++++++++++--------
 src/backend/utils/adt/timestamp.c               | 38 ++++++++++++++++++++
 src/include/utils/timestamp.h                   |  2 ++
 5 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/doc/src/sgml/recovery-config.sgml b/doc/src/sgml/recovery-config.sgml
index 0c64ff2..d4babbd 100644
--- a/doc/src/sgml/recovery-config.sgml
+++ b/doc/src/sgml/recovery-config.sgml
@@ -145,6 +145,23 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
       </listitem>
      </varlistentry>
 
+     <varlistentry id="wal-retrieve-retry-interval" xreflabel="wal_retrieve_retry_interval">
+      <term><varname>wal_retrieve_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+        <primary><varname>wal_retrieve_retry_interval</> recovery parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        This parameter specifies the amount of time to wait when a failure
+        occurred when reading WAL from a source (be it via streaming
+        replication, local <filename>pg_xlog</> or WAL archive) for a node
+        in standby mode, or when WAL is expected from a source. Default
+        value is <literal>5s</>.
+       </para>
+      </listitem>
+     </varlistentry>
+
     </variablelist>
 
   </sect1>
diff --git a/src/backend/access/transam/recovery.conf.sample b/src/backend/access/transam/recovery.conf.sample
index b777400..458308c 100644
--- a/src/backend/access/transam/recovery.conf.sample
+++ b/src/backend/access/transam/recovery.conf.sample
@@ -58,6 +58,15 @@
 #
 #recovery_end_command = ''
 #
+#
+# wal_retrieve_retry_interval
+#
+# specifies an optional internal to wait for WAL to become available when
+# a failure occurred when reading WAL from a source for a node in standby
+# mode, or when WAL is expected from a source.
+#
+#wal_retrieve_retry_interval = '5s'
+#
 #---------------------------------------------------------------------------
 # RECOVERY TARGET PARAMETERS
 #---------------------------------------------------------------------------
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 629a457..111e53d 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -235,6 +235,7 @@ static TimestampTz recoveryTargetTime;
 static char *recoveryTargetName;
 static int	recovery_min_apply_delay = 0;
 static TimestampTz recoveryDelayUntilTime;
+static int	wal_retrieve_retry_interval = 5000;
 
 /* options taken from recovery.conf for XLOG streaming */
 static bool StandbyModeRequested = false;
@@ -4881,6 +4882,26 @@ readRecoveryCommandFile(void)
 					(errmsg_internal("trigger_file = '%s'",
 									 TriggerFile)));
 		}
+		else if (strcmp(item->name, "wal_retrieve_retry_interval") == 0)
+		{
+			const char *hintmsg;
+
+			if (!parse_int(item->value, &wal_retrieve_retry_interval, GUC_UNIT_MS,
+						   &hintmsg))
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("parameter \"%s\" requires a temporal value",
+								"wal_retrieve_retry_interval"),
+						 hintmsg ? errhint("%s", _(hintmsg)) : 0));
+			ereport(DEBUG2,
+					(errmsg_internal("wal_retrieve_retry_interval = '%s'", item->value)));
+
+			if (wal_retrieve_retry_interval <= 0)
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("\"%s\" must have a value strictly positive",
+								"wal_retrieve_retry_interval")));
+		}
 		else if (strcmp(item->name, "recovery_min_apply_delay") == 0)
 		{
 			const char *hintmsg;
@@ -10340,8 +10361,8 @@ static bool
 WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool fetching_ckpt, XLogRecPtr tliRecPtr)
 {
-	static pg_time_t last_fail_time = 0;
-	pg_time_t	now;
+	TimestampTz now = GetCurrentTimestamp();
+	TimestampTz	last_fail_time = now;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -10490,15 +10511,13 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					 * machine, so we've exhausted all the options for
 					 * obtaining the requested WAL. We're going to loop back
 					 * and retry from the archive, but if it hasn't been long
-					 * since last attempt, sleep 5 seconds to avoid
-					 * busy-waiting.
+					 * since last attempt, sleep the amount of time specified
+					 * by wal_retrieve_retry_interval to avoid busy-waiting.
 					 */
-					now = (pg_time_t) time(NULL);
-					if ((now - last_fail_time) < 5)
-					{
-						pg_usleep(1000000L * (5 - (now - last_fail_time)));
-						now = (pg_time_t) time(NULL);
-					}
+					now = GetCurrentTimestamp();
+					if (TimestampSleepDifference(last_fail_time, now,
+											wal_retrieve_retry_interval))
+						now = GetCurrentTimestamp();
 					last_fail_time = now;
 					currentSource = XLOG_FROM_ARCHIVE;
 					break;
@@ -10653,13 +10672,13 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					}
 
 					/*
-					 * Wait for more WAL to arrive. Time out after 5 seconds,
-					 * like when polling the archive, to react to a trigger
-					 * file promptly.
+					 * Wait for more WAL to arrive. Time out after the amount of
+					 * time specified by wal_retrieve_retry_interval, like
+					 * when polling the archive, to react to a trigger file promptly.
 					 */
 					WaitLatch(&XLogCtl->recoveryWakeupLatch,
 							  WL_LATCH_SET | WL_TIMEOUT,
-							  5000L);
+							  wal_retrieve_retry_interval * 1000L);
 					ResetLatch(&XLogCtl->recoveryWakeupLatch);
 					break;
 				}
diff --git a/src/backend/utils/adt/timestamp.c b/src/backend/utils/adt/timestamp.c
index 67e0cf9..a4cf717 100644
--- a/src/backend/utils/adt/timestamp.c
+++ b/src/backend/utils/adt/timestamp.c
@@ -1674,6 +1674,44 @@ TimestampDifferenceExceeds(TimestampTz start_time,
 }
 
 /*
+ * TimestampSleepDifference -- sleep for the amout of interval time
+ *		specified, reduced by the difference between two timestamps.
+ *		Returns true if sleep is done, false otherwise.
+ *
+ * Both inputs must be ordinary finite timestamps (in current usage,
+ * they'll be results from GetCurrentTimestamp()). Sleep is done
+ * per steps of 1s to be able to handle process interruptions without
+ * having to wait for a too long time.
+ */
+bool
+TimestampSleepDifference(TimestampTz start_time,
+						 TimestampTz stop_time,
+						 int interval_msec)
+{
+	long        secs, total_time;
+	int         microsecs;
+
+	if (TimestampDifferenceExceeds(start_time, stop_time, interval_msec))
+		return false;
+
+	TimestampDifference(start_time, stop_time, &secs, &microsecs);
+	total_time = interval_msec * 1000L - (1000000L * secs + 1L * microsecs);
+
+	while (total_time > 0)
+	{
+		long wait_time = 1000000L;	/* 1s */
+
+		if (total_time < wait_time)
+			wait_time = total_time;
+
+		pg_usleep(wait_time);
+		HandleStartupProcInterrupts();
+		total_time -= wait_time;
+	}
+	return true;
+}
+
+/*
  * Convert a time_t to TimestampTz.
  *
  * We do not use time_t internally in Postgres, but this is provided for use
diff --git a/src/include/utils/timestamp.h b/src/include/utils/timestamp.h
index 70118f5..e4a7673 100644
--- a/src/include/utils/timestamp.h
+++ b/src/include/utils/timestamp.h
@@ -217,6 +217,8 @@ extern void TimestampDifference(TimestampTz start_time, TimestampTz stop_time,
 extern bool TimestampDifferenceExceeds(TimestampTz start_time,
 						   TimestampTz stop_time,
 						   int msec);
+extern bool TimestampSleepDifference(TimestampTz start_time,
+					TimestampTz stop_time, int interval_msec);
 
 /*
  * Prototypes for functions to deal with integer timestamps, when the native
-- 
2.2.2

From 594db45eda43c0abc13ad0dbca81d6ed3f4650e7 Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@otacoo.com>
Date: Mon, 19 Jan 2015 16:08:48 +0900
Subject: [PATCH] Add wal_retrieve_retry_interval

This parameter aids to control at which timing WAL availability is checked
when a node is in recovery, particularly  when successive failures happen when
fetching WAL archives, or when fetching WAL records from a streaming source.

Default value is 5s.
---
 doc/src/sgml/config.sgml                      | 17 +++++
 src/backend/access/transam/xlog.c             | 91 ++++++++++++++++++---------
 src/backend/utils/misc/guc.c                  | 12 ++++
 src/backend/utils/misc/postgresql.conf.sample |  3 +
 src/include/access/xlog.h                     |  1 +
 5 files changed, 93 insertions(+), 31 deletions(-)

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 6bcb106..d82b26a 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -2985,6 +2985,23 @@ include_dir 'conf.d'
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-wal-retrieve-retry-interval" xreflabel="wal_retrieve_retry_interval">
+      <term><varname>wal_retrieve_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>wal_retrieve_retry_interval</> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specify the amount of time to wait when WAL is not available from
+        any sources (streaming replication, local <filename>pg_xlog</> or
+        WAL archive) before retrying to retrieve WAL.  This parameter can
+        only be set in the <filename>postgresql.conf</> file or on the
+        server command line.  The default value is 5 seconds.
+       </para>
+      </listitem>
+     </varlistentry>
+
      </variablelist>
     </sect2>
    </sect1>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 629a457..769fbf3 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -93,6 +93,7 @@ int			sync_method = DEFAULT_SYNC_METHOD;
 int			wal_level = WAL_LEVEL_MINIMAL;
 int			CommitDelay = 0;	/* precommit delay in microseconds */
 int			CommitSiblings = 5; /* # concurrent xacts needed to sleep */
+int			wal_retrieve_retry_interval = 5000;
 
 #ifdef WAL_DEBUG
 bool		XLOG_DEBUG = false;
@@ -10340,8 +10341,8 @@ static bool
 WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool fetching_ckpt, XLogRecPtr tliRecPtr)
 {
-	static pg_time_t last_fail_time = 0;
-	pg_time_t	now;
+	TimestampTz now = GetCurrentTimestamp();
+	TimestampTz	last_fail_time = now;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -10351,7 +10352,9 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * 2. Check trigger file
 	 * 3. Read from primary server via walreceiver (XLOG_FROM_STREAM)
 	 * 4. Rescan timelines
-	 * 5. Sleep 5 seconds, and loop back to 1.
+	 * 5. Sleep the amount of time defined by wal_retrieve_retry_interval
+	 *    while checking periodically for the presence of user-defined
+	 *    trigger file and loop back to 1.
 	 *
 	 * Failure to read from the current source advances the state machine to
 	 * the next state.
@@ -10490,14 +10493,27 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					 * machine, so we've exhausted all the options for
 					 * obtaining the requested WAL. We're going to loop back
 					 * and retry from the archive, but if it hasn't been long
-					 * since last attempt, sleep 5 seconds to avoid
-					 * busy-waiting.
+					 * since last attempt, sleep the amount of time specified
+					 * by wal_retrieve_retry_interval to avoid busy-waiting.
 					 */
-					now = (pg_time_t) time(NULL);
-					if ((now - last_fail_time) < 5)
+					now = GetCurrentTimestamp();
+					if (!TimestampDifferenceExceeds(last_fail_time, now,
+													wal_retrieve_retry_interval))
 					{
-						pg_usleep(1000000L * (5 - (now - last_fail_time)));
-						now = (pg_time_t) time(NULL);
+						long		secs, wait_time;
+						int			microsecs;
+						TimestampDifference(last_fail_time, now, &secs, &microsecs);
+
+						wait_time = wal_retrieve_retry_interval * 1000L -
+							(1000000L * secs + 1L * microsecs);
+						if (wait_time > 5000000L) /* 5s */
+							wait_time = 5000000L;
+
+						WaitLatch(&XLogCtl->recoveryWakeupLatch,
+								  WL_LATCH_SET | WL_TIMEOUT,
+								  wait_time / 1000);
+						ResetLatch(&XLogCtl->recoveryWakeupLatch);
+						now = GetCurrentTimestamp();
 					}
 					last_fail_time = now;
 					currentSource = XLOG_FROM_ARCHIVE;
@@ -10562,6 +10578,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 			case XLOG_FROM_STREAM:
 				{
 					bool		havedata;
+					long		remaining_wait_time;
 
 					/*
 					 * Check if WAL receiver is still active.
@@ -10634,33 +10651,45 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					}
 
 					/*
-					 * Data not here yet. Check for trigger, then wait for
-					 * walreceiver to wake us up when new WAL arrives.
+					 * Wait for more WAL for a total of time defined by
+					 * wal_retrieve_retry_interval and check the presence of a
+					 * user-defined trigger file every 5s to ensure quick
+					 * responsiveness of system.
 					 */
-					if (CheckForStandbyTrigger())
+					remaining_wait_time = wal_retrieve_retry_interval * 1L;
+					while (remaining_wait_time > 0)
 					{
+						long wait_time = 5000L; /* 5s */
+
+						if (remaining_wait_time < wait_time)
+							wait_time = remaining_wait_time;
+
 						/*
-						 * Note that we don't "return false" immediately here.
-						 * After being triggered, we still want to replay all
-						 * the WAL that was already streamed. It's in pg_xlog
-						 * now, so we just treat this as a failure, and the
-						 * state machine will move on to replay the streamed
-						 * WAL from pg_xlog, and then recheck the trigger and
-						 * exit replay.
+						 * Data not here yet. Check for trigger, then wait for
+						 * walreceiver to wake us up when new WAL arrives.
 						 */
-						lastSourceFailed = true;
-						break;
-					}
+						if (CheckForStandbyTrigger())
+						{
+							/*
+							 * Note that we don't "return false" immediately here.
+							 * After being triggered, we still want to replay all
+							 * the WAL that was already streamed. It's in pg_xlog
+							 * now, so we just treat this as a failure, and the
+							 * state machine will move on to replay the streamed
+							 * WAL from pg_xlog, and then recheck the trigger and
+							 * exit replay.
+							 */
+							lastSourceFailed = true;
+							break;
+						}
 
-					/*
-					 * Wait for more WAL to arrive. Time out after 5 seconds,
-					 * like when polling the archive, to react to a trigger
-					 * file promptly.
-					 */
-					WaitLatch(&XLogCtl->recoveryWakeupLatch,
-							  WL_LATCH_SET | WL_TIMEOUT,
-							  5000L);
-					ResetLatch(&XLogCtl->recoveryWakeupLatch);
+						/* wait a bit */
+						WaitLatch(&XLogCtl->recoveryWakeupLatch,
+								  WL_LATCH_SET | WL_TIMEOUT,
+								  wait_time);
+						ResetLatch(&XLogCtl->recoveryWakeupLatch);
+						remaining_wait_time -= wait_time;
+					}
 					break;
 				}
 
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 9572777..c4598ed 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -2364,6 +2364,18 @@ static struct config_int ConfigureNamesInt[] =
 	},
 
 	{
+		{"wal_retrieve_retry_interval", PGC_SIGHUP, WAL_SETTINGS,
+			gettext_noop("Specifies the amount of time to wait when WAL is not "
+						 "available from a source."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&wal_retrieve_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the number of pages per write ahead log segment."),
 			NULL,
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index b053659..73e2bca 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -260,6 +260,9 @@
 #wal_receiver_timeout = 60s		# time that receiver waits for
 					# communication from master
 					# in milliseconds; 0 disables
+#wal_retrieve_retry_interval = 5s	# time to wait before retrying to
+					# retrieve WAL from a source (streaming
+					# replication, archive or local pg_xlog)
 
 
 #------------------------------------------------------------------------------
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 138deaf..be27a85 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -93,6 +93,7 @@ extern int	CheckPointSegments;
 extern int	wal_keep_segments;
 extern int	XLOGbuffers;
 extern int	XLogArchiveTimeout;
+extern int	wal_retrieve_retry_interval;
 extern bool XLogArchiveMode;
 extern char *XLogArchiveCommand;
 extern bool EnableHotStandby;
-- 
2.3.0

From 9b7e3bb32c744b328a0d99db3040cadfcba606aa Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@otacoo.com>
Date: Mon, 19 Jan 2015 16:08:48 +0900
Subject: [PATCH] Add wal_retrieve_retry_interval

This parameter aids to control at which timing WAL availability is checked
when a node is in recovery, particularly  when successive failures happen when
fetching WAL archives, or when fetching WAL records from a streaming source.

Default value is 5s.
---
 doc/src/sgml/config.sgml                      | 17 ++++++++++
 src/backend/access/transam/xlog.c             | 46 +++++++++++++++++++--------
 src/backend/utils/misc/guc.c                  | 12 +++++++
 src/backend/utils/misc/postgresql.conf.sample |  3 ++
 src/include/access/xlog.h                     |  1 +
 5 files changed, 66 insertions(+), 13 deletions(-)

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 6bcb106..d82b26a 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -2985,6 +2985,23 @@ include_dir 'conf.d'
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-wal-retrieve-retry-interval" xreflabel="wal_retrieve_retry_interval">
+      <term><varname>wal_retrieve_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>wal_retrieve_retry_interval</> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specify the amount of time to wait when WAL is not available from
+        any sources (streaming replication, local <filename>pg_xlog</> or
+        WAL archive) before retrying to retrieve WAL.  This parameter can
+        only be set in the <filename>postgresql.conf</> file or on the
+        server command line.  The default value is 5 seconds.
+       </para>
+      </listitem>
+     </varlistentry>
+
      </variablelist>
     </sect2>
    </sect1>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 629a457..1f9c3c4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -93,6 +93,7 @@ int			sync_method = DEFAULT_SYNC_METHOD;
 int			wal_level = WAL_LEVEL_MINIMAL;
 int			CommitDelay = 0;	/* precommit delay in microseconds */
 int			CommitSiblings = 5; /* # concurrent xacts needed to sleep */
+int			wal_retrieve_retry_interval = 5000;
 
 #ifdef WAL_DEBUG
 bool		XLOG_DEBUG = false;
@@ -10340,8 +10341,8 @@ static bool
 WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 							bool fetching_ckpt, XLogRecPtr tliRecPtr)
 {
-	static pg_time_t last_fail_time = 0;
-	pg_time_t	now;
+	TimestampTz now = GetCurrentTimestamp();
+	TimestampTz	last_fail_time = now;
 
 	/*-------
 	 * Standby mode is implemented by a state machine:
@@ -10351,7 +10352,9 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 	 * 2. Check trigger file
 	 * 3. Read from primary server via walreceiver (XLOG_FROM_STREAM)
 	 * 4. Rescan timelines
-	 * 5. Sleep 5 seconds, and loop back to 1.
+	 * 5. Sleep the amount of time defined by wal_retrieve_retry_interval
+	 *    while checking periodically for the presence of user-defined
+	 *    trigger file and loop back to 1.
 	 *
 	 * Failure to read from the current source advances the state machine to
 	 * the next state.
@@ -10490,14 +10493,25 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					 * machine, so we've exhausted all the options for
 					 * obtaining the requested WAL. We're going to loop back
 					 * and retry from the archive, but if it hasn't been long
-					 * since last attempt, sleep 5 seconds to avoid
-					 * busy-waiting.
+					 * since last attempt, sleep the amount of time specified
+					 * by wal_retrieve_retry_interval.
 					 */
-					now = (pg_time_t) time(NULL);
-					if ((now - last_fail_time) < 5)
+					now = GetCurrentTimestamp();
+					if (!TimestampDifferenceExceeds(last_fail_time, now,
+													wal_retrieve_retry_interval))
 					{
-						pg_usleep(1000000L * (5 - (now - last_fail_time)));
-						now = (pg_time_t) time(NULL);
+						long		secs, wait_time;
+						int			microsecs;
+						TimestampDifference(last_fail_time, now, &secs, &microsecs);
+
+						wait_time = wal_retrieve_retry_interval * 1000L -
+							(1000000L * secs + 1L * microsecs);
+
+						WaitLatch(&XLogCtl->recoveryWakeupLatch,
+								  WL_LATCH_SET | WL_TIMEOUT,
+								  wait_time / 1000);
+						ResetLatch(&XLogCtl->recoveryWakeupLatch);
+						now = GetCurrentTimestamp();
 					}
 					last_fail_time = now;
 					currentSource = XLOG_FROM_ARCHIVE;
@@ -10562,6 +10576,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 			case XLOG_FROM_STREAM:
 				{
 					bool		havedata;
+					long		wait_time = wal_retrieve_retry_interval;
 
 					/*
 					 * Check if WAL receiver is still active.
@@ -10653,13 +10668,18 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
 					}
 
 					/*
-					 * Wait for more WAL to arrive. Time out after 5 seconds,
-					 * like when polling the archive, to react to a trigger
-					 * file promptly.
+					 * Wait for more WAL to arrive. Time out after the amount
+					 * of time defined by wal_retrieve_retry_interval like when
+					 * polling the archive for no more than 5s to ensure quick
+					 * responsiveness of system.
 					 */
+					if (wal_retrieve_retry_interval >= 5000)
+						wait_time = 5000;
+
+					/* wait a bit */
 					WaitLatch(&XLogCtl->recoveryWakeupLatch,
 							  WL_LATCH_SET | WL_TIMEOUT,
-							  5000L);
+							  wait_time * 1L);
 					ResetLatch(&XLogCtl->recoveryWakeupLatch);
 					break;
 				}
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 9572777..c4598ed 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -2364,6 +2364,18 @@ static struct config_int ConfigureNamesInt[] =
 	},
 
 	{
+		{"wal_retrieve_retry_interval", PGC_SIGHUP, WAL_SETTINGS,
+			gettext_noop("Specifies the amount of time to wait when WAL is not "
+						 "available from a source."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&wal_retrieve_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the number of pages per write ahead log segment."),
 			NULL,
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index b053659..73e2bca 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -260,6 +260,9 @@
 #wal_receiver_timeout = 60s		# time that receiver waits for
 					# communication from master
 					# in milliseconds; 0 disables
+#wal_retrieve_retry_interval = 5s	# time to wait before retrying to
+					# retrieve WAL from a source (streaming
+					# replication, archive or local pg_xlog)
 
 
 #------------------------------------------------------------------------------
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 138deaf..be27a85 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -93,6 +93,7 @@ extern int	CheckPointSegments;
 extern int	wal_keep_segments;
 extern int	XLOGbuffers;
 extern int	XLogArchiveTimeout;
+extern int	wal_retrieve_retry_interval;
 extern bool XLogArchiveMode;
 extern char *XLogArchiveCommand;
 extern bool EnableHotStandby;
-- 
2.3.0

*** a/doc/src/sgml/config.sgml
--- b/doc/src/sgml/config.sgml
***************
*** 2985,2990 **** include_dir 'conf.d'
--- 2985,3008 ----
        </listitem>
       </varlistentry>
  
+      <varlistentry id="guc-wal-retrieve-retry-interval" xreflabel="wal_retrieve_retry_interval">
+       <term><varname>wal_retrieve_retry_interval</varname> (<type>integer</type>)
+       <indexterm>
+        <primary><varname>wal_retrieve_retry_interval</> configuration parameter</primary>
+       </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specify the amount of time, in milliseconds, to wait when WAL is not
+         available from any sources (streaming replication,
+         local <filename>pg_xlog</> or WAL archive) before retrying to
+         retrieve WAL.  This parameter can only be set in the
+         <filename>postgresql.conf</> file or on the server command line.
+         The default value is 5 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+ 
       </variablelist>
      </sect2>
     </sect1>
*** a/src/backend/access/transam/xlog.c
--- b/src/backend/access/transam/xlog.c
***************
*** 93,98 **** int			sync_method = DEFAULT_SYNC_METHOD;
--- 93,99 ----
  int			wal_level = WAL_LEVEL_MINIMAL;
  int			CommitDelay = 0;	/* precommit delay in microseconds */
  int			CommitSiblings = 5; /* # concurrent xacts needed to sleep */
+ int			wal_retrieve_retry_interval = 5000;
  
  #ifdef WAL_DEBUG
  bool		XLOG_DEBUG = false;
***************
*** 10340,10347 **** static bool
  WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
  							bool fetching_ckpt, XLogRecPtr tliRecPtr)
  {
! 	static pg_time_t last_fail_time = 0;
! 	pg_time_t	now;
  
  	/*-------
  	 * Standby mode is implemented by a state machine:
--- 10341,10348 ----
  WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
  							bool fetching_ckpt, XLogRecPtr tliRecPtr)
  {
! 	static TimestampTz	last_fail_time = 0;
! 	TimestampTz	now;
  
  	/*-------
  	 * Standby mode is implemented by a state machine:
***************
*** 10351,10357 **** WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
  	 * 2. Check trigger file
  	 * 3. Read from primary server via walreceiver (XLOG_FROM_STREAM)
  	 * 4. Rescan timelines
! 	 * 5. Sleep 5 seconds, and loop back to 1.
  	 *
  	 * Failure to read from the current source advances the state machine to
  	 * the next state.
--- 10352,10358 ----
  	 * 2. Check trigger file
  	 * 3. Read from primary server via walreceiver (XLOG_FROM_STREAM)
  	 * 4. Rescan timelines
! 	 * 5. Sleep wal_retrieve_retry_interval milliseconds, and loop back to 1.
  	 *
  	 * Failure to read from the current source advances the state machine to
  	 * the next state.
***************
*** 10490,10503 **** WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
  					 * machine, so we've exhausted all the options for
  					 * obtaining the requested WAL. We're going to loop back
  					 * and retry from the archive, but if it hasn't been long
! 					 * since last attempt, sleep 5 seconds to avoid
! 					 * busy-waiting.
  					 */
! 					now = (pg_time_t) time(NULL);
! 					if ((now - last_fail_time) < 5)
  					{
! 						pg_usleep(1000000L * (5 - (now - last_fail_time)));
! 						now = (pg_time_t) time(NULL);
  					}
  					last_fail_time = now;
  					currentSource = XLOG_FROM_ARCHIVE;
--- 10491,10515 ----
  					 * machine, so we've exhausted all the options for
  					 * obtaining the requested WAL. We're going to loop back
  					 * and retry from the archive, but if it hasn't been long
! 					 * since last attempt, sleep wal_retrieve_retry_interval
! 					 * milliseconds to avoid busy-waiting.
  					 */
! 					now = GetCurrentTimestamp();
! 					if (!TimestampDifferenceExceeds(last_fail_time, now,
! 													wal_retrieve_retry_interval))
  					{
! 						long		secs, wait_time;
! 						int			usecs;
! 
! 						TimestampDifference(last_fail_time, now, &secs, &usecs);
! 						wait_time = wal_retrieve_retry_interval -
! 							(secs * 1000 + usecs / 1000);
! 
! 						WaitLatch(&XLogCtl->recoveryWakeupLatch,
! 								  WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
! 								  wait_time);
! 						ResetLatch(&XLogCtl->recoveryWakeupLatch);
! 						now = GetCurrentTimestamp();
  					}
  					last_fail_time = now;
  					currentSource = XLOG_FROM_ARCHIVE;
***************
*** 10653,10664 **** WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
  					}
  
  					/*
! 					 * Wait for more WAL to arrive. Time out after 5 seconds,
! 					 * like when polling the archive, to react to a trigger
! 					 * file promptly.
  					 */
  					WaitLatch(&XLogCtl->recoveryWakeupLatch,
! 							  WL_LATCH_SET | WL_TIMEOUT,
  							  5000L);
  					ResetLatch(&XLogCtl->recoveryWakeupLatch);
  					break;
--- 10665,10675 ----
  					}
  
  					/*
! 					 * Wait for more WAL to arrive. Time out after 5 seconds
! 					 * to react to a trigger file promptly.
  					 */
  					WaitLatch(&XLogCtl->recoveryWakeupLatch,
! 							  WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
  							  5000L);
  					ResetLatch(&XLogCtl->recoveryWakeupLatch);
  					break;
*** a/src/backend/utils/misc/guc.c
--- b/src/backend/utils/misc/guc.c
***************
*** 2364,2369 **** static struct config_int ConfigureNamesInt[] =
--- 2364,2381 ----
  	},
  
  	{
+ 		{"wal_retrieve_retry_interval", PGC_SIGHUP, REPLICATION_STANDBY,
+ 			gettext_noop("Sets the time to wait before retrying to retrieve WAL"
+ 						 "after a failed attempt."),
+ 			NULL,
+ 			GUC_UNIT_MS
+ 		},
+ 		&wal_retrieve_retry_interval,
+ 		5000, 1, INT_MAX,
+ 		NULL, NULL, NULL
+ 	},
+ 
+ 	{
  		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
  			gettext_noop("Shows the number of pages per write ahead log segment."),
  			NULL,
*** a/src/backend/utils/misc/postgresql.conf.sample
--- b/src/backend/utils/misc/postgresql.conf.sample
***************
*** 260,265 ****
--- 260,267 ----
  #wal_receiver_timeout = 60s		# time that receiver waits for
  					# communication from master
  					# in milliseconds; 0 disables
+ #wal_retrieve_retry_interval = 5s	# time to wait before retrying to
+ 					# retrieve WAL after a failed attempt
  
  
  #------------------------------------------------------------------------------
*** a/src/include/access/xlog.h
--- b/src/include/access/xlog.h
***************
*** 93,98 **** extern int	CheckPointSegments;
--- 93,99 ----
  extern int	wal_keep_segments;
  extern int	XLOGbuffers;
  extern int	XLogArchiveTimeout;
+ extern int	wal_retrieve_retry_interval;
  extern bool XLogArchiveMode;
  extern char *XLogArchiveCommand;
  extern bool EnableHotStandby;