pg_receivexlog --status-interval add fsync feedback
Hi all,
This patch is to add setting to send status packets after fsync to --status-interval of pg_receivexlog.
If -1 is specified to --status-interval, status packets is sent as soon as after fsync.
Others are the same as when 0 is specified to --status-interval.
When requested by the server, send the status packet.
To use replication slot, and specify -1 to --fsync-interval.
As a result, simple WAL synchronous replication can be performed.
In simple WAL synchronization, it is possible to test the replication.
If there was F/O in synchronous replication,
it is available in the substitute until standby is restored.
Regards,
--
Furuya Osamu
Attachments:
pg_receivexlog-fsync-feedback-v1.patchapplication/octet-stream; name=pg_receivexlog-fsync-feedback-v1.patchDownload
*** a/doc/src/sgml/ref/pg_receivexlog.sgml
--- b/doc/src/sgml/ref/pg_receivexlog.sgml
***************
*** 191,196 **** PostgreSQL documentation
--- 191,198 ----
A value of zero disables the periodic status updates completely,
although an update will still be sent when requested by the server, to
avoid timeout disconnect. The default value is 10 seconds.
+ If -1 is specified, status packets sent to the server ,
+ after WAL file is flushed.
</para>
</listitem>
</varlistentry>
*** a/src/bin/pg_basebackup/pg_receivexlog.c
--- b/src/bin/pg_basebackup/pg_receivexlog.c
***************
*** 428,434 **** main(int argc, char **argv)
break;
case 's':
standby_message_timeout = atoi(optarg) * 1000;
! if (standby_message_timeout < 0)
{
fprintf(stderr, _("%s: invalid status interval \"%s\"\n"),
progname, optarg);
--- 428,434 ----
break;
case 's':
standby_message_timeout = atoi(optarg) * 1000;
! if (standby_message_timeout < -1000)
{
fprintf(stderr, _("%s: invalid status interval \"%s\"\n"),
progname, optarg);
*** a/src/bin/pg_basebackup/receivelog.c
--- b/src/bin/pg_basebackup/receivelog.c
***************
*** 46,59 **** static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
static bool ProcessXLogDataMsg(PGconn *conn, char *copybuf, int len,
XLogRecPtr *blockpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
! char *partial_suffix);
static PGresult *HandleEndOfCopyStream(PGconn *conn, char *copybuf,
XLogRecPtr blockpos, char *basedir, char *partial_suffix,
! XLogRecPtr *stoppos);
static bool CheckCopyStreamStop(PGconn *conn, XLogRecPtr blockpos,
uint32 timeline, char *basedir,
stream_stop_callback stream_stop,
! char *partial_suffix, XLogRecPtr *stoppos);
static long CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
int64 last_status, int fsync_interval,
XLogRecPtr blockpos);
--- 46,60 ----
static bool ProcessXLogDataMsg(PGconn *conn, char *copybuf, int len,
XLogRecPtr *blockpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
! char *partial_suffix, int standby_message_timeout);
static PGresult *HandleEndOfCopyStream(PGconn *conn, char *copybuf,
XLogRecPtr blockpos, char *basedir, char *partial_suffix,
! XLogRecPtr *stoppos, int standby_message_timeout);
static bool CheckCopyStreamStop(PGconn *conn, XLogRecPtr blockpos,
uint32 timeline, char *basedir,
stream_stop_callback stream_stop,
! char *partial_suffix, XLogRecPtr *stoppos,
! int standby_message_timeout);
static long CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
int64 last_status, int fsync_interval,
XLogRecPtr blockpos);
***************
*** 61,66 **** static long CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
--- 62,70 ----
static bool ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos,
uint32 *timeline);
+ static bool sendFeedback(PGconn *conn, XLogRecPtr blockpos, int64 now,
+ bool replyRequested);
+
/*
* Open a new WAL file in the specified directory.
*
***************
*** 154,160 **** open_walfile(XLogRecPtr startpoint, uint32 timeline, char *basedir,
* and returns false, otherwise returns true.
*/
static bool
! close_walfile(char *basedir, char *partial_suffix, XLogRecPtr pos)
{
off_t currpos;
--- 158,164 ----
* and returns false, otherwise returns true.
*/
static bool
! close_walfile(char *basedir, char *partial_suffix, XLogRecPtr pos, int standby_message_timeout)
{
off_t currpos;
***************
*** 210,215 **** close_walfile(char *basedir, char *partial_suffix, XLogRecPtr pos)
--- 214,223 ----
lastFlushPosition = pos;
last_fsync = feGetCurrentTimestamp();
+ /* Send feedback! */
+ if (standby_message_timeout < 0)
+ if (!sendFeedback(conn, pos, last_fsync, false))
+ return false;
return true;
}
***************
*** 778,784 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
* Check if we should continue streaming, or abort at this point.
*/
if (!CheckCopyStreamStop(conn, blockpos, timeline, basedir,
! stream_stop, partial_suffix, stoppos))
goto error;
now = feGetCurrentTimestamp();
--- 786,792 ----
* Check if we should continue streaming, or abort at this point.
*/
if (!CheckCopyStreamStop(conn, blockpos, timeline, basedir,
! stream_stop, partial_suffix, stoppos, standby_message_timeout))
goto error;
now = feGetCurrentTimestamp();
***************
*** 802,807 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
--- 810,820 ----
lastFlushPosition = blockpos;
last_fsync = now;
+
+ /* Send feedback! */
+ if (standby_message_timeout < 0)
+ if (!sendFeedback(conn, blockpos, now, false))
+ goto error;
}
/*
***************
*** 831,837 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
if (r == -2)
{
PGresult *res = HandleEndOfCopyStream(conn, copybuf, blockpos,
! basedir, partial_suffix, stoppos);
if (res == NULL)
goto error;
else
--- 844,850 ----
if (r == -2)
{
PGresult *res = HandleEndOfCopyStream(conn, copybuf, blockpos,
! basedir, partial_suffix, stoppos, standby_message_timeout);
if (res == NULL)
goto error;
else
***************
*** 848,861 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
else if (copybuf[0] == 'w')
{
if (!ProcessXLogDataMsg(conn, copybuf, r, &blockpos,
! timeline, basedir, stream_stop, partial_suffix))
goto error;
/*
* Check if we should continue streaming, or abort at this point.
*/
if (!CheckCopyStreamStop(conn, blockpos, timeline, basedir,
! stream_stop, partial_suffix, stoppos))
goto error;
}
else
--- 861,874 ----
else if (copybuf[0] == 'w')
{
if (!ProcessXLogDataMsg(conn, copybuf, r, &blockpos,
! timeline, basedir, stream_stop, partial_suffix, standby_message_timeout))
goto error;
/*
* Check if we should continue streaming, or abort at this point.
*/
if (!CheckCopyStreamStop(conn, blockpos, timeline, basedir,
! stream_stop, partial_suffix, stoppos, standby_message_timeout))
goto error;
}
else
***************
*** 1037,1043 **** static bool
ProcessXLogDataMsg(PGconn *conn, char *copybuf, int len,
XLogRecPtr *blockpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
! char *partial_suffix)
{
int xlogoff;
int bytes_left;
--- 1050,1056 ----
ProcessXLogDataMsg(PGconn *conn, char *copybuf, int len,
XLogRecPtr *blockpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
! char *partial_suffix, int standby_message_timeout)
{
int xlogoff;
int bytes_left;
***************
*** 1145,1151 **** ProcessXLogDataMsg(PGconn *conn, char *copybuf, int len,
/* Did we reach the end of a WAL segment? */
if (*blockpos % XLOG_SEG_SIZE == 0)
{
! if (!close_walfile(basedir, partial_suffix, *blockpos))
/* Error message written in close_walfile() */
return false;
--- 1158,1164 ----
/* Did we reach the end of a WAL segment? */
if (*blockpos % XLOG_SEG_SIZE == 0)
{
! if (!close_walfile(basedir, partial_suffix, *blockpos, standby_message_timeout))
/* Error message written in close_walfile() */
return false;
***************
*** 1175,1181 **** ProcessXLogDataMsg(PGconn *conn, char *copybuf, int len,
static PGresult *
HandleEndOfCopyStream(PGconn *conn, char *copybuf,
XLogRecPtr blockpos, char *basedir, char *partial_suffix,
! XLogRecPtr *stoppos)
{
PGresult *res = PQgetResult(conn);
--- 1188,1194 ----
static PGresult *
HandleEndOfCopyStream(PGconn *conn, char *copybuf,
XLogRecPtr blockpos, char *basedir, char *partial_suffix,
! XLogRecPtr *stoppos, int standby_message_timeout)
{
PGresult *res = PQgetResult(conn);
***************
*** 1186,1192 **** HandleEndOfCopyStream(PGconn *conn, char *copybuf,
*/
if (still_sending)
{
! if (!close_walfile(basedir, partial_suffix, blockpos))
{
/* Error message written in close_walfile() */
PQclear(res);
--- 1199,1205 ----
*/
if (still_sending)
{
! if (!close_walfile(basedir, partial_suffix, blockpos, standby_message_timeout))
{
/* Error message written in close_walfile() */
PQclear(res);
***************
*** 1218,1228 **** HandleEndOfCopyStream(PGconn *conn, char *copybuf,
static bool
CheckCopyStreamStop(PGconn *conn, XLogRecPtr blockpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
! char *partial_suffix, XLogRecPtr *stoppos)
{
if (still_sending && stream_stop(blockpos, timeline, false))
{
! if (!close_walfile(basedir, partial_suffix, blockpos))
{
/* Potential error message is written by close_walfile */
return false;
--- 1231,1242 ----
static bool
CheckCopyStreamStop(PGconn *conn, XLogRecPtr blockpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
! char *partial_suffix, XLogRecPtr *stoppos,
! int standby_message_timeout)
{
if (still_sending && stream_stop(blockpos, timeline, false))
{
! if (!close_walfile(basedir, partial_suffix, blockpos, standby_message_timeout))
{
/* Potential error message is written by close_walfile */
return false;
On Tue, Aug 12, 2014 at 6:19 PM, <furuyao@pm.nttdata.co.jp> wrote:
Hi all,
This patch is to add setting to send status packets after fsync to --status-interval of pg_receivexlog.
If -1 is specified to --status-interval, status packets is sent as soon as after fsync.
I don't think that it's good idea to control that behavior by using
--status-interval. I'm sure that there are some users who both want
that behavior and want set the maximum interval between a feedback
is sent back to the server because these settings are available in
walreceiver. But your version of --status-interval doesn't allow those
settings at all. That is, if set to -1, the maximum interval between
each feedback cannot be set. OTOH, if set to the positive number,
feedback-as-soon-as-fsync behavior cannot be enabled. Therefore,
I'm thinking that it's better to introduce new separate option for that
behavior.
Regards,
--
Fujii Masao
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
I don't think that it's good idea to control that behavior by using
--status-interval. I'm sure that there are some users who both want that
behavior and want set the maximum interval between a feedback is sent
back to the server because these settings are available in walreceiver.
But your version of --status-interval doesn't allow those settings at
all. That is, if set to -1, the maximum interval between each feedback
cannot be set. OTOH, if set to the positive number,
feedback-as-soon-as-fsync behavior cannot be enabled. Therefore, I'm
thinking that it's better to introduce new separate option for that
behavior.
Thanks for the review!
This patch was split option as you pointed out.
If -r option is specified, status packets sent to server as soon as after fsync.
Otherwise to send server status packet in the spacing of the --status-interval.
Regards,
--
Furuya Osamu
Attachments:
pg_receivexlog-fsync-feedback-v2.patchapplication/octet-stream; name=pg_receivexlog-fsync-feedback-v2.patchDownload
*** a/doc/src/sgml/ref/pg_receivexlog.sgml
--- b/doc/src/sgml/ref/pg_receivexlog.sgml
***************
*** 196,201 **** PostgreSQL documentation
--- 196,211 ----
</varlistentry>
<varlistentry>
+ <term><option>-r</></term>
+ <term><option>--reply-fsync</></term>
+ <listitem>
+ <para>
+ status packets sent to server as soon as after fsync.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
<term><option>-U <replaceable>username</replaceable></option></term>
<term><option>--username=<replaceable class="parameter">username</replaceable></option></term>
<listitem>
*** a/src/bin/pg_basebackup/pg_basebackup.c
--- b/src/bin/pg_basebackup/pg_basebackup.c
***************
*** 371,377 **** LogStreamerMain(logstreamer_param *param)
if (!ReceiveXlogStream(param->bgconn, param->startptr, param->timeline,
param->sysidentifier, param->xlogdir,
reached_end_position, standby_message_timeout,
! NULL, 0))
/*
* Any errors will already have been reported in the function process,
--- 371,377 ----
if (!ReceiveXlogStream(param->bgconn, param->startptr, param->timeline,
param->sysidentifier, param->xlogdir,
reached_end_position, standby_message_timeout,
! NULL, 0, false))
/*
* Any errors will already have been reported in the function process,
*** a/src/bin/pg_basebackup/pg_receivexlog.c
--- b/src/bin/pg_basebackup/pg_receivexlog.c
***************
*** 38,44 **** static int noloop = 0;
static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
static int fsync_interval = 0; /* 0 = default */
static volatile bool time_to_abort = false;
!
static void usage(void);
static XLogRecPtr FindStreamingStart(uint32 *tli);
--- 38,44 ----
static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
static int fsync_interval = 0; /* 0 = default */
static volatile bool time_to_abort = false;
! static volatile bool reply_fsync = false;
static void usage(void);
static XLogRecPtr FindStreamingStart(uint32 *tli);
***************
*** 74,79 **** usage(void)
--- 74,80 ----
printf(_(" -p, --port=PORT database server port number\n"));
printf(_(" -s, --status-interval=INTERVAL\n"
" time between status packets sent to server (in seconds)\n"));
+ printf(_(" -r, --reply-fsync status packets sent to server as soon as after fsync\n"));
printf(_(" -U, --username=NAME connect as specified database user\n"));
printf(_(" -w, --no-password never prompt for password\n"));
printf(_(" -W, --password force password prompt (should happen automatically)\n"));
***************
*** 334,340 **** StreamLog(void)
ReceiveXlogStream(conn, startpos, starttli, NULL, basedir,
stop_streaming, standby_message_timeout, ".partial",
! fsync_interval);
PQfinish(conn);
}
--- 335,341 ----
ReceiveXlogStream(conn, startpos, starttli, NULL, basedir,
stop_streaming, standby_message_timeout, ".partial",
! fsync_interval, reply_fsync);
PQfinish(conn);
}
***************
*** 368,373 **** main(int argc, char **argv)
--- 369,375 ----
{"no-password", no_argument, NULL, 'w'},
{"password", no_argument, NULL, 'W'},
{"status-interval", required_argument, NULL, 's'},
+ {"reply-fsync", no_argument, NULL, 'r'},
{"slot", required_argument, NULL, 'S'},
{"verbose", no_argument, NULL, 'v'},
{NULL, 0, NULL, 0}
***************
*** 394,400 **** main(int argc, char **argv)
}
}
! while ((c = getopt_long(argc, argv, "D:d:h:p:U:s:S:nF:wWv",
long_options, &option_index)) != -1)
{
switch (c)
--- 396,402 ----
}
}
! while ((c = getopt_long(argc, argv, "D:d:h:p:U:s:rS:nF:wWv",
long_options, &option_index)) != -1)
{
switch (c)
***************
*** 435,440 **** main(int argc, char **argv)
--- 437,445 ----
exit(1);
}
break;
+ case 'r':
+ reply_fsync = true;
+ break;
case 'S':
replication_slot = pg_strdup(optarg);
break;
*** a/src/bin/pg_basebackup/receivelog.c
--- b/src/bin/pg_basebackup/receivelog.c
***************
*** 33,65 **** static XLogRecPtr lastFlushPosition = InvalidXLogRecPtr;
static int64 last_fsync = -1; /* timestamp of last WAL file flush */
static bool still_sending = true; /* feedback still needs to be sent? */
static PGresult *HandleCopyStream(PGconn *conn, XLogRecPtr startpos,
uint32 timeline, char *basedir,
stream_stop_callback stream_stop, int standby_message_timeout,
char *partial_suffix, XLogRecPtr *stoppos,
! int fsync_interval);
static int CopyStreamPoll(PGconn *conn, long timeout_ms);
static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
! XLogRecPtr blockpos, int64 *last_status);
static bool ProcessXLogDataMsg(PGconn *conn, char *copybuf, int len,
XLogRecPtr *blockpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
! char *partial_suffix);
static PGresult *HandleEndOfCopyStream(PGconn *conn, char *copybuf,
XLogRecPtr blockpos, char *basedir, char *partial_suffix,
! XLogRecPtr *stoppos);
static bool CheckCopyStreamStop(PGconn *conn, XLogRecPtr blockpos,
uint32 timeline, char *basedir,
stream_stop_callback stream_stop,
! char *partial_suffix, XLogRecPtr *stoppos);
static long CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
! int64 last_status, int fsync_interval,
XLogRecPtr blockpos);
static bool ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos,
uint32 *timeline);
/*
* Open a new WAL file in the specified directory.
--- 33,69 ----
static int64 last_fsync = -1; /* timestamp of last WAL file flush */
static bool still_sending = true; /* feedback still needs to be sent? */
+ static int64 last_status = -1; /* timestamp of last feedback message */
static PGresult *HandleCopyStream(PGconn *conn, XLogRecPtr startpos,
uint32 timeline, char *basedir,
stream_stop_callback stream_stop, int standby_message_timeout,
char *partial_suffix, XLogRecPtr *stoppos,
! int fsync_interval, bool reply_fsync);
static int CopyStreamPoll(PGconn *conn, long timeout_ms);
static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
! XLogRecPtr blockpos);
static bool ProcessXLogDataMsg(PGconn *conn, char *copybuf, int len,
XLogRecPtr *blockpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
! char *partial_suffix, bool reply_fsync);
static PGresult *HandleEndOfCopyStream(PGconn *conn, char *copybuf,
XLogRecPtr blockpos, char *basedir, char *partial_suffix,
! XLogRecPtr *stoppos, bool reply_fsync);
static bool CheckCopyStreamStop(PGconn *conn, XLogRecPtr blockpos,
uint32 timeline, char *basedir,
stream_stop_callback stream_stop,
! char *partial_suffix, XLogRecPtr *stoppos,
! bool reply_fsync);
static long CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
! int fsync_interval,
XLogRecPtr blockpos);
static bool ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos,
uint32 *timeline);
+ static bool sendFeedback(PGconn *conn, XLogRecPtr blockpos, int64 now,
+ bool replyRequested);
/*
* Open a new WAL file in the specified directory.
***************
*** 154,160 **** open_walfile(XLogRecPtr startpoint, uint32 timeline, char *basedir,
* and returns false, otherwise returns true.
*/
static bool
! close_walfile(char *basedir, char *partial_suffix, XLogRecPtr pos)
{
off_t currpos;
--- 158,164 ----
* and returns false, otherwise returns true.
*/
static bool
! close_walfile(char *basedir, char *partial_suffix, XLogRecPtr pos, bool reply_fsync)
{
off_t currpos;
***************
*** 210,215 **** close_walfile(char *basedir, char *partial_suffix, XLogRecPtr pos)
--- 214,226 ----
lastFlushPosition = pos;
last_fsync = feGetCurrentTimestamp();
+ /* Send feedback! */
+ if (reply_fsync)
+ {
+ if (!sendFeedback(conn, pos, last_fsync, false))
+ return false;
+ last_status = last_fsync;
+ }
return true;
}
***************
*** 443,448 **** CheckServerVersionForStreaming(PGconn *conn)
--- 454,462 ----
* fsync_interval controls how often we flush to the received WAL file,
* in milliseconds.
*
+ * If 'reply_fsync' is true, status packets sent to server as soon as
+ * after fsync.
+ *
* Note: The log position *must* be at a log segment start!
*/
bool
***************
*** 450,456 **** ReceiveXlogStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *sysidentifier, char *basedir,
stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! int fsync_interval)
{
char query[128];
char slotcmd[128];
--- 464,470 ----
char *sysidentifier, char *basedir,
stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! int fsync_interval, bool reply_fsync)
{
char query[128];
char slotcmd[128];
***************
*** 595,601 **** ReceiveXlogStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
/* Stream the WAL */
res = HandleCopyStream(conn, startpos, timeline, basedir, stream_stop,
standby_message_timeout, partial_suffix,
! &stoppos, fsync_interval);
if (res == NULL)
goto error;
--- 609,615 ----
/* Stream the WAL */
res = HandleCopyStream(conn, startpos, timeline, basedir, stream_stop,
standby_message_timeout, partial_suffix,
! &stoppos, fsync_interval, reply_fsync);
if (res == NULL)
goto error;
***************
*** 760,769 **** static PGresult *
HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! XLogRecPtr *stoppos, int fsync_interval)
{
char *copybuf = NULL;
- int64 last_status = -1;
XLogRecPtr blockpos = startpos;
still_sending = true;
--- 774,782 ----
HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! XLogRecPtr *stoppos, int fsync_interval, bool reply_fsync)
{
char *copybuf = NULL;
XLogRecPtr blockpos = startpos;
still_sending = true;
***************
*** 778,784 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
* Check if we should continue streaming, or abort at this point.
*/
if (!CheckCopyStreamStop(conn, blockpos, timeline, basedir,
! stream_stop, partial_suffix, stoppos))
goto error;
now = feGetCurrentTimestamp();
--- 791,797 ----
* Check if we should continue streaming, or abort at this point.
*/
if (!CheckCopyStreamStop(conn, blockpos, timeline, basedir,
! stream_stop, partial_suffix, stoppos, reply_fsync))
goto error;
now = feGetCurrentTimestamp();
***************
*** 802,807 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
--- 815,827 ----
lastFlushPosition = blockpos;
last_fsync = now;
+ /* Send feedback! */
+ if (reply_fsync)
+ {
+ if (!sendFeedback(conn, blockpos, now, false))
+ goto error;
+ last_status = now;
+ }
}
/*
***************
*** 821,827 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
* Calculate how long send/receive loops should sleep
*/
sleeptime = CalculateCopyStreamSleeptime(now, standby_message_timeout,
! last_status, fsync_interval, blockpos);
r = CopyStreamReceive(conn, sleeptime, ©buf);
while (r != 0)
--- 841,847 ----
* Calculate how long send/receive loops should sleep
*/
sleeptime = CalculateCopyStreamSleeptime(now, standby_message_timeout,
! fsync_interval, blockpos);
r = CopyStreamReceive(conn, sleeptime, ©buf);
while (r != 0)
***************
*** 831,837 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
if (r == -2)
{
PGresult *res = HandleEndOfCopyStream(conn, copybuf, blockpos,
! basedir, partial_suffix, stoppos);
if (res == NULL)
goto error;
else
--- 851,857 ----
if (r == -2)
{
PGresult *res = HandleEndOfCopyStream(conn, copybuf, blockpos,
! basedir, partial_suffix, stoppos, reply_fsync);
if (res == NULL)
goto error;
else
***************
*** 841,861 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
/* Check the message type. */
if (copybuf[0] == 'k')
{
! if (!ProcessKeepaliveMsg(conn, copybuf, r, blockpos,
! &last_status))
goto error;
}
else if (copybuf[0] == 'w')
{
if (!ProcessXLogDataMsg(conn, copybuf, r, &blockpos,
! timeline, basedir, stream_stop, partial_suffix))
goto error;
/*
* Check if we should continue streaming, or abort at this point.
*/
if (!CheckCopyStreamStop(conn, blockpos, timeline, basedir,
! stream_stop, partial_suffix, stoppos))
goto error;
}
else
--- 861,880 ----
/* Check the message type. */
if (copybuf[0] == 'k')
{
! if (!ProcessKeepaliveMsg(conn, copybuf, r, blockpos))
goto error;
}
else if (copybuf[0] == 'w')
{
if (!ProcessXLogDataMsg(conn, copybuf, r, &blockpos,
! timeline, basedir, stream_stop, partial_suffix, reply_fsync))
goto error;
/*
* Check if we should continue streaming, or abort at this point.
*/
if (!CheckCopyStreamStop(conn, blockpos, timeline, basedir,
! stream_stop, partial_suffix, stoppos, reply_fsync))
goto error;
}
else
***************
*** 995,1001 **** CopyStreamReceive(PGconn *conn, long timeout, char **buffer)
*/
static bool
ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
! XLogRecPtr blockpos, int64 *last_status)
{
int pos;
bool replyRequested;
--- 1014,1020 ----
*/
static bool
ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
! XLogRecPtr blockpos)
{
int pos;
bool replyRequested;
***************
*** 1024,1030 **** ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
now = feGetCurrentTimestamp();
if (!sendFeedback(conn, blockpos, now, false))
return false;
! *last_status = now;
}
return true;
--- 1043,1049 ----
now = feGetCurrentTimestamp();
if (!sendFeedback(conn, blockpos, now, false))
return false;
! last_status = now;
}
return true;
***************
*** 1037,1043 **** static bool
ProcessXLogDataMsg(PGconn *conn, char *copybuf, int len,
XLogRecPtr *blockpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
! char *partial_suffix)
{
int xlogoff;
int bytes_left;
--- 1056,1062 ----
ProcessXLogDataMsg(PGconn *conn, char *copybuf, int len,
XLogRecPtr *blockpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
! char *partial_suffix, bool reply_fsync)
{
int xlogoff;
int bytes_left;
***************
*** 1145,1151 **** ProcessXLogDataMsg(PGconn *conn, char *copybuf, int len,
/* Did we reach the end of a WAL segment? */
if (*blockpos % XLOG_SEG_SIZE == 0)
{
! if (!close_walfile(basedir, partial_suffix, *blockpos))
/* Error message written in close_walfile() */
return false;
--- 1164,1170 ----
/* Did we reach the end of a WAL segment? */
if (*blockpos % XLOG_SEG_SIZE == 0)
{
! if (!close_walfile(basedir, partial_suffix, *blockpos, reply_fsync))
/* Error message written in close_walfile() */
return false;
***************
*** 1175,1181 **** ProcessXLogDataMsg(PGconn *conn, char *copybuf, int len,
static PGresult *
HandleEndOfCopyStream(PGconn *conn, char *copybuf,
XLogRecPtr blockpos, char *basedir, char *partial_suffix,
! XLogRecPtr *stoppos)
{
PGresult *res = PQgetResult(conn);
--- 1194,1200 ----
static PGresult *
HandleEndOfCopyStream(PGconn *conn, char *copybuf,
XLogRecPtr blockpos, char *basedir, char *partial_suffix,
! XLogRecPtr *stoppos, bool reply_fsync)
{
PGresult *res = PQgetResult(conn);
***************
*** 1186,1192 **** HandleEndOfCopyStream(PGconn *conn, char *copybuf,
*/
if (still_sending)
{
! if (!close_walfile(basedir, partial_suffix, blockpos))
{
/* Error message written in close_walfile() */
PQclear(res);
--- 1205,1211 ----
*/
if (still_sending)
{
! if (!close_walfile(basedir, partial_suffix, blockpos, reply_fsync))
{
/* Error message written in close_walfile() */
PQclear(res);
***************
*** 1218,1228 **** HandleEndOfCopyStream(PGconn *conn, char *copybuf,
static bool
CheckCopyStreamStop(PGconn *conn, XLogRecPtr blockpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
! char *partial_suffix, XLogRecPtr *stoppos)
{
if (still_sending && stream_stop(blockpos, timeline, false))
{
! if (!close_walfile(basedir, partial_suffix, blockpos))
{
/* Potential error message is written by close_walfile */
return false;
--- 1237,1248 ----
static bool
CheckCopyStreamStop(PGconn *conn, XLogRecPtr blockpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
! char *partial_suffix, XLogRecPtr *stoppos,
! bool reply_fsync)
{
if (still_sending && stream_stop(blockpos, timeline, false))
{
! if (!close_walfile(basedir, partial_suffix, blockpos, reply_fsync))
{
/* Potential error message is written by close_walfile */
return false;
***************
*** 1244,1250 **** CheckCopyStreamStop(PGconn *conn, XLogRecPtr blockpos, uint32 timeline,
*/
static long
CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
! int64 last_status, int fsync_interval, XLogRecPtr blockpos)
{
int64 targettime = 0;
int64 status_targettime = 0;
--- 1264,1270 ----
*/
static long
CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
! int fsync_interval, XLogRecPtr blockpos)
{
int64 targettime = 0;
int64 status_targettime = 0;
*** a/src/bin/pg_basebackup/receivelog.h
--- b/src/bin/pg_basebackup/receivelog.h
***************
*** 17,20 **** extern bool ReceiveXlogStream(PGconn *conn,
stream_stop_callback stream_stop,
int standby_message_timeout,
char *partial_suffix,
! int fsync_interval);
--- 17,21 ----
stream_stop_callback stream_stop,
int standby_message_timeout,
char *partial_suffix,
! int fsync_interval,
! bool reply_fsync);
On Wed, Aug 13, 2014 at 5:55 PM, <furuyao@pm.nttdata.co.jp> wrote:
I don't think that it's good idea to control that behavior by using
--status-interval. I'm sure that there are some users who both want that
behavior and want set the maximum interval between a feedback is sent
back to the server because these settings are available in walreceiver.
But your version of --status-interval doesn't allow those settings at
all. That is, if set to -1, the maximum interval between each feedback
cannot be set. OTOH, if set to the positive number,
feedback-as-soon-as-fsync behavior cannot be enabled. Therefore, I'm
thinking that it's better to introduce new separate option for that
behavior.Thanks for the review!
This patch was split option as you pointed out.If -r option is specified, status packets sent to server as soon as after fsync.
Otherwise to send server status packet in the spacing of the --status-interval.
Hi,
I took a look at this patch.
I applied patch to master successfully, and did not get error with compiling.
Also it works fine.
One question is why reply_fsync is defined as volatile variable?
Sorry I could not understand reason of that.
Currently patch modifies argument of some function (e.g., Handle
CopyStream, Process LogDate Msg), and add the similar code to each
function.
I don't think it is good approach.
For example, I think that we should gather these code into one function.
Thought?
Regards,
-------
Sawada Masahiko
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
Thanks for the review!
One question is why reply_fsync is defined as volatile variable?
Sorry I could not understand reason of that.
It was affected to time_to_abort -- since it is unnecessary, it deletes.
Currently patch modifies argument of some function (e.g., Handle
CopyStream, Process LogDate Msg), and add the similar code to each
function.
I don't think it is good approach.
For example, I think that we should gather these code into one function.
Feedback was judged immediately after each fsync until now.
I revised it in reference to walreceiver.
Feedback of fsync is judged together with the judgment of --status-interval.
Thereby, the specification to an argument became minimum.
Regards,
--
Furuya Osamu
Attachments:
pg_receivexlog-fsync-feedback-v3.patchapplication/octet-stream; name=pg_receivexlog-fsync-feedback-v3.patchDownload
*** a/doc/src/sgml/ref/pg_receivexlog.sgml
--- b/doc/src/sgml/ref/pg_receivexlog.sgml
***************
*** 196,201 **** PostgreSQL documentation
--- 196,211 ----
</varlistentry>
<varlistentry>
+ <term><option>-r</></term>
+ <term><option>--reply-fsync</></term>
+ <listitem>
+ <para>
+ status packets sent to server as soon as after fsync.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
<term><option>-U <replaceable>username</replaceable></option></term>
<term><option>--username=<replaceable class="parameter">username</replaceable></option></term>
<listitem>
*** a/src/bin/pg_basebackup/pg_basebackup.c
--- b/src/bin/pg_basebackup/pg_basebackup.c
***************
*** 371,377 **** LogStreamerMain(logstreamer_param *param)
if (!ReceiveXlogStream(param->bgconn, param->startptr, param->timeline,
param->sysidentifier, param->xlogdir,
reached_end_position, standby_message_timeout,
! NULL, 0))
/*
* Any errors will already have been reported in the function process,
--- 371,377 ----
if (!ReceiveXlogStream(param->bgconn, param->startptr, param->timeline,
param->sysidentifier, param->xlogdir,
reached_end_position, standby_message_timeout,
! NULL, 0, false))
/*
* Any errors will already have been reported in the function process,
*** a/src/bin/pg_basebackup/pg_receivexlog.c
--- b/src/bin/pg_basebackup/pg_receivexlog.c
***************
*** 38,44 **** static int noloop = 0;
static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
static int fsync_interval = 0; /* 0 = default */
static volatile bool time_to_abort = false;
!
static void usage(void);
static XLogRecPtr FindStreamingStart(uint32 *tli);
--- 38,44 ----
static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
static int fsync_interval = 0; /* 0 = default */
static volatile bool time_to_abort = false;
! static bool reply_fsync = false;
static void usage(void);
static XLogRecPtr FindStreamingStart(uint32 *tli);
***************
*** 74,79 **** usage(void)
--- 74,80 ----
printf(_(" -p, --port=PORT database server port number\n"));
printf(_(" -s, --status-interval=INTERVAL\n"
" time between status packets sent to server (in seconds)\n"));
+ printf(_(" -r, --reply-fsync status packets sent to server as soon as after fsync\n"));
printf(_(" -U, --username=NAME connect as specified database user\n"));
printf(_(" -w, --no-password never prompt for password\n"));
printf(_(" -W, --password force password prompt (should happen automatically)\n"));
***************
*** 334,340 **** StreamLog(void)
ReceiveXlogStream(conn, startpos, starttli, NULL, basedir,
stop_streaming, standby_message_timeout, ".partial",
! fsync_interval);
PQfinish(conn);
}
--- 335,341 ----
ReceiveXlogStream(conn, startpos, starttli, NULL, basedir,
stop_streaming, standby_message_timeout, ".partial",
! fsync_interval, reply_fsync);
PQfinish(conn);
}
***************
*** 368,373 **** main(int argc, char **argv)
--- 369,375 ----
{"no-password", no_argument, NULL, 'w'},
{"password", no_argument, NULL, 'W'},
{"status-interval", required_argument, NULL, 's'},
+ {"reply-fsync", no_argument, NULL, 'r'},
{"slot", required_argument, NULL, 'S'},
{"verbose", no_argument, NULL, 'v'},
{NULL, 0, NULL, 0}
***************
*** 394,400 **** main(int argc, char **argv)
}
}
! while ((c = getopt_long(argc, argv, "D:d:h:p:U:s:S:nF:wWv",
long_options, &option_index)) != -1)
{
switch (c)
--- 396,402 ----
}
}
! while ((c = getopt_long(argc, argv, "D:d:h:p:U:s:rS:nF:wWv",
long_options, &option_index)) != -1)
{
switch (c)
***************
*** 435,440 **** main(int argc, char **argv)
--- 437,445 ----
exit(1);
}
break;
+ case 'r':
+ reply_fsync = true;
+ break;
case 'S':
replication_slot = pg_strdup(optarg);
break;
*** a/src/bin/pg_basebackup/receivelog.c
--- b/src/bin/pg_basebackup/receivelog.c
***************
*** 38,44 **** static PGresult *HandleCopyStream(PGconn *conn, XLogRecPtr startpos,
uint32 timeline, char *basedir,
stream_stop_callback stream_stop, int standby_message_timeout,
char *partial_suffix, XLogRecPtr *stoppos,
! int fsync_interval);
static int CopyStreamPoll(PGconn *conn, long timeout_ms);
static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
--- 38,44 ----
uint32 timeline, char *basedir,
stream_stop_callback stream_stop, int standby_message_timeout,
char *partial_suffix, XLogRecPtr *stoppos,
! int fsync_interval, bool reply_fsync);
static int CopyStreamPoll(PGconn *conn, long timeout_ms);
static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
***************
*** 443,448 **** CheckServerVersionForStreaming(PGconn *conn)
--- 443,451 ----
* fsync_interval controls how often we flush to the received WAL file,
* in milliseconds.
*
+ * If 'reply_fsync' is true, status packets sent to server as soon as
+ * after fsync.
+ *
* Note: The log position *must* be at a log segment start!
*/
bool
***************
*** 450,456 **** ReceiveXlogStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *sysidentifier, char *basedir,
stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! int fsync_interval)
{
char query[128];
char slotcmd[128];
--- 453,459 ----
char *sysidentifier, char *basedir,
stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! int fsync_interval, bool reply_fsync)
{
char query[128];
char slotcmd[128];
***************
*** 595,601 **** ReceiveXlogStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
/* Stream the WAL */
res = HandleCopyStream(conn, startpos, timeline, basedir, stream_stop,
standby_message_timeout, partial_suffix,
! &stoppos, fsync_interval);
if (res == NULL)
goto error;
--- 598,604 ----
/* Stream the WAL */
res = HandleCopyStream(conn, startpos, timeline, basedir, stream_stop,
standby_message_timeout, partial_suffix,
! &stoppos, fsync_interval, reply_fsync);
if (res == NULL)
goto error;
***************
*** 760,770 **** static PGresult *
HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! XLogRecPtr *stoppos, int fsync_interval)
{
char *copybuf = NULL;
int64 last_status = -1;
XLogRecPtr blockpos = startpos;
still_sending = true;
--- 763,774 ----
HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! XLogRecPtr *stoppos, int fsync_interval, bool reply_fsync)
{
char *copybuf = NULL;
int64 last_status = -1;
XLogRecPtr blockpos = startpos;
+ XLogRecPtr lastFeedbackFlushPosition = startpos;
still_sending = true;
***************
*** 807,820 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
/*
* Potentially send a status message to the master
*/
! if (still_sending && standby_message_timeout > 0 &&
! feTimestampDifferenceExceeds(last_status, now,
! standby_message_timeout))
{
/* Time to send feedback! */
if (!sendFeedback(conn, blockpos, now, false))
goto error;
last_status = now;
}
/*
--- 811,826 ----
/*
* Potentially send a status message to the master
*/
! if ((still_sending && standby_message_timeout > 0 &&
! feTimestampDifferenceExceeds(last_status, now, standby_message_timeout)) ||
! (reply_fsync && lastFeedbackFlushPosition != lastFlushPosition))
{
/* Time to send feedback! */
if (!sendFeedback(conn, blockpos, now, false))
goto error;
last_status = now;
+ if (reply_fsync)
+ lastFeedbackFlushPosition = lastFlushPosition;
}
/*
*** a/src/bin/pg_basebackup/receivelog.h
--- b/src/bin/pg_basebackup/receivelog.h
***************
*** 17,20 **** extern bool ReceiveXlogStream(PGconn *conn,
stream_stop_callback stream_stop,
int standby_message_timeout,
char *partial_suffix,
! int fsync_interval);
--- 17,21 ----
stream_stop_callback stream_stop,
int standby_message_timeout,
char *partial_suffix,
! int fsync_interval,
! bool reply_fsync);
On Mon, Aug 18, 2014 at 7:55 PM, <furuyao@pm.nttdata.co.jp> wrote:
Thanks for the review!
One question is why reply_fsync is defined as volatile variable?
Sorry I could not understand reason of that.It was affected to time_to_abort -- since it is unnecessary, it deletes.
Currently patch modifies argument of some function (e.g., Handle
CopyStream, Process LogDate Msg), and add the similar code to each
function.
I don't think it is good approach.
For example, I think that we should gather these code into one function.Feedback was judged immediately after each fsync until now.
I revised it in reference to walreceiver.
Feedback of fsync is judged together with the judgment of --status-interval.
Thereby, the specification to an argument became minimum.
Thank you for updating the patch.
I did not get error with applying, and compiling.
It works fine. I think this function code has no problem.
Could you please submit patch to commit fest app?
Regards,
-------
Sawada Masahiko
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
Thank you for updating the patch.
I did not get error with applying, and compiling.
It works fine. I think this function code has no problem.
Could you please submit patch to commit fest app?
Thanks for the review!
As you pointed out, submitted patch to commit fest app.
Regards,
--
Furuya Osamu
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On Tue, Aug 19, 2014 at 9:52 AM, <furuyao@pm.nttdata.co.jp> wrote:
Thank you for updating the patch.
I did not get error with applying, and compiling.
It works fine. I think this function code has no problem.
Could you please submit patch to commit fest app?Thanks for the review!
As you pointed out, submitted patch to commit fest app.
When replication slot is not specified in pg_receivexlog, the flush location
in the feedback message always indicates invalid. So there seems to be
no need to send the feedback as soon as fsync is issued, in that case.
How should this option work when replication slot is not specified?
Regards,
--
Fujii Masao
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
When replication slot is not specified in pg_receivexlog, the flush
location in the feedback message always indicates invalid. So there seems
to be no need to send the feedback as soon as fsync is issued, in that
case.
How should this option work when replication slot is not specified?
Thanks for the review!
The present is not checking the existence of specification of -S.
The use case when replication slot is not specified.
Because it does fsync, it isn't an original intention.
remote_write is set in synchronous_commit.
To call attention to the user, append following documents.
"If you want to report the flush position to the server, should use -S option."
Regards,
--
Furuya Osamu
Attachments:
pg_receivexlog-fsync-feedback-v4.patchapplication/octet-stream; name=pg_receivexlog-fsync-feedback-v4.patchDownload
*** a/doc/src/sgml/ref/pg_receivexlog.sgml
--- b/doc/src/sgml/ref/pg_receivexlog.sgml
***************
*** 196,201 **** PostgreSQL documentation
--- 196,212 ----
</varlistentry>
<varlistentry>
+ <term><option>-r</></term>
+ <term><option>--reply-fsync</></term>
+ <listitem>
+ <para>
+ status packets sent to server as soon as after fsync.
+ If you want to report the flush position to the server, should use -S option.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
<term><option>-U <replaceable>username</replaceable></option></term>
<term><option>--username=<replaceable class="parameter">username</replaceable></option></term>
<listitem>
*** a/src/bin/pg_basebackup/pg_basebackup.c
--- b/src/bin/pg_basebackup/pg_basebackup.c
***************
*** 371,377 **** LogStreamerMain(logstreamer_param *param)
if (!ReceiveXlogStream(param->bgconn, param->startptr, param->timeline,
param->sysidentifier, param->xlogdir,
reached_end_position, standby_message_timeout,
! NULL, 0))
/*
* Any errors will already have been reported in the function process,
--- 371,377 ----
if (!ReceiveXlogStream(param->bgconn, param->startptr, param->timeline,
param->sysidentifier, param->xlogdir,
reached_end_position, standby_message_timeout,
! NULL, 0, false))
/*
* Any errors will already have been reported in the function process,
*** a/src/bin/pg_basebackup/pg_receivexlog.c
--- b/src/bin/pg_basebackup/pg_receivexlog.c
***************
*** 38,44 **** static int noloop = 0;
static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
static int fsync_interval = 0; /* 0 = default */
static volatile bool time_to_abort = false;
!
static void usage(void);
static XLogRecPtr FindStreamingStart(uint32 *tli);
--- 38,44 ----
static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
static int fsync_interval = 0; /* 0 = default */
static volatile bool time_to_abort = false;
! static bool reply_fsync = false;
static void usage(void);
static XLogRecPtr FindStreamingStart(uint32 *tli);
***************
*** 74,79 **** usage(void)
--- 74,81 ----
printf(_(" -p, --port=PORT database server port number\n"));
printf(_(" -s, --status-interval=INTERVAL\n"
" time between status packets sent to server (in seconds)\n"));
+ printf(_(" -r, --reply-fsync status packets sent to server as soon as after fsync\n"
+ " If you want to report the flush position to the server, should use -S option.\n"));
printf(_(" -U, --username=NAME connect as specified database user\n"));
printf(_(" -w, --no-password never prompt for password\n"));
printf(_(" -W, --password force password prompt (should happen automatically)\n"));
***************
*** 334,340 **** StreamLog(void)
ReceiveXlogStream(conn, startpos, starttli, NULL, basedir,
stop_streaming, standby_message_timeout, ".partial",
! fsync_interval);
PQfinish(conn);
}
--- 336,342 ----
ReceiveXlogStream(conn, startpos, starttli, NULL, basedir,
stop_streaming, standby_message_timeout, ".partial",
! fsync_interval, reply_fsync);
PQfinish(conn);
}
***************
*** 368,373 **** main(int argc, char **argv)
--- 370,376 ----
{"no-password", no_argument, NULL, 'w'},
{"password", no_argument, NULL, 'W'},
{"status-interval", required_argument, NULL, 's'},
+ {"reply-fsync", no_argument, NULL, 'r'},
{"slot", required_argument, NULL, 'S'},
{"verbose", no_argument, NULL, 'v'},
{NULL, 0, NULL, 0}
***************
*** 394,400 **** main(int argc, char **argv)
}
}
! while ((c = getopt_long(argc, argv, "D:d:h:p:U:s:S:nF:wWv",
long_options, &option_index)) != -1)
{
switch (c)
--- 397,403 ----
}
}
! while ((c = getopt_long(argc, argv, "D:d:h:p:U:s:rS:nF:wWv",
long_options, &option_index)) != -1)
{
switch (c)
***************
*** 435,440 **** main(int argc, char **argv)
--- 438,446 ----
exit(1);
}
break;
+ case 'r':
+ reply_fsync = true;
+ break;
case 'S':
replication_slot = pg_strdup(optarg);
break;
*** a/src/bin/pg_basebackup/receivelog.c
--- b/src/bin/pg_basebackup/receivelog.c
***************
*** 38,44 **** static PGresult *HandleCopyStream(PGconn *conn, XLogRecPtr startpos,
uint32 timeline, char *basedir,
stream_stop_callback stream_stop, int standby_message_timeout,
char *partial_suffix, XLogRecPtr *stoppos,
! int fsync_interval);
static int CopyStreamPoll(PGconn *conn, long timeout_ms);
static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
--- 38,44 ----
uint32 timeline, char *basedir,
stream_stop_callback stream_stop, int standby_message_timeout,
char *partial_suffix, XLogRecPtr *stoppos,
! int fsync_interval, bool reply_fsync);
static int CopyStreamPoll(PGconn *conn, long timeout_ms);
static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
***************
*** 443,448 **** CheckServerVersionForStreaming(PGconn *conn)
--- 443,451 ----
* fsync_interval controls how often we flush to the received WAL file,
* in milliseconds.
*
+ * If 'reply_fsync' is true, status packets sent to server as soon as
+ * after fsync.
+ *
* Note: The log position *must* be at a log segment start!
*/
bool
***************
*** 450,456 **** ReceiveXlogStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *sysidentifier, char *basedir,
stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! int fsync_interval)
{
char query[128];
char slotcmd[128];
--- 453,459 ----
char *sysidentifier, char *basedir,
stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! int fsync_interval, bool reply_fsync)
{
char query[128];
char slotcmd[128];
***************
*** 595,601 **** ReceiveXlogStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
/* Stream the WAL */
res = HandleCopyStream(conn, startpos, timeline, basedir, stream_stop,
standby_message_timeout, partial_suffix,
! &stoppos, fsync_interval);
if (res == NULL)
goto error;
--- 598,604 ----
/* Stream the WAL */
res = HandleCopyStream(conn, startpos, timeline, basedir, stream_stop,
standby_message_timeout, partial_suffix,
! &stoppos, fsync_interval, reply_fsync);
if (res == NULL)
goto error;
***************
*** 760,770 **** static PGresult *
HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! XLogRecPtr *stoppos, int fsync_interval)
{
char *copybuf = NULL;
int64 last_status = -1;
XLogRecPtr blockpos = startpos;
still_sending = true;
--- 763,774 ----
HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! XLogRecPtr *stoppos, int fsync_interval, bool reply_fsync)
{
char *copybuf = NULL;
int64 last_status = -1;
XLogRecPtr blockpos = startpos;
+ XLogRecPtr lastFeedbackFlushPosition = startpos;
still_sending = true;
***************
*** 807,820 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
/*
* Potentially send a status message to the master
*/
! if (still_sending && standby_message_timeout > 0 &&
! feTimestampDifferenceExceeds(last_status, now,
! standby_message_timeout))
{
/* Time to send feedback! */
if (!sendFeedback(conn, blockpos, now, false))
goto error;
last_status = now;
}
/*
--- 811,826 ----
/*
* Potentially send a status message to the master
*/
! if ((still_sending && standby_message_timeout > 0 &&
! feTimestampDifferenceExceeds(last_status, now, standby_message_timeout)) ||
! (reply_fsync && lastFeedbackFlushPosition != lastFlushPosition))
{
/* Time to send feedback! */
if (!sendFeedback(conn, blockpos, now, false))
goto error;
last_status = now;
+ if (reply_fsync)
+ lastFeedbackFlushPosition = lastFlushPosition;
}
/*
*** a/src/bin/pg_basebackup/receivelog.h
--- b/src/bin/pg_basebackup/receivelog.h
***************
*** 17,20 **** extern bool ReceiveXlogStream(PGconn *conn,
stream_stop_callback stream_stop,
int standby_message_timeout,
char *partial_suffix,
! int fsync_interval);
--- 17,21 ----
stream_stop_callback stream_stop,
int standby_message_timeout,
char *partial_suffix,
! int fsync_interval,
! bool reply_fsync);
On Thu, Aug 21, 2014 at 2:54 PM, <furuyao@pm.nttdata.co.jp> wrote:
When replication slot is not specified in pg_receivexlog, the flush
location in the feedback message always indicates invalid. So there seems
to be no need to send the feedback as soon as fsync is issued, in that
case.
How should this option work when replication slot is not specified?Thanks for the review!
The present is not checking the existence of specification of -S.
The use case when replication slot is not specified.
Because it does fsync, it isn't an original intention.
remote_write is set in synchronous_commit.To call attention to the user, append following documents.
"If you want to report the flush position to the server, should use -S option."
Thank you for updating the patch.
I reviewed the patch.
First of all, I think that we should not append the above message to
section of '-r' option.
(Or these message might not be needed at all)
Whether flush location in feedback message is valid, is not depend on
'-r' option.
If we use '-r' option and 'S' option (i.g., replication slot) then
pg_receivexlog informs valid flush
location to primary server at the same time as doing fsync.
But, if we don't specify replication slot then the flush location in
feedback message always invalid.
So I think Fujii-san pointed out that sending of invalid flush
location is not needed
if pg_receivexlog does not use replication slot.
Regards,
-------
Sawada Masahiko
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
Thank you for updating the patch.
I reviewed the patch.First of all, I think that we should not append the above message to
section of '-r' option.
(Or these message might not be needed at all) Whether flush location in
feedback message is valid, is not depend on '-r' option.If we use '-r' option and 'S' option (i.g., replication slot) then
pg_receivexlog informs valid flush location to primary server at the same
time as doing fsync.
But, if we don't specify replication slot then the flush location in
feedback message always invalid.
So I think Fujii-san pointed out that sending of invalid flush location
is not needed if pg_receivexlog does not use replication slot.
Thanks for the review!
I understand the attention message wasn't appropriate.
To report the write location, even If you do not specify a replication slot.
So the fix only appended messages.
There was a description of the flush location section of '-S' option,
but I intended to catch eye more and added a message.
Is it better to make specification of the -S option indispensable?
Regards,
--
Furuya Osamu
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On Fri, Aug 22, 2014 at 1:35 PM, <furuyao@pm.nttdata.co.jp> wrote:
Thank you for updating the patch.
I reviewed the patch.First of all, I think that we should not append the above message to
section of '-r' option.
(Or these message might not be needed at all) Whether flush location in
feedback message is valid, is not depend on '-r' option.If we use '-r' option and 'S' option (i.g., replication slot) then
pg_receivexlog informs valid flush location to primary server at the same
time as doing fsync.
But, if we don't specify replication slot then the flush location in
feedback message always invalid.
So I think Fujii-san pointed out that sending of invalid flush location
is not needed if pg_receivexlog does not use replication slot.Thanks for the review!
I understand the attention message wasn't appropriate.
To report the write location, even If you do not specify a replication slot.
So the fix only appended messages.There was a description of the flush location section of '-S' option,
but I intended to catch eye more and added a message.Is it better to make specification of the -S option indispensable?
The patch cannot be applied to HEAD cleanly. Could you update the patch?
Regards,
--
Fujii Masao
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
Thanks for the review!
I understand the attention message wasn't appropriate.
To report the write location, even If you do not specify a replication
slot.
So the fix only appended messages.
There was a description of the flush location section of '-S' option,
but I intended to catch eye more and added a message.Is it better to make specification of the -S option indispensable?
The patch cannot be applied to HEAD cleanly. Could you update the patch?
Thank you for pointing out.
Updated the patch.
Regards,
--
Furuya Osamu
Attachments:
pg_receivexlog-fsync-feedback-v5.patchapplication/octet-stream; name=pg_receivexlog-fsync-feedback-v5.patchDownload
*** a/doc/src/sgml/ref/pg_receivexlog.sgml
--- b/doc/src/sgml/ref/pg_receivexlog.sgml
***************
*** 196,201 **** PostgreSQL documentation
--- 196,212 ----
</varlistentry>
<varlistentry>
+ <term><option>-r</></term>
+ <term><option>--reply-fsync</></term>
+ <listitem>
+ <para>
+ status packets sent to server as soon as after fsync.
+ If you want to report the flush position to the server, should use -S option.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
<term><option>-U <replaceable>username</replaceable></option></term>
<term><option>--username=<replaceable class="parameter">username</replaceable></option></term>
<listitem>
*** a/src/bin/pg_basebackup/pg_basebackup.c
--- b/src/bin/pg_basebackup/pg_basebackup.c
***************
*** 370,376 **** LogStreamerMain(logstreamer_param *param)
if (!ReceiveXlogStream(param->bgconn, param->startptr, param->timeline,
param->sysidentifier, param->xlogdir,
reached_end_position, standby_message_timeout,
! NULL, 0))
/*
* Any errors will already have been reported in the function process,
--- 370,376 ----
if (!ReceiveXlogStream(param->bgconn, param->startptr, param->timeline,
param->sysidentifier, param->xlogdir,
reached_end_position, standby_message_timeout,
! NULL, 0, false))
/*
* Any errors will already have been reported in the function process,
*** a/src/bin/pg_basebackup/pg_receivexlog.c
--- b/src/bin/pg_basebackup/pg_receivexlog.c
***************
*** 38,44 **** static int noloop = 0;
static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
static int fsync_interval = 0; /* 0 = default */
static volatile bool time_to_abort = false;
!
static void usage(void);
static XLogRecPtr FindStreamingStart(uint32 *tli);
--- 38,44 ----
static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
static int fsync_interval = 0; /* 0 = default */
static volatile bool time_to_abort = false;
! static bool reply_fsync = false;
static void usage(void);
static XLogRecPtr FindStreamingStart(uint32 *tli);
***************
*** 74,79 **** usage(void)
--- 74,81 ----
printf(_(" -p, --port=PORT database server port number\n"));
printf(_(" -s, --status-interval=INTERVAL\n"
" time between status packets sent to server (in seconds)\n"));
+ printf(_(" -r, --reply-fsync status packets sent to server as soon as after fsync\n"
+ " If you want to report the flush position to the server, should use -S option.\n"));
printf(_(" -U, --username=NAME connect as specified database user\n"));
printf(_(" -w, --no-password never prompt for password\n"));
printf(_(" -W, --password force password prompt (should happen automatically)\n"));
***************
*** 334,340 **** StreamLog(void)
ReceiveXlogStream(conn, startpos, starttli, NULL, basedir,
stop_streaming, standby_message_timeout, ".partial",
! fsync_interval);
PQfinish(conn);
}
--- 336,342 ----
ReceiveXlogStream(conn, startpos, starttli, NULL, basedir,
stop_streaming, standby_message_timeout, ".partial",
! fsync_interval, reply_fsync);
PQfinish(conn);
}
***************
*** 368,373 **** main(int argc, char **argv)
--- 370,376 ----
{"no-password", no_argument, NULL, 'w'},
{"password", no_argument, NULL, 'W'},
{"status-interval", required_argument, NULL, 's'},
+ {"reply-fsync", no_argument, NULL, 'r'},
{"slot", required_argument, NULL, 'S'},
{"verbose", no_argument, NULL, 'v'},
{NULL, 0, NULL, 0}
***************
*** 394,400 **** main(int argc, char **argv)
}
}
! while ((c = getopt_long(argc, argv, "D:d:h:p:U:s:S:nF:wWv",
long_options, &option_index)) != -1)
{
switch (c)
--- 397,403 ----
}
}
! while ((c = getopt_long(argc, argv, "D:d:h:p:U:s:rS:nF:wWv",
long_options, &option_index)) != -1)
{
switch (c)
***************
*** 435,440 **** main(int argc, char **argv)
--- 438,446 ----
exit(1);
}
break;
+ case 'r':
+ reply_fsync = true;
+ break;
case 'S':
replication_slot = pg_strdup(optarg);
break;
*** a/src/bin/pg_basebackup/receivelog.c
--- b/src/bin/pg_basebackup/receivelog.c
***************
*** 38,44 **** static PGresult *HandleCopyStream(PGconn *conn, XLogRecPtr startpos,
uint32 timeline, char *basedir,
stream_stop_callback stream_stop, int standby_message_timeout,
char *partial_suffix, XLogRecPtr *stoppos,
! int fsync_interval);
static int CopyStreamPoll(PGconn *conn, long timeout_ms);
static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
--- 38,44 ----
uint32 timeline, char *basedir,
stream_stop_callback stream_stop, int standby_message_timeout,
char *partial_suffix, XLogRecPtr *stoppos,
! int fsync_interval, bool reply_fsync);
static int CopyStreamPoll(PGconn *conn, long timeout_ms);
static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
***************
*** 443,448 **** CheckServerVersionForStreaming(PGconn *conn)
--- 443,451 ----
* fsync_interval controls how often we flush to the received WAL file,
* in milliseconds.
*
+ * If 'reply_fsync' is true, status packets sent to server as soon as
+ * after fsync.
+ *
* Note: The log position *must* be at a log segment start!
*/
bool
***************
*** 450,456 **** ReceiveXlogStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *sysidentifier, char *basedir,
stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! int fsync_interval)
{
char query[128];
char slotcmd[128];
--- 453,459 ----
char *sysidentifier, char *basedir,
stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! int fsync_interval, bool reply_fsync)
{
char query[128];
char slotcmd[128];
***************
*** 595,601 **** ReceiveXlogStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
/* Stream the WAL */
res = HandleCopyStream(conn, startpos, timeline, basedir, stream_stop,
standby_message_timeout, partial_suffix,
! &stoppos, fsync_interval);
if (res == NULL)
goto error;
--- 598,604 ----
/* Stream the WAL */
res = HandleCopyStream(conn, startpos, timeline, basedir, stream_stop,
standby_message_timeout, partial_suffix,
! &stoppos, fsync_interval, reply_fsync);
if (res == NULL)
goto error;
***************
*** 760,770 **** static PGresult *
HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! XLogRecPtr *stoppos, int fsync_interval)
{
char *copybuf = NULL;
int64 last_status = -1;
XLogRecPtr blockpos = startpos;
still_sending = true;
--- 763,774 ----
HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! XLogRecPtr *stoppos, int fsync_interval, bool reply_fsync)
{
char *copybuf = NULL;
int64 last_status = -1;
XLogRecPtr blockpos = startpos;
+ XLogRecPtr lastFeedbackFlushPosition = startpos;
still_sending = true;
***************
*** 807,820 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
/*
* Potentially send a status message to the master
*/
! if (still_sending && standby_message_timeout > 0 &&
! feTimestampDifferenceExceeds(last_status, now,
! standby_message_timeout))
{
/* Time to send feedback! */
if (!sendFeedback(conn, blockpos, now, false))
goto error;
last_status = now;
}
/*
--- 811,826 ----
/*
* Potentially send a status message to the master
*/
! if ((still_sending && standby_message_timeout > 0 &&
! feTimestampDifferenceExceeds(last_status, now, standby_message_timeout)) ||
! (reply_fsync && lastFeedbackFlushPosition != lastFlushPosition))
{
/* Time to send feedback! */
if (!sendFeedback(conn, blockpos, now, false))
goto error;
last_status = now;
+ if (reply_fsync)
+ lastFeedbackFlushPosition = lastFlushPosition;
}
/*
*** a/src/bin/pg_basebackup/receivelog.h
--- b/src/bin/pg_basebackup/receivelog.h
***************
*** 31,36 **** extern bool ReceiveXlogStream(PGconn *conn,
stream_stop_callback stream_stop,
int standby_message_timeout,
char *partial_suffix,
! int fsync_interval);
#endif /* RECEIVELOG_H */
--- 31,37 ----
stream_stop_callback stream_stop,
int standby_message_timeout,
char *partial_suffix,
! int fsync_interval,
! bool reply_fsync);
#endif /* RECEIVELOG_H */
On 09/05/2014 08:51 AM, furuyao@pm.nttdata.co.jp wrote:
Thanks for the review!
I understand the attention message wasn't appropriate.
To report the write location, even If you do not specify a replication
slot.
So the fix only appended messages.
There was a description of the flush location section of '-S' option,
but I intended to catch eye more and added a message.Is it better to make specification of the -S option indispensable?
The patch cannot be applied to HEAD cleanly. Could you update the patch?
Thank you for pointing out.
Updated the patch.
I don't understand what this patch does. When would you want to use the
new --reply-fsync option? Is there any reason *not* to use it? In other
words, do we need an option for this, couldn't you just always send the
feedback message after fsync?
- Heikki
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 09/05/2014 08:51 AM, furuyao@pm.nttdata.co.jp wrote:
Thanks for the review!
I understand the attention message wasn't appropriate.
To report the write location, even If you do not specify a
replicationslot.
So the fix only appended messages.
There was a description of the flush location section of '-S'
option, but I intended to catch eye more and added a message.Is it better to make specification of the -S option indispensable?
The patch cannot be applied to HEAD cleanly. Could you update the
patch?
Thank you for pointing out.
Updated the patch.I don't understand what this patch does. When would you want to use the
new --reply-fsync option? Is there any reason *not* to use it? In other
words, do we need an option for this, couldn't you just always send the
feedback message after fsync?
Thanks for the comment.
--reply-fsync option is intended for use in synchronous mode.
By specifying -F option and --slot option, process calls fsync() when it received the WAL, and flush location would be set in feedback message.
Interval of sending feedback message depends on -s option in this state, so in the case of synchronous mode, waiting for feedback message would occure.
therefore, --reply-fsync option is necessary. because it can send the feedback message after fsync without waiting for the interval of -s option.
The reason for not sending the feedback message after fsync without waiting for the interval of -s option always, is to answer the needs who want to use fsync only (NOT using synchronous mode).
Regards,
--
Furuya Osamu
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 09/29/2014 01:13 PM, furuyao@pm.nttdata.co.jp wrote:
I don't understand what this patch does. When would you want to use
the new --reply-fsync option? Is there any reason *not* to use it?
In other words, do we need an option for this, couldn't you just
always send the feedback message after fsync?Thanks for the comment.
--reply-fsync option is intended for use in synchronous mode.
By specifying -F option and --slot option, process calls fsync() when
it received the WAL, and flush location would be set in feedback
message.Interval of sending feedback message depends on -s option in this
state, so in the case of synchronous mode, waiting for feedback
message would occure.therefore, --reply-fsync option is necessary. because it can send the
feedback message after fsync without waiting for the interval of -s
option.The reason for not sending the feedback message after fsync without
waiting for the interval of -s option always, is to answer the needs
who want to use fsync only (NOT using synchronous mode).
I still don't get it. AFAICS there are two ways to use pg_receivexlog.
Either you use it as a synchronous standby, or not.
What set of options would you pass if you want to use it as a
synchronous standby? And if you don't? Could we just have a single
"--synchronous" flag, instead of -F and --reply-fsync?
- Heikki
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 09/29/2014 01:13 PM, furuyao@pm.nttdata.co.jp wrote:
I don't understand what this patch does. When would you want to use
the new --reply-fsync option? Is there any reason *not* to use it?
In other words, do we need an option for this, couldn't you just
always send the feedback message after fsync?Thanks for the comment.
--reply-fsync option is intended for use in synchronous mode.
By specifying -F option and --slot option, process calls fsync() when
it received the WAL, and flush location would be set in feedback
message.Interval of sending feedback message depends on -s option in this
state, so in the case of synchronous mode, waiting for feedback
message would occure.therefore, --reply-fsync option is necessary. because it can send the
feedback message after fsync without waiting for the interval of -s
option.The reason for not sending the feedback message after fsync without
waiting for the interval of -s option always, is to answer the needs
who want to use fsync only (NOT using synchronous mode).I still don't get it. AFAICS there are two ways to use pg_receivexlog.
Either you use it as a synchronous standby, or not.What set of options would you pass if you want to use it as a synchronous
standby? And if you don't? Could we just have a single "--synchronous"
flag, instead of -F and --reply-fsync?
Thanks for comment.
If you set "synchronous_commit" as "remote_write", the options would be different .
The set of options in each case, see the following.
Synchronous standby(synchronous_commit=on)
--fsync-interval=-1
--reply-fsync
--slot=slotname
Synchronous standby(synchronous_commit=remote_write)
--fsync-interval=-1
--reply-fsync
Asynchronous
There are no relative options.
Well, if the response time delay(value of "--status-interval=interval") is acceptable, "--reply-fsync" is unnecessary.
Instead of "--reply-fsync", using "--synchronous"(which summarizes the "--reply-fsync" and "fsync-interval = -1") might be easy to understand. Although, in that case, "--fsync-interval=interval" would be fixed value. Isn't there any problem ?
Regards,
--
Furuya Osamu
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 10/08/2014 07:23 AM, furuyao@pm.nttdata.co.jp wrote:
What set of options would you pass if you want to use it as a synchronous
standby? And if you don't? Could we just have a single "--synchronous"
flag, instead of -F and --reply-fsync?If you set "synchronous_commit" as "remote_write", the options would be different .
The set of options in each case, see the following.Synchronous standby(synchronous_commit=on)
--fsync-interval=-1
--reply-fsync
--slot=slotnameSynchronous standby(synchronous_commit=remote_write)
--fsync-interval=-1
--reply-fsyncAsynchronous
There are no relative options.Well, if the response time delay(value of "--status-interval=interval") is acceptable, "--reply-fsync" is unnecessary.
Instead of "--reply-fsync", using "--synchronous"(which summarizes the "--reply-fsync" and "fsync-interval = -1") might be easy to understand. Although, in that case, "--fsync-interval=interval" would be fixed value. Isn't there any problem ?
I think we should remove --fsync-interval and --reply-fsync, and just
have a --synchronous option, which would imply the same behavior you get
with --fsync-interval=-1 --reply--fsync.
That leaves the question of whether pg_receivexlog should do fsyncs when
it's not acting as a synchronous standby. There isn't any real need to
do so. In asynchronous mode, there are no guarantees anyway, and even
without an fsync, the OS will eventually flush the data to disk.
But we could do even better than that. It would be best if you didn't
need even the --synchronous flag. The server knows whether the client is
a synchronous standby or not. It could tell the client.
- Heikki
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 10/08/2014 07:23 AM, furuyao@pm.nttdata.co.jp wrote:
What set of options would you pass if you want to use it as a
synchronous standby? And if you don't? Could we just have a single"--synchronous"
flag, instead of -F and --reply-fsync?
If you set "synchronous_commit" as "remote_write", the options would
be different .
The set of options in each case, see the following.
Synchronous standby(synchronous_commit=on)
--fsync-interval=-1
--reply-fsync
--slot=slotnameSynchronous standby(synchronous_commit=remote_write)
--fsync-interval=-1
--reply-fsyncAsynchronous
There are no relative options.Well, if the response time delay(value of
"--status-interval=interval") is acceptable, "--reply-fsync" is
unnecessary.Instead of "--reply-fsync", using "--synchronous"(which summarizes
the "--reply-fsync" and "fsync-interval = -1") might be easy to
understand. Although, in that case, "--fsync-interval=interval" would
be fixed value. Isn't there any problem ?I think we should remove --fsync-interval and --reply-fsync, and just
have a --synchronous option, which would imply the same behavior you get
with --fsync-interval=-1 --reply--fsync.That leaves the question of whether pg_receivexlog should do fsyncs when
it's not acting as a synchronous standby. There isn't any real need to
do so. In asynchronous mode, there are no guarantees anyway, and even
without an fsync, the OS will eventually flush the data to disk.But we could do even better than that. It would be best if you didn't
need even the --synchronous flag. The server knows whether the client
is a synchronous standby or not. It could tell the client.
Thanks for the reply.
If we remove --fsync-interval, resoponse time to user will not be delay.
Although, fsync will be executed multiple times in a short period.
And there is no way to solve the problem without --fsync-interval, what should we do about it?
In asynchronous mode, I think there is no problem since the specification is same with release version.
The server knows whether the client is a synchronous standby or not.
It could tell the client.
When notifying the synchronous/asynchronous mode to the client from the server,
do we need to change the format of the Message ?
Regards,
--
Furuya Osamu
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 10/08/2014 11:47 AM, furuyao@pm.nttdata.co.jp wrote:
On 10/08/2014 07:23 AM, furuyao@pm.nttdata.co.jp wrote:
What set of options would you pass if you want to use it as a
synchronous standby? And if you don't? Could we just have a single"--synchronous"
flag, instead of -F and --reply-fsync?
If you set "synchronous_commit" as "remote_write", the options would
be different .
The set of options in each case, see the following.
Synchronous standby(synchronous_commit=on)
--fsync-interval=-1
--reply-fsync
--slot=slotnameSynchronous standby(synchronous_commit=remote_write)
--fsync-interval=-1
--reply-fsyncAsynchronous
There are no relative options.Well, if the response time delay(value of
"--status-interval=interval") is acceptable, "--reply-fsync" is
unnecessary.Instead of "--reply-fsync", using "--synchronous"(which summarizes
the "--reply-fsync" and "fsync-interval = -1") might be easy to
understand. Although, in that case, "--fsync-interval=interval" would
be fixed value. Isn't there any problem ?I think we should remove --fsync-interval and --reply-fsync, and just
have a --synchronous option, which would imply the same behavior you get
with --fsync-interval=-1 --reply--fsync.That leaves the question of whether pg_receivexlog should do fsyncs when
it's not acting as a synchronous standby. There isn't any real need to
do so. In asynchronous mode, there are no guarantees anyway, and even
without an fsync, the OS will eventually flush the data to disk.But we could do even better than that. It would be best if you didn't
need even the --synchronous flag. The server knows whether the client
is a synchronous standby or not. It could tell the client.If we remove --fsync-interval, resoponse time to user will not be delay.
Although, fsync will be executed multiple times in a short period.
And there is no way to solve the problem without --fsync-interval, what should we do about it?
I'm sorry, I didn't understand that.
In asynchronous mode, I think there is no problem since the specification is same with release version.
The server knows whether the client is a synchronous standby or not.
It could tell the client.When notifying the synchronous/asynchronous mode to the client from the server,
do we need to change the format of the Message ?
Yeah. Or rather, add a new message type, to indicate the
synchronous/asynchronous status.
- Heikki
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
What set of options would you pass if you want to use it as a
synchronous standby? And if you don't? Could we just have a single"--synchronous"
flag, instead of -F and --reply-fsync?
If you set "synchronous_commit" as "remote_write", the options would
be different .
The set of options in each case, see the following.
Synchronous standby(synchronous_commit=on)
--fsync-interval=-1
--reply-fsync
--slot=slotnameSynchronous standby(synchronous_commit=remote_write)
--fsync-interval=-1
--reply-fsyncAsynchronous
There are no relative options.Well, if the response time delay(value of
"--status-interval=interval") is acceptable, "--reply-fsync" is
unnecessary.Instead of "--reply-fsync", using "--synchronous"(which summarizes
the "--reply-fsync" and "fsync-interval = -1") might be easy to
understand. Although, in that case, "--fsync-interval=interval"
would be fixed value. Isn't there any problem ?I think we should remove --fsync-interval and --reply-fsync, and just
have a --synchronous option, which would imply the same behavior you
get with --fsync-interval=-1 --reply--fsync.That leaves the question of whether pg_receivexlog should do fsyncs
when it's not acting as a synchronous standby. There isn't any real
need to do so. In asynchronous mode, there are no guarantees anyway,
and even without an fsync, the OS will eventually flush the data todisk.
But we could do even better than that. It would be best if you didn't
need even the --synchronous flag. The server knows whether the client
is a synchronous standby or not. It could tell the client.If we remove --fsync-interval, resoponse time to user will not be delay.
Although, fsync will be executed multiple times in a short period.
And there is no way to solve the problem without --fsync-interval, whatshould we do about it?
I'm sorry, I didn't understand that.
In asynchronous mode, I think there is no problem since the
specification is same with release version.
The server knows whether the client is a synchronous standby or not.
It could tell the client.When notifying the synchronous/asynchronous mode to the client from
the server, do we need to change the format of the Message ?Yeah. Or rather, add a new message type, to indicate the
synchronous/asynchronous status.
Thanks for the reply.
If we remove --fsync-interval, resoponse time to user will not be delay.
Although, fsync will be executed multiple times in a short period.
And there is no way to solve the problem without --fsync-interval, whatshould we do about it?
I'm sorry, I didn't understand that.
Here is an example.
When WAL is sent at 100ms intervals, fsync() is executed 10 times per second.
If --fsync-interval is set by 1 second, we have to wait SQL responce(kind of making WAL record) for 1 second, though, fsync() won't be executed several times in 1 second.
I think --fsync-interval meets the demands of people who wants to restrain fsync() happens for several time in short period, but what do you think?
Is it ok to delete --fsync-interval ?
The server knows whether the client is a synchronous standby or not.
It could tell the client.When notifying the synchronous/asynchronous mode to the client from
the server, do we need to change the format of the Message ?Yeah. Or rather, add a new message type, to indicate the
synchronous/asynchronous status.
What kind of 'message type' we have to add ?
Do we need to separate 'w' into two types ? synchronous and asynchronous ?
OR
Add a new message type, kind of 'notify synchronous',
and inform pg_receivexlog of synchronous status when it connect to the server.
Regards,
--
Furuya Osamu
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 10/09/2014 07:47 AM, furuyao@pm.nttdata.co.jp wrote:
If we remove --fsync-interval, resoponse time to user will not be delay.
Although, fsync will be executed multiple times in a short period.
And there is no way to solve the problem without --fsync-interval, whatshould we do about it?
I'm sorry, I didn't understand that.
Here is an example.
When WAL is sent at 100ms intervals, fsync() is executed 10 times per second.
If --fsync-interval is set by 1 second, we have to wait SQL responce(kind of making WAL record) for 1 second, though, fsync() won't be executed several times in 1 second.
I think --fsync-interval meets the demands of people who wants to restrain fsync() happens for several time in short period, but what do you think?
Is it ok to delete --fsync-interval ?
I still don't see the problem.
In synchronous mode, pg_receivexlog should have similar logic as
walreceiver does. It should read as much WAL from the socket as it can
without blocking, and fsync() and send reply after that. And also fsync
whenever switching to new segment. If the master sends WAL every 100ms,
then pg_recevexlog will indeed fsync() 10 times per second. There's
nothing wrong with that.
In asynchronous mode, only fsync whenever switching to new segment.
Yeah. Or rather, add a new message type, to indicate the
synchronous/asynchronous status.What kind of 'message type' we have to add ?
Do we need to separate 'w' into two types ? synchronous and asynchronous ?
OR
Add a new message type, kind of 'notify synchronous',
and inform pg_receivexlog of synchronous status when it connect to the server.
Better to add a new "notify" message type. And pg_recevexlog should be
prepared to receive it at any time. The status might change on the fly,
if the server's configuration is reloaded.
- Heikki
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
If we remove --fsync-interval, resoponse time to user will not be
delay.
Although, fsync will be executed multiple times in a short period.
And there is no way to solve the problem without --fsync-interval,
whatshould we do about it?
I'm sorry, I didn't understand that.
Here is an example.
When WAL is sent at 100ms intervals, fsync() is executed 10 times persecond.
If --fsync-interval is set by 1 second, we have to wait SQL
responce(kind of making WAL record) for 1 second, though, fsync() won't
be executed several times in 1 second.I think --fsync-interval meets the demands of people who wants to
restrain fsync() happens for several time in short period, but what do
you think?Is it ok to delete --fsync-interval ?
I still don't see the problem.
In synchronous mode, pg_receivexlog should have similar logic as
walreceiver does. It should read as much WAL from the socket as it can
without blocking, and fsync() and send reply after that. And also fsync
whenever switching to new segment. If the master sends WAL every 100ms,
then pg_recevexlog will indeed fsync() 10 times per second. There's
nothing wrong with that.In asynchronous mode, only fsync whenever switching to new segment.
Yeah. Or rather, add a new message type, to indicate the
synchronous/asynchronous status.What kind of 'message type' we have to add ?
Do we need to separate 'w' into two types ? synchronous and
asynchronous ?
OR
Add a new message type, kind of 'notify synchronous', and inform
pg_receivexlog of synchronous status when it connect to the server.Better to add a new "notify" message type. And pg_recevexlog should be
prepared to receive it at any time. The status might change on the fly,
if the server's configuration is reloaded.
Thanks for the reply.
In synchronous mode, pg_receivexlog should have similar logic as walreceiver does.
OK. I understand that removing --fsync-interval has no problem.
Better to add a new "notify" message type. And pg_recevexlog should be prepared to receive it at any time. The status might change on the fly, if the server's configuration is reloaded.
OK. I'll consider it.
Regards,
--
Furuya Osamu
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On Thu, Oct 9, 2014 at 6:42 PM, <furuyao@pm.nttdata.co.jp> wrote:
If we remove --fsync-interval, resoponse time to user will not be
delay.
Although, fsync will be executed multiple times in a short period.
And there is no way to solve the problem without --fsync-interval,
whatshould we do about it?
I'm sorry, I didn't understand that.
Here is an example.
When WAL is sent at 100ms intervals, fsync() is executed 10 times persecond.
If --fsync-interval is set by 1 second, we have to wait SQL
responce(kind of making WAL record) for 1 second, though, fsync() won't
be executed several times in 1 second.I think --fsync-interval meets the demands of people who wants to
restrain fsync() happens for several time in short period, but what do
you think?Is it ok to delete --fsync-interval ?
I still don't see the problem.
In synchronous mode, pg_receivexlog should have similar logic as
walreceiver does. It should read as much WAL from the socket as it can
without blocking, and fsync() and send reply after that. And also fsync
whenever switching to new segment. If the master sends WAL every 100ms,
then pg_recevexlog will indeed fsync() 10 times per second. There's
nothing wrong with that.In asynchronous mode, only fsync whenever switching to new segment.
Yeah. Or rather, add a new message type, to indicate the
synchronous/asynchronous status.What kind of 'message type' we have to add ?
Do we need to separate 'w' into two types ? synchronous and
asynchronous ?
OR
Add a new message type, kind of 'notify synchronous', and inform
pg_receivexlog of synchronous status when it connect to the server.Better to add a new "notify" message type. And pg_recevexlog should be
prepared to receive it at any time. The status might change on the fly,
if the server's configuration is reloaded.Thanks for the reply.
In synchronous mode, pg_receivexlog should have similar logic as walreceiver does.
OK. I understand that removing --fsync-interval has no problem.
+1 for adding something like --synchronous option. To me,
it sounds walreceiver-compatible mode rather than synchronous.
Better to add a new "notify" message type. And pg_recevexlog should be prepared to receive it at any time. The status might change on the fly, if the server's configuration is reloaded.
OK. I'll consider it.
I don't think that's good idea because it prevents us from using
pg_receivexlog as async walreceiver (i.e., received WAL data is
fsynced and feedback is sent back to the server soon,
but transaction commit in the server doesn't wait for the feedback).
Regards,
--
Fujii Masao
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
If we remove --fsync-interval, resoponse time to user will not
be
delay.
Although, fsync will be executed multiple times in a short period.
And there is no way to solve the problem without
--fsync-interval, whatshould we do about it?
I'm sorry, I didn't understand that.
Here is an example.
When WAL is sent at 100ms intervals, fsync() is executed 10 times
persecond.
If --fsync-interval is set by 1 second, we have to wait SQL
responce(kind of making WAL record) for 1 second, though, fsync()
won't be executed several times in 1 second.I think --fsync-interval meets the demands of people who wants to
restrain fsync() happens for several time in short period, but what
do you think?Is it ok to delete --fsync-interval ?
I still don't see the problem.
In synchronous mode, pg_receivexlog should have similar logic as
walreceiver does. It should read as much WAL from the socket as it
can without blocking, and fsync() and send reply after that. And also
fsync whenever switching to new segment. If the master sends WAL
every 100ms, then pg_recevexlog will indeed fsync() 10 times per
second. There's nothing wrong with that.In asynchronous mode, only fsync whenever switching to new segment.
Yeah. Or rather, add a new message type, to indicate the
synchronous/asynchronous status.What kind of 'message type' we have to add ?
Do we need to separate 'w' into two types ? synchronous and
asynchronous ?
OR
Add a new message type, kind of 'notify synchronous', and inform
pg_receivexlog of synchronous status when it connect to the server.Better to add a new "notify" message type. And pg_recevexlog should
be prepared to receive it at any time. The status might change on the
fly, if the server's configuration is reloaded.Thanks for the reply.
In synchronous mode, pg_receivexlog should have similar logic as
walreceiver does.
OK. I understand that removing --fsync-interval has no problem.
+1 for adding something like --synchronous option. To me,
it sounds walreceiver-compatible mode rather than synchronous.Better to add a new "notify" message type. And pg_recevexlog should
be prepared to receive it at any time. The status might change on the
fly, if the server's configuration is reloaded.OK. I'll consider it.
I don't think that's good idea because it prevents us from using
pg_receivexlog as async walreceiver (i.e., received WAL data is fsynced
and feedback is sent back to the server soon, but transaction commit in
the server doesn't wait for the feedback).
Thanks for the comment.
it prevents us from using pg_receivexlog as async walreceiver.
Yes. If sync or async is switched by message,
in case pg_receivexlog is async, it won't work like a walreceiver.
User don't have to set options if synchronous status can be distinguished by message, though they cannot specify the action.
adding something like --synchronous option
Integrate required options for sync like above, is more appropriate ?
Regards,
--
Furuya Osamu
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 10 October 2014 09:28, Fujii Masao <masao.fujii@gmail.com> wrote:
In synchronous mode, pg_receivexlog should have similar logic as walreceiver does.
OK. I understand that removing --fsync-interval has no problem.
+1 for adding something like --synchronous option. To me,
it sounds walreceiver-compatible mode rather than synchronous.Better to add a new "notify" message type. And pg_recevexlog should be prepared to receive it at any time. The status might change on the fly, if the server's configuration is reloaded.
OK. I'll consider it.
I don't think that's good idea because it prevents us from using
pg_receivexlog as async walreceiver (i.e., received WAL data is
fsynced and feedback is sent back to the server soon,
but transaction commit in the server doesn't wait for the feedback).
Sync rep works by setting parameters on the master. Standby servers
send replies by default, though you can turn replies off.
pg_receivexlog should work the same, but can't do this because it
doesn't know the fsync position unless it fsyncs.
So its not appropriate to have an option called "--synchronous" in the
same way that there is no parameter called "synchronous" on the
standby, for good reason.
A new parameter to send feedback should be called --feedback
A second parameter to decide whether to fsync should be called --fsync
if (feedback && fsync)
send fsynced LSN
else if (feedback)
send received LSN
; /* else send no feedback */
--
Simon Riggs http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
In synchronous mode, pg_receivexlog should have similar logic as
walreceiver does.
OK. I understand that removing --fsync-interval has no problem.
+1 for adding something like --synchronous option. To me,
it sounds walreceiver-compatible mode rather than synchronous.Better to add a new "notify" message type. And pg_recevexlog should
be prepared to receive it at any time. The status might change on the
fly, if the server's configuration is reloaded.OK. I'll consider it.
I don't think that's good idea because it prevents us from using
pg_receivexlog as async walreceiver (i.e., received WAL data is
fsynced and feedback is sent back to the server soon, but transaction
commit in the server doesn't wait for the feedback).Sync rep works by setting parameters on the master. Standby servers send
replies by default, though you can turn replies off.pg_receivexlog should work the same, but can't do this because it doesn't
know the fsync position unless it fsyncs.So its not appropriate to have an option called "--synchronous" in the
same way that there is no parameter called "synchronous" on the standby,
for good reason.A new parameter to send feedback should be called --feedback A second
parameter to decide whether to fsync should be called --fsyncif (feedback && fsync)
send fsynced LSN
else if (feedback)
send received LSN
; /* else send no feedback */
Thanks for the comment.
The patch cannot be applied to HEAD cleanly so I updated.
So its not appropriate to have an option called "--synchronous" in the same way that there is no >parameter called "synchronous" on the standby, for good reason.
In case of gathering options to one option,
change the name "--synchronous" to other name solves the problem ?
A new parameter to send feedback should be called --feedback
A second parameter to decide whether to fsync should be called --fsync
I think keep using "--reply-fsync" and "--fsync-interval" is better than make new options.
Thought?
Regards,
--
Furuya Osamu
Attachments:
pg_receivexlog-fsync-feedback-v6.patchapplication/octet-stream; name=pg_receivexlog-fsync-feedback-v6.patchDownload
*** a/doc/src/sgml/ref/pg_receivexlog.sgml
--- b/doc/src/sgml/ref/pg_receivexlog.sgml
***************
*** 196,201 **** PostgreSQL documentation
--- 196,212 ----
</varlistentry>
<varlistentry>
+ <term><option>-r</></term>
+ <term><option>--reply-fsync</></term>
+ <listitem>
+ <para>
+ status packets sent to server as soon as after fsync.
+ If you want to report the flush position to the server, should use -S option.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
<term><option>-U <replaceable>username</replaceable></option></term>
<term><option>--username=<replaceable class="parameter">username</replaceable></option></term>
<listitem>
*** a/src/bin/pg_basebackup/pg_basebackup.c
--- b/src/bin/pg_basebackup/pg_basebackup.c
***************
*** 370,376 **** LogStreamerMain(logstreamer_param *param)
if (!ReceiveXlogStream(param->bgconn, param->startptr, param->timeline,
param->sysidentifier, param->xlogdir,
reached_end_position, standby_message_timeout,
! NULL, 0))
/*
* Any errors will already have been reported in the function process,
--- 370,376 ----
if (!ReceiveXlogStream(param->bgconn, param->startptr, param->timeline,
param->sysidentifier, param->xlogdir,
reached_end_position, standby_message_timeout,
! NULL, 0, false))
/*
* Any errors will already have been reported in the function process,
*** a/src/bin/pg_basebackup/pg_receivexlog.c
--- b/src/bin/pg_basebackup/pg_receivexlog.c
***************
*** 40,46 **** static int fsync_interval = 0; /* 0 = default */
static volatile bool time_to_abort = false;
static bool do_create_slot = false;
static bool do_drop_slot = false;
!
static void usage(void);
static DIR* get_destination_dir(char *dest_folder);
--- 40,46 ----
static volatile bool time_to_abort = false;
static bool do_create_slot = false;
static bool do_drop_slot = false;
! static bool reply_fsync = false;
static void usage(void);
static DIR* get_destination_dir(char *dest_folder);
***************
*** 71,76 **** usage(void)
--- 71,78 ----
printf(_(" -n, --no-loop do not loop on connection lost\n"));
printf(_(" -s, --status-interval=SECS\n"
" time between status packets sent to server (default: %d)\n"), (standby_message_timeout / 1000));
+ printf(_(" -r, --reply-fsync status packets sent to server as soon as after fsync\n"
+ " If you want to report the flush position to the server, should use -S option.\n"));
printf(_(" -S, --slot=SLOTNAME replication slot to use\n"));
printf(_(" -v, --verbose output verbose messages\n"));
printf(_(" -V, --version output version information, then exit\n"));
***************
*** 342,348 **** StreamLog(void)
ReceiveXlogStream(conn, startpos, starttli, NULL, basedir,
stop_streaming, standby_message_timeout, ".partial",
! fsync_interval);
PQfinish(conn);
}
--- 344,350 ----
ReceiveXlogStream(conn, startpos, starttli, NULL, basedir,
stop_streaming, standby_message_timeout, ".partial",
! fsync_interval, reply_fsync);
PQfinish(conn);
}
***************
*** 376,381 **** main(int argc, char **argv)
--- 378,384 ----
{"no-password", no_argument, NULL, 'w'},
{"password", no_argument, NULL, 'W'},
{"status-interval", required_argument, NULL, 's'},
+ {"reply-fsync", no_argument, NULL, 'r'},
{"slot", required_argument, NULL, 'S'},
{"verbose", no_argument, NULL, 'v'},
/* action */
***************
*** 406,412 **** main(int argc, char **argv)
}
}
! while ((c = getopt_long(argc, argv, "D:d:h:p:U:s:S:nF:wWv",
long_options, &option_index)) != -1)
{
switch (c)
--- 409,415 ----
}
}
! while ((c = getopt_long(argc, argv, "D:d:h:p:U:s:rS:nF:wWv",
long_options, &option_index)) != -1)
{
switch (c)
***************
*** 447,452 **** main(int argc, char **argv)
--- 450,458 ----
exit(1);
}
break;
+ case 'r':
+ reply_fsync = true;
+ break;
case 'S':
replication_slot = pg_strdup(optarg);
break;
*** a/src/bin/pg_basebackup/receivelog.c
--- b/src/bin/pg_basebackup/receivelog.c
***************
*** 38,44 **** static PGresult *HandleCopyStream(PGconn *conn, XLogRecPtr startpos,
uint32 timeline, char *basedir,
stream_stop_callback stream_stop, int standby_message_timeout,
char *partial_suffix, XLogRecPtr *stoppos,
! int fsync_interval);
static int CopyStreamPoll(PGconn *conn, long timeout_ms);
static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
--- 38,44 ----
uint32 timeline, char *basedir,
stream_stop_callback stream_stop, int standby_message_timeout,
char *partial_suffix, XLogRecPtr *stoppos,
! int fsync_interval, bool reply_fsync);
static int CopyStreamPoll(PGconn *conn, long timeout_ms);
static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
***************
*** 443,448 **** CheckServerVersionForStreaming(PGconn *conn)
--- 443,451 ----
* fsync_interval controls how often we flush to the received WAL file,
* in milliseconds.
*
+ * If 'reply_fsync' is true, status packets sent to server as soon as
+ * after fsync.
+ *
* Note: The log position *must* be at a log segment start!
*/
bool
***************
*** 450,456 **** ReceiveXlogStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *sysidentifier, char *basedir,
stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! int fsync_interval)
{
char query[128];
char slotcmd[128];
--- 453,459 ----
char *sysidentifier, char *basedir,
stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! int fsync_interval, bool reply_fsync)
{
char query[128];
char slotcmd[128];
***************
*** 595,601 **** ReceiveXlogStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
/* Stream the WAL */
res = HandleCopyStream(conn, startpos, timeline, basedir, stream_stop,
standby_message_timeout, partial_suffix,
! &stoppos, fsync_interval);
if (res == NULL)
goto error;
--- 598,604 ----
/* Stream the WAL */
res = HandleCopyStream(conn, startpos, timeline, basedir, stream_stop,
standby_message_timeout, partial_suffix,
! &stoppos, fsync_interval, reply_fsync);
if (res == NULL)
goto error;
***************
*** 760,770 **** static PGresult *
HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! XLogRecPtr *stoppos, int fsync_interval)
{
char *copybuf = NULL;
int64 last_status = -1;
XLogRecPtr blockpos = startpos;
still_sending = true;
--- 763,774 ----
HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! XLogRecPtr *stoppos, int fsync_interval, bool reply_fsync)
{
char *copybuf = NULL;
int64 last_status = -1;
XLogRecPtr blockpos = startpos;
+ XLogRecPtr lastFeedbackFlushPosition = startpos;
still_sending = true;
***************
*** 807,820 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
/*
* Potentially send a status message to the master
*/
! if (still_sending && standby_message_timeout > 0 &&
! feTimestampDifferenceExceeds(last_status, now,
! standby_message_timeout))
{
/* Time to send feedback! */
if (!sendFeedback(conn, blockpos, now, false))
goto error;
last_status = now;
}
/*
--- 811,826 ----
/*
* Potentially send a status message to the master
*/
! if ((still_sending && standby_message_timeout > 0 &&
! feTimestampDifferenceExceeds(last_status, now, standby_message_timeout)) ||
! (reply_fsync && lastFeedbackFlushPosition != lastFlushPosition))
{
/* Time to send feedback! */
if (!sendFeedback(conn, blockpos, now, false))
goto error;
last_status = now;
+ if (reply_fsync)
+ lastFeedbackFlushPosition = lastFlushPosition;
}
/*
*** a/src/bin/pg_basebackup/receivelog.h
--- b/src/bin/pg_basebackup/receivelog.h
***************
*** 31,36 **** extern bool ReceiveXlogStream(PGconn *conn,
stream_stop_callback stream_stop,
int standby_message_timeout,
char *partial_suffix,
! int fsync_interval);
#endif /* RECEIVELOG_H */
--- 31,37 ----
stream_stop_callback stream_stop,
int standby_message_timeout,
char *partial_suffix,
! int fsync_interval,
! bool reply_fsync);
#endif /* RECEIVELOG_H */
On 17 October 2014 09:55, <furuyao@pm.nttdata.co.jp> wrote:
A new parameter to send feedback should be called --feedback
A second parameter to decide whether to fsync should be called --fsync
I think keep using "--reply-fsync" and "--fsync-interval" is better than make new options.
Thought?
We already have hot_standby_feedback, so using the name feedback is best idea.
I am suggesting that we send feedback even if we do not fsync, to
allow the master to track our progress. Hence the name of the second
parameter was just fsync.
So both names were suggested because of links to those terms already
being used for similar reasons elsewhere in Postgres.
--
Simon Riggs http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 10/17/2014 01:59 PM, Simon Riggs wrote:
On 17 October 2014 09:55, <furuyao@pm.nttdata.co.jp> wrote:
A new parameter to send feedback should be called --feedback
A second parameter to decide whether to fsync should be called --fsync
I think keep using "--reply-fsync" and "--fsync-interval" is better than make new options.
Thought?We already have hot_standby_feedback, so using the name feedback is best idea.
I am suggesting that we send feedback even if we do not fsync, to
allow the master to track our progress. Hence the name of the second
parameter was just fsync.So both names were suggested because of links to those terms already
being used for similar reasons elsewhere in Postgres.
We seem to be going in circles. You suggested having two options,
--feedback, and --fsync, which is almost exactly what Furuya posted
originally. I objected to that, because I think that user interface is
too complicated. Instead, I suggested having just a single option called
--synchronous, or even better, have no option at all and have the server
tell the client if it's participating in synchronous replication, and
have pg_receivexlog automatically fsync when it is, and not otherwise
[1]: /messages/by-id/5434E0EF.9050304@vmware.com
did you think of that idea?
[1]: /messages/by-id/5434E0EF.9050304@vmware.com
- Heikki
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 22 October 2014 14:26, Heikki Linnakangas <hlinnakangas@vmware.com> wrote:
We seem to be going in circles. You suggested having two options,
--feedback, and --fsync, which is almost exactly what Furuya posted
originally. I objected to that, because I think that user interface is too
complicated. Instead, I suggested having just a single option called
--synchronous, or even better, have no option at all and have the server
tell the client if it's participating in synchronous replication, and have
pg_receivexlog automatically fsync when it is, and not otherwise [1]. That
way you don't need to expose any new options to the user. What did you think
of that idea?
Sorry, if we're going in circles.
Yes, I like the idea.
The master setting of synchronous_standby_names defines which
standbys/rep clients will have their feedback used to release waiting
COMMITs. That uses application_name, which is set at the replication
client connection. (That has a default value, but lets assume the user
sets this). So when a replication client connects, the WALSender knows
its priority, as defined in sync_standby_names.
I guess we can have WALSender send a protocol message to the client
every time the sync priority changes (as a result of a changed value
of sync_standby_names). Which is the function SyncRepInitConfig()
--
Simon Riggs http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On Wed, Oct 22, 2014 at 10:47 PM, Simon Riggs <simon@2ndquadrant.com> wrote:
On 22 October 2014 14:26, Heikki Linnakangas <hlinnakangas@vmware.com> wrote:
We seem to be going in circles. You suggested having two options,
--feedback, and --fsync, which is almost exactly what Furuya posted
originally. I objected to that, because I think that user interface is too
complicated. Instead, I suggested having just a single option called
--synchronous
I'm OK with this. But the option name "synchronous" seems confusing
because pg_receivexlog with sync option can work as async standby.
So the better name is needed.
or even better, have no option at all and have the server
tell the client if it's participating in synchronous replication, and have
pg_receivexlog automatically fsync when it is, and not otherwise [1]. That
way you don't need to expose any new options to the user. What did you think
of that idea?Sorry, if we're going in circles.
Yes, I like the idea.
The master setting of synchronous_standby_names defines which
standbys/rep clients will have their feedback used to release waiting
COMMITs. That uses application_name, which is set at the replication
client connection. (That has a default value, but lets assume the user
sets this). So when a replication client connects, the WALSender knows
its priority, as defined in sync_standby_names.I guess we can have WALSender send a protocol message to the client
every time the sync priority changes (as a result of a changed value
of sync_standby_names). Which is the function SyncRepInitConfig()
Sorry, I'm going around in the circle. But I'd like to say again, I don't think
this is good idea. It prevents asynchronous pg_receivexlog from fsyncing
WAL data and sending feedbacks more frequently at all. They are useful,
for example, when we want to monitor the write location of asynchronous
pg_receivexlog in almost realtime. But if we adopt the idea, since feedback
cannot be sent soon in async mode, pg_stat_replication always returns
the not-up-to-date location.
Regards,
--
Fujii Masao
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 23 October 2014 15:39, Fujii Masao <masao.fujii@gmail.com> wrote:
Sorry, I'm going around in the circle. But I'd like to say again, I don't think
this is good idea. It prevents asynchronous pg_receivexlog from fsyncing
WAL data and sending feedbacks more frequently at all. They are useful,
for example, when we want to monitor the write location of asynchronous
pg_receivexlog in almost realtime. But if we adopt the idea, since feedback
cannot be sent soon in async mode, pg_stat_replication always returns
the not-up-to-date location.
Why not send a message every 10 seconds when its not sync rep?
--
Simon Riggs http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 10/23/2014 06:01 PM, Simon Riggs wrote:
On 23 October 2014 15:39, Fujii Masao <masao.fujii@gmail.com> wrote:
Sorry, I'm going around in the circle. But I'd like to say again, I don't think
this is good idea. It prevents asynchronous pg_receivexlog from fsyncing
WAL data and sending feedbacks more frequently at all. They are useful,
for example, when we want to monitor the write location of asynchronous
pg_receivexlog in almost realtime. But if we adopt the idea, since feedback
cannot be sent soon in async mode, pg_stat_replication always returns
the not-up-to-date location.Why not send a message every 10 seconds when its not sync rep?
Or even after every write(). It's a tiny amount of network traffic anyway.
- Heikki
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
Sorry, I'm going around in the circle. But I'd like to say again, I
don't think this is good idea. It prevents asynchronous
pg_receivexlog from fsyncing WAL data and sending feedbacks more
frequently at all. They are useful, for example, when we want to
monitor the write location of asynchronous pg_receivexlog in almost
realtime. But if we adopt the idea, since feedback cannot be sent
soon in async mode, pg_stat_replication always returns thenot-up-to-date location.
Why not send a message every 10 seconds when its not sync rep?
Or even after every write(). It's a tiny amount of network traffic anyway.
I understand that send feedback message frequently will keep pg_stat_replication up-to-date state.
Are there really no needs who wants to fsync even in async mode ?
I think the people who dislike Data lost will like that idea.
Thought?
Nevertheless in sync or async, returning feedback and executing fsync() same as like walreceiver is such a problem?
--
Furuya Osamu
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 10/24/2014 01:24 PM, furuyao@pm.nttdata.co.jp wrote:
Sorry, I'm going around in the circle. But I'd like to say again, I
don't think this is good idea. It prevents asynchronous
pg_receivexlog from fsyncing WAL data and sending feedbacks more
frequently at all. They are useful, for example, when we want to
monitor the write location of asynchronous pg_receivexlog in almost
realtime. But if we adopt the idea, since feedback cannot be sent
soon in async mode, pg_stat_replication always returns thenot-up-to-date location.
Why not send a message every 10 seconds when its not sync rep?
Or even after every write(). It's a tiny amount of network traffic anyway.
I understand that send feedback message frequently will keep pg_stat_replication up-to-date state.
Are there really no needs who wants to fsync even in async mode ?
I think the people who dislike Data lost will like that idea.
The OS will not sit on the written but not fsync'd data forever, it will
get flushed to disk in a few seconds even without the fsync. It's just
that there is no guarantee on when it will hit the disk, but there are
no strong guarantees in asynchronous replication anyway.
Nevertheless in sync or async, returning feedback and executing
fsync() same as like walreceiver is such a problem?
Probably would be OK. It would increase the I/O a lot, thanks to a lot
of small writes and fsyncs, which might be surprising for a tool like
pg_receivexlog.
One idea is to change the default behavior to be like walreceiver, and
fsync() after every write. But provide an option to get the old
behavior, "--no-fsync".
- Heikki
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On Fri, Oct 24, 2014 at 11:21 PM, Heikki Linnakangas
<hlinnakangas@vmware.com> wrote:
On 10/24/2014 01:24 PM, furuyao@pm.nttdata.co.jp wrote:
Sorry, I'm going around in the circle. But I'd like to say again, I
don't think this is good idea. It prevents asynchronous
pg_receivexlog from fsyncing WAL data and sending feedbacks more
frequently at all. They are useful, for example, when we want to
monitor the write location of asynchronous pg_receivexlog in almost
realtime. But if we adopt the idea, since feedback cannot be sent
soon in async mode, pg_stat_replication always returns thenot-up-to-date location.
Why not send a message every 10 seconds when its not sync rep?
Or even after every write(). It's a tiny amount of network traffic
anyway.I understand that send feedback message frequently will keep
pg_stat_replication up-to-date state.Are there really no needs who wants to fsync even in async mode ?
I think the people who dislike Data lost will like that idea.The OS will not sit on the written but not fsync'd data forever, it will get
flushed to disk in a few seconds even without the fsync. It's just that
there is no guarantee on when it will hit the disk, but there are no strong
guarantees in asynchronous replication anyway.
This makes me come up with the idea about adding new GUC parameter like
wal_receiver_fsync so that a user can disable walreceiver to fsync WAL after
every write(). IOW, it can allow walreceiver to work like current
pg_receivexlog.
One problem of the above idea is that the startup process can replay WAL
which has not been fsync'd yet. This violates the WAL rule. To resolve this,
the startup process would need to ensure that WAL to replay has already been
fsync'd, and otherwise issue the fsync. This basically makes the WAL replay
longer, but we can reduce the amount of I/O by walreceiver.
This seems also useful even with synchronous replication if synchronous_commit
is set to remote_write. Since walreceiver doesn't issue the fsync and can focus
on receiving and writing WAL, the performance of replication would be improved.
Nevertheless in sync or async, returning feedback and executing
fsync() same as like walreceiver is such a problem?Probably would be OK. It would increase the I/O a lot, thanks to a lot of
small writes and fsyncs, which might be surprising for a tool like
pg_receivexlog.One idea is to change the default behavior to be like walreceiver, and
fsync() after every write. But provide an option to get the old behavior,
"--no-fsync".
I'm OK with this. That is, by default, pg_receivexlog issues the fsync and
sends the feedback message back after every write(). But something like
--no-fsync is specified, WAL file is fsync'd only when it's closed and
a feedback message is sent back only when --status-interval time is elapsed.
Regards,
--
Fujii Masao
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On Wed, Oct 22, 2014 at 9:26 AM, Heikki Linnakangas
<hlinnakangas@vmware.com> wrote:
We seem to be going in circles. You suggested having two options,
--feedback, and --fsync, which is almost exactly what Furuya posted
originally. I objected to that, because I think that user interface is too
complicated. Instead, I suggested having just a single option called
--synchronous, or even better, have no option at all and have the server
tell the client if it's participating in synchronous replication, and have
pg_receivexlog automatically fsync when it is, and not otherwise [1]. That
way you don't need to expose any new options to the user. What did you think
of that idea?
I think it's pretty weird to make the fsync behavior of the client is
controlled by the server.
I also don't think that fsync() on the client side is useless in
asynchronous replication. Yeah, it's true that there are no
*guarantees* with asynchronous replication, but the bound on how long
the data can take to get out to disk is a heck of a lot shorter if you
fsync frequently than if you don't. And on the flip side, that has a
performance impact.
So I actually think the design you proposed is not as good as what was
proposed by Furuya and Simon. But I don't feel incredibly strongly
about it.
--
Robert Haas
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
We seem to be going in circles. You suggested having two options,
--feedback, and --fsync, which is almost exactly what Furuya posted
originally. I objected to that, because I think that user interfaceis
too complicated. Instead, I suggested having just a single option
called --synchronous, or even better, have no option at all and have
the server tell the client if it's participating in synchronous
replication, and have pg_receivexlog automatically fsync when it is,
and not otherwise [1]. That way you don't need to expose any new
options to the user. What did you think of that idea?I think it's pretty weird to make the fsync behavior of the client is
controlled by the server.I also don't think that fsync() on the client side is useless in
asynchronous replication. Yeah, it's true that there are no
*guarantees* with asynchronous replication, but the bound on how long
the data can take to get out to disk is a heck of a lot shorter if you
fsync frequently than if you don't. And on the flip side, that has a
performance impact.So I actually think the design you proposed is not as good as what was
proposed by Furuya and Simon. But I don't feel incredibly strongly about
it.
Thanks for lots of comments!!
I fixed the patch.
As a default, it behave like a walreceiver.
Same as walreceiver, it fsync and send a feedback after fsync.
When --no-fsync is specified, it will fsync only wal has switched.
feedback message is specified by --status-interval.
Regards,
--
Furuya Osamu
Attachments:
pg_receivexlog-fsync-feedback-v7.patchapplication/octet-stream; name=pg_receivexlog-fsync-feedback-v7.patchDownload
*** a/doc/src/sgml/ref/pg_receivexlog.sgml
--- b/doc/src/sgml/ref/pg_receivexlog.sgml
***************
*** 204,209 **** PostgreSQL documentation
--- 204,219 ----
</varlistentry>
<varlistentry>
+ <term><option>-N</></term>
+ <term><option>--no-fsync</></term>
+ <listitem>
+ <para>
+ WAL file is flushed only it has changed.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
<term><option>-U <replaceable>username</replaceable></option></term>
<term><option>--username=<replaceable class="parameter">username</replaceable></option></term>
<listitem>
*** a/src/bin/pg_basebackup/pg_basebackup.c
--- b/src/bin/pg_basebackup/pg_basebackup.c
***************
*** 370,376 **** LogStreamerMain(logstreamer_param *param)
if (!ReceiveXlogStream(param->bgconn, param->startptr, param->timeline,
param->sysidentifier, param->xlogdir,
reached_end_position, standby_message_timeout,
! NULL, 0))
/*
* Any errors will already have been reported in the function process,
--- 370,376 ----
if (!ReceiveXlogStream(param->bgconn, param->startptr, param->timeline,
param->sysidentifier, param->xlogdir,
reached_end_position, standby_message_timeout,
! NULL, true))
/*
* Any errors will already have been reported in the function process,
*** a/src/bin/pg_basebackup/pg_receivexlog.c
--- b/src/bin/pg_basebackup/pg_receivexlog.c
***************
*** 36,46 **** static char *basedir = NULL;
static int verbose = 0;
static int noloop = 0;
static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
- static int fsync_interval = 0; /* 0 = default */
static volatile bool time_to_abort = false;
static bool do_create_slot = false;
static bool do_drop_slot = false;
!
static void usage(void);
static DIR* get_destination_dir(char *dest_folder);
--- 36,45 ----
static int verbose = 0;
static int noloop = 0;
static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
static volatile bool time_to_abort = false;
static bool do_create_slot = false;
static bool do_drop_slot = false;
! static bool no_fsync = false;
static void usage(void);
static DIR* get_destination_dir(char *dest_folder);
***************
*** 66,76 **** usage(void)
printf(_(" %s [OPTION]...\n"), progname);
printf(_("\nOptions:\n"));
printf(_(" -D, --directory=DIR receive transaction log files into this directory\n"));
- printf(_(" -F --fsync-interval=SECS\n"
- " time between fsyncs to transaction log files (default: %d)\n"), (fsync_interval / 1000));
printf(_(" -n, --no-loop do not loop on connection lost\n"));
printf(_(" -s, --status-interval=SECS\n"
" time between status packets sent to server (default: %d)\n"), (standby_message_timeout / 1000));
printf(_(" -S, --slot=SLOTNAME replication slot to use\n"));
printf(_(" -v, --verbose output verbose messages\n"));
printf(_(" -V, --version output version information, then exit\n"));
--- 65,75 ----
printf(_(" %s [OPTION]...\n"), progname);
printf(_("\nOptions:\n"));
printf(_(" -D, --directory=DIR receive transaction log files into this directory\n"));
printf(_(" -n, --no-loop do not loop on connection lost\n"));
printf(_(" -s, --status-interval=SECS\n"
" time between status packets sent to server (default: %d)\n"), (standby_message_timeout / 1000));
+ printf(_(" -N, --no-fsync fsync only wal has changed\n"
+ " If you want to report the flush position to the server, should use -S option.\n"));
printf(_(" -S, --slot=SLOTNAME replication slot to use\n"));
printf(_(" -v, --verbose output verbose messages\n"));
printf(_(" -V, --version output version information, then exit\n"));
***************
*** 343,349 **** StreamLog(void)
ReceiveXlogStream(conn, startpos, starttli, NULL, basedir,
stop_streaming, standby_message_timeout, ".partial",
! fsync_interval);
PQfinish(conn);
conn = NULL;
--- 342,348 ----
ReceiveXlogStream(conn, startpos, starttli, NULL, basedir,
stop_streaming, standby_message_timeout, ".partial",
! no_fsync);
PQfinish(conn);
conn = NULL;
***************
*** 374,383 **** main(int argc, char **argv)
{"port", required_argument, NULL, 'p'},
{"username", required_argument, NULL, 'U'},
{"no-loop", no_argument, NULL, 'n'},
- {"fsync-interval", required_argument, NULL, 'F'},
{"no-password", no_argument, NULL, 'w'},
{"password", no_argument, NULL, 'W'},
{"status-interval", required_argument, NULL, 's'},
{"slot", required_argument, NULL, 'S'},
{"verbose", no_argument, NULL, 'v'},
/* action */
--- 373,382 ----
{"port", required_argument, NULL, 'p'},
{"username", required_argument, NULL, 'U'},
{"no-loop", no_argument, NULL, 'n'},
{"no-password", no_argument, NULL, 'w'},
{"password", no_argument, NULL, 'W'},
{"status-interval", required_argument, NULL, 's'},
+ {"no-fsync", no_argument, NULL, 'N'},
{"slot", required_argument, NULL, 'S'},
{"verbose", no_argument, NULL, 'v'},
/* action */
***************
*** 408,414 **** main(int argc, char **argv)
}
}
! while ((c = getopt_long(argc, argv, "D:d:h:p:U:s:S:nF:wWv",
long_options, &option_index)) != -1)
{
switch (c)
--- 407,413 ----
}
}
! while ((c = getopt_long(argc, argv, "D:d:h:p:U:s:NS:nwWv",
long_options, &option_index)) != -1)
{
switch (c)
***************
*** 449,469 **** main(int argc, char **argv)
exit(1);
}
break;
case 'S':
replication_slot = pg_strdup(optarg);
break;
case 'n':
noloop = 1;
break;
- case 'F':
- fsync_interval = atoi(optarg) * 1000;
- if (fsync_interval < -1000)
- {
- fprintf(stderr, _("%s: invalid fsync interval \"%s\"\n"),
- progname, optarg);
- exit(1);
- }
- break;
case 'v':
verbose++;
break;
--- 448,462 ----
exit(1);
}
break;
+ case 'N':
+ no_fsync = true;
+ break;
case 'S':
replication_slot = pg_strdup(optarg);
break;
case 'n':
noloop = 1;
break;
case 'v':
verbose++;
break;
*** a/src/bin/pg_basebackup/receivelog.c
--- b/src/bin/pg_basebackup/receivelog.c
***************
*** 31,44 **** static char current_walfile_name[MAXPGPATH] = "";
static bool reportFlushPosition = false;
static XLogRecPtr lastFlushPosition = InvalidXLogRecPtr;
- static int64 last_fsync = -1; /* timestamp of last WAL file flush */
static bool still_sending = true; /* feedback still needs to be sent? */
static PGresult *HandleCopyStream(PGconn *conn, XLogRecPtr startpos,
uint32 timeline, char *basedir,
stream_stop_callback stream_stop, int standby_message_timeout,
char *partial_suffix, XLogRecPtr *stoppos,
! int fsync_interval);
static int CopyStreamPoll(PGconn *conn, long timeout_ms);
static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
--- 31,43 ----
static bool reportFlushPosition = false;
static XLogRecPtr lastFlushPosition = InvalidXLogRecPtr;
static bool still_sending = true; /* feedback still needs to be sent? */
static PGresult *HandleCopyStream(PGconn *conn, XLogRecPtr startpos,
uint32 timeline, char *basedir,
stream_stop_callback stream_stop, int standby_message_timeout,
char *partial_suffix, XLogRecPtr *stoppos,
! bool no_fsync);
static int CopyStreamPoll(PGconn *conn, long timeout_ms);
static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
***************
*** 55,62 **** static bool CheckCopyStreamStop(PGconn *conn, XLogRecPtr blockpos,
stream_stop_callback stream_stop,
char *partial_suffix, XLogRecPtr *stoppos);
static long CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
! int64 last_status, int fsync_interval,
! XLogRecPtr blockpos);
static bool ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos,
uint32 *timeline);
--- 54,60 ----
stream_stop_callback stream_stop,
char *partial_suffix, XLogRecPtr *stoppos);
static long CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
! int64 last_status, XLogRecPtr blockpos);
static bool ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos,
uint32 *timeline);
***************
*** 209,215 **** close_walfile(char *basedir, char *partial_suffix, XLogRecPtr pos)
progname, current_walfile_name, partial_suffix);
lastFlushPosition = pos;
- last_fsync = feGetCurrentTimestamp();
return true;
}
--- 207,212 ----
***************
*** 440,447 **** CheckServerVersionForStreaming(PGconn *conn)
* allows you to tell the difference between partial and completed files,
* so that you can continue later where you left.
*
! * fsync_interval controls how often we flush to the received WAL file,
! * in milliseconds.
*
* Note: The log position *must* be at a log segment start!
*/
--- 437,443 ----
* allows you to tell the difference between partial and completed files,
* so that you can continue later where you left.
*
! * If 'no_fsync' is true, fsync is only wal change.
*
* Note: The log position *must* be at a log segment start!
*/
***************
*** 450,456 **** ReceiveXlogStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *sysidentifier, char *basedir,
stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! int fsync_interval)
{
char query[128];
char slotcmd[128];
--- 446,452 ----
char *sysidentifier, char *basedir,
stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! bool no_fsync)
{
char query[128];
char slotcmd[128];
***************
*** 595,601 **** ReceiveXlogStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
/* Stream the WAL */
res = HandleCopyStream(conn, startpos, timeline, basedir, stream_stop,
standby_message_timeout, partial_suffix,
! &stoppos, fsync_interval);
if (res == NULL)
goto error;
--- 591,597 ----
/* Stream the WAL */
res = HandleCopyStream(conn, startpos, timeline, basedir, stream_stop,
standby_message_timeout, partial_suffix,
! &stoppos, no_fsync);
if (res == NULL)
goto error;
***************
*** 760,770 **** static PGresult *
HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! XLogRecPtr *stoppos, int fsync_interval)
{
char *copybuf = NULL;
int64 last_status = -1;
XLogRecPtr blockpos = startpos;
still_sending = true;
--- 756,767 ----
HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! XLogRecPtr *stoppos, bool no_fsync)
{
char *copybuf = NULL;
int64 last_status = -1;
XLogRecPtr blockpos = startpos;
+ XLogRecPtr lastFeedbackFlushPosition = startpos;
still_sending = true;
***************
*** 789,797 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
*/
if (lastFlushPosition < blockpos &&
walfile != -1 &&
! ((fsync_interval > 0 &&
! feTimestampDifferenceExceeds(last_fsync, now, fsync_interval)) ||
! fsync_interval < 0))
{
if (fsync(walfile) != 0)
{
--- 786,792 ----
*/
if (lastFlushPosition < blockpos &&
walfile != -1 &&
! ! no_fsync)
{
if (fsync(walfile) != 0)
{
***************
*** 801,827 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
}
lastFlushPosition = blockpos;
- last_fsync = now;
}
/*
* Potentially send a status message to the master
*/
! if (still_sending && standby_message_timeout > 0 &&
! feTimestampDifferenceExceeds(last_status, now,
! standby_message_timeout))
{
/* Time to send feedback! */
if (!sendFeedback(conn, blockpos, now, false))
goto error;
last_status = now;
}
/*
* Calculate how long send/receive loops should sleep
*/
sleeptime = CalculateCopyStreamSleeptime(now, standby_message_timeout,
! last_status, fsync_interval, blockpos);
r = CopyStreamReceive(conn, sleeptime, ©buf);
while (r != 0)
--- 796,823 ----
}
lastFlushPosition = blockpos;
}
/*
* Potentially send a status message to the master
*/
! if ((still_sending && standby_message_timeout > 0 &&
! feTimestampDifferenceExceeds(last_status, now, standby_message_timeout)) ||
! (! no_fsync && lastFeedbackFlushPosition != lastFlushPosition))
{
/* Time to send feedback! */
if (!sendFeedback(conn, blockpos, now, false))
goto error;
last_status = now;
+ if (! no_fsync)
+ lastFeedbackFlushPosition = lastFlushPosition;
}
/*
* Calculate how long send/receive loops should sleep
*/
sleeptime = CalculateCopyStreamSleeptime(now, standby_message_timeout,
! last_status, blockpos);
r = CopyStreamReceive(conn, sleeptime, ©buf);
while (r != 0)
***************
*** 1244,1277 **** CheckCopyStreamStop(PGconn *conn, XLogRecPtr blockpos, uint32 timeline,
*/
static long
CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
! int64 last_status, int fsync_interval, XLogRecPtr blockpos)
{
- int64 targettime = 0;
int64 status_targettime = 0;
- int64 fsync_targettime = 0;
long sleeptime;
if (standby_message_timeout && still_sending)
status_targettime = last_status +
(standby_message_timeout - 1) * ((int64) 1000);
! if (fsync_interval > 0 && lastFlushPosition < blockpos)
! fsync_targettime = last_fsync +
! (fsync_interval - 1) * ((int64) 1000);
!
! if ((status_targettime < fsync_targettime && status_targettime > 0) ||
! fsync_targettime == 0)
! targettime = status_targettime;
! else
! targettime = fsync_targettime;
!
! if (targettime > 0)
{
long secs;
int usecs;
feTimestampDifference(now,
! targettime,
&secs,
&usecs);
/* Always sleep at least 1 sec */
--- 1240,1261 ----
*/
static long
CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
! int64 last_status, XLogRecPtr blockpos)
{
int64 status_targettime = 0;
long sleeptime;
if (standby_message_timeout && still_sending)
status_targettime = last_status +
(standby_message_timeout - 1) * ((int64) 1000);
! if (status_targettime > 0)
{
long secs;
int usecs;
feTimestampDifference(now,
! status_targettime,
&secs,
&usecs);
/* Always sleep at least 1 sec */
*** a/src/bin/pg_basebackup/receivelog.h
--- b/src/bin/pg_basebackup/receivelog.h
***************
*** 31,36 **** extern bool ReceiveXlogStream(PGconn *conn,
stream_stop_callback stream_stop,
int standby_message_timeout,
char *partial_suffix,
! int fsync_interval);
#endif /* RECEIVELOG_H */
--- 31,36 ----
stream_stop_callback stream_stop,
int standby_message_timeout,
char *partial_suffix,
! bool no_fsync);
#endif /* RECEIVELOG_H */
On Fri, Oct 31, 2014 at 5:46 PM, <furuyao@pm.nttdata.co.jp> wrote:
We seem to be going in circles. You suggested having two options,
--feedback, and --fsync, which is almost exactly what Furuya posted
originally. I objected to that, because I think that user interfaceis
too complicated. Instead, I suggested having just a single option
called --synchronous, or even better, have no option at all and have
the server tell the client if it's participating in synchronous
replication, and have pg_receivexlog automatically fsync when it is,
and not otherwise [1]. That way you don't need to expose any new
options to the user. What did you think of that idea?I think it's pretty weird to make the fsync behavior of the client is
controlled by the server.I also don't think that fsync() on the client side is useless in
asynchronous replication. Yeah, it's true that there are no
*guarantees* with asynchronous replication, but the bound on how long
the data can take to get out to disk is a heck of a lot shorter if you
fsync frequently than if you don't. And on the flip side, that has a
performance impact.So I actually think the design you proposed is not as good as what was
proposed by Furuya and Simon. But I don't feel incredibly strongly about
it.Thanks for lots of comments!!
I fixed the patch.
As a default, it behave like a walreceiver.
Same as walreceiver, it fsync and send a feedback after fsync.
On second thought, flipping the default behavior seems not worthwhile here.
Which might surprise the existing users and cause some troubles to them.
I'd like to back to the Heikki's original suggestion like just adding
--synchronous
option. That is, only when --synchronous is specified, WAL is flushed and
feedback is sent back as soon as WAL is received.
I changed the patch heavily in that way. Please find the attached patch.
By default, pg_receivexlog flushes WAL data only when WAL file is closed.
If --synchronous is specified, like the standby's WAL receiver,
sync commands are issued as soon as there is WAL data which has not been
flushed yet. Also status packets are sent back to the server just after
WAL data is flushed whatever --status-interval is set to. I added
the description of this behavior to the doc.
Thought?
Regards,
--
Fujii Masao
Attachments:
pg_receivexlog-fsync-feedback-v8.patchtext/x-patch; charset=US-ASCII; name=pg_receivexlog-fsync-feedback-v8.patchDownload
*** a/doc/src/sgml/ref/pg_receivexlog.sgml
--- b/doc/src/sgml/ref/pg_receivexlog.sgml
***************
*** 49,54 **** PostgreSQL documentation
--- 49,61 ----
</para>
<para>
+ Unlike the standby's WAL receiver, <application>pg_receivexlog</>
+ flushes WAL data only when WAL file is closed, by default.
+ <literal>--synchronous</> option must be specified to flush WAL data
+ in real time and ensure it's safely flushed to disk.
+ </para>
+
+ <para>
The transaction log is streamed over a regular
<productname>PostgreSQL</productname> connection, and uses the replication
protocol. The connection must be made with a superuser or a user
***************
*** 86,106 **** PostgreSQL documentation
</varlistentry>
<varlistentry>
- <term><option>-F <replaceable class="parameter">interval</replaceable></option></term>
- <term><option>--fsync-interval=<replaceable class="parameter">interval</replaceable></option></term>
- <listitem>
- <para>
- Specifies the maximum time to issue sync commands to ensure the
- received WAL file is safely flushed to disk, in seconds. The default
- value is zero, which disables issuing fsyncs except when WAL file is
- closed. If <literal>-1</literal> is specified, WAL file is flushed as
- soon as possible, that is, as soon as there are WAL data which has
- not been flushed yet.
- </para>
- </listitem>
- </varlistentry>
-
- <varlistentry>
<term><option>-n</option></term>
<term><option>--no-loop</option></term>
<listitem>
--- 93,98 ----
***************
*** 135,150 **** PostgreSQL documentation
When this option is used, <application>pg_receivexlog</> will report
a flush position to the server, indicating when each segment has been
synchronized to disk so that the server can remove that segment if it
! is not otherwise needed. When using this parameter, it is important
! to make sure that <application>pg_receivexlog</> cannot become the
! synchronous standby through an incautious setting of
! <xref linkend="guc-synchronous-standby-names">; it does not flush
! data frequently enough for this to work correctly.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>-v</option></term>
<term><option>--verbose</option></term>
<listitem>
--- 127,152 ----
When this option is used, <application>pg_receivexlog</> will report
a flush position to the server, indicating when each segment has been
synchronized to disk so that the server can remove that segment if it
! is not otherwise needed. <literal>--synchronous</literal> option must
! be specified when making <application>pg_receivexlog</> run as
! synchronous standby by using replication slot. Otherwise WAL data
! cannot be flushed frequently enough for this to work correctly.
</para>
</listitem>
</varlistentry>
<varlistentry>
+ <term><option>--synchronous</option></term>
+ <listitem>
+ <para>
+ Issue sync commands as soon as there is WAL data which has not been
+ flushed yet. Also status packets are sent back to the server just after
+ WAL data is flushed whatever <literal>--status-interval</> is set to.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
<term><option>-v</option></term>
<term><option>--verbose</option></term>
<listitem>
*** a/src/bin/pg_basebackup/pg_basebackup.c
--- b/src/bin/pg_basebackup/pg_basebackup.c
***************
*** 370,376 **** LogStreamerMain(logstreamer_param *param)
if (!ReceiveXlogStream(param->bgconn, param->startptr, param->timeline,
param->sysidentifier, param->xlogdir,
reached_end_position, standby_message_timeout,
! NULL, 0))
/*
* Any errors will already have been reported in the function process,
--- 370,376 ----
if (!ReceiveXlogStream(param->bgconn, param->startptr, param->timeline,
param->sysidentifier, param->xlogdir,
reached_end_position, standby_message_timeout,
! NULL, false))
/*
* Any errors will already have been reported in the function process,
*** a/src/bin/pg_basebackup/pg_receivexlog.c
--- b/src/bin/pg_basebackup/pg_receivexlog.c
***************
*** 36,45 **** static char *basedir = NULL;
static int verbose = 0;
static int noloop = 0;
static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
- static int fsync_interval = 0; /* 0 = default */
static volatile bool time_to_abort = false;
static bool do_create_slot = false;
static bool do_drop_slot = false;
static void usage(void);
--- 36,45 ----
static int verbose = 0;
static int noloop = 0;
static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
static volatile bool time_to_abort = false;
static bool do_create_slot = false;
static bool do_drop_slot = false;
+ static bool synchronous = false;
static void usage(void);
***************
*** 66,77 **** usage(void)
printf(_(" %s [OPTION]...\n"), progname);
printf(_("\nOptions:\n"));
printf(_(" -D, --directory=DIR receive transaction log files into this directory\n"));
- printf(_(" -F --fsync-interval=SECS\n"
- " time between fsyncs to transaction log files (default: %d)\n"), (fsync_interval / 1000));
printf(_(" -n, --no-loop do not loop on connection lost\n"));
printf(_(" -s, --status-interval=SECS\n"
" time between status packets sent to server (default: %d)\n"), (standby_message_timeout / 1000));
printf(_(" -S, --slot=SLOTNAME replication slot to use\n"));
printf(_(" -v, --verbose output verbose messages\n"));
printf(_(" -V, --version output version information, then exit\n"));
printf(_(" -?, --help show this help, then exit\n"));
--- 66,76 ----
printf(_(" %s [OPTION]...\n"), progname);
printf(_("\nOptions:\n"));
printf(_(" -D, --directory=DIR receive transaction log files into this directory\n"));
printf(_(" -n, --no-loop do not loop on connection lost\n"));
printf(_(" -s, --status-interval=SECS\n"
" time between status packets sent to server (default: %d)\n"), (standby_message_timeout / 1000));
printf(_(" -S, --slot=SLOTNAME replication slot to use\n"));
+ printf(_(" --synchronous flush transaction log in real time\n"));
printf(_(" -v, --verbose output verbose messages\n"));
printf(_(" -V, --version output version information, then exit\n"));
printf(_(" -?, --help show this help, then exit\n"));
***************
*** 343,349 **** StreamLog(void)
ReceiveXlogStream(conn, startpos, starttli, NULL, basedir,
stop_streaming, standby_message_timeout, ".partial",
! fsync_interval);
PQfinish(conn);
conn = NULL;
--- 342,348 ----
ReceiveXlogStream(conn, startpos, starttli, NULL, basedir,
stop_streaming, standby_message_timeout, ".partial",
! synchronous);
PQfinish(conn);
conn = NULL;
***************
*** 374,380 **** main(int argc, char **argv)
{"port", required_argument, NULL, 'p'},
{"username", required_argument, NULL, 'U'},
{"no-loop", no_argument, NULL, 'n'},
- {"fsync-interval", required_argument, NULL, 'F'},
{"no-password", no_argument, NULL, 'w'},
{"password", no_argument, NULL, 'W'},
{"status-interval", required_argument, NULL, 's'},
--- 373,378 ----
***************
*** 383,388 **** main(int argc, char **argv)
--- 381,387 ----
/* action */
{"create-slot", no_argument, NULL, 1},
{"drop-slot", no_argument, NULL, 2},
+ {"synchronous", no_argument, NULL, 3},
{NULL, 0, NULL, 0}
};
***************
*** 408,414 **** main(int argc, char **argv)
}
}
! while ((c = getopt_long(argc, argv, "D:d:h:p:U:s:S:nF:wWv",
long_options, &option_index)) != -1)
{
switch (c)
--- 407,413 ----
}
}
! while ((c = getopt_long(argc, argv, "D:d:h:p:U:s:S:nwWv",
long_options, &option_index)) != -1)
{
switch (c)
***************
*** 455,469 **** main(int argc, char **argv)
case 'n':
noloop = 1;
break;
- case 'F':
- fsync_interval = atoi(optarg) * 1000;
- if (fsync_interval < -1000)
- {
- fprintf(stderr, _("%s: invalid fsync interval \"%s\"\n"),
- progname, optarg);
- exit(1);
- }
- break;
case 'v':
verbose++;
break;
--- 454,459 ----
***************
*** 474,479 **** main(int argc, char **argv)
--- 464,472 ----
case 2:
do_drop_slot = true;
break;
+ case 3:
+ synchronous = true;
+ break;
default:
/*
*** a/src/bin/pg_basebackup/receivelog.c
--- b/src/bin/pg_basebackup/receivelog.c
***************
*** 31,44 **** static char current_walfile_name[MAXPGPATH] = "";
static bool reportFlushPosition = false;
static XLogRecPtr lastFlushPosition = InvalidXLogRecPtr;
- static int64 last_fsync = -1; /* timestamp of last WAL file flush */
static bool still_sending = true; /* feedback still needs to be sent? */
static PGresult *HandleCopyStream(PGconn *conn, XLogRecPtr startpos,
uint32 timeline, char *basedir,
stream_stop_callback stream_stop, int standby_message_timeout,
char *partial_suffix, XLogRecPtr *stoppos,
! int fsync_interval);
static int CopyStreamPoll(PGconn *conn, long timeout_ms);
static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
--- 31,43 ----
static bool reportFlushPosition = false;
static XLogRecPtr lastFlushPosition = InvalidXLogRecPtr;
static bool still_sending = true; /* feedback still needs to be sent? */
static PGresult *HandleCopyStream(PGconn *conn, XLogRecPtr startpos,
uint32 timeline, char *basedir,
stream_stop_callback stream_stop, int standby_message_timeout,
char *partial_suffix, XLogRecPtr *stoppos,
! bool synchronous);
static int CopyStreamPoll(PGconn *conn, long timeout_ms);
static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
***************
*** 55,62 **** static bool CheckCopyStreamStop(PGconn *conn, XLogRecPtr blockpos,
stream_stop_callback stream_stop,
char *partial_suffix, XLogRecPtr *stoppos);
static long CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
! int64 last_status, int fsync_interval,
! XLogRecPtr blockpos);
static bool ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos,
uint32 *timeline);
--- 54,60 ----
stream_stop_callback stream_stop,
char *partial_suffix, XLogRecPtr *stoppos);
static long CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
! int64 last_status);
static bool ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos,
uint32 *timeline);
***************
*** 209,215 **** close_walfile(char *basedir, char *partial_suffix, XLogRecPtr pos)
progname, current_walfile_name, partial_suffix);
lastFlushPosition = pos;
- last_fsync = feGetCurrentTimestamp();
return true;
}
--- 207,212 ----
***************
*** 440,447 **** CheckServerVersionForStreaming(PGconn *conn)
* allows you to tell the difference between partial and completed files,
* so that you can continue later where you left.
*
! * fsync_interval controls how often we flush to the received WAL file,
! * in milliseconds.
*
* Note: The log position *must* be at a log segment start!
*/
--- 437,444 ----
* allows you to tell the difference between partial and completed files,
* so that you can continue later where you left.
*
! * If 'synchronous' is true, the received WAL is flushed as soon as written,
! * otherwise only when the WAL file is closed.
*
* Note: The log position *must* be at a log segment start!
*/
***************
*** 450,456 **** ReceiveXlogStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *sysidentifier, char *basedir,
stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! int fsync_interval)
{
char query[128];
char slotcmd[128];
--- 447,453 ----
char *sysidentifier, char *basedir,
stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! bool synchronous)
{
char query[128];
char slotcmd[128];
***************
*** 595,601 **** ReceiveXlogStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
/* Stream the WAL */
res = HandleCopyStream(conn, startpos, timeline, basedir, stream_stop,
standby_message_timeout, partial_suffix,
! &stoppos, fsync_interval);
if (res == NULL)
goto error;
--- 592,598 ----
/* Stream the WAL */
res = HandleCopyStream(conn, startpos, timeline, basedir, stream_stop,
standby_message_timeout, partial_suffix,
! &stoppos, synchronous);
if (res == NULL)
goto error;
***************
*** 760,766 **** static PGresult *
HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! XLogRecPtr *stoppos, int fsync_interval)
{
char *copybuf = NULL;
int64 last_status = -1;
--- 757,763 ----
HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! XLogRecPtr *stoppos, bool synchronous)
{
char *copybuf = NULL;
int64 last_status = -1;
***************
*** 784,797 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
now = feGetCurrentTimestamp();
/*
! * If fsync_interval has elapsed since last WAL flush and we've written
! * some WAL data, flush them to disk.
*/
if (lastFlushPosition < blockpos &&
! walfile != -1 &&
! ((fsync_interval > 0 &&
! feTimestampDifferenceExceeds(last_fsync, now, fsync_interval)) ||
! fsync_interval < 0))
{
if (fsync(walfile) != 0)
{
--- 781,791 ----
now = feGetCurrentTimestamp();
/*
! * Issue sync command as soon as there are WAL data which
! * has not been flushed yet if synchronous option is true.
*/
if (lastFlushPosition < blockpos &&
! walfile != -1 && synchronous)
{
if (fsync(walfile) != 0)
{
***************
*** 799,807 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
progname, current_walfile_name, strerror(errno));
goto error;
}
-
lastFlushPosition = blockpos;
! last_fsync = now;
}
/*
--- 793,807 ----
progname, current_walfile_name, strerror(errno));
goto error;
}
lastFlushPosition = blockpos;
!
! /*
! * Send feedback so that the server sees the latest WAL locations
! * immediately if synchronous option is true.
! */
! if (!sendFeedback(conn, blockpos, now, false))
! goto error;
! last_status = now;
}
/*
***************
*** 821,827 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
* Calculate how long send/receive loops should sleep
*/
sleeptime = CalculateCopyStreamSleeptime(now, standby_message_timeout,
! last_status, fsync_interval, blockpos);
r = CopyStreamReceive(conn, sleeptime, ©buf);
while (r != 0)
--- 821,827 ----
* Calculate how long send/receive loops should sleep
*/
sleeptime = CalculateCopyStreamSleeptime(now, standby_message_timeout,
! last_status);
r = CopyStreamReceive(conn, sleeptime, ©buf);
while (r != 0)
***************
*** 1244,1277 **** CheckCopyStreamStop(PGconn *conn, XLogRecPtr blockpos, uint32 timeline,
*/
static long
CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
! int64 last_status, int fsync_interval, XLogRecPtr blockpos)
{
- int64 targettime = 0;
int64 status_targettime = 0;
- int64 fsync_targettime = 0;
long sleeptime;
if (standby_message_timeout && still_sending)
status_targettime = last_status +
(standby_message_timeout - 1) * ((int64) 1000);
! if (fsync_interval > 0 && lastFlushPosition < blockpos)
! fsync_targettime = last_fsync +
! (fsync_interval - 1) * ((int64) 1000);
!
! if ((status_targettime < fsync_targettime && status_targettime > 0) ||
! fsync_targettime == 0)
! targettime = status_targettime;
! else
! targettime = fsync_targettime;
!
! if (targettime > 0)
{
long secs;
int usecs;
feTimestampDifference(now,
! targettime,
&secs,
&usecs);
/* Always sleep at least 1 sec */
--- 1244,1265 ----
*/
static long
CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
! int64 last_status)
{
int64 status_targettime = 0;
long sleeptime;
if (standby_message_timeout && still_sending)
status_targettime = last_status +
(standby_message_timeout - 1) * ((int64) 1000);
! if (status_targettime > 0)
{
long secs;
int usecs;
feTimestampDifference(now,
! status_targettime,
&secs,
&usecs);
/* Always sleep at least 1 sec */
*** a/src/bin/pg_basebackup/receivelog.h
--- b/src/bin/pg_basebackup/receivelog.h
***************
*** 31,36 **** extern bool ReceiveXlogStream(PGconn *conn,
stream_stop_callback stream_stop,
int standby_message_timeout,
char *partial_suffix,
! int fsync_interval);
#endif /* RECEIVELOG_H */
--- 31,36 ----
stream_stop_callback stream_stop,
int standby_message_timeout,
char *partial_suffix,
! bool synchronous);
#endif /* RECEIVELOG_H */
On Fri, Oct 31, 2014 at 5:46 PM, <furuyao@pm.nttdata.co.jp> wrote:
We seem to be going in circles. You suggested having two options,
--feedback, and --fsync, which is almost exactly what Furuya posted
originally. I objected to that, because I think that user interfaceis
too complicated. Instead, I suggested having just a single option
called --synchronous, or even better, have no option at all and
have the server tell the client if it's participating in
synchronous replication, and have pg_receivexlog automatically
fsync when it is, and not otherwise [1]. That way you don't needto
expose any new options to the user. What did you think of that idea?
I think it's pretty weird to make the fsync behavior of the client
is
controlled by the server.
I also don't think that fsync() on the client side is useless in
asynchronous replication. Yeah, it's true that there are no
*guarantees* with asynchronous replication, but the bound on how long
the data can take to get out to disk is a heck of a lot shorter if
you fsync frequently than if you don't. And on the flip side, that
has a performance impact.So I actually think the design you proposed is not as good as what
was proposed by Furuya and Simon. But I don't feel incredibly
strongly about it.Thanks for lots of comments!!
I fixed the patch.
As a default, it behave like a walreceiver.
Same as walreceiver, it fsync and send a feedback after fsync.On second thought, flipping the default behavior seems not worthwhile
here.
Which might surprise the existing users and cause some troubles to them.
I'd like to back to the Heikki's original suggestion like just adding
--synchronous option. That is, only when --synchronous is specified, WAL
is flushed and feedback is sent back as soon as WAL is received.I changed the patch heavily in that way. Please find the attached patch.
By default, pg_receivexlog flushes WAL data only when WAL file is closed.
If --synchronous is specified, like the standby's WAL receiver, sync
commands are issued as soon as there is WAL data which has not been flushed
yet. Also status packets are sent back to the server just after WAL data
is flushed whatever --status-interval is set to. I added the description
of this behavior to the doc.Thought?
I think it's as you pointed out.
Thank you for the patch.
I did a review of the patch.
There was no problem.
I confirmed the following.
1. applied cleanly and compilation was without warnings and errors
2. all regress tests was passed ok
3. when --synchronous is not specified, do not fsync except wal has switched
4. it get normal backup when pg_basebackup has executed with '-X s'
5. when --synchronous is specified, it fsync every time when it receive WAL data
6. when --synchronous is specified, in spite of -s option, feedback are sent back when it fsync
7. when --slog is not specified, it wouldn't feedback fsync location
8. no problem in document
Regards,
--
Furuya Osamu
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On Mon, Nov 10, 2014 at 7:19 PM, <furuyao@pm.nttdata.co.jp> wrote:
On Fri, Oct 31, 2014 at 5:46 PM, <furuyao@pm.nttdata.co.jp> wrote:
We seem to be going in circles. You suggested having two options,
--feedback, and --fsync, which is almost exactly what Furuya posted
originally. I objected to that, because I think that user interfaceis
too complicated. Instead, I suggested having just a single option
called --synchronous, or even better, have no option at all and
have the server tell the client if it's participating in
synchronous replication, and have pg_receivexlog automatically
fsync when it is, and not otherwise [1]. That way you don't needto
expose any new options to the user. What did you think of that idea?
I think it's pretty weird to make the fsync behavior of the client
is
controlled by the server.
I also don't think that fsync() on the client side is useless in
asynchronous replication. Yeah, it's true that there are no
*guarantees* with asynchronous replication, but the bound on how long
the data can take to get out to disk is a heck of a lot shorter if
you fsync frequently than if you don't. And on the flip side, that
has a performance impact.So I actually think the design you proposed is not as good as what
was proposed by Furuya and Simon. But I don't feel incredibly
strongly about it.Thanks for lots of comments!!
I fixed the patch.
As a default, it behave like a walreceiver.
Same as walreceiver, it fsync and send a feedback after fsync.On second thought, flipping the default behavior seems not worthwhile
here.
Which might surprise the existing users and cause some troubles to them.
I'd like to back to the Heikki's original suggestion like just adding
--synchronous option. That is, only when --synchronous is specified, WAL
is flushed and feedback is sent back as soon as WAL is received.I changed the patch heavily in that way. Please find the attached patch.
By default, pg_receivexlog flushes WAL data only when WAL file is closed.
If --synchronous is specified, like the standby's WAL receiver, sync
commands are issued as soon as there is WAL data which has not been flushed
yet. Also status packets are sent back to the server just after WAL data
is flushed whatever --status-interval is set to. I added the description
of this behavior to the doc.Thought?
I think it's as you pointed out.
Thank you for the patch.
I did a review of the patch.
There was no problem.
So I'm thinking to push the patch. Does anyone object to that?
Regards,
--
Fujii Masao
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
Fujii Masao wrote:
--- 127,152 ---- When this option is used, <application>pg_receivexlog</> will report a flush position to the server, indicating when each segment has been synchronized to disk so that the server can remove that segment if it ! is not otherwise needed. <literal>--synchronous</literal> option must ! be specified when making <application>pg_receivexlog</> run as ! synchronous standby by using replication slot. Otherwise WAL data ! cannot be flushed frequently enough for this to work correctly. </para> </listitem> </varlistentry>
Whitespace damage here.
+ printf(_(" --synchronous flush transaction log in real time\n"));
"in real time" sounds odd. How about "flush transaction log
immediately after writing", or maybe "have transaction log writes be
synchronous".
--- 781,791 ---- now = feGetCurrentTimestamp();/*
! * Issue sync command as soon as there are WAL data which
! * has not been flushed yet if synchronous option is true.
*/
if (lastFlushPosition < blockpos &&
! walfile != -1 && synchronous)
I'd put the "synchronous" condition first in the if(), and start the
comment with it rather than putting it at the end. Both seem weird.
--- 793,807 ---- progname, current_walfile_name, strerror(errno)); goto error; } lastFlushPosition = blockpos; ! ! /* ! * Send feedback so that the server sees the latest WAL locations ! * immediately if synchronous option is true. ! */ ! if (!sendFeedback(conn, blockpos, now, false)) ! goto error; ! last_status = now;
I'm not clear about this comment .. why does it say "if synchronous
option is true" when it's not checking the condition?
--
�lvaro Herrera http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
Thanks for reviewing the patch!
On Thu, Nov 13, 2014 at 4:05 AM, Alvaro Herrera
<alvherre@2ndquadrant.com> wrote:
Fujii Masao wrote:
--- 127,152 ---- When this option is used, <application>pg_receivexlog</> will report a flush position to the server, indicating when each segment has been synchronized to disk so that the server can remove that segment if it ! is not otherwise needed. <literal>--synchronous</literal> option must ! be specified when making <application>pg_receivexlog</> run as ! synchronous standby by using replication slot. Otherwise WAL data ! cannot be flushed frequently enough for this to work correctly. </para> </listitem> </varlistentry>Whitespace damage here.
Fixed.
+ printf(_(" --synchronous flush transaction log in real time\n"));
"in real time" sounds odd. How about "flush transaction log
immediately after writing", or maybe "have transaction log writes be
synchronous".
The former sounds better to me. So I chose it.
--- 781,791 ---- now = feGetCurrentTimestamp();/*
! * Issue sync command as soon as there are WAL data which
! * has not been flushed yet if synchronous option is true.
*/
if (lastFlushPosition < blockpos &&
! walfile != -1 && synchronous)I'd put the "synchronous" condition first in the if(), and start the
comment with it rather than putting it at the end. Both seem weird.
Fixed, i.e., moved the "synchronous" condition first in the if()'s test
and also moved the comment "If synchronous option is true" also first
in the comment.
progname, current_walfile_name, strerror(errno));
goto error;
}
lastFlushPosition = blockpos;
!
! /*
! * Send feedback so that the server sees the latest WAL locations
! * immediately if synchronous option is true.
! */
! if (!sendFeedback(conn, blockpos, now, false))
! goto error;
! last_status = now;I'm not clear about this comment .. why does it say "if synchronous
option is true" when it's not checking the condition?
I added that comment because the code exists with the if() block
checking "synchronous" condition. But it seems confusing. Just removed
that part from the comment.
Attached is the updated version of the patch.
Regards,
--
Fujii Masao
Attachments:
pg_receivexlog-fsync-feedback-v9.patchtext/x-patch; charset=US-ASCII; name=pg_receivexlog-fsync-feedback-v9.patchDownload
*** a/doc/src/sgml/ref/pg_receivexlog.sgml
--- b/doc/src/sgml/ref/pg_receivexlog.sgml
***************
*** 49,54 **** PostgreSQL documentation
--- 49,61 ----
</para>
<para>
+ Unlike the standby's WAL receiver, <application>pg_receivexlog</>
+ flushes WAL data only when WAL file is closed, by default.
+ <literal>--synchronous</> option must be specified to flush WAL data
+ in real time and ensure it's safely flushed to disk.
+ </para>
+
+ <para>
The transaction log is streamed over a regular
<productname>PostgreSQL</productname> connection, and uses the replication
protocol. The connection must be made with a superuser or a user
***************
*** 86,106 **** PostgreSQL documentation
</varlistentry>
<varlistentry>
- <term><option>-F <replaceable class="parameter">interval</replaceable></option></term>
- <term><option>--fsync-interval=<replaceable class="parameter">interval</replaceable></option></term>
- <listitem>
- <para>
- Specifies the maximum time to issue sync commands to ensure the
- received WAL file is safely flushed to disk, in seconds. The default
- value is zero, which disables issuing fsyncs except when WAL file is
- closed. If <literal>-1</literal> is specified, WAL file is flushed as
- soon as possible, that is, as soon as there are WAL data which has
- not been flushed yet.
- </para>
- </listitem>
- </varlistentry>
-
- <varlistentry>
<term><option>-n</option></term>
<term><option>--no-loop</option></term>
<listitem>
--- 93,98 ----
***************
*** 135,150 **** PostgreSQL documentation
When this option is used, <application>pg_receivexlog</> will report
a flush position to the server, indicating when each segment has been
synchronized to disk so that the server can remove that segment if it
! is not otherwise needed. When using this parameter, it is important
! to make sure that <application>pg_receivexlog</> cannot become the
! synchronous standby through an incautious setting of
! <xref linkend="guc-synchronous-standby-names">; it does not flush
! data frequently enough for this to work correctly.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>-v</option></term>
<term><option>--verbose</option></term>
<listitem>
--- 127,152 ----
When this option is used, <application>pg_receivexlog</> will report
a flush position to the server, indicating when each segment has been
synchronized to disk so that the server can remove that segment if it
! is not otherwise needed. <literal>--synchronous</literal> option must
! be specified when making <application>pg_receivexlog</> run as
! synchronous standby by using replication slot. Otherwise WAL data
! cannot be flushed frequently enough for this to work correctly.
</para>
</listitem>
</varlistentry>
<varlistentry>
+ <term><option>--synchronous</option></term>
+ <listitem>
+ <para>
+ Issue sync commands as soon as there is WAL data which has not been
+ flushed yet. Also status packets are sent back to the server just after
+ WAL data is flushed whatever <literal>--status-interval</> is set to.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
<term><option>-v</option></term>
<term><option>--verbose</option></term>
<listitem>
*** a/src/bin/pg_basebackup/pg_basebackup.c
--- b/src/bin/pg_basebackup/pg_basebackup.c
***************
*** 370,376 **** LogStreamerMain(logstreamer_param *param)
if (!ReceiveXlogStream(param->bgconn, param->startptr, param->timeline,
param->sysidentifier, param->xlogdir,
reached_end_position, standby_message_timeout,
! NULL, 0))
/*
* Any errors will already have been reported in the function process,
--- 370,376 ----
if (!ReceiveXlogStream(param->bgconn, param->startptr, param->timeline,
param->sysidentifier, param->xlogdir,
reached_end_position, standby_message_timeout,
! NULL, false))
/*
* Any errors will already have been reported in the function process,
*** a/src/bin/pg_basebackup/pg_receivexlog.c
--- b/src/bin/pg_basebackup/pg_receivexlog.c
***************
*** 36,45 **** static char *basedir = NULL;
static int verbose = 0;
static int noloop = 0;
static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
- static int fsync_interval = 0; /* 0 = default */
static volatile bool time_to_abort = false;
static bool do_create_slot = false;
static bool do_drop_slot = false;
static void usage(void);
--- 36,45 ----
static int verbose = 0;
static int noloop = 0;
static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
static volatile bool time_to_abort = false;
static bool do_create_slot = false;
static bool do_drop_slot = false;
+ static bool synchronous = false;
static void usage(void);
***************
*** 66,77 **** usage(void)
printf(_(" %s [OPTION]...\n"), progname);
printf(_("\nOptions:\n"));
printf(_(" -D, --directory=DIR receive transaction log files into this directory\n"));
- printf(_(" -F --fsync-interval=SECS\n"
- " time between fsyncs to transaction log files (default: %d)\n"), (fsync_interval / 1000));
printf(_(" -n, --no-loop do not loop on connection lost\n"));
printf(_(" -s, --status-interval=SECS\n"
" time between status packets sent to server (default: %d)\n"), (standby_message_timeout / 1000));
printf(_(" -S, --slot=SLOTNAME replication slot to use\n"));
printf(_(" -v, --verbose output verbose messages\n"));
printf(_(" -V, --version output version information, then exit\n"));
printf(_(" -?, --help show this help, then exit\n"));
--- 66,76 ----
printf(_(" %s [OPTION]...\n"), progname);
printf(_("\nOptions:\n"));
printf(_(" -D, --directory=DIR receive transaction log files into this directory\n"));
printf(_(" -n, --no-loop do not loop on connection lost\n"));
printf(_(" -s, --status-interval=SECS\n"
" time between status packets sent to server (default: %d)\n"), (standby_message_timeout / 1000));
printf(_(" -S, --slot=SLOTNAME replication slot to use\n"));
+ printf(_(" --synchronous flush transaction log immediately after writing\n"));
printf(_(" -v, --verbose output verbose messages\n"));
printf(_(" -V, --version output version information, then exit\n"));
printf(_(" -?, --help show this help, then exit\n"));
***************
*** 343,349 **** StreamLog(void)
ReceiveXlogStream(conn, startpos, starttli, NULL, basedir,
stop_streaming, standby_message_timeout, ".partial",
! fsync_interval);
PQfinish(conn);
conn = NULL;
--- 342,348 ----
ReceiveXlogStream(conn, startpos, starttli, NULL, basedir,
stop_streaming, standby_message_timeout, ".partial",
! synchronous);
PQfinish(conn);
conn = NULL;
***************
*** 374,380 **** main(int argc, char **argv)
{"port", required_argument, NULL, 'p'},
{"username", required_argument, NULL, 'U'},
{"no-loop", no_argument, NULL, 'n'},
- {"fsync-interval", required_argument, NULL, 'F'},
{"no-password", no_argument, NULL, 'w'},
{"password", no_argument, NULL, 'W'},
{"status-interval", required_argument, NULL, 's'},
--- 373,378 ----
***************
*** 383,388 **** main(int argc, char **argv)
--- 381,387 ----
/* action */
{"create-slot", no_argument, NULL, 1},
{"drop-slot", no_argument, NULL, 2},
+ {"synchronous", no_argument, NULL, 3},
{NULL, 0, NULL, 0}
};
***************
*** 408,414 **** main(int argc, char **argv)
}
}
! while ((c = getopt_long(argc, argv, "D:d:h:p:U:s:S:nF:wWv",
long_options, &option_index)) != -1)
{
switch (c)
--- 407,413 ----
}
}
! while ((c = getopt_long(argc, argv, "D:d:h:p:U:s:S:nwWv",
long_options, &option_index)) != -1)
{
switch (c)
***************
*** 455,469 **** main(int argc, char **argv)
case 'n':
noloop = 1;
break;
- case 'F':
- fsync_interval = atoi(optarg) * 1000;
- if (fsync_interval < -1000)
- {
- fprintf(stderr, _("%s: invalid fsync interval \"%s\"\n"),
- progname, optarg);
- exit(1);
- }
- break;
case 'v':
verbose++;
break;
--- 454,459 ----
***************
*** 474,479 **** main(int argc, char **argv)
--- 464,472 ----
case 2:
do_drop_slot = true;
break;
+ case 3:
+ synchronous = true;
+ break;
default:
/*
*** a/src/bin/pg_basebackup/receivelog.c
--- b/src/bin/pg_basebackup/receivelog.c
***************
*** 31,44 **** static char current_walfile_name[MAXPGPATH] = "";
static bool reportFlushPosition = false;
static XLogRecPtr lastFlushPosition = InvalidXLogRecPtr;
- static int64 last_fsync = -1; /* timestamp of last WAL file flush */
static bool still_sending = true; /* feedback still needs to be sent? */
static PGresult *HandleCopyStream(PGconn *conn, XLogRecPtr startpos,
uint32 timeline, char *basedir,
stream_stop_callback stream_stop, int standby_message_timeout,
char *partial_suffix, XLogRecPtr *stoppos,
! int fsync_interval);
static int CopyStreamPoll(PGconn *conn, long timeout_ms);
static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
--- 31,43 ----
static bool reportFlushPosition = false;
static XLogRecPtr lastFlushPosition = InvalidXLogRecPtr;
static bool still_sending = true; /* feedback still needs to be sent? */
static PGresult *HandleCopyStream(PGconn *conn, XLogRecPtr startpos,
uint32 timeline, char *basedir,
stream_stop_callback stream_stop, int standby_message_timeout,
char *partial_suffix, XLogRecPtr *stoppos,
! bool synchronous);
static int CopyStreamPoll(PGconn *conn, long timeout_ms);
static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
***************
*** 55,62 **** static bool CheckCopyStreamStop(PGconn *conn, XLogRecPtr blockpos,
stream_stop_callback stream_stop,
char *partial_suffix, XLogRecPtr *stoppos);
static long CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
! int64 last_status, int fsync_interval,
! XLogRecPtr blockpos);
static bool ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos,
uint32 *timeline);
--- 54,60 ----
stream_stop_callback stream_stop,
char *partial_suffix, XLogRecPtr *stoppos);
static long CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
! int64 last_status);
static bool ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos,
uint32 *timeline);
***************
*** 209,215 **** close_walfile(char *basedir, char *partial_suffix, XLogRecPtr pos)
progname, current_walfile_name, partial_suffix);
lastFlushPosition = pos;
- last_fsync = feGetCurrentTimestamp();
return true;
}
--- 207,212 ----
***************
*** 440,447 **** CheckServerVersionForStreaming(PGconn *conn)
* allows you to tell the difference between partial and completed files,
* so that you can continue later where you left.
*
! * fsync_interval controls how often we flush to the received WAL file,
! * in milliseconds.
*
* Note: The log position *must* be at a log segment start!
*/
--- 437,444 ----
* allows you to tell the difference between partial and completed files,
* so that you can continue later where you left.
*
! * If 'synchronous' is true, the received WAL is flushed as soon as written,
! * otherwise only when the WAL file is closed.
*
* Note: The log position *must* be at a log segment start!
*/
***************
*** 450,456 **** ReceiveXlogStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *sysidentifier, char *basedir,
stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! int fsync_interval)
{
char query[128];
char slotcmd[128];
--- 447,453 ----
char *sysidentifier, char *basedir,
stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! bool synchronous)
{
char query[128];
char slotcmd[128];
***************
*** 595,601 **** ReceiveXlogStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
/* Stream the WAL */
res = HandleCopyStream(conn, startpos, timeline, basedir, stream_stop,
standby_message_timeout, partial_suffix,
! &stoppos, fsync_interval);
if (res == NULL)
goto error;
--- 592,598 ----
/* Stream the WAL */
res = HandleCopyStream(conn, startpos, timeline, basedir, stream_stop,
standby_message_timeout, partial_suffix,
! &stoppos, synchronous);
if (res == NULL)
goto error;
***************
*** 760,766 **** static PGresult *
HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! XLogRecPtr *stoppos, int fsync_interval)
{
char *copybuf = NULL;
int64 last_status = -1;
--- 757,763 ----
HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *basedir, stream_stop_callback stream_stop,
int standby_message_timeout, char *partial_suffix,
! XLogRecPtr *stoppos, bool synchronous)
{
char *copybuf = NULL;
int64 last_status = -1;
***************
*** 784,797 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
now = feGetCurrentTimestamp();
/*
! * If fsync_interval has elapsed since last WAL flush and we've written
! * some WAL data, flush them to disk.
*/
! if (lastFlushPosition < blockpos &&
! walfile != -1 &&
! ((fsync_interval > 0 &&
! feTimestampDifferenceExceeds(last_fsync, now, fsync_interval)) ||
! fsync_interval < 0))
{
if (fsync(walfile) != 0)
{
--- 781,790 ----
now = feGetCurrentTimestamp();
/*
! * If synchronous option is true, issue sync command as soon as
! * there are WAL data which has not been flushed yet.
*/
! if (synchronous && lastFlushPosition < blockpos && walfile != -1)
{
if (fsync(walfile) != 0)
{
***************
*** 799,807 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
progname, current_walfile_name, strerror(errno));
goto error;
}
-
lastFlushPosition = blockpos;
! last_fsync = now;
}
/*
--- 792,806 ----
progname, current_walfile_name, strerror(errno));
goto error;
}
lastFlushPosition = blockpos;
!
! /*
! * Send feedback so that the server sees the latest WAL locations
! * immediately.
! */
! if (!sendFeedback(conn, blockpos, now, false))
! goto error;
! last_status = now;
}
/*
***************
*** 821,827 **** HandleCopyStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
* Calculate how long send/receive loops should sleep
*/
sleeptime = CalculateCopyStreamSleeptime(now, standby_message_timeout,
! last_status, fsync_interval, blockpos);
r = CopyStreamReceive(conn, sleeptime, ©buf);
while (r != 0)
--- 820,826 ----
* Calculate how long send/receive loops should sleep
*/
sleeptime = CalculateCopyStreamSleeptime(now, standby_message_timeout,
! last_status);
r = CopyStreamReceive(conn, sleeptime, ©buf);
while (r != 0)
***************
*** 1244,1277 **** CheckCopyStreamStop(PGconn *conn, XLogRecPtr blockpos, uint32 timeline,
*/
static long
CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
! int64 last_status, int fsync_interval, XLogRecPtr blockpos)
{
- int64 targettime = 0;
int64 status_targettime = 0;
- int64 fsync_targettime = 0;
long sleeptime;
if (standby_message_timeout && still_sending)
status_targettime = last_status +
(standby_message_timeout - 1) * ((int64) 1000);
! if (fsync_interval > 0 && lastFlushPosition < blockpos)
! fsync_targettime = last_fsync +
! (fsync_interval - 1) * ((int64) 1000);
!
! if ((status_targettime < fsync_targettime && status_targettime > 0) ||
! fsync_targettime == 0)
! targettime = status_targettime;
! else
! targettime = fsync_targettime;
!
! if (targettime > 0)
{
long secs;
int usecs;
feTimestampDifference(now,
! targettime,
&secs,
&usecs);
/* Always sleep at least 1 sec */
--- 1243,1264 ----
*/
static long
CalculateCopyStreamSleeptime(int64 now, int standby_message_timeout,
! int64 last_status)
{
int64 status_targettime = 0;
long sleeptime;
if (standby_message_timeout && still_sending)
status_targettime = last_status +
(standby_message_timeout - 1) * ((int64) 1000);
! if (status_targettime > 0)
{
long secs;
int usecs;
feTimestampDifference(now,
! status_targettime,
&secs,
&usecs);
/* Always sleep at least 1 sec */
*** a/src/bin/pg_basebackup/receivelog.h
--- b/src/bin/pg_basebackup/receivelog.h
***************
*** 31,36 **** extern bool ReceiveXlogStream(PGconn *conn,
stream_stop_callback stream_stop,
int standby_message_timeout,
char *partial_suffix,
! int fsync_interval);
#endif /* RECEIVELOG_H */
--- 31,36 ----
stream_stop_callback stream_stop,
int standby_message_timeout,
char *partial_suffix,
! bool synchronous);
#endif /* RECEIVELOG_H */
On Thu, Nov 13, 2014 at 12:38 PM, Fujii Masao <masao.fujii@gmail.com> wrote:
Thanks for reviewing the patch!
On Thu, Nov 13, 2014 at 4:05 AM, Alvaro Herrera
<alvherre@2ndquadrant.com> wrote:Fujii Masao wrote:
--- 127,152 ---- When this option is used, <application>pg_receivexlog</> will report a flush position to the server, indicating when each segment has been synchronized to disk so that the server can remove that segment if it ! is not otherwise needed. <literal>--synchronous</literal> option must ! be specified when making <application>pg_receivexlog</> run as ! synchronous standby by using replication slot. Otherwise WAL data ! cannot be flushed frequently enough for this to work correctly. </para> </listitem> </varlistentry>Whitespace damage here.
Fixed.
+ printf(_(" --synchronous flush transaction log in real time\n"));
"in real time" sounds odd. How about "flush transaction log
immediately after writing", or maybe "have transaction log writes be
synchronous".The former sounds better to me. So I chose it.
--- 781,791 ---- now = feGetCurrentTimestamp();/*
! * Issue sync command as soon as there are WAL data which
! * has not been flushed yet if synchronous option is true.
*/
if (lastFlushPosition < blockpos &&
! walfile != -1 && synchronous)I'd put the "synchronous" condition first in the if(), and start the
comment with it rather than putting it at the end. Both seem weird.Fixed, i.e., moved the "synchronous" condition first in the if()'s test
and also moved the comment "If synchronous option is true" also first
in the comment.progname, current_walfile_name, strerror(errno));
goto error;
}
lastFlushPosition = blockpos;
!
! /*
! * Send feedback so that the server sees the latest WAL locations
! * immediately if synchronous option is true.
! */
! if (!sendFeedback(conn, blockpos, now, false))
! goto error;
! last_status = now;I'm not clear about this comment .. why does it say "if synchronous
option is true" when it's not checking the condition?I added that comment because the code exists with the if() block
checking "synchronous" condition. But it seems confusing. Just removed
that part from the comment.Attached is the updated version of the patch.
I've just pushed this.
Regards,
--
Fujii Masao
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On Tue, Nov 18, 2014 at 2:36 AM, Fujii Masao <masao.fujii@gmail.com> wrote:
On Thu, Nov 13, 2014 at 12:38 PM, Fujii Masao <masao.fujii@gmail.com>
wrote:Thanks for reviewing the patch!
On Thu, Nov 13, 2014 at 4:05 AM, Alvaro Herrera
<alvherre@2ndquadrant.com> wrote:Fujii Masao wrote:
--- 127,152 ---- When this option is used, <application>pg_receivexlog</>will report
a flush position to the server, indicating when each
segment has been
synchronized to disk so that the server can remove that
segment if it
! is not otherwise needed. <literal>--synchronous</literal>
option must
! be specified when making <application>pg_receivexlog</> run
as
! synchronous standby by using replication slot. Otherwise WAL
data
! cannot be flushed frequently enough for this to work
correctly.
</para>
</listitem>
</varlistentry>Whitespace damage here.
Fixed.
+ printf(_(" --synchronous flush transaction log in real
time\n"));
"in real time" sounds odd. How about "flush transaction log
immediately after writing", or maybe "have transaction log writes be
synchronous".The former sounds better to me. So I chose it.
--- 781,791 ---- now = feGetCurrentTimestamp();/*
! * Issue sync command as soon as there are WAL data which
! * has not been flushed yet if synchronous option istrue.
*/
if (lastFlushPosition < blockpos &&
! walfile != -1 && synchronous)I'd put the "synchronous" condition first in the if(), and start the
comment with it rather than putting it at the end. Both seem weird.Fixed, i.e., moved the "synchronous" condition first in the if()'s test
and also moved the comment "If synchronous option is true" also first
in the comment.progname,
current_walfile_name, strerror(errno));
goto error;
}
lastFlushPosition = blockpos;
!
! /*
! * Send feedback so that the server sees thelatest WAL locations
! * immediately if synchronous option is true.
! */
! if (!sendFeedback(conn, blockpos, now, false))
! goto error;
! last_status = now;I'm not clear about this comment .. why does it say "if synchronous
option is true" when it's not checking the condition?I added that comment because the code exists with the if() block
checking "synchronous" condition. But it seems confusing. Just removed
that part from the comment.Attached is the updated version of the patch.
I've just pushed this.
Marked this patch as committed on the commit fest app.
--
Michael
On Wed, Nov 19, 2014 at 10:20 AM, Michael Paquier
<michael.paquier@gmail.com> wrote:
On Tue, Nov 18, 2014 at 2:36 AM, Fujii Masao <masao.fujii@gmail.com> wrote:
On Thu, Nov 13, 2014 at 12:38 PM, Fujii Masao <masao.fujii@gmail.com>
wrote:Thanks for reviewing the patch!
On Thu, Nov 13, 2014 at 4:05 AM, Alvaro Herrera
<alvherre@2ndquadrant.com> wrote:Fujii Masao wrote:
--- 127,152 ---- When this option is used, <application>pg_receivexlog</> will report a flush position to the server, indicating when each segment has been synchronized to disk so that the server can remove that segment if it ! is not otherwise needed. <literal>--synchronous</literal> option must ! be specified when making <application>pg_receivexlog</> run as ! synchronous standby by using replication slot. Otherwise WAL data ! cannot be flushed frequently enough for this to work correctly. </para> </listitem> </varlistentry>Whitespace damage here.
Fixed.
+ printf(_(" --synchronous flush transaction log in real
time\n"));"in real time" sounds odd. How about "flush transaction log
immediately after writing", or maybe "have transaction log writes be
synchronous".The former sounds better to me. So I chose it.
--- 781,791 ---- now = feGetCurrentTimestamp();/*
! * Issue sync command as soon as there are WAL data
which
! * has not been flushed yet if synchronous option is
true.
*/
if (lastFlushPosition < blockpos &&
! walfile != -1 && synchronous)I'd put the "synchronous" condition first in the if(), and start the
comment with it rather than putting it at the end. Both seem weird.Fixed, i.e., moved the "synchronous" condition first in the if()'s test
and also moved the comment "If synchronous option is true" also first
in the comment.progname,
current_walfile_name, strerror(errno));
goto error;
}
lastFlushPosition = blockpos;
!
! /*
! * Send feedback so that the server sees the
latest WAL locations
! * immediately if synchronous option is true.
! */
! if (!sendFeedback(conn, blockpos, now, false))
! goto error;
! last_status = now;I'm not clear about this comment .. why does it say "if synchronous
option is true" when it's not checking the condition?I added that comment because the code exists with the if() block
checking "synchronous" condition. But it seems confusing. Just removed
that part from the comment.Attached is the updated version of the patch.
I've just pushed this.
Marked this patch as committed on the commit fest app.
Thanks!
Regards,
--
Fujii Masao
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers