Re: protocol version negotiation (Re: Libpq PGRES_COPY_BOTH - version compatibility)

From 442f54a04fa814fb99f3bf46dc66060d1c58555e Mon Sep 17 00:00:00 2001
From: Badrul Chowdhury <bachow@microsoft.com>
Date: Mon, 25 Sep 2017 13:42:44 -0700
Subject: [PATCH] =?UTF-8?q?Introducing=20pgwire=20v3.1:=201.=20adds=20supp?=
 =?UTF-8?q?ort=20for=20backend=20(BE)=20to=20accept=20optional=20parameter?=
 =?UTF-8?q?s,=20ie=20parameters=20that=20have=20=E2=80=9C=5Fpq=5F=E2=80=9D?=
 =?UTF-8?q?=20as=20a=20proper=20prefix=20in=20their=20names=202.=20creates?=
 =?UTF-8?q?=20data=20structure=20for=20passing=20in=20additional=20paramet?=
 =?UTF-8?q?ers=20in=20FE,=20adding=20command=20line=20support=20is=20out?=
 =?UTF-8?q?=20of=20scope=20of=20this=20item=203.=20enhances=20FE->BE=20pro?=
 =?UTF-8?q?tocol=20negotiation:=20adds=20support=20for=20newer=20FE=20to?=
 =?UTF-8?q?=20connect=20to=20older=20BE=20while=20maintaining=20back-compa?=
 =?UTF-8?q?tibility,=20ie=20same=20FE=20<->=20BE,=20older=20FE=20to=20newe?=
 =?UTF-8?q?r=20BE=20work=20as=20before=204.=20adds=20support=20for=20newer?=
 =?UTF-8?q?=20FE=20to=20connect=20to=20old=20BE=20that=20doesn't=20support?=
 =?UTF-8?q?=20protocol=20negotiation:=20FE=20downgrades=20to=20v3.0=20if?=
 =?UTF-8?q?=20sending=20startup=20parameters=20fails?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/Makefile                        |   2 +-
 src/Makefile.global.in              |   2 +-
 src/backend/postmaster/postmaster.c | 112 +++++++++++++++---
 src/backend/utils/misc/guc.c        |  22 +++-
 src/bin/pg_dump/pg_dump.c           |  10 +-
 src/bin/scripts/clusterdb.c         |   2 +-
 src/bin/scripts/createuser.c        |   2 +-
 src/bin/scripts/reindexdb.c         |   6 +-
 src/bin/scripts/vacuumdb.c          |   4 +-
 src/common/Makefile                 |   2 +-
 src/fe_utils/Makefile               |   2 +-
 src/fe_utils/simple_list.c          |   4 +
 src/include/fe_utils/simple_list.h  |   1 +
 src/include/libpq/pqcomm.h          |   2 +-
 src/include/postmaster/postmaster.h |   7 ++
 src/interfaces/libpq/fe-connect.c   | 218 ++++++++++++++++++++++++++++++++++--
 src/interfaces/libpq/fe-protocol3.c |  42 ++++++-
 src/interfaces/libpq/libpq-fe.h     |   1 +
 src/interfaces/libpq/libpq-int.h    |  10 +-
 src/tools/msvc/Mkvcbuild.pm         |   2 +-
 20 files changed, 404 insertions(+), 49 deletions(-)

diff --git a/src/Makefile b/src/Makefile
index 380da92c75..048d2d7ca8 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -20,10 +20,10 @@ SUBDIRS = \
 	backend/utils/mb/conversion_procs \
 	backend/snowball \
 	include \
+	fe_utils \
 	interfaces \
 	backend/replication/libpqwalreceiver \
 	backend/replication/pgoutput \
-	fe_utils \
 	bin \
 	pl \
 	makefiles \
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index e8b3a519cb..c3b45ce650 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -480,7 +480,7 @@ endif
 
 # This macro is for use by libraries linking to libpq.  (Because libpgport
 # isn't created with the same link flags as libpq, it can't be used.)
-libpq = -L$(libpq_builddir) -lpq
+libpq = -L$(libpq_builddir) -lpq -L$(top_builddir)/src/common -lpgcommon -L$(top_builddir)/src/fe_utils -lpgfeutils
 
 # This macro is for use by client executables (not libraries) that use libpq.
 # We force clients to pull symbols from the non-shared libraries libpgport
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 95180b2ef5..21d77333be 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -103,6 +103,7 @@
 #include "lib/ilist.h"
 #include "libpq/auth.h"
 #include "libpq/libpq.h"
+#include "libpq/pqformat.h"
 #include "libpq/pqsignal.h"
 #include "miscadmin.h"
 #include "pg_getopt.h"
@@ -567,6 +568,77 @@ HANDLE		PostmasterHandle;
 #endif
 
 /*
+ * Initiate protocol negotiation phase;
+ * protocol negotiation is only supported for pgwire version 3.x, x>0.
+ *
+ * Ensure that the packet write buffer is flushed.
+ */
+int
+NegotiateServerProtocol(Port *port)
+{
+	if (whereToSendOutput != DestRemote ||
+		PG_PROTOCOL_MAJOR(FrontendProtocol) < 3)
+		return -1;
+
+	int sendStatus = 0;
+
+	/* NegotiateServerProtocol packet structure
+	 *
+	 * [ 'Y' | msgLength | min_version | max_version | param_list_len | list of param names ]
+	 */
+
+	sendStatus = SendServerProtocolVersionMessage(port);
+
+	/* Ensure that the message buffer is flushed */
+	pq_flush();
+
+	return sendStatus;
+}
+
+/*
+ * Construct and send a ServerProtocolVersion message.
+ *
+ * Message contains:
+ * 1. minimum, maximum versions supported by the BE,
+ * 2. number of parameters that were honored by the BE from startup packet,
+ * 3. a list of strings consisting of the parameter names accepted by BE.
+ */
+int
+SendServerProtocolVersionMessage(Port *port)
+{
+	StringInfoData buf;
+
+	/* PG message type*/
+	pq_beginmessage(&buf, 'Y');
+
+	/* Protocol version numbers */
+	pq_sendint(&buf, PG_PROTOCOL_EARLIEST, sizeof(int32)); /* min */
+	pq_sendint(&buf, PG_PROTOCOL_LATEST, sizeof(int32));   /* max */
+
+	/* Length of parameter list; parameter list consists of (key, value) pairs */
+	pq_sendint(&buf, list_length(port->guc_options) / 2, sizeof(int32));
+
+	ListCell *gucopts = list_head(port->guc_options);
+	while (gucopts)
+	{
+		char	   *name;
+
+		/* First comes key, which we need. */
+		name = lfirst(gucopts);
+		gucopts = lnext(gucopts);
+
+		/* Then comes value, which we don't need. */
+		gucopts = lnext(gucopts);
+
+		pq_sendstring(&buf, name);
+	}
+
+	pq_endmessage(&buf);
+
+	return 0;
+}
+
+/*
  * Postmaster main entry point
  */
 void
@@ -2050,20 +2122,6 @@ retry1:
 	 */
 	FrontendProtocol = proto;
 
-	/* Check we can handle the protocol the frontend is using. */
-
-	if (PG_PROTOCOL_MAJOR(proto) < PG_PROTOCOL_MAJOR(PG_PROTOCOL_EARLIEST) ||
-		PG_PROTOCOL_MAJOR(proto) > PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST) ||
-		(PG_PROTOCOL_MAJOR(proto) == PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST) &&
-		 PG_PROTOCOL_MINOR(proto) > PG_PROTOCOL_MINOR(PG_PROTOCOL_LATEST)))
-		ereport(FATAL,
-				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-				 errmsg("unsupported frontend protocol %u.%u: server supports %u.0 to %u.%u",
-						PG_PROTOCOL_MAJOR(proto), PG_PROTOCOL_MINOR(proto),
-						PG_PROTOCOL_MAJOR(PG_PROTOCOL_EARLIEST),
-						PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST),
-						PG_PROTOCOL_MINOR(PG_PROTOCOL_LATEST))));
-
 	/*
 	 * Now fetch parameters out of startup packet and save them into the Port
 	 * structure.  All data structures attached to the Port struct must be
@@ -2145,9 +2203,35 @@ retry1:
 			ereport(FATAL,
 					(errcode(ERRCODE_PROTOCOL_VIOLATION),
 					 errmsg("invalid startup packet layout: expected terminator as last byte")));
+
+		/*
+		 * Need to negotiate pgwire protocol if
+		 * 1. FE version is not the same as BE version
+		 * 2. FE version is not 3.0
+		 */
+		if (FrontendProtocol != PG_PROTOCOL_LATEST
+			&& FrontendProtocol != PG_PROTOCOL(3, 0))
+		{
+			/* Negotiate parameters after all the error-checking is done */
+			if (NegotiateServerProtocol(port))
+				return STATUS_ERROR;
+		}
 	}
 	else
 	{
+		/* Check we can handle the protocol the frontend is using. */
+		if (PG_PROTOCOL_MAJOR(proto) < PG_PROTOCOL_MAJOR(PG_PROTOCOL_EARLIEST) ||
+			PG_PROTOCOL_MAJOR(proto) > PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST) ||
+			(PG_PROTOCOL_MAJOR(proto) == PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST) &&
+			 PG_PROTOCOL_MINOR(proto) > PG_PROTOCOL_MINOR(PG_PROTOCOL_LATEST)))
+			ereport(FATAL,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					 errmsg("unsupported frontend protocol %u.%u: server supports %u.0 to %u.%u",
+					 PG_PROTOCOL_MAJOR(proto), PG_PROTOCOL_MINOR(proto),
+					 PG_PROTOCOL_MAJOR(PG_PROTOCOL_EARLIEST),
+					 PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST),
+					 PG_PROTOCOL_MINOR(PG_PROTOCOL_LATEST))));
+
 		/*
 		 * Get the parameters from the old-style, fixed-width-fields startup
 		 * packet as C strings.  The packet destination was cleared first so a
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 246fea8693..69441f3f86 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -3961,6 +3961,7 @@ static int	GUCNestLevel = 0;	/* 1 when in main transaction */
 
 static int	guc_var_compare(const void *a, const void *b);
 static int	guc_name_compare(const char *namea, const char *nameb);
+static bool is_optional(const char *guc_name);
 static void InitializeGUCOptionsFromEnvironment(void);
 static void InitializeOneGUCOption(struct config_generic *gconf);
 static void push_old_value(struct config_generic *gconf, GucAction action);
@@ -4416,7 +4417,7 @@ find_option(const char *name, bool create_placeholders, int elevel)
 		/*
 		 * Check if the name is qualified, and if so, add a placeholder.
 		 */
-		if (strchr(name, GUC_QUALIFIER_SEPARATOR) != NULL)
+		if (strchr(name, GUC_QUALIFIER_SEPARATOR) != NULL || is_optional(name))
 			return add_placeholder_variable(name, elevel);
 	}
 
@@ -4467,6 +4468,25 @@ guc_name_compare(const char *namea, const char *nameb)
 	return 0;
 }
 
+/*
+ * A GUC variable is deemed optional if the name contains "_pq_" as a proper prefix.
+ *
+ * It takes the whole struct as input in case we want to move away from name-based
+ * tagging of optional variables.
+ */
+bool
+is_optional(const char *guc_name)
+{
+	const char *optionalPrefix = "_pq_";
+	bool isOptional = false;
+
+	/* "_pq_" must be a proper prefix of the guc name in all encodings */
+	if (guc_name_compare(guc_name, optionalPrefix) == 1 &&
+		strstr(guc_name, optionalPrefix))
+		isOptional = true;
+
+	return isOptional;
+}
 
 /*
  * Initialize GUC options during program startup.
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 75f08cd792..9d3feb0a0a 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -110,16 +110,16 @@ static int	strict_names = 0;
  * The string lists record the patterns given by command-line switches,
  * which we then convert to lists of OIDs of matching objects.
  */
-static SimpleStringList schema_include_patterns = {NULL, NULL};
+static SimpleStringList schema_include_patterns = {NULL, NULL, NULL};
 static SimpleOidList schema_include_oids = {NULL, NULL};
-static SimpleStringList schema_exclude_patterns = {NULL, NULL};
+static SimpleStringList schema_exclude_patterns = {NULL, NULL, NULL};
 static SimpleOidList schema_exclude_oids = {NULL, NULL};
 
-static SimpleStringList table_include_patterns = {NULL, NULL};
+static SimpleStringList table_include_patterns = {NULL, NULL, NULL};
 static SimpleOidList table_include_oids = {NULL, NULL};
-static SimpleStringList table_exclude_patterns = {NULL, NULL};
+static SimpleStringList table_exclude_patterns = {NULL, NULL, NULL};
 static SimpleOidList table_exclude_oids = {NULL, NULL};
-static SimpleStringList tabledata_exclude_patterns = {NULL, NULL};
+static SimpleStringList tabledata_exclude_patterns = {NULL, NULL, NULL};
 static SimpleOidList tabledata_exclude_oids = {NULL, NULL};
 
 
diff --git a/src/bin/scripts/clusterdb.c b/src/bin/scripts/clusterdb.c
index a6640aa57b..79fa46afd2 100644
--- a/src/bin/scripts/clusterdb.c
+++ b/src/bin/scripts/clusterdb.c
@@ -60,7 +60,7 @@ main(int argc, char *argv[])
 	bool		quiet = false;
 	bool		alldb = false;
 	bool		verbose = false;
-	SimpleStringList tables = {NULL, NULL};
+	SimpleStringList tables = {NULL, NULL, NULL};
 
 	progname = get_progname(argv[0]);
 	set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pgscripts"));
diff --git a/src/bin/scripts/createuser.c b/src/bin/scripts/createuser.c
index 0e36edcc5d..ff33ec63c3 100644
--- a/src/bin/scripts/createuser.c
+++ b/src/bin/scripts/createuser.c
@@ -58,7 +58,7 @@ main(int argc, char *argv[])
 	char	   *host = NULL;
 	char	   *port = NULL;
 	char	   *username = NULL;
-	SimpleStringList roles = {NULL, NULL};
+	SimpleStringList roles = {NULL, NULL, NULL};
 	enum trivalue prompt_password = TRI_DEFAULT;
 	bool		echo = false;
 	bool		interactive = false;
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index ffd611e7bb..2aede05156 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -68,9 +68,9 @@ main(int argc, char *argv[])
 	bool		echo = false;
 	bool		quiet = false;
 	bool		verbose = false;
-	SimpleStringList indexes = {NULL, NULL};
-	SimpleStringList tables = {NULL, NULL};
-	SimpleStringList schemas = {NULL, NULL};
+	SimpleStringList indexes = {NULL, NULL, NULL};
+	SimpleStringList tables = {NULL, NULL, NULL};
+	SimpleStringList schemas = {NULL, NULL, NULL};
 
 	progname = get_progname(argv[0]);
 	set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pgscripts"));
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index 5d2869ea6b..df77eccf5f 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -123,7 +123,7 @@ main(int argc, char *argv[])
 	vacuumingOptions vacopts;
 	bool		analyze_in_stages = false;
 	bool		alldb = false;
-	SimpleStringList tables = {NULL, NULL};
+	SimpleStringList tables = {NULL, NULL, NULL};
 	int			concurrentCons = 1;
 	int			tbl_count = 0;
 
@@ -342,7 +342,7 @@ vacuum_one_database(const char *dbname, vacuumingOptions *vacopts,
 	PGconn	   *conn;
 	SimpleStringListCell *cell;
 	ParallelSlot *slots = NULL;
-	SimpleStringList dbtables = {NULL, NULL};
+	SimpleStringList dbtables = {NULL, NULL, NULL};
 	int			i;
 	bool		failed = false;
 	bool		parallel = concurrentCons > 1;
diff --git a/src/common/Makefile b/src/common/Makefile
index 80e78d72fe..fa29c3e0ac 100644
--- a/src/common/Makefile
+++ b/src/common/Makefile
@@ -24,7 +24,7 @@ subdir = src/common
 top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 
-override CPPFLAGS := -DFRONTEND $(CPPFLAGS)
+override CPPFLAGS := -DFRONTEND $(CPPFLAGS) -fPIC
 LIBS += $(PTHREAD_LIBS)
 
 # don't include subdirectory-path-dependent -I and -L switches
diff --git a/src/fe_utils/Makefile b/src/fe_utils/Makefile
index ebce38ceb4..7706bc5fa9 100644
--- a/src/fe_utils/Makefile
+++ b/src/fe_utils/Makefile
@@ -17,7 +17,7 @@ subdir = src/fe_utils
 top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 
-override CPPFLAGS := -DFRONTEND -I$(libpq_srcdir) $(CPPFLAGS)
+override CPPFLAGS := -DFRONTEND -I$(libpq_srcdir) $(CPPFLAGS) -fPIC
 
 OBJS = mbprint.o print.o psqlscan.o simple_list.o string_utils.o
 
diff --git a/src/fe_utils/simple_list.c b/src/fe_utils/simple_list.c
index 21a2e57297..ed67f254f3 100644
--- a/src/fe_utils/simple_list.c
+++ b/src/fe_utils/simple_list.c
@@ -64,6 +64,10 @@ simple_string_list_append(SimpleStringList *list, const char *val)
 {
 	SimpleStringListCell *cell;
 
+	/* Cannot append to immutable list */
+	if (list->is_immutable)
+		return;
+
 	cell = (SimpleStringListCell *)
 		pg_malloc(offsetof(SimpleStringListCell, val) + strlen(val) + 1);
 
diff --git a/src/include/fe_utils/simple_list.h b/src/include/fe_utils/simple_list.h
index 97bb34f191..ea5e9af7b4 100644
--- a/src/include/fe_utils/simple_list.h
+++ b/src/include/fe_utils/simple_list.h
@@ -41,6 +41,7 @@ typedef struct SimpleStringList
 {
 	SimpleStringListCell *head;
 	SimpleStringListCell *tail;
+	bool is_immutable;
 } SimpleStringList;
 
 
diff --git a/src/include/libpq/pqcomm.h b/src/include/libpq/pqcomm.h
index 10c7434c41..3a8ea400ac 100644
--- a/src/include/libpq/pqcomm.h
+++ b/src/include/libpq/pqcomm.h
@@ -108,7 +108,7 @@ typedef struct
 /* The earliest and latest frontend/backend protocol version supported. */
 
 #define PG_PROTOCOL_EARLIEST	PG_PROTOCOL(2,0)
-#define PG_PROTOCOL_LATEST		PG_PROTOCOL(3,0)
+#define PG_PROTOCOL_LATEST		PG_PROTOCOL(3,1)
 
 typedef uint32 ProtocolVersion; /* FE/BE protocol version number */
 
diff --git a/src/include/postmaster/postmaster.h b/src/include/postmaster/postmaster.h
index 0f85908b09..da0a3a79c3 100644
--- a/src/include/postmaster/postmaster.h
+++ b/src/include/postmaster/postmaster.h
@@ -13,6 +13,10 @@
 #ifndef _POSTMASTER_H
 #define _POSTMASTER_H
 
+#include "postgres.h"
+
+#include "libpq/libpq-be.h"
+
 /* GUC options */
 extern bool EnableSSL;
 extern int	ReservedBackends;
@@ -46,6 +50,9 @@ extern int	postmaster_alive_fds[2];
 
 extern const char *progname;
 
+extern int NegotiateServerProtocol(Port *port);
+extern int SendServerProtocolVersionMessage(Port *port);
+
 extern void PostmasterMain(int argc, char *argv[]) pg_attribute_noreturn();
 extern void ClosePostmasterPorts(bool am_syslogger);
 
diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c
index d0e97ecdd4..db22fca57a 100644
--- a/src/interfaces/libpq/fe-connect.c
+++ b/src/interfaces/libpq/fe-connect.c
@@ -86,8 +86,11 @@ static int ldapServiceLookup(const char *purl, PQconninfoOption *options,
  * application_name in a startup packet.  We hard-wire the value rather
  * than looking into errcodes.h since it reflects historical behavior
  * rather than that of the current code.
+ *
+ * Note that application_name is a startup parameter.
+ * The same error code is used for all unrecognized startup parameters.
  */
-#define ERRCODE_APPNAME_UNKNOWN "42704"
+#define ERRCODE_PARAMETER_UNKNOWN "42704"
 
 /* This is part of the protocol so just define it */
 #define ERRCODE_INVALID_PASSWORD "28P01"
@@ -350,6 +353,11 @@ static const PQEnvironmentOption EnvironmentOptions[] =
 static const char uri_designator[] = "postgresql://";
 static const char short_uri_designator[] = "postgres://";
 
+static const char *optional_parameter_prefix = "_pq_";
+
+/* A list of string options to add as startup options */
+static SimpleStringList startup_parameters = {NULL, NULL, NULL};
+
 static bool connectOptions1(PGconn *conn, const char *conninfo);
 static bool connectOptions2(PGconn *conn);
 static int	connectDBStart(PGconn *conn);
@@ -1803,8 +1811,9 @@ connectDBStart(PGconn *conn)
 	 */
 	conn->whichhost = 0;
 	conn->addr_cur = conn->connhost[0].addrlist;
-	conn->pversion = PG_PROTOCOL(3, 0);
+	conn->pversion = PG_PROTOCOL(3, 1);
 	conn->send_appname = true;
+	conn->send_startup_params = true;
 	conn->status = CONNECTION_NEEDED;
 
 	/*
@@ -2001,6 +2010,9 @@ PQconnectPoll(PGconn *conn)
 	int			optval;
 	PQExpBufferData savedMessage;
 
+	/* Flag to check if newer FE is connecting to older BE. */
+	bool server_is_older = false;
+
 	if (conn == NULL)
 		return PGRES_POLLING_FAILED;
 
@@ -2018,6 +2030,7 @@ PQconnectPoll(PGconn *conn)
 
 			/* These are reading states */
 		case CONNECTION_AWAITING_RESPONSE:
+		case CONNECTION_NEGOTIATING:
 		case CONNECTION_AUTH_OK:
 			{
 				/* Load waiting data */
@@ -2462,7 +2475,8 @@ keep_going:						/* We will come back to here until there is
 				 */
 				if (PG_PROTOCOL_MAJOR(conn->pversion) >= 3)
 					startpacket = pqBuildStartupPacket3(conn, &packetlen,
-														EnvironmentOptions);
+														EnvironmentOptions,
+														&startup_parameters);
 				else
 					startpacket = pqBuildStartupPacket2(conn, &packetlen,
 														EnvironmentOptions);
@@ -2649,6 +2663,12 @@ keep_going:						/* We will come back to here until there is
 					return PGRES_POLLING_READING;
 				}
 
+				if (beresp == 'Y')
+				{
+					conn->status = CONNECTION_NEGOTIATING;
+					goto keep_going;
+				}
+
 				/*
 				 * Validate message type: we expect only an authentication
 				 * request or an error here.  Anything else probably means
@@ -2716,10 +2736,19 @@ keep_going:						/* We will come back to here until there is
 					appendPQExpBufferChar(&conn->errorMessage, '\n');
 
 					/*
-					 * If we tried to open the connection in 3.0 protocol,
-					 * fall back to 2.0 protocol.
+					 * If we tried to open the connection in 3.1 protocol,
+					 * fall back to 3.0 protocol; fall back to 2.0 from 3.0.
 					 */
-					if (PG_PROTOCOL_MAJOR(conn->pversion) >= 3)
+					if (PG_PROTOCOL_MAJOR(conn->pversion) >= 3 &&
+						PG_PROTOCOL_MINOR(conn->pversion) >= 1)
+					{
+						conn->pversion = PG_PROTOCOL(3, 0);
+						/* Must drop the old connection */
+						pqDropConnection(conn, true);
+						conn->status = CONNECTION_NEEDED;
+						goto keep_going;
+					}
+					else if (PG_PROTOCOL_MAJOR(conn->pversion) >= 3)
 					{
 						conn->pversion = PG_PROTOCOL(2, 0);
 						/* Must drop the old connection */
@@ -2893,6 +2922,161 @@ keep_going:						/* We will come back to here until there is
 				goto keep_going;
 			}
 
+		case CONNECTION_NEGOTIATING:
+			{
+				int minServerVersion;	/* Min pgwire protocol supported by server */
+				int maxServerVersion;	/* Max pgwire protocol supported by server */
+				int param_list_len;		/* Length of list of parameters honored by server */
+				int proper_prefix_len;	/* Length of required prefix in optional parameter name */
+				int i;					/* Index for loop */
+				PQExpBuffer buf;		/* Buffer for data */
+				char *rejected_param;	/* Parameter that was rejected by the BE */
+				int originalMsgLen;		/* Length of message sans msg type */
+				int runningMsgLength;	/* Copy of originalMsgLen, will not be preserved */
+				int available;			/* Bytes available in message body */
+
+				/* Mark 'Y' as consumed: 1 byte for message type */
+				conn->inCursor = conn->inStart + 1;
+
+				/*
+				* Block until message length is read.
+				*
+				* No need to account for 2.x fixed-length message because this state cannot
+				* be reached by pre-3.0 server.
+				*/
+				if (pqGetInt(&originalMsgLen, sizeof(int32), conn))
+					return PGRES_POLLING_READING;
+
+				runningMsgLength = originalMsgLen;
+
+				/* Block until each of the fields in the packet is read */
+				if (pqGetInt(&minServerVersion, sizeof(int32), conn) ||
+					pqGetInt(&maxServerVersion, sizeof(int32), conn) ||
+					pqGetInt(&param_list_len, sizeof(int32), conn))
+				{
+					return PGRES_POLLING_READING;
+				}
+
+				/* 4 bytes each for msgLength, min, max, and length of list of param names */
+				runningMsgLength -= sizeof(int32) * 4;
+
+				/* Create empty buffer */
+				buf = createPQExpBuffer();
+				for (i = 0; i < param_list_len; ++i)
+				{
+					if (pqGets(buf, conn))
+					{
+						available = conn->inEnd - conn->inCursor;
+						if (available < runningMsgLength)
+						{
+							/* Enlarge buffer if required */
+							if (pqCheckInBufferSpace(conn->inCursor + (size_t)runningMsgLength, conn))
+								goto error_return;
+
+							return PGRES_POLLING_READING;
+						}
+						else
+						{
+							appendPQExpBuffer(&conn->errorMessage,
+											  libpq_gettext("encountered error while attempting "
+															"to read parameter name from connection\n"));
+							/* Free buffer */
+							destroyPQExpBuffer(buf);
+
+							goto error_return;
+						}
+					}
+
+					/* Read string successfully, decrement message length to reflect this */
+					runningMsgLength -= buf->len + 1; /* +1 for NULL */
+
+					simple_string_list_member(&startup_parameters, buf->data);
+
+					/* pqGets() resets the buffer, so buffer is clean for next iteration */
+				}
+
+				/* Free buffer */
+				destroyPQExpBuffer(buf);
+
+				/* Check for extraneous data in packet, +1 to account for message type char */
+				if (conn->inCursor != conn->inStart + 1 + originalMsgLen)
+				{
+					appendPQExpBuffer(&conn->errorMessage,
+									  libpq_gettext("Extraneous data in ServerProtocolMessage packet\n"));
+					goto error_return;
+				}
+
+				/* Mark read data consumed */
+				conn->inStart = conn->inCursor;
+
+				/*
+				* Emit error if FE is older than min supported by BE.
+				*
+				* This check will be effective once the minimum BE version is >= 3.0;
+				* otherwise, older FEs will be turned away when parsing startup packet.
+				*/
+				if (PG_PROTOCOL_MAJOR(conn->pversion) < PG_PROTOCOL_MAJOR(minServerVersion) ||
+					(PG_PROTOCOL_MAJOR(conn->pversion) == PG_PROTOCOL_MAJOR(minServerVersion) &&
+					 PG_PROTOCOL_MINOR(conn->pversion) < PG_PROTOCOL_MINOR(minServerVersion)))
+				{
+					appendPQExpBuffer(&conn->errorMessage,
+									  libpq_gettext("unsupported frontend protocol %u.%u: "
+													"server supports %u.%u to %u.%u"),
+													PG_PROTOCOL_MAJOR(conn->pversion),
+													PG_PROTOCOL_MINOR(conn->pversion),
+													PG_PROTOCOL_MAJOR(PG_PROTOCOL_EARLIEST),
+													PG_PROTOCOL_MINOR(PG_PROTOCOL_EARLIEST),
+													PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST),
+													PG_PROTOCOL_MINOR(PG_PROTOCOL_LATEST));
+					goto error_return;
+				}
+
+				/* Set flag to enable parameter check if newer FE is connecting to older BE
+				* In this case, check if the non-optional parameters sent
+				* in StartupMsg by FE were accepted by the BE.
+				*
+				* If the FE version is the same as BE or it falls in the [min, max]
+				* range of BE, all the parameters sent by FE should be accepted by BE,
+				* so skip the accepted parameter check.
+				*/
+				else if (PG_PROTOCOL_MAJOR(conn->pversion) > PG_PROTOCOL_MAJOR(maxServerVersion) ||
+						(PG_PROTOCOL_MAJOR(conn->pversion) == PG_PROTOCOL_MAJOR(maxServerVersion) &&
+						 PG_PROTOCOL_MINOR(conn->pversion) > PG_PROTOCOL_MINOR(maxServerVersion)))
+				{
+					server_is_older = true;
+				}
+
+				/*
+				* Check whether all required parameters sent by newer FE were accepted by older BE.
+				* Do not error out if optional parameters were rejected by the BE.
+				*/
+				while (server_is_older && (rejected_param = simple_string_list_not_touched(&startup_parameters)) != NULL)
+				{
+					/*
+					* Terminate connection if the rejected parameter is not optional.
+					* This is not encoding-aware, which is okay because it is all on client-side
+					* optional_parameter_prefix must be a proper prefix of the rejected paramteter's name.
+					*/
+					proper_prefix_len = strlen(optional_parameter_prefix);
+					if (strlen(rejected_param) <= proper_prefix_len ||
+						strncmp(rejected_param, optional_parameter_prefix, proper_prefix_len) != 0)
+					{
+						appendPQExpBuffer(&conn->errorMessage,
+										  libpq_gettext("Non-optional startup parameter \'%s\'"
+														"was rejected by server\n"),
+														rejected_param);
+						goto error_return;
+
+					}
+
+					/* Mark member as visited */
+					simple_string_list_member(&startup_parameters, rejected_param);
+				}
+
+				conn->status = CONNECTION_AWAITING_RESPONSE;
+				goto keep_going;
+			}
+
 		case CONNECTION_AUTH_OK:
 			{
 				/*
@@ -2921,8 +3105,9 @@ keep_going:						/* We will come back to here until there is
 					if (res->resultStatus != PGRES_FATAL_ERROR)
 						appendPQExpBufferStr(&conn->errorMessage,
 											 libpq_gettext("unexpected message from server during startup\n"));
-					else if (conn->send_appname &&
-							 (conn->appname || conn->fbappname))
+					else if ((conn->send_appname &&
+							 (conn->appname || conn->fbappname)) ||
+							 (conn->send_startup_params && startup_parameters.head))
 					{
 						/*
 						 * If we tried to send application_name, check to see
@@ -2938,10 +3123,23 @@ keep_going:						/* We will come back to here until there is
 
 						sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
 						if (sqlstate &&
-							strcmp(sqlstate, ERRCODE_APPNAME_UNKNOWN) == 0)
+							strcmp(sqlstate, ERRCODE_PARAMETER_UNKNOWN) == 0)
 						{
 							PQclear(res);
-							conn->send_appname = false;
+
+							if (conn->send_startup_params && startup_parameters.head)
+							{
+								conn->send_startup_params = false;
+								/*
+								 * Sending optional parameters is the only FE feature of pgwire v3.1;
+								 * downgrade to v3.0 if this failed.
+								 */
+								conn->pversion = PG_PROTOCOL(3, 0);
+							}
+
+							if (conn->send_appname && (conn->appname || conn->fbappname))
+								conn->send_appname = false;
+
 							/* Must drop the old connection */
 							pqDropConnection(conn, true);
 							conn->status = CONNECTION_NEEDED;
diff --git a/src/interfaces/libpq/fe-protocol3.c b/src/interfaces/libpq/fe-protocol3.c
index a484fe80a1..7f5c74b11f 100644
--- a/src/interfaces/libpq/fe-protocol3.c
+++ b/src/interfaces/libpq/fe-protocol3.c
@@ -13,6 +13,7 @@
  *-------------------------------------------------------------------------
  */
 #include "postgres_fe.h"
+#include "fe_utils/simple_list.h"
 
 #include <ctype.h>
 #include <fcntl.h>
@@ -54,7 +55,8 @@ static int	getReadyForQuery(PGconn *conn);
 static void reportErrorPosition(PQExpBuffer msg, const char *query,
 					int loc, int encoding);
 static int build_startup_packet(const PGconn *conn, char *packet,
-					 const PQEnvironmentOption *options);
+								const PQEnvironmentOption *options,
+								SimpleStringList *startup_parameters);
 
 
 /*
@@ -2116,15 +2118,16 @@ pqFunctionCall3(PGconn *conn, Oid fnid,
  */
 char *
 pqBuildStartupPacket3(PGconn *conn, int *packetlen,
-					  const PQEnvironmentOption *options)
+					  const PQEnvironmentOption *options,
+					  SimpleStringList *startup_parameters)
 {
 	char	   *startpacket;
 
-	*packetlen = build_startup_packet(conn, NULL, options);
+	*packetlen = build_startup_packet(conn, NULL, options, startup_parameters);
 	startpacket = (char *) malloc(*packetlen);
 	if (!startpacket)
 		return NULL;
-	*packetlen = build_startup_packet(conn, startpacket, options);
+	*packetlen = build_startup_packet(conn, startpacket, options, startup_parameters);
 	return startpacket;
 }
 
@@ -2139,11 +2142,15 @@ pqBuildStartupPacket3(PGconn *conn, int *packetlen,
  */
 static int
 build_startup_packet(const PGconn *conn, char *packet,
-					 const PQEnvironmentOption *options)
+					 const PQEnvironmentOption *options,
+					 SimpleStringList *startup_parameters)
 {
 	int			packet_len = 0;
 	const PQEnvironmentOption *next_eo;
 	const char *val;
+	int nCells = 0;
+	char *name, *value;
+	SimpleStringListCell *pCurr;
 
 	/* Protocol version comes first. */
 	if (packet)
@@ -2195,6 +2202,31 @@ build_startup_packet(const PGconn *conn, char *packet,
 		}
 	}
 
+	if (conn->send_startup_params && startup_parameters->head)
+	{
+		for (pCurr = startup_parameters->head; pCurr != NULL; pCurr = pCurr->next, ++nCells)
+		{
+			if ((nCells % 2) == 0)
+				name = pCurr->val;
+			else
+			{
+				value = pCurr->val;
+				ADD_STARTUP_OPTION(name, value);
+
+				/* Mark value consumed */
+				pCurr->touched = true;
+			}
+		}
+
+		if ((nCells % 2) != 0)
+		{
+			printfPQExpBuffer(&conn->errorMessage,
+				libpq_gettext("at least one parameter specified in"
+				"StartupMessage does not have a value\n"));
+			pqSaveErrorResult(conn);
+		}
+	}
+
 	/* Add trailing terminator */
 	if (packet)
 		packet[packet_len] = '\0';
diff --git a/src/interfaces/libpq/libpq-fe.h b/src/interfaces/libpq/libpq-fe.h
index 1d915e7915..d04f48544c 100644
--- a/src/interfaces/libpq/libpq-fe.h
+++ b/src/interfaces/libpq/libpq-fe.h
@@ -58,6 +58,7 @@ typedef enum
 	CONNECTION_MADE,			/* Connection OK; waiting to send.     */
 	CONNECTION_AWAITING_RESPONSE,	/* Waiting for a response from the
 									 * postmaster.        */
+	CONNECTION_NEGOTIATING,		/* Negotiating pgwire protocol between FE/BE */
 	CONNECTION_AUTH_OK,			/* Received authentication; waiting for
 								 * backend startup. */
 	CONNECTION_SETENV,			/* Negotiating environment. */
diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h
index 42913604e3..4a1839186f 100644
--- a/src/interfaces/libpq/libpq-int.h
+++ b/src/interfaces/libpq/libpq-int.h
@@ -43,6 +43,12 @@
 /* include stuff found in fe only */
 #include "pqexpbuffer.h"
 
+/*
+ * Simple string list data structure to track
+ * startup parameters that were accepted.
+ */
+#include "fe_utils/simple_list.h"
+
 #ifdef ENABLE_GSS
 #if defined(HAVE_GSSAPI_H)
 #include <gssapi.h>
@@ -414,6 +420,7 @@ struct pg_conn
 	PGSetenvStatusType setenv_state;	/* for 2.0 protocol only */
 	const PQEnvironmentOption *next_eo;
 	bool		send_appname;	/* okay to send application_name? */
+	bool		send_startup_params;	/* okay to send startup parameters? */
 
 	/* Miscellaneous stuff */
 	int			be_pid;			/* PID of backend --- needed for cancels */
@@ -598,7 +605,8 @@ extern PGresult *pqFunctionCall2(PGconn *conn, Oid fnid,
 /* === in fe-protocol3.c === */
 
 extern char *pqBuildStartupPacket3(PGconn *conn, int *packetlen,
-					  const PQEnvironmentOption *options);
+								   const PQEnvironmentOption *options,
+								   SimpleStringList *parameters);
 extern void pqParseInput3(PGconn *conn);
 extern int	pqGetErrorNotice3(PGconn *conn, bool isError);
 extern void pqBuildErrorMessage3(PQExpBuffer msg, const PGresult *res,
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index 686c7369f6..a440ca303f 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -234,7 +234,7 @@ sub mkvcbuild
 	$libpq->UseDef('src/interfaces/libpq/libpqdll.def');
 	$libpq->ReplaceFile('src/interfaces/libpq/libpqrc.c',
 		'src/interfaces/libpq/libpq.rc');
-	$libpq->AddReference($libpgport);
+	$libpq->AddReference($libpgcommon, $libpgfeutils, $libpgport);
 
    # The OBJS scraper doesn't know about ifdefs, so remove fe-secure-openssl.c
    # and sha2_openssl.c if building without OpenSSL, and remove sha2.c if
-- 
2.13.2.windows.1

From a5ef6cc738123b6ad2032a20ab0a8cc983cf2f0e Mon Sep 17 00:00:00 2001
From: Badrul Chowdhury <bachow@microsoft.com>
Date: Wed, 18 Oct 2017 15:26:33 -0700
Subject: [PATCH] BE changes for pgwire v3.1.

---
 src/backend/postmaster/postmaster.c | 80 +++++++++++++++++++++++++++++++++++--
 src/include/libpq/libpq-be.h        |  1 +
 2 files changed, 77 insertions(+), 4 deletions(-)

diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 95180b2ef5..1eafdea87b 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -387,6 +387,7 @@ static DNSServiceRef bonjour_sdref = NULL;
 /*
  * postmaster.c - function prototypes
  */
+static int	NegotiateServerProtocol(Port *port);
 static void CloseServerPorts(int status, Datum arg);
 static void unlink_external_pid_file(int status, Datum arg);
 static void getInstallationPaths(const char *argv0);
@@ -1368,6 +1369,48 @@ PostmasterMain(int argc, char *argv[])
 	abort();					/* not reached */
 }
 
+static int
+NegotiateServerProtocol(Port *port)
+{
+	StringInfoData buf;
+	char	   *name;
+	ListCell   *optional_params;
+
+	/*
+	 * NegotiateServerProtocol packet structure
+	 *
+	 * [ 'Y' | msgLength | min_version | max_version | param_list_len | list
+	 * of param names ]
+	 */
+
+	/* PG message type */
+	pq_beginmessage(&buf, 'Y');
+
+	/* Protocol version numbers */
+	pq_sendint(&buf, PG_PROTOCOL_EARLIEST, sizeof(int32));	/* min */
+	pq_sendint(&buf, PG_PROTOCOL_LATEST, sizeof(int32));	/* max */
+
+	/* Length of parameter list; parameter list consists of (key, value) pairs */
+	pq_sendint(&buf, list_length(port->optional_parameters) / 2, sizeof(int32));
+
+	optional_params = list_head(port->optional_parameters);
+	while (optional_params)
+	{
+		/* First comes key, which we need. */
+		name = lfirst(optional_params);
+		optional_params = lnext(optional_params);
+
+		/* Then comes value, which we don't need. */
+		optional_params = lnext(optional_params);
+
+		pq_sendstring(&buf, name);
+	}
+
+	pq_endmessage(&buf);
+
+	/* Ensure that the message buffer is flushed */
+	pq_flush();
+}
 
 /*
  * on_proc_exit callback to close server's listen sockets
@@ -2050,12 +2093,10 @@ retry1:
 	 */
 	FrontendProtocol = proto;
 
-	/* Check we can handle the protocol the frontend is using. */
+	/* Check we can handle the major protocol the frontend is using. */
 
 	if (PG_PROTOCOL_MAJOR(proto) < PG_PROTOCOL_MAJOR(PG_PROTOCOL_EARLIEST) ||
-		PG_PROTOCOL_MAJOR(proto) > PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST) ||
-		(PG_PROTOCOL_MAJOR(proto) == PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST) &&
-		 PG_PROTOCOL_MINOR(proto) > PG_PROTOCOL_MINOR(PG_PROTOCOL_LATEST)))
+		PG_PROTOCOL_MAJOR(proto) > PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST))
 		ereport(FATAL,
 				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
 				 errmsg("unsupported frontend protocol %u.%u: server supports %u.0 to %u.%u",
@@ -2126,6 +2167,13 @@ retry1:
 									valptr),
 							 errhint("Valid values are: \"false\", 0, \"true\", 1, \"database\".")));
 			}
+			else if (strlen(nameptr) > 4 && strncmp(nameptr, "_pq_", 4) == 0)
+			{
+				port->optional_parameters = lappend(port->optional_parameters,
+													pstrdup(nameptr));
+				port->optional_parameters = lappend(port->optional_parameters,
+													pstrdup(valptr));
+			}
 			else
 			{
 				/* Assume it's a generic GUC option */
@@ -2145,9 +2193,33 @@ retry1:
 			ereport(FATAL,
 					(errcode(ERRCODE_PROTOCOL_VIOLATION),
 					 errmsg("invalid startup packet layout: expected terminator as last byte")));
+
+		/*
+		 * Need to negotiate pgwire protocol if FE version is not the same
+		 * as BE version and FE version is not 3.0
+		 */
+		if (FrontendProtocol != PG_PROTOCOL_LATEST
+			&& FrontendProtocol != PG_PROTOCOL(3, 0))
+		{
+			/* Negotiate parameters after all the error-checking is done */
+			if (NegotiateServerProtocol(port))
+				return STATUS_ERROR;
+		}
 	}
 	else
 	{
+		/* Check we can handle the minor protocol the frontend is using. */
+
+		if (PG_PROTOCOL_MAJOR(proto) == PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST) &&
+			PG_PROTOCOL_MINOR(proto) > PG_PROTOCOL_MINOR(PG_PROTOCOL_LATEST))
+			ereport(FATAL,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					 errmsg("unsupported frontend protocol %u.%u: server supports %u.0 to %u.%u",
+							PG_PROTOCOL_MAJOR(proto), PG_PROTOCOL_MINOR(proto),
+							PG_PROTOCOL_MAJOR(PG_PROTOCOL_EARLIEST),
+							PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST),
+							PG_PROTOCOL_MINOR(PG_PROTOCOL_LATEST))));
+
 		/*
 		 * Get the parameters from the old-style, fixed-width-fields startup
 		 * packet as C strings.  The packet destination was cleared first so a
diff --git a/src/include/libpq/libpq-be.h b/src/include/libpq/libpq-be.h
index 7bde744d51..8b2ef62580 100644
--- a/src/include/libpq/libpq-be.h
+++ b/src/include/libpq/libpq-be.h
@@ -137,6 +137,7 @@ typedef struct Port
 	char	   *user_name;
 	char	   *cmdline_options;
 	List	   *guc_options;
+	List	   *optional_parameters;
 
 	/*
 	 * Information that needs to be held during the authentication cycle.
-- 
2.13.2.windows.1

From 85e04f460f378cdb53da137e82ba62e8a83833a5 Mon Sep 17 00:00:00 2001
From: Robert Haas <rhaas@postgresql.org>
Date: Fri, 27 Oct 2017 14:20:32 +0200
Subject: [PATCH 1/2] Minor protocol versions + extension parameters.

---
 doc/src/sgml/protocol.sgml          | 115 ++++++++++++++++++++++++++++++++----
 src/backend/postmaster/postmaster.c |  58 ++++++++++++++++--
 2 files changed, 157 insertions(+), 16 deletions(-)

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index 15108baf71..f0816c3b16 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -22,10 +22,18 @@
   <productname>PostgreSQL</productname> 7.4 and later.  For descriptions
   of the earlier protocol versions, see previous releases of the
   <productname>PostgreSQL</productname> documentation.  A single server
-  can support multiple protocol versions.  The initial
-  startup-request message tells the server which protocol version the
-  client is attempting to use, and then the server follows that protocol
-  if it is able.
+  can support multiple protocol versions.  The initial startup-request
+  message tells the server which protocol version the client is attempting to
+  use.  If the major version requested by the client is not supported by
+  the server, the connection will be rejected (for example, this would occur
+  if the client requested protocol version 4.0, which does not exist as of
+  this writing).  If the minor version requested by the client is not
+  supported by the server (e.g. the client requests version 3.1, but the
+  server supports only 3.0), the server may either reject the connection or
+  may respond with a NegotiateProtocolVersion message containing the highest
+  minor protocol version which it supports.  The client may then choose either
+  to continue with the connection using the specified protocol version or
+  to abort the connection.
  </para>
 
   <para>
@@ -406,6 +414,19 @@
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term>NegotiateProtocolVersion</term>
+      <listitem>
+       <para>
+        The server does not support the minor protocol version requested
+        by the client, but does support an earlier version of the protocol;
+        this message indicates the highest supported minor version.
+        This message will be followed by an ErrorResponse or a message
+        indicating the success or failure of authentication.
+       </para>
+      </listitem>
+     </varlistentry>
+
     </variablelist>
    </para>
 
@@ -420,8 +441,10 @@
     for further messages from the server.  In this phase a backend process
     is being started, and the frontend is just an interested bystander.
     It is still possible for the startup attempt
-    to fail (ErrorResponse), but in the normal case the backend will send
-    some ParameterStatus messages, BackendKeyData, and finally ReadyForQuery.
+    to fail (ErrorResponse) or the server to decline support for the requested
+    minor protocol version (NegotiateProtocolVersion), but in the normal case
+    the backend will send some ParameterStatus messages, BackendKeyData, and
+    finally ReadyForQuery.
    </para>
 
    <para>
@@ -4699,6 +4722,74 @@ GSSResponse (F)
 </listitem>
 </varlistentry>
 
+<varlistentry>
+<term>
+NegotiateProtocolVersion (B)
+</term>
+<listitem>
+<para>
+
+<variablelist>
+<varlistentry>
+<term>
+        Byte1('v')
+</term>
+<listitem>
+<para>
+                Identifies the message as a protocol version negotiation
+                message.
+</para>
+</listitem>
+</varlistentry>
+<varlistentry>
+<term>
+        Int32
+</term>
+<listitem>
+<para>
+                Length of message contents in bytes, including self.
+</para>
+</listitem>
+</varlistentry>
+<varlistentry>
+<term>
+        Int32
+</term>
+<listitem>
+<para>
+                Newest minor protocol version supported by the server
+                for the major protocol version requested by the client.
+</para>
+</listitem>
+</varlistentry>
+<varlistentry>
+<term>
+        Int32
+</term>
+<listitem>
+<para>
+                Number of protocol options not recognized by the server.
+</para>
+</listitem>
+</varlistentry>
+</variablelist>
+        Then, for protocol option not recognized by the server, there
+        is the following:
+<variablelist>
+<varlistentry>
+<term>
+        String
+</term>
+<listitem>
+<para>
+                The option name.
+</para>
+</listitem>
+</varlistentry>
+</variablelist>
+</para>
+</listitem>
+</varlistentry>
 
 <varlistentry>
 <term>
@@ -5654,11 +5745,13 @@ StartupMessage (F)
 </varlistentry>
 </variablelist>
 
-                In addition to the above, any run-time parameter that can be
-                set at backend start time might be listed.  Such settings
-                will be applied during backend start (after parsing the
-                command-line arguments if any).  The values will act as
-                session defaults.
+                In addition to the above, others parameter may be listed.
+                Parameter names beginning with <literal>_pq_.</literal> are
+                reserved for use as protocol extensions, while others are
+                treated as run-time parameters to be set at backend start
+                time.  Such settings will be applied during backend start
+                (after parsing the command-line arguments if any) and will
+                act as session defaults.
 </para>
 </listitem>
 </varlistentry>
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 2b2b993e2c..3963e97386 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -101,6 +101,7 @@
 #include "lib/ilist.h"
 #include "libpq/auth.h"
 #include "libpq/libpq.h"
+#include "libpq/pqformat.h"
 #include "libpq/pqsignal.h"
 #include "miscadmin.h"
 #include "pg_getopt.h"
@@ -412,6 +413,7 @@ static void ExitPostmaster(int status) pg_attribute_noreturn();
 static int	ServerLoop(void);
 static int	BackendStartup(Port *port);
 static int	ProcessStartupPacket(Port *port, bool SSLdone);
+static void SendNegotiateProtocolVersion(List *unrecognized_protocol_options);
 static void processCancelRequest(Port *port, void *pkt);
 static int	initMasks(fd_set *rmask);
 static void report_fork_failure_to_client(Port *port, int errnum);
@@ -2052,12 +2054,9 @@ retry1:
 	 */
 	FrontendProtocol = proto;
 
-	/* Check we can handle the protocol the frontend is using. */
-
+	/* Check that the major protocol version is in range. */
 	if (PG_PROTOCOL_MAJOR(proto) < PG_PROTOCOL_MAJOR(PG_PROTOCOL_EARLIEST) ||
-		PG_PROTOCOL_MAJOR(proto) > PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST) ||
-		(PG_PROTOCOL_MAJOR(proto) == PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST) &&
-		 PG_PROTOCOL_MINOR(proto) > PG_PROTOCOL_MINOR(PG_PROTOCOL_LATEST)))
+		PG_PROTOCOL_MAJOR(proto) > PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST))
 		ereport(FATAL,
 				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
 				 errmsg("unsupported frontend protocol %u.%u: server supports %u.0 to %u.%u",
@@ -2079,6 +2078,7 @@ retry1:
 	if (PG_PROTOCOL_MAJOR(proto) >= 3)
 	{
 		int32		offset = sizeof(ProtocolVersion);
+		List	   *unrecognized_protocol_options = NIL;
 
 		/*
 		 * Scan packet body for name/option pairs.  We can assume any string
@@ -2128,6 +2128,16 @@ retry1:
 									valptr),
 							 errhint("Valid values are: \"false\", 0, \"true\", 1, \"database\".")));
 			}
+			else if (strncmp(nameptr, "_pq_.", 5) == 0)
+			{
+				/*
+				 * Any option beginning with _pq_. is reserved for use as a
+				 * protocol-level option, but at present no such options are
+				 * defined.
+				 */
+				unrecognized_protocol_options =
+					lappend(unrecognized_protocol_options, pstrdup(nameptr));
+			}
 			else
 			{
 				/* Assume it's a generic GUC option */
@@ -2147,6 +2157,16 @@ retry1:
 			ereport(FATAL,
 					(errcode(ERRCODE_PROTOCOL_VIOLATION),
 					 errmsg("invalid startup packet layout: expected terminator as last byte")));
+
+
+		/*
+		 * If necessary, tell the client that we don't speak the minor
+		 * protocol version they have requested, and tell them about the
+		 * newest version we do speak and any protocol options we didn't
+		 * recognize.
+		 */
+		if (PG_PROTOCOL_MINOR(proto) > PG_PROTOCOL_MINOR(PG_PROTOCOL_LATEST))
+			SendNegotiateProtocolVersion(unrecognized_protocol_options);
 	}
 	else
 	{
@@ -2260,6 +2280,34 @@ retry1:
 	return STATUS_OK;
 }
 
+/*
+ * Send a NegotiateProtocolVersion to the client.  This lets the client know
+ * that they have requested a newer minor protocol version than we are able
+ * to speak.  We'll speak the highest version we know about; the client can,
+ * of course, abandon the connection if that's a problem.
+ *
+ * We also include in the response a list of protocol options we didn't
+ * understand.  This allows clients to include optional parameters that might
+ * be present either in newer protocol versions or third-party protocol
+ * extensions without fear of having to reconnect if those options are not
+ * understood, while at the same time making certain that the client is aware
+ * of which options were actually accepted.
+ */
+static void
+SendNegotiateProtocolVersion(List *unrecognized_protocol_options)
+{
+	StringInfoData buf;
+	ListCell   *lc;
+
+	pq_beginmessage(&buf, 'v'); /* NegotiateProtocolVersion */
+	pq_sendint32(&buf, PG_PROTOCOL_LATEST);
+	pq_sendint32(&buf, list_length(unrecognized_protocol_options));
+	foreach(lc, unrecognized_protocol_options)
+		pq_sendstring(&buf, lfirst(lc));
+	pq_endmessage(&buf);
+
+	/* no need to flush, some other message will follow */
+}
 
 /*
  * The client has sent a cancel request packet, not a normal
-- 
2.13.5 (Apple Git-94)

From 4d3d56f19dd4673daa7510cfd0d463802f9d77f2 Mon Sep 17 00:00:00 2001
From: Robert Haas <rhaas@postgresql.org>
Date: Fri, 27 Oct 2017 14:21:16 +0200
Subject: [PATCH 2/2] libpq test framework

---
 src/interfaces/libpq/fe-connect.c | 38 +++++++++++++++++++++++++++++++++-----
 1 file changed, 33 insertions(+), 5 deletions(-)

diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c
index 6bcf60a712..2c859d7617 100644
--- a/src/interfaces/libpq/fe-connect.c
+++ b/src/interfaces/libpq/fe-connect.c
@@ -1798,11 +1798,11 @@ connectDBStart(PGconn *conn)
 #endif
 
 	/*
-	 * Set up to try to connect, with protocol 3.0 as the first attempt.
+	 * Set up to try to connect, with protocol 3.1 as the first attempt.
 	 */
 	conn->whichhost = 0;
 	conn->addr_cur = conn->connhost[0].addrlist;
-	conn->pversion = PG_PROTOCOL(3, 0);
+	conn->pversion = PG_PROTOCOL(3, 1);
 	conn->send_appname = true;
 	conn->status = CONNECTION_NEEDED;
 
@@ -2653,13 +2653,13 @@ keep_going:						/* We will come back to here until there is
 				 * request or an error here.  Anything else probably means
 				 * it's not Postgres on the other end at all.
 				 */
-				if (!(beresp == 'R' || beresp == 'E'))
+				if (!(beresp == 'R' || beresp == 'E' || beresp == 'v'))
 				{
 					appendPQExpBuffer(&conn->errorMessage,
 									  libpq_gettext(
 													"expected authentication request from "
-													"server, but received %c\n"),
-									  beresp);
+													"server, but received %d\n"),
+									  (int) beresp);
 					goto error_return;
 				}
 
@@ -2813,6 +2813,34 @@ keep_going:						/* We will come back to here until there is
 					goto error_return;
 				}
 
+				/*
+				 * Parse and discard any NegotiateProtocolVersion message.
+				 */
+				if (beresp == 'v')
+				{
+					int			server_max_version;
+					int			num_unrecognized_parameters;
+
+					if (pqGetInt(&server_max_version, 4, conn))
+						return PGRES_POLLING_READING;
+					if (pqGetInt(&num_unrecognized_parameters, 4, conn))
+						return PGRES_POLLING_READING;
+					if (num_unrecognized_parameters > 0)
+					{
+						PQExpBufferData buf;
+						int			i;
+
+						initPQExpBuffer(&buf);
+						for (i = 0; i < num_unrecognized_parameters; ++i)
+							(void) pqGets(&buf, conn);
+						termPQExpBuffer(&buf);
+					}
+
+					conn->inStart = conn->inCursor;
+
+					goto keep_going;
+				}
+
 				/* It is an authentication request. */
 				conn->auth_req_received = true;
 
-- 
2.13.5 (Apple Git-94)

commit e63395bb924d480fa8184be957df54b3975229c7
Author: Robert Haas <rhaas@postgresql.org>
Date:   Wed Nov 15 15:11:21 2017 -0500

    Provide for forward compatibility with future minor protocol versions.
    
    Previously, any attempt to request a 3.x protocol version other than
    3.0 would lead to a hard connection failure, which made the minor
    protocol version really no different from the major protocol version
    and precluded gentle protocol version breaks.  Instead, when the
    client requests a 3.x protocol version where x is greater than 0, send
    the new NegotiateProtocolVersion message to convey that we support
    only 3.0.  This makes it possible to introduce new minor protocol
    versions without requiring a connection retry when the server is
    older.
    
    In addition, if the startup packet includes name/value pairs where
    the name starts with "_pq_.", assume that those are protocol options,
    not GUCs.  Include those we don't support (i.e. all of them, at
    present) in the NegotiateProtocolVersion message so that the client
    knows they were not understood.  This makes it possible for the
    client to request previously-unsupported features without bumping
    the protocol version at all; the client can tell from the server's
    response whether the option was understood.
    
    It will take some time before servers that support these new
    facilities become common in the wild; to speed things up and make
    things easier for a future 3.1 protocol version, back-patch to all
    supported releases.
    
    Robert Haas and Badrul Chowdhury
    
    Discussion: http://postgr.es/m/BN6PR21MB0772FFA0CBD298B76017744CD1730@BN6PR21MB0772.namprd21.prod.outlook.com
    Discussion: http://postgr.es/m/30788.1498672033@sss.pgh.pa.us

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index 6d4dcf83ac..4fd7420f85 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -22,10 +22,18 @@
   <productname>PostgreSQL</productname> 7.4 and later.  For descriptions
   of the earlier protocol versions, see previous releases of the
   <productname>PostgreSQL</productname> documentation.  A single server
-  can support multiple protocol versions.  The initial
-  startup-request message tells the server which protocol version the
-  client is attempting to use, and then the server follows that protocol
-  if it is able.
+  can support multiple protocol versions.  The initial startup-request
+  message tells the server which protocol version the client is attempting to
+  use.  If the major version requested by the client is not supported by
+  the server, the connection will be rejected (for example, this would occur
+  if the client requested protocol version 4.0, which does not exist as of
+  this writing).  If the minor version requested by the client is not
+  supported by the server (e.g. the client requests version 3.1, but the
+  server supports only 3.0), the server may either reject the connection or
+  may respond with a NegotiateProtocolVersion message containing the highest
+  minor protocol version which it supports.  The client may then choose either
+  to continue with the connection using the specified protocol version or
+  to abort the connection.
  </para>
 
   <para>
@@ -406,6 +414,21 @@
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term>NegotiateProtocolVersion</term>
+      <listitem>
+       <para>
+        The server does not support the minor protocol version requested
+        by the client, but does support an earlier version of the protocol;
+        this message indicates the highest supported minor version.  This
+        message will also be sent if the client requested unsupported protocol
+        options (i.e. beginning with <literal>_pq_.</literal>) in the
+        startup packet.  This message will be followed by an ErrorResponse or
+        a message indicating the success or failure of authentication.
+       </para>
+      </listitem>
+     </varlistentry>
+
     </variablelist>
    </para>
 
@@ -420,8 +443,10 @@
     for further messages from the server.  In this phase a backend process
     is being started, and the frontend is just an interested bystander.
     It is still possible for the startup attempt
-    to fail (ErrorResponse), but in the normal case the backend will send
-    some ParameterStatus messages, BackendKeyData, and finally ReadyForQuery.
+    to fail (ErrorResponse) or the server to decline support for the requested
+    minor protocol version (NegotiateProtocolVersion), but in the normal case
+    the backend will send some ParameterStatus messages, BackendKeyData, and
+    finally ReadyForQuery.
    </para>
 
    <para>
@@ -4704,6 +4729,74 @@ GSSResponse (F)
 </listitem>
 </varlistentry>
 
+<varlistentry>
+<term>
+NegotiateProtocolVersion (B)
+</term>
+<listitem>
+<para>
+
+<variablelist>
+<varlistentry>
+<term>
+        Byte1('v')
+</term>
+<listitem>
+<para>
+                Identifies the message as a protocol version negotiation
+                message.
+</para>
+</listitem>
+</varlistentry>
+<varlistentry>
+<term>
+        Int32
+</term>
+<listitem>
+<para>
+                Length of message contents in bytes, including self.
+</para>
+</listitem>
+</varlistentry>
+<varlistentry>
+<term>
+        Int32
+</term>
+<listitem>
+<para>
+                Newest minor protocol version supported by the server
+                for the major protocol version requested by the client.
+</para>
+</listitem>
+</varlistentry>
+<varlistentry>
+<term>
+        Int32
+</term>
+<listitem>
+<para>
+                Number of protocol options not recognized by the server.
+</para>
+</listitem>
+</varlistentry>
+</variablelist>
+        Then, for protocol option not recognized by the server, there
+        is the following:
+<variablelist>
+<varlistentry>
+<term>
+        String
+</term>
+<listitem>
+<para>
+                The option name.
+</para>
+</listitem>
+</varlistentry>
+</variablelist>
+</para>
+</listitem>
+</varlistentry>
 
 <varlistentry>
 <term>
@@ -5659,11 +5752,13 @@ StartupMessage (F)
 </varlistentry>
 </variablelist>
 
-                In addition to the above, any run-time parameter that can be
-                set at backend start time might be listed.  Such settings
-                will be applied during backend start (after parsing the
-                command-line arguments if any).  The values will act as
-                session defaults.
+                In addition to the above, others parameter may be listed.
+                Parameter names beginning with <literal>_pq_.</literal> are
+                reserved for use as protocol extensions, while others are
+                treated as run-time parameters to be set at backend start
+                time.  Such settings will be applied during backend start
+                (after parsing the command-line arguments if any) and will
+                act as session defaults.
 </para>
 </listitem>
 </varlistentry>
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 9906a85bc0..a3d4917173 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -101,6 +101,7 @@
 #include "lib/ilist.h"
 #include "libpq/auth.h"
 #include "libpq/libpq.h"
+#include "libpq/pqformat.h"
 #include "libpq/pqsignal.h"
 #include "miscadmin.h"
 #include "pg_getopt.h"
@@ -412,6 +413,7 @@ static void ExitPostmaster(int status) pg_attribute_noreturn();
 static int	ServerLoop(void);
 static int	BackendStartup(Port *port);
 static int	ProcessStartupPacket(Port *port, bool SSLdone);
+static void SendNegotiateProtocolVersion(List *unrecognized_protocol_options);
 static void processCancelRequest(Port *port, void *pkt);
 static int	initMasks(fd_set *rmask);
 static void report_fork_failure_to_client(Port *port, int errnum);
@@ -2052,12 +2054,9 @@ retry1:
 	 */
 	FrontendProtocol = proto;
 
-	/* Check we can handle the protocol the frontend is using. */
-
+	/* Check that the major protocol version is in range. */
 	if (PG_PROTOCOL_MAJOR(proto) < PG_PROTOCOL_MAJOR(PG_PROTOCOL_EARLIEST) ||
-		PG_PROTOCOL_MAJOR(proto) > PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST) ||
-		(PG_PROTOCOL_MAJOR(proto) == PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST) &&
-		 PG_PROTOCOL_MINOR(proto) > PG_PROTOCOL_MINOR(PG_PROTOCOL_LATEST)))
+		PG_PROTOCOL_MAJOR(proto) > PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST))
 		ereport(FATAL,
 				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
 				 errmsg("unsupported frontend protocol %u.%u: server supports %u.0 to %u.%u",
@@ -2079,6 +2078,7 @@ retry1:
 	if (PG_PROTOCOL_MAJOR(proto) >= 3)
 	{
 		int32		offset = sizeof(ProtocolVersion);
+		List	   *unrecognized_protocol_options = NIL;
 
 		/*
 		 * Scan packet body for name/option pairs.  We can assume any string
@@ -2128,6 +2128,16 @@ retry1:
 									valptr),
 							 errhint("Valid values are: \"false\", 0, \"true\", 1, \"database\".")));
 			}
+			else if (strncmp(nameptr, "_pq_.", 5) == 0)
+			{
+				/*
+				 * Any option beginning with _pq_. is reserved for use as a
+				 * protocol-level option, but at present no such options are
+				 * defined.
+				 */
+				unrecognized_protocol_options =
+					lappend(unrecognized_protocol_options, pstrdup(nameptr));
+			}
 			else
 			{
 				/* Assume it's a generic GUC option */
@@ -2147,6 +2157,16 @@ retry1:
 			ereport(FATAL,
 					(errcode(ERRCODE_PROTOCOL_VIOLATION),
 					 errmsg("invalid startup packet layout: expected terminator as last byte")));
+
+		/*
+		 * If the client requested a newer protocol version or if the client
+		 * requested any protocol options we didn't recognize, let them know
+		 * the newest minor protocol version we do support and the names of any
+		 * unrecognized options.
+		 */
+		if (PG_PROTOCOL_MINOR(proto) > PG_PROTOCOL_MINOR(PG_PROTOCOL_LATEST) ||
+			unrecognized_protocol_options != NIL)
+			SendNegotiateProtocolVersion(unrecognized_protocol_options);
 	}
 	else
 	{
@@ -2260,6 +2280,34 @@ retry1:
 	return STATUS_OK;
 }
 
+/*
+ * Send a NegotiateProtocolVersion to the client.  This lets the client know
+ * that they have requested a newer minor protocol version than we are able
+ * to speak.  We'll speak the highest version we know about; the client can,
+ * of course, abandon the connection if that's a problem.
+ *
+ * We also include in the response a list of protocol options we didn't
+ * understand.  This allows clients to include optional parameters that might
+ * be present either in newer protocol versions or third-party protocol
+ * extensions without fear of having to reconnect if those options are not
+ * understood, while at the same time making certain that the client is aware
+ * of which options were actually accepted.
+ */
+static void
+SendNegotiateProtocolVersion(List *unrecognized_protocol_options)
+{
+	StringInfoData buf;
+	ListCell   *lc;
+
+	pq_beginmessage(&buf, 'v'); /* NegotiateProtocolVersion */
+	pq_sendint32(&buf, PG_PROTOCOL_LATEST);
+	pq_sendint32(&buf, list_length(unrecognized_protocol_options));
+	foreach(lc, unrecognized_protocol_options)
+		pq_sendstring(&buf, lfirst(lc));
+	pq_endmessage(&buf);
+
+	/* no need to flush, some other message will follow */
+}
 
 /*
  * The client has sent a cancel request packet, not a normal